From 81f789cc3050c02a23bce75491a8babf9590cba6 Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 17:35:11 +0100 Subject: [PATCH] Update all CI config --- tests/{launch_test.sh => launch_test_b3d.sh} | 2 +- tests/main.py | 30 +++++--- tests/requirements.txt | 3 +- tests/utils/blender.py | 56 +++----------- tests/utils/blender_addon.py | 19 +++-- tests/utils/container.py | 80 +++++++++++++++++--- tests/utils/forge.py | 36 +++++++++ tests/utils/issue.py | 4 + tests/utils/properties.py | 63 +++++++++++++++ 9 files changed, 215 insertions(+), 78 deletions(-) rename tests/{launch_test.sh => launch_test_b3d.sh} (52%) create mode 100644 tests/utils/forge.py create mode 100644 tests/utils/properties.py diff --git a/tests/launch_test.sh b/tests/launch_test_b3d.sh similarity index 52% rename from tests/launch_test.sh rename to tests/launch_test_b3d.sh index 04fbb61..a9bf8a0 100644 --- a/tests/launch_test.sh +++ b/tests/launch_test_b3d.sh @@ -1,3 +1,3 @@ #!/bin/bash -/opt/blender/blender --background --python-exit-code 1 --python "$FOLDER_TEST/tests/utils/blender_addon.py" || exit 1 > /dev/null 2>&1 +/opt/blender/blender --background --python-exit-code 1 --python "$FOLDER_TEST/tests/utils/blender_addon.py" > /dev/null 2>&1 || exit 1 /opt/blender/blender --background -noaudio --disable-autoexec --python-exit-code 1 --python "$1" -- --verbose || exit 1 \ No newline at end of file diff --git a/tests/main.py b/tests/main.py index 3cdcfa8..10092e3 100644 --- a/tests/main.py +++ b/tests/main.py @@ -1,31 +1,38 @@ import sys import os +import enum from pathlib import Path -from utils.blender import b3d_launch_blender_test -from utils.container import clear_container_test +from utils.blender import get_b3d_addon_dependency +from utils.container import VirtualMachine from utils.misc import ordering_test_file, generate_archive +from utils.properties import ContainerObject -def launch_unit_test(tag: str, test: [str] = None): +class Container(enum.Enum): + """Enumerate about the Geometry node""" + BLENDER = ContainerObject(name='Blender', image='stilobique/blender') + + +def launch_unit_test(test: [str] = None): """Start all Unit Test, Blender and Unreal if needed""" # Ordering Unit test Blender/Unreal if test is None: test = ordering_test_file() - test = test['blender'] + b3d = test['blender'] + else: + b3d = test - b3d_launch_blender_test(test=test, tag=tag) + vm_b3d = VirtualMachine(Container.BLENDER.value) + vm_b3d.launch_unit_test(tests=b3d) if __name__ == '__main__': # Initialize Variable and module request archives = [] - docker_tag = 'stilobique/blender:latest' - - # Clear blender container - clear_container_test(tag=docker_tag) # Prepare Blender and Unreal dependency + get_b3d_addon_dependency(archives) generate_archive(archives, 'blender_addon_folder') # Generate Unit Test, a specific call or execute all Unit Test @@ -40,8 +47,9 @@ if __name__ == '__main__': test_list.append(item) # Launch Unit Test - launch_unit_test(test=test_list, tag=docker_tag) + launch_unit_test(test=test_list) # Clear archive file and container for archive in archives: - os.remove(Path(os.getcwd(), archive)) + if Path(Path(os.getcwd(), archive)).exists(): + os.remove(Path(os.getcwd(), archive)) diff --git a/tests/requirements.txt b/tests/requirements.txt index 36011d0..161b063 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ docker -PyGithub \ No newline at end of file +PyGithub +numpy \ No newline at end of file diff --git a/tests/utils/blender.py b/tests/utils/blender.py index dea0178..1e7d5f4 100644 --- a/tests/utils/blender.py +++ b/tests/utils/blender.py @@ -1,49 +1,17 @@ +import json import os -import docker -import sys -from pathlib import PurePosixPath, Path +from pathlib import Path +from .forge import get_release_file -class ErrorTest(Exception): - """ Failed to generate the test """ +def get_b3d_addon_dependency(archive: list): + """From json resources, get all blender dependency and download-it""" + dependency = Path(os.getcwd(), "tests", "dependency.json") + with open(dependency) as f: + data = json.load(f) - -def b3d_launch_blender_test(client: docker = docker.from_env(), test: list = None, - tag: str = 'stilobique/blender:latest'): - """Launch blender and a list of Test, you can set a specific test file with his arg.""" - unit_test_folder = Path(os.getcwd(), "tests", "unit_test") - - if os.environ.get('GITHUB_WORKSPACE'): - local_path = os.environ.get('GITHUB_WORKSPACE') - else: - local_path = os.getcwd() - container_folder = '/addon_moderlab' - volume = [f'{local_path}:{container_folder}'] - image_name = tag - if test is None: - unit_test = os.listdir(unit_test_folder) - else: - if type(test) is not list: - test = [test] - unit_test = test - - try: - for test in unit_test: - print(f'Start this test : {test}') - command = [f"/bin/sh", - f"{PurePosixPath(container_folder, 'tests', 'launch_test.sh')}", - f"{PurePosixPath(container_folder, 'tests', 'unit_test', test)}"] - - docker_test = client.containers.run(image_name, command=command, volumes=volume, privileged=True, - environment=[f'FOLDER_TEST={container_folder}'], detach=True, name=test) - exit_docker = docker_test.wait() - if exit_docker['StatusCode'] != 0: - print(f'Container error "{exit_docker["StatusCode"]}".\n\t' - f'Show log : \n\t' - f'{docker_test.logs()}') - raise ErrorTest - - except ErrorTest as exception: - print(f'{exception.__doc__}') - sys.exit(1) + b3d_dependency = data['blender'] + for key, value in b3d_dependency.items(): + get_release_file(value[0], value[1]) + archive.append(value[0]) diff --git a/tests/utils/blender_addon.py b/tests/utils/blender_addon.py index d51ade7..5bbd770 100644 --- a/tests/utils/blender_addon.py +++ b/tests/utils/blender_addon.py @@ -1,18 +1,17 @@ -import bpy import os +import bpy import pathlib - -# Paste this variable in the blender.py -dependency = { - # 'moderlab_plugin': ['moderlab_plugin.zip', 'Moderlab-Production/BlenderPlugin'], - # 'moderlab_type': ['moderlab_type.zip', 'Moderlab-Production/BlenderObjectType'], - # 'moderlab_pie': ['moderlab_pie.zip', 'Moderlab-Production/BlenderPieMenu'], - # 'uv-packer': ['uv-packer.zip', 'Moderlab-Production/UvPacker'], -} +import json def b3d_install_addon(): env_name = 'FOLDER_TEST' + dependency = pathlib.Path(os.environ[env_name], "tests", "dependency.json") + with open(dependency) as f: + data = json.load(f) + + b3d_dependency = data['blender'] + try: if not os.environ.get(env_name): raise KeyError @@ -20,7 +19,7 @@ def b3d_install_addon(): if not env_path.exists(): raise FileNotFoundError - for key, value in dependency.items(): + for key, value in b3d_dependency.items(): bpy.ops.preferences.addon_install(filepath=f'{env_path}/{value[0]}') bpy.ops.preferences.addon_enable(module=key) bpy.ops.wm.save_userpref() diff --git a/tests/utils/container.py b/tests/utils/container.py index 39bf072..c6189f7 100644 --- a/tests/utils/container.py +++ b/tests/utils/container.py @@ -1,17 +1,75 @@ import docker +import os +import sys + +from pathlib import PurePosixPath +from docker.errors import DockerException +from .issue import ContainerErrorTest +from .properties import ContainerObject -def clear_container_test(tag: str = 'stilobique/blender:latest'): - client = docker.from_env() - containers = client.containers.list(all=True) +class VirtualMachine: + def __init__(self, container: ContainerObject): + self.client = self.start_docker() - for container in containers: - image_tag = container.image.tags[0] - status = container.status - if image_tag == tag and status == 'exited': - container.remove() + self.container = container + self.base_command = self.container.commands + self.clear_containers() + @staticmethod + def start_docker(): + """Generate docker client information""" + try: + client = docker.from_env() + return client -if __name__ == '__main__': - # Clear all unused blender container - clear_container_test() + except DockerException: + print('Docker isn\'t start or installed') + exit(1) + + def clear_containers(self): + """Look all docker containers, and remove-it if the task are used with the unit test.""" + containers = self.client.containers.list(all=True) + + for container in containers: + image_tag = container.image.tags[0] + status = container.status + if image_tag in self.container.ref and status in 'exited': + container.remove() + + @staticmethod + def workspace(): + """Define the workspace folder""" + if os.environ.get('GITHUB_WORKSPACE'): + return os.environ['GITHUB_WORKSPACE'] + else: + return os.getcwd() + + def launch_unit_test(self, tests: list[str]): + """Execute a docker container to start unit test dedicated""" + if tests is None: + unit_test = os.listdir(self.workspace()) + else: + if type(tests) is not list: + tests = [tests] + unit_test = tests + + try: + for test in unit_test: + print(f'Launch unit test "{test}"') + cmds = self.base_command + [f'{PurePosixPath(self.container.local_folder, "tests", "unit_test", test)}'] + + docker_test = self.client.containers.run(self.container.ref, command=cmds, + volumes=self.container.volumes, privileged=True, + environment=self.container.environments, detach=True, + name=test, tty=True) + exit_docker = docker_test.wait() + if exit_docker['StatusCode'] != 0: + print(f'Container error "{exit_docker["StatusCode"]}".\n\t' + f'Show log : \n\t' + f'{docker_test.logs()}') + raise ContainerErrorTest + + except ContainerErrorTest as exception: + print(f'{exception.__doc__}') + sys.exit(1) diff --git a/tests/utils/forge.py b/tests/utils/forge.py new file mode 100644 index 0000000..f30d7c4 --- /dev/null +++ b/tests/utils/forge.py @@ -0,0 +1,36 @@ +import os +import requests + +from pathlib import Path +from github import Github + + +def read_token(): + token_file = Path(os.getcwd(), 'tests', 'token.txt') + with open(token_file, 'r') as f: + token = f.read() + + return token + + +def get_release_file(filename: str, repo: str): + g = Github(read_token()) + repository = g.get_repo(repo) + latest = repository.get_latest_release() + latest.get_assets() + for file in latest.get_assets(): + if file.name == filename: + dl_file(file.url, file.name) + + +def dl_file(url: str, filename: str): + """From specific asset, download it""" + headers = { + 'Authorization': f'token {read_token()}', + 'Accept': 'application/octet-stream' + } + session = requests.Session() + link = session.get(url, stream=True, headers=headers) + with open(filename, 'wb') as f: + for chunk in link.iter_content(1024*1024): + f.write(chunk) diff --git a/tests/utils/issue.py b/tests/utils/issue.py index 1106ce9..962f897 100644 --- a/tests/utils/issue.py +++ b/tests/utils/issue.py @@ -6,3 +6,7 @@ class Archive(Exception): class ArchiveFolderSourceNotFound(Archive): def __str__(self): return f'Can\'t find the folder source "{self.source}".' + + +class ContainerErrorTest(Exception): + """ Failed to generate the test """ diff --git a/tests/utils/properties.py b/tests/utils/properties.py new file mode 100644 index 0000000..b586215 --- /dev/null +++ b/tests/utils/properties.py @@ -0,0 +1,63 @@ +import os + +from pathlib import PurePosixPath + + +class ContainerObject(object): + """All data request to launch one dedicated container""" + + def __init__(self, name: str, image: str): + self.label: str = name.lower() + self.image: str = image + self.tag: str = self.set_tag() + self.local_folder: str = '' + self.volumes: list[str] = self.set_volumes() + self.environments: list[str] = self.set_environments() + self.commands: list[str] = self.set_commands() + self.ref = f'{self.image}:{self.tag}' + + def set_tag(self): + """Determine the tag use with the docker image call""" + if 'blender' in self.label: + return 'latest' + elif 'unreal' in self.label: + return 'dev-slim-4.27.2' + else: + return None + + def set_volumes(self): + """Define folder/path with mount with each docker container""" + if 'blender' in self.label: + self.local_folder = '/addon_moderlab' + elif 'unreal' in self.label: + self.local_folder = '/project' + else: + return None + return [f'{self.workspace()}:{self.local_folder}'] + + def set_environments(self): + """Add environment variable start with the docker container""" + if 'blender' in self.label: + return [f'FOLDER_TEST={self.local_folder}'] + else: + return [] + + def set_commands(self): + """Write all commands execute on the docker container start""" + cmds = ['/bin/sh'] + if 'blender' in self.label: + cmds.append(str(PurePosixPath(self.local_folder, "tests", "launch_test_b3d.sh"))) + else: + cmds.append(str(PurePosixPath(self.local_folder, "tests", "launch_test_ue.sh"))) + cmds.append(str(PurePosixPath(self.local_folder, "tests", "unreal_sample", "empty_project", + "EmptyProject.uproject"))) + + return cmds + + @staticmethod + def workspace(): + """Define the workspace folder""" + if os.environ.get('GITHUB_WORKSPACE'): + return os.environ['GITHUB_WORKSPACE'] + else: + return os.getcwd()