From 181f45b051f5f995d19bc848456bbb2768a58ad4 Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 16:34:30 +0100 Subject: [PATCH 1/5] Clear info in the readme file --- readme.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/readme.md b/readme.md index e65a5ac..62a2cc6 100644 --- a/readme.md +++ b/readme.md @@ -2,21 +2,16 @@ Template repository about blender addon. To used-it, clone this repository and rename the folder "blender_addon_folder" with your addon name. Update the file "tests/main.py", line 29, set your addon name. -````python +```python # Prepare Blender and Unreal dependency generate_archive(archives, 'blender_addon_folder') -```` +``` -> ⚠️ It's more easy to use the "_" with your addon folder name, the "-" character can be problematic with python used. - - -# Setup Variable -The Github Workflow (CI chain) request some update with each new repository : -- [ ] Env. variable : request the package ex name. Get a better solution to automatically used the folder name ? +> ⚠️ It's more easy to use the "_" with your addon folder name, the "-" character can be problematic with python use. # Unit Test -All unit test call the blender docker [stilobique/blender:latest](https://hub.docker.com/repository/docker/stilobique/blender). +All unit tests call docker image [stilobique/blender:latest](https://hub.docker.com/repository/docker/stilobique/blender). #### Blender addon dependency With all unit test, if your addon request some dependency, add the repo inside the list in `blender_addon.py` and `blender.py`. This system are not perfect and need to be improved. \ No newline at end of file From f6ebc92d2b84c5c7410456294d19c5b2d503a20d Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 16:45:29 +0100 Subject: [PATCH 2/5] Add dependency json file --- tests/dependency.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/dependency.json diff --git a/tests/dependency.json b/tests/dependency.json new file mode 100644 index 0000000..9e4e9f5 --- /dev/null +++ b/tests/dependency.json @@ -0,0 +1,11 @@ +{ + "blender": { + "moderlab_plugin": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPlugin"], + "moderlab_type": ["moderlab_type.zip", "Moderlab-Production/BlenderObjectType"], + "moderlab_pie": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPieMenu"] + }, + "unreal": { + "unreal-pipeline": ["unreal-moderlab-pipeline.zip", "Moderlab-Production/UnrealPipeline"], + "unreal-type": ["Moderlab-Unreal-Type.zip", "Moderlab-Production/UnrealType"] + } +} \ No newline at end of file From 0b2e1d702faf92430afa5391f985ed5af5bb61ff Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 17:00:20 +0100 Subject: [PATCH 3/5] Quick update about dependency --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 62a2cc6..dbf156d 100644 --- a/readme.md +++ b/readme.md @@ -13,5 +13,5 @@ Update the file "tests/main.py", line 29, set your addon name. # Unit Test All unit tests call docker image [stilobique/blender:latest](https://hub.docker.com/repository/docker/stilobique/blender). -#### Blender addon dependency -With all unit test, if your addon request some dependency, add the repo inside the list in `blender_addon.py` and `blender.py`. This system are not perfect and need to be improved. \ No newline at end of file +# Addons/Plugins dependency +Update json file `tests/dependency.json` with name, archive and repository Github path. \ No newline at end of file From 3bcee6f017c37fc3f9ffb236213482b1f66a4c47 Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 17:26:07 +0100 Subject: [PATCH 4/5] Add warning about dependency, and reordering entry --- readme.md | 4 +++- tests/dependency.json | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index dbf156d..1e53a80 100644 --- a/readme.md +++ b/readme.md @@ -14,4 +14,6 @@ Update the file "tests/main.py", line 29, set your addon name. All unit tests call docker image [stilobique/blender:latest](https://hub.docker.com/repository/docker/stilobique/blender). # Addons/Plugins dependency -Update json file `tests/dependency.json` with name, archive and repository Github path. \ No newline at end of file +Update json file `tests/dependency.json` with name, archive and repository Github path. + +> ⛔ The `moderlab_plugin` need to be on last entry. diff --git a/tests/dependency.json b/tests/dependency.json index 9e4e9f5..2b91c4d 100644 --- a/tests/dependency.json +++ b/tests/dependency.json @@ -1,8 +1,8 @@ { "blender": { - "moderlab_plugin": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPlugin"], "moderlab_type": ["moderlab_type.zip", "Moderlab-Production/BlenderObjectType"], - "moderlab_pie": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPieMenu"] + "moderlab_pie": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPieMenu"], + "moderlab_plugin": ["moderlab_plugin.zip", "Moderlab-Production/BlenderPlugin"] }, "unreal": { "unreal-pipeline": ["unreal-moderlab-pipeline.zip", "Moderlab-Production/UnrealPipeline"], From 81f789cc3050c02a23bce75491a8babf9590cba6 Mon Sep 17 00:00:00 2001 From: Aurelien Vaillant Date: Wed, 23 Feb 2022 17:35:11 +0100 Subject: [PATCH 5/5] Update all CI config --- tests/{launch_test.sh => launch_test_b3d.sh} | 2 +- tests/main.py | 30 +++++--- tests/requirements.txt | 3 +- tests/utils/blender.py | 56 +++----------- tests/utils/blender_addon.py | 19 +++-- tests/utils/container.py | 80 +++++++++++++++++--- tests/utils/forge.py | 36 +++++++++ tests/utils/issue.py | 4 + tests/utils/properties.py | 63 +++++++++++++++ 9 files changed, 215 insertions(+), 78 deletions(-) rename tests/{launch_test.sh => launch_test_b3d.sh} (52%) create mode 100644 tests/utils/forge.py create mode 100644 tests/utils/properties.py diff --git a/tests/launch_test.sh b/tests/launch_test_b3d.sh similarity index 52% rename from tests/launch_test.sh rename to tests/launch_test_b3d.sh index 04fbb61..a9bf8a0 100644 --- a/tests/launch_test.sh +++ b/tests/launch_test_b3d.sh @@ -1,3 +1,3 @@ #!/bin/bash -/opt/blender/blender --background --python-exit-code 1 --python "$FOLDER_TEST/tests/utils/blender_addon.py" || exit 1 > /dev/null 2>&1 +/opt/blender/blender --background --python-exit-code 1 --python "$FOLDER_TEST/tests/utils/blender_addon.py" > /dev/null 2>&1 || exit 1 /opt/blender/blender --background -noaudio --disable-autoexec --python-exit-code 1 --python "$1" -- --verbose || exit 1 \ No newline at end of file diff --git a/tests/main.py b/tests/main.py index 3cdcfa8..10092e3 100644 --- a/tests/main.py +++ b/tests/main.py @@ -1,31 +1,38 @@ import sys import os +import enum from pathlib import Path -from utils.blender import b3d_launch_blender_test -from utils.container import clear_container_test +from utils.blender import get_b3d_addon_dependency +from utils.container import VirtualMachine from utils.misc import ordering_test_file, generate_archive +from utils.properties import ContainerObject -def launch_unit_test(tag: str, test: [str] = None): +class Container(enum.Enum): + """Enumerate about the Geometry node""" + BLENDER = ContainerObject(name='Blender', image='stilobique/blender') + + +def launch_unit_test(test: [str] = None): """Start all Unit Test, Blender and Unreal if needed""" # Ordering Unit test Blender/Unreal if test is None: test = ordering_test_file() - test = test['blender'] + b3d = test['blender'] + else: + b3d = test - b3d_launch_blender_test(test=test, tag=tag) + vm_b3d = VirtualMachine(Container.BLENDER.value) + vm_b3d.launch_unit_test(tests=b3d) if __name__ == '__main__': # Initialize Variable and module request archives = [] - docker_tag = 'stilobique/blender:latest' - - # Clear blender container - clear_container_test(tag=docker_tag) # Prepare Blender and Unreal dependency + get_b3d_addon_dependency(archives) generate_archive(archives, 'blender_addon_folder') # Generate Unit Test, a specific call or execute all Unit Test @@ -40,8 +47,9 @@ if __name__ == '__main__': test_list.append(item) # Launch Unit Test - launch_unit_test(test=test_list, tag=docker_tag) + launch_unit_test(test=test_list) # Clear archive file and container for archive in archives: - os.remove(Path(os.getcwd(), archive)) + if Path(Path(os.getcwd(), archive)).exists(): + os.remove(Path(os.getcwd(), archive)) diff --git a/tests/requirements.txt b/tests/requirements.txt index 36011d0..161b063 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ docker -PyGithub \ No newline at end of file +PyGithub +numpy \ No newline at end of file diff --git a/tests/utils/blender.py b/tests/utils/blender.py index dea0178..1e7d5f4 100644 --- a/tests/utils/blender.py +++ b/tests/utils/blender.py @@ -1,49 +1,17 @@ +import json import os -import docker -import sys -from pathlib import PurePosixPath, Path +from pathlib import Path +from .forge import get_release_file -class ErrorTest(Exception): - """ Failed to generate the test """ +def get_b3d_addon_dependency(archive: list): + """From json resources, get all blender dependency and download-it""" + dependency = Path(os.getcwd(), "tests", "dependency.json") + with open(dependency) as f: + data = json.load(f) - -def b3d_launch_blender_test(client: docker = docker.from_env(), test: list = None, - tag: str = 'stilobique/blender:latest'): - """Launch blender and a list of Test, you can set a specific test file with his arg.""" - unit_test_folder = Path(os.getcwd(), "tests", "unit_test") - - if os.environ.get('GITHUB_WORKSPACE'): - local_path = os.environ.get('GITHUB_WORKSPACE') - else: - local_path = os.getcwd() - container_folder = '/addon_moderlab' - volume = [f'{local_path}:{container_folder}'] - image_name = tag - if test is None: - unit_test = os.listdir(unit_test_folder) - else: - if type(test) is not list: - test = [test] - unit_test = test - - try: - for test in unit_test: - print(f'Start this test : {test}') - command = [f"/bin/sh", - f"{PurePosixPath(container_folder, 'tests', 'launch_test.sh')}", - f"{PurePosixPath(container_folder, 'tests', 'unit_test', test)}"] - - docker_test = client.containers.run(image_name, command=command, volumes=volume, privileged=True, - environment=[f'FOLDER_TEST={container_folder}'], detach=True, name=test) - exit_docker = docker_test.wait() - if exit_docker['StatusCode'] != 0: - print(f'Container error "{exit_docker["StatusCode"]}".\n\t' - f'Show log : \n\t' - f'{docker_test.logs()}') - raise ErrorTest - - except ErrorTest as exception: - print(f'{exception.__doc__}') - sys.exit(1) + b3d_dependency = data['blender'] + for key, value in b3d_dependency.items(): + get_release_file(value[0], value[1]) + archive.append(value[0]) diff --git a/tests/utils/blender_addon.py b/tests/utils/blender_addon.py index d51ade7..5bbd770 100644 --- a/tests/utils/blender_addon.py +++ b/tests/utils/blender_addon.py @@ -1,18 +1,17 @@ -import bpy import os +import bpy import pathlib - -# Paste this variable in the blender.py -dependency = { - # 'moderlab_plugin': ['moderlab_plugin.zip', 'Moderlab-Production/BlenderPlugin'], - # 'moderlab_type': ['moderlab_type.zip', 'Moderlab-Production/BlenderObjectType'], - # 'moderlab_pie': ['moderlab_pie.zip', 'Moderlab-Production/BlenderPieMenu'], - # 'uv-packer': ['uv-packer.zip', 'Moderlab-Production/UvPacker'], -} +import json def b3d_install_addon(): env_name = 'FOLDER_TEST' + dependency = pathlib.Path(os.environ[env_name], "tests", "dependency.json") + with open(dependency) as f: + data = json.load(f) + + b3d_dependency = data['blender'] + try: if not os.environ.get(env_name): raise KeyError @@ -20,7 +19,7 @@ def b3d_install_addon(): if not env_path.exists(): raise FileNotFoundError - for key, value in dependency.items(): + for key, value in b3d_dependency.items(): bpy.ops.preferences.addon_install(filepath=f'{env_path}/{value[0]}') bpy.ops.preferences.addon_enable(module=key) bpy.ops.wm.save_userpref() diff --git a/tests/utils/container.py b/tests/utils/container.py index 39bf072..c6189f7 100644 --- a/tests/utils/container.py +++ b/tests/utils/container.py @@ -1,17 +1,75 @@ import docker +import os +import sys + +from pathlib import PurePosixPath +from docker.errors import DockerException +from .issue import ContainerErrorTest +from .properties import ContainerObject -def clear_container_test(tag: str = 'stilobique/blender:latest'): - client = docker.from_env() - containers = client.containers.list(all=True) +class VirtualMachine: + def __init__(self, container: ContainerObject): + self.client = self.start_docker() - for container in containers: - image_tag = container.image.tags[0] - status = container.status - if image_tag == tag and status == 'exited': - container.remove() + self.container = container + self.base_command = self.container.commands + self.clear_containers() + @staticmethod + def start_docker(): + """Generate docker client information""" + try: + client = docker.from_env() + return client -if __name__ == '__main__': - # Clear all unused blender container - clear_container_test() + except DockerException: + print('Docker isn\'t start or installed') + exit(1) + + def clear_containers(self): + """Look all docker containers, and remove-it if the task are used with the unit test.""" + containers = self.client.containers.list(all=True) + + for container in containers: + image_tag = container.image.tags[0] + status = container.status + if image_tag in self.container.ref and status in 'exited': + container.remove() + + @staticmethod + def workspace(): + """Define the workspace folder""" + if os.environ.get('GITHUB_WORKSPACE'): + return os.environ['GITHUB_WORKSPACE'] + else: + return os.getcwd() + + def launch_unit_test(self, tests: list[str]): + """Execute a docker container to start unit test dedicated""" + if tests is None: + unit_test = os.listdir(self.workspace()) + else: + if type(tests) is not list: + tests = [tests] + unit_test = tests + + try: + for test in unit_test: + print(f'Launch unit test "{test}"') + cmds = self.base_command + [f'{PurePosixPath(self.container.local_folder, "tests", "unit_test", test)}'] + + docker_test = self.client.containers.run(self.container.ref, command=cmds, + volumes=self.container.volumes, privileged=True, + environment=self.container.environments, detach=True, + name=test, tty=True) + exit_docker = docker_test.wait() + if exit_docker['StatusCode'] != 0: + print(f'Container error "{exit_docker["StatusCode"]}".\n\t' + f'Show log : \n\t' + f'{docker_test.logs()}') + raise ContainerErrorTest + + except ContainerErrorTest as exception: + print(f'{exception.__doc__}') + sys.exit(1) diff --git a/tests/utils/forge.py b/tests/utils/forge.py new file mode 100644 index 0000000..f30d7c4 --- /dev/null +++ b/tests/utils/forge.py @@ -0,0 +1,36 @@ +import os +import requests + +from pathlib import Path +from github import Github + + +def read_token(): + token_file = Path(os.getcwd(), 'tests', 'token.txt') + with open(token_file, 'r') as f: + token = f.read() + + return token + + +def get_release_file(filename: str, repo: str): + g = Github(read_token()) + repository = g.get_repo(repo) + latest = repository.get_latest_release() + latest.get_assets() + for file in latest.get_assets(): + if file.name == filename: + dl_file(file.url, file.name) + + +def dl_file(url: str, filename: str): + """From specific asset, download it""" + headers = { + 'Authorization': f'token {read_token()}', + 'Accept': 'application/octet-stream' + } + session = requests.Session() + link = session.get(url, stream=True, headers=headers) + with open(filename, 'wb') as f: + for chunk in link.iter_content(1024*1024): + f.write(chunk) diff --git a/tests/utils/issue.py b/tests/utils/issue.py index 1106ce9..962f897 100644 --- a/tests/utils/issue.py +++ b/tests/utils/issue.py @@ -6,3 +6,7 @@ class Archive(Exception): class ArchiveFolderSourceNotFound(Archive): def __str__(self): return f'Can\'t find the folder source "{self.source}".' + + +class ContainerErrorTest(Exception): + """ Failed to generate the test """ diff --git a/tests/utils/properties.py b/tests/utils/properties.py new file mode 100644 index 0000000..b586215 --- /dev/null +++ b/tests/utils/properties.py @@ -0,0 +1,63 @@ +import os + +from pathlib import PurePosixPath + + +class ContainerObject(object): + """All data request to launch one dedicated container""" + + def __init__(self, name: str, image: str): + self.label: str = name.lower() + self.image: str = image + self.tag: str = self.set_tag() + self.local_folder: str = '' + self.volumes: list[str] = self.set_volumes() + self.environments: list[str] = self.set_environments() + self.commands: list[str] = self.set_commands() + self.ref = f'{self.image}:{self.tag}' + + def set_tag(self): + """Determine the tag use with the docker image call""" + if 'blender' in self.label: + return 'latest' + elif 'unreal' in self.label: + return 'dev-slim-4.27.2' + else: + return None + + def set_volumes(self): + """Define folder/path with mount with each docker container""" + if 'blender' in self.label: + self.local_folder = '/addon_moderlab' + elif 'unreal' in self.label: + self.local_folder = '/project' + else: + return None + return [f'{self.workspace()}:{self.local_folder}'] + + def set_environments(self): + """Add environment variable start with the docker container""" + if 'blender' in self.label: + return [f'FOLDER_TEST={self.local_folder}'] + else: + return [] + + def set_commands(self): + """Write all commands execute on the docker container start""" + cmds = ['/bin/sh'] + if 'blender' in self.label: + cmds.append(str(PurePosixPath(self.local_folder, "tests", "launch_test_b3d.sh"))) + else: + cmds.append(str(PurePosixPath(self.local_folder, "tests", "launch_test_ue.sh"))) + cmds.append(str(PurePosixPath(self.local_folder, "tests", "unreal_sample", "empty_project", + "EmptyProject.uproject"))) + + return cmds + + @staticmethod + def workspace(): + """Define the workspace folder""" + if os.environ.get('GITHUB_WORKSPACE'): + return os.environ['GITHUB_WORKSPACE'] + else: + return os.getcwd()