Blender 5.1.2 (#1)
Refactoring to works with the new blender depot config Reviewed-on: #1 Co-authored-by: stilobique <contact@aurelien-vaillant.net>
This commit was merged in pull request #1.
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
import os
|
||||
import requests
|
||||
import tempfile
|
||||
import importlib.util
|
||||
|
||||
from blender.version import BlenderVersion
|
||||
|
||||
# "Wayland Protocols" has no "Library" in its name but is still a
|
||||
# build-critical dependency, so it's matched by name too.
|
||||
_EXTRA_PACKAGE_NAMES = {'wayland protocols'}
|
||||
|
||||
|
||||
class BlenderDependency:
|
||||
def __init__(self, blender: BlenderVersion):
|
||||
self._blender_version = blender
|
||||
self._url = rf'https://projects.blender.org/blender/blender/raw/tag/v{self._blender_version.tag_name()}/build_files/build_environment/install_linux_packages.py'
|
||||
self.module = self._get_source_file()
|
||||
|
||||
self.packages = list()
|
||||
self._get_list()
|
||||
|
||||
self.packages_complete = list()
|
||||
self._get_complete_list()
|
||||
|
||||
def _get_source_file(self):
|
||||
"""Download the Blender dependency script and import it at runtime."""
|
||||
r = requests.get(self._url)
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='wb', suffix='.py', delete=False) as t:
|
||||
t.write(r.content)
|
||||
path = t.name
|
||||
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location('install_linux_packages', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.remove(path)
|
||||
|
||||
def _get_list(self):
|
||||
"""Keep the mandatory 'library' packages (and Wayland Protocols) as Debian/Ubuntu apt names."""
|
||||
sources = self.module.BUILD_MANDATORY_SUBPACKAGES + self.module.DEPS_CRITICAL_SUBPACKAGES
|
||||
for package in self._flatten(sources):
|
||||
if 'library' not in package.name.lower() and package.name.lower() not in _EXTRA_PACKAGE_NAMES:
|
||||
continue
|
||||
name = package.distro_package_names.get(self.module.DISTRO_ID_DEBIAN)
|
||||
if isinstance(name, str):
|
||||
self.packages.append(name)
|
||||
|
||||
def _get_complete_list(self):
|
||||
"""All packages required to build Blender (PACKAGES_BASICS_BUILD) as Debian/Ubuntu apt names."""
|
||||
for package in self._flatten(self.module.PACKAGES_BASICS_BUILD):
|
||||
name = package.distro_package_names.get(self.module.DISTRO_ID_DEBIAN)
|
||||
if isinstance(name, str):
|
||||
self.packages_complete.append(name)
|
||||
|
||||
def _flatten(self, packages):
|
||||
"""Yield every leaf Package, descending into is_group sub_packages."""
|
||||
for package in packages:
|
||||
if package.is_group:
|
||||
yield from self._flatten(package.sub_packages)
|
||||
else:
|
||||
yield package
|
||||
|
||||
def __str__(self):
|
||||
return ' '.join(self.packages)
|
||||
|
||||
def complete_str(self):
|
||||
return ' '.join(self.packages_complete)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
bld = BlenderVersion(version='5.0.0')
|
||||
dep = BlenderDependency(blender=bld)
|
||||
|
||||
print('Show All packages:')
|
||||
print(dep.__str__())
|
||||
@@ -0,0 +1,264 @@
|
||||
import os
|
||||
import re
|
||||
import docker
|
||||
import logging
|
||||
import datetime
|
||||
import requests
|
||||
|
||||
from .version import BlenderVersion
|
||||
from .dependency import BlenderDependency
|
||||
from pathlib import Path
|
||||
from docker.errors import BuildError, APIError, ContainerError, ImageNotFound
|
||||
|
||||
|
||||
_ANSI_ESCAPE_RE = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]')
|
||||
|
||||
|
||||
def _clean_log_line(text: str) -> str:
|
||||
"""Strip ANSI escape codes and keep only the last update of a \r-rewritten line."""
|
||||
text = _ANSI_ESCAPE_RE.sub('', text)
|
||||
return text.split('\r')[-1].strip()
|
||||
|
||||
|
||||
class BlenderDocker:
|
||||
def __init__(self, blender: BlenderVersion, severity: int = logging.WARNING):
|
||||
self._blender_version = blender
|
||||
# Todo get a user config
|
||||
self._docker_client = docker.DockerClient(base_url=f'unix:///var/run/docker.sock')
|
||||
self.login()
|
||||
self._docker_image = None
|
||||
self._repository = 'stilobique/blender'
|
||||
self._docker_image_name = self._blender_version.format_to_tag(self._repository)
|
||||
|
||||
self._docker_file = Path(os.getcwd(), 'Dockerfile')
|
||||
if not self._docker_file.exists():
|
||||
print(f'Dockerfile not found: {self._docker_file}')
|
||||
exit(1)
|
||||
|
||||
self._packages = BlenderDependency(blender)
|
||||
|
||||
self._args = {
|
||||
'b3d_vs_major': f'{self._blender_version.major}',
|
||||
'b3d_vs_minor': f'{self._blender_version.minor}.{self._blender_version.hotfix}',
|
||||
'b3d_dependency': self._packages.__str__(),
|
||||
'python_version': self._get_required_python_version(),
|
||||
}
|
||||
|
||||
self._severity = severity
|
||||
|
||||
def _get_required_python_version(self) -> str:
|
||||
"""
|
||||
Fetch the exact Python version (major.minor) Blender's own CMake pins for this
|
||||
release, so the build stage's base image can match it instead of drifting with
|
||||
whatever a floating `python` tag currently resolves to.
|
||||
"""
|
||||
url = (f'https://projects.blender.org/blender/blender/raw/tag/'
|
||||
f'v{self._blender_version.tag_name()}/build_files/cmake/Modules/FindPythonLibsUnix.cmake')
|
||||
r = requests.get(url)
|
||||
match = re.search(r'set\(_PYTHON_VERSION_SUPPORTED\s+([0-9]+\.[0-9]+)\)', r.text)
|
||||
if not match:
|
||||
print(f'Could not determine required Python version from {url}')
|
||||
exit(1)
|
||||
return match.group(1)
|
||||
|
||||
def _get_logger(self, name: str) -> logging.Logger:
|
||||
"""
|
||||
Private Method to set a logger for a specific method set (by his name)
|
||||
:param name: Make the logger file name.
|
||||
:return:
|
||||
"""
|
||||
log_dir = Path('logs')
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger(f'blender_docker.{name}')
|
||||
logger.setLevel(self._severity)
|
||||
logger.propagate = False
|
||||
|
||||
if not logger.handlers:
|
||||
handler = logging.FileHandler(log_dir / f'{name}.log')
|
||||
handler.setFormatter(logging.Formatter(
|
||||
'%(asctime)s | [%(levelname)s] %(message)s'
|
||||
))
|
||||
logger.addHandler(handler)
|
||||
|
||||
return logger
|
||||
|
||||
def login(self):
|
||||
"""Login to the Docker Hub"""
|
||||
try:
|
||||
print('Login to Hub Docker.')
|
||||
self._docker_client.login(
|
||||
username=os.environ['DOCKER_HUB_USER'],
|
||||
password=os.environ['DOCKER_HUB_PASSWORD']
|
||||
)
|
||||
|
||||
except APIError as e:
|
||||
print(f'Login failed with error: {e}')
|
||||
exit(1)
|
||||
|
||||
def build(self):
|
||||
"""Build the Blender Docker Image"""
|
||||
time = datetime.datetime.now()
|
||||
logger = self._get_logger(f'build-{time.strftime("%Y-%m-%d-%H-%M")}')
|
||||
collected_logs = []
|
||||
try:
|
||||
print('Building Blender Docker Image')
|
||||
logger.info('Building Blender Docker Image')
|
||||
logger.info(f'\tBuilds withs args: {self._args}')
|
||||
logger.info(f'\tTag generate: {self._docker_image_name}')
|
||||
|
||||
# Use the low-level API: images.build() only returns once the whole
|
||||
# build is finished, so it gives no live progress. api.build() with
|
||||
# decode=True streams each step (including cache hits) as it happens.
|
||||
build_logs = self._docker_client.api.build(
|
||||
path=os.getcwd(),
|
||||
# nocache=True,
|
||||
buildargs=self._args,
|
||||
tag=self._docker_image_name,
|
||||
rm=True,
|
||||
forcerm=True,
|
||||
decode=True,
|
||||
)
|
||||
|
||||
for chunk in build_logs:
|
||||
collected_logs.append(chunk)
|
||||
|
||||
if 'stream' in chunk:
|
||||
for raw_line in chunk['stream'].split('\n'):
|
||||
line = _clean_log_line(raw_line)
|
||||
if not line:
|
||||
continue
|
||||
if 'Using cache' in line:
|
||||
line = f'[CACHE] {line}'
|
||||
logger.info(line)
|
||||
|
||||
elif 'status' in chunk:
|
||||
status = _clean_log_line(chunk['status'])
|
||||
progress = _clean_log_line(chunk.get('progress', ''))
|
||||
line = f"{status} {chunk.get('id', '')} {progress}".strip()
|
||||
if 'Already exists' in status:
|
||||
line = f'[CACHE] {line}'
|
||||
logger.info(line)
|
||||
|
||||
elif 'error' in chunk:
|
||||
logger.error(chunk['error'])
|
||||
raise BuildError(chunk['error'], collected_logs)
|
||||
|
||||
self._docker_image = self._docker_client.images.get(self._docker_image_name)
|
||||
|
||||
except BuildError as e:
|
||||
logger.exception(f'Build docker image failed: {e}')
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def tag(self):
|
||||
print('\tSet Tags version and/or Latest')
|
||||
pass
|
||||
|
||||
def test(self) -> bool:
|
||||
"""
|
||||
This method request an image (call the method build before) and try to execute the blender inside the container
|
||||
to check if the program can be correctly executed.
|
||||
Based on `docker run --rm stilobique/blender:5.1.2 ./blender --version`
|
||||
:return: True if no issue are returned
|
||||
"""
|
||||
logger = self._get_logger('test')
|
||||
container_name = f"Blender-Docker-Testing-{self._blender_version.tag_name_slugify()}"
|
||||
command = ['./blender', '--version']
|
||||
container = None
|
||||
try:
|
||||
print(f'Testing your Blender Docker image {self._docker_image_name}')
|
||||
logger.info(f'Testing image "{self._docker_image_name}" in container "{container_name}" '
|
||||
f'with command: {" ".join(command)}')
|
||||
|
||||
# auto_remove is intentionally NOT used: Docker can remove the container
|
||||
# right after it exits, racing with the logs() call below and turning a
|
||||
# real container-side error into an unrelated 409 "dead or marked for
|
||||
# removal" API error. We remove the container ourselves once logs are read.
|
||||
container = self._docker_client.containers.run(
|
||||
name=container_name,
|
||||
image=self._docker_image_name,
|
||||
command=command,
|
||||
detach=True,
|
||||
stdout=True,
|
||||
stderr=True,
|
||||
)
|
||||
result = container.wait()
|
||||
exit_code = result.get('StatusCode')
|
||||
|
||||
container_logs = container.logs(stdout=True, stderr=True).decode('utf-8', errors='replace').strip()
|
||||
for raw_line in container_logs.split('\n'):
|
||||
line = _clean_log_line(raw_line)
|
||||
if line:
|
||||
logger.info(f'[container] {line}')
|
||||
|
||||
if exit_code == 0:
|
||||
logger.info(f'Test succeeded (exit code {exit_code}) for image "{self._docker_image_name}".')
|
||||
return True
|
||||
|
||||
logger.error(f'Test failed (exit code {exit_code}) for image "{self._docker_image_name}".')
|
||||
print(f'Testing your Blender Docker image {self._docker_image_name} failed.')
|
||||
return False
|
||||
|
||||
except ContainerError as e:
|
||||
logger.exception(f'Container error while testing "{self._docker_image_name}": {e}')
|
||||
print(f'Container error: {e}')
|
||||
exit(1)
|
||||
|
||||
except ImageNotFound as e:
|
||||
logger.exception(f'Image "{self._docker_image_name}" not found: {e}')
|
||||
print(f'Image not found: {e}')
|
||||
exit(1)
|
||||
|
||||
except APIError as e:
|
||||
logger.exception(f'API error while testing "{self._docker_image_name}": {e}')
|
||||
print(f'API error: {e}')
|
||||
exit(1)
|
||||
|
||||
finally:
|
||||
if container is not None:
|
||||
try:
|
||||
container.remove(force=True)
|
||||
except APIError:
|
||||
pass
|
||||
|
||||
def push(self):
|
||||
"""Push the Blender Docker Image"""
|
||||
time = datetime.datetime.now()
|
||||
logger = self._get_logger(f'push')
|
||||
collected_logs = []
|
||||
try:
|
||||
print(f'Pushing Blender Docker image {self._docker_image_name}')
|
||||
logger.info(f'Pushing Blender Docker Image')
|
||||
logger.info(f'\tTag pushed: {self._docker_image_name}')
|
||||
|
||||
# Use the low-level API: images.push() only returns once the whole
|
||||
# push is finished, so it gives no live progress. api.push() with
|
||||
# decode=True streams each layer's progress as it happens.
|
||||
push_logs = self._docker_client.api.push(
|
||||
repository=self._repository,
|
||||
tag=self._blender_version.tag_name(),
|
||||
stream=True,
|
||||
decode=True,
|
||||
)
|
||||
|
||||
for chunk in push_logs:
|
||||
collected_logs.append(chunk)
|
||||
|
||||
if 'status' in chunk:
|
||||
status = _clean_log_line(chunk['status'])
|
||||
progress = _clean_log_line(chunk.get('progress', ''))
|
||||
line = f"{status} {chunk.get('id', '')} {progress}".strip()
|
||||
if 'Layer already exists' in status:
|
||||
line = f'[CACHE] {line}'
|
||||
logger.info(line)
|
||||
|
||||
elif 'error' in chunk:
|
||||
logger.error(chunk['error'])
|
||||
raise APIError(chunk['error'])
|
||||
|
||||
logger.info(f'Push succeeded for image "{self._docker_image_name}".')
|
||||
|
||||
except APIError as e:
|
||||
logger.exception(f'Push docker image failed: {e}')
|
||||
raise
|
||||
@@ -0,0 +1,92 @@
|
||||
import logging
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
|
||||
from pathlib import Path
|
||||
from subprocess import run, CalledProcessError
|
||||
|
||||
|
||||
def clean_all_dockers():
|
||||
"""
|
||||
Nettoie Docker pour tous les contextes disponibles (default, podman, ...).
|
||||
Equivalent a, pour chaque contexte :
|
||||
docker context use <context>
|
||||
docker system prune -a -f
|
||||
Puis remet le contexte d'origine en place a la fin.
|
||||
"""
|
||||
logs_dir = Path(os.getcwd(), 'logs')
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger('blender_docker.clean')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.propagate = False
|
||||
if not logger.handlers:
|
||||
handler = logging.FileHandler(logs_dir / 'clean.log')
|
||||
handler.setFormatter(logging.Formatter(
|
||||
'%(asctime)s|%(name)s [%(levelname)s] %(message)s'
|
||||
))
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Contexte courant, pour le restaurer a la fin
|
||||
original_context = run(
|
||||
['docker', 'context', 'show'],
|
||||
capture_output=True, text=True, check=True
|
||||
).stdout.strip()
|
||||
logger.info(f'Contexte d\'origine: {original_context}')
|
||||
|
||||
# Liste de tous les contextes disponibles (format json, un objet par ligne)
|
||||
contexts_raw = run(
|
||||
['docker', 'context', 'ls', '--format', '{{json .}}'],
|
||||
capture_output=True, text=True, check=True
|
||||
).stdout.strip().splitlines()
|
||||
contexts = [json.loads(line)['Name'] for line in contexts_raw if line]
|
||||
|
||||
print(f'\tContextes trouves: {contexts}')
|
||||
logger.info(f'Contextes trouves: {contexts}')
|
||||
|
||||
for context in contexts:
|
||||
try:
|
||||
print(f'\tNettoyage du contexte: {context}')
|
||||
logger.info(f'Nettoyage du contexte: {context}')
|
||||
|
||||
run(['docker', 'context', 'use', context], check=True)
|
||||
|
||||
result = run(
|
||||
['docker', 'system', 'prune', '-a', '-f'],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
|
||||
print(f'\t{result.stdout.strip()}')
|
||||
logger.info(result.stdout.strip())
|
||||
|
||||
except CalledProcessError as e:
|
||||
print(f'\tErreur sur le contexte {context}: {e.stderr.strip()}')
|
||||
logger.error(f'Erreur sur le contexte {context}: {e.stderr.strip()}')
|
||||
|
||||
# On remet le contexte d'origine
|
||||
run(['docker', 'context', 'use', original_context], check=True)
|
||||
print(f'\tContexte restaure: {original_context}')
|
||||
logger.info(f'Contexte restaure: {original_context}')
|
||||
|
||||
|
||||
def get_all_tags_from_blender():
|
||||
"""From the Gitea api, get all blender tags release"""
|
||||
r = requests.get(r"https://projects.blender.org/api/v1/repos/blender/blender/tags")
|
||||
print(r.text)
|
||||
j = r.json()
|
||||
|
||||
# All tags data:
|
||||
for release in j:
|
||||
print(release)
|
||||
|
||||
|
||||
def get_all_tags_from_docker():
|
||||
"""From the Gitea api, get all blender tags release"""
|
||||
r = requests.get(r"https://hub.docker.com/v2/namespaces/stilobique/repositories/blender/tags")
|
||||
print(r.text)
|
||||
j = r.json()
|
||||
|
||||
# All tags data:
|
||||
for release in j:
|
||||
print(release)
|
||||
@@ -0,0 +1,43 @@
|
||||
import re
|
||||
|
||||
|
||||
class BlenderVersion:
|
||||
def __init__(self, version: str = '5.1.0'):
|
||||
"""
|
||||
Need an object like 'X.Y.Z' to work correctly, build a pythonic object about a blender version.
|
||||
:param version:
|
||||
"""
|
||||
self._check_pattern(version)
|
||||
self.major : str = ''
|
||||
self.minor : str = ''
|
||||
self.hotfix : str = ''
|
||||
self.version_object(version)
|
||||
|
||||
def version_object(self, version: str):
|
||||
"""Set all version properties to a string"""
|
||||
version = version.split('.')
|
||||
self.major = version[0]
|
||||
self.minor = version[1]
|
||||
self.hotfix = version[2]
|
||||
|
||||
def format_to_tag(self, repository: str) -> str:
|
||||
"""From version information, format to return a string"""
|
||||
return f'{repository}:{self.major}.{self.minor}.{self.hotfix}'
|
||||
|
||||
def tag_name(self) -> str:
|
||||
"""Return the tag name 'latest', 'X.Y.Z'..."""
|
||||
return f'{self.major}.{self.minor}.{self.hotfix}'
|
||||
|
||||
def tag_name_slugify(self):
|
||||
return f'{self.major}-{self.minor}-{self.hotfix}'
|
||||
|
||||
@staticmethod
|
||||
def _check_pattern(tested: str):
|
||||
"""Simple test to look if the string give can be a blender version name."""
|
||||
test = re.match(r'^[0-9]{1}.[0-9]+.[0-9]+$', tested)
|
||||
if test is None:
|
||||
print(f'Bad version name set: {tested}')
|
||||
exit(1)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.major + '.' + self.minor + '.' + self.hotfix)
|
||||
Reference in New Issue
Block a user