diff --git a/docker_templates/create.py b/docker_templates/create.py index e879c4a..cfa0c0d 100644 --- a/docker_templates/create.py +++ b/docker_templates/create.py @@ -21,6 +21,8 @@ from ros_buildfarm.templates import get_template_path from ros_buildfarm.templates import expand_template, get_wrapper_scripts +from docker_templates.packages import indexUrlTemplateLookup + default_template_prefix_path = ros_buildfarm.templates.template_prefix_path def expand_template_prefix_path(template_packages): @@ -44,9 +46,25 @@ def create_files(data, verbose=False): # generate Dockerfile create_dockerfile(template_name, data, dockerfile_dir, verbose) + create_lockfiles(data) + if 'entrypoint_name' in data: create_entrypoint(data) +def create_lockfiles(data): + dockerfile_dir = data['dockerfile_dir'] + for arch_name, arch_data in data['archs'].items(): + for package_type, package_list in arch_data.items(): + lockfile_dir = os.path.join(dockerfile_dir, package_type) + if not os.path.exists(lockfile_dir): + os.makedirs(lockfile_dir) + lockfile_path = os.path.join(lockfile_dir, arch_name + '.txt') + if package_list: + with open(lockfile_path, 'w') as h: + for package in package_list: + line = f"{package['name']}{package['version']}" + h.write(line) + def create_entrypoint(data): # find entrypoint path entrypoint_name = data['entrypoint_name'] diff --git a/docker_templates/folders.py b/docker_templates/folders.py index 720d890..f2f3afd 100644 --- a/docker_templates/folders.py +++ b/docker_templates/folders.py @@ -2,20 +2,33 @@ import shutil import string +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO +from em import Interpreter + class AltTemplate(string.Template): delimiter = '@' idpattern = r'[a-z][_a-z0-9]*' -def interpret_tempate(tempate, data, tempate_class=string.Template): - # Read image perams using platform perams - with open(tempate, 'r') as f: - value = tempate_class(f.read()) +def interpret_tempate(tempate, data): - value = value.substitute(data) + output = StringIO() + try: + interpreter = Interpreter(output=output) + interpreter.file(open(tempate, 'r'), locals=data) + output = output.getvalue() + except Exception as e: + print("Error processing %s" % tempate) + raise + finally: + interpreter.shutdown() + interpreter = None - return value + return output def populate_path(data, path): @@ -37,21 +50,6 @@ def populate_path(data, path): shutil.copy(templates['images'], path) -def populate_hooks(data, path): - if not os.path.exists(path): - os.makedirs(path) - for hook_name, hook_tempate in data['hook_names'].items(): - tags = ' '.join([data['release_name'] + '-' + data['tag_name']]) - hookfile = interpret_tempate( - hook_tempate, - {'tags': tags, 'release_name': data['release_name']}, - AltTemplate) - hookfile_path = os.path.join(path, hook_name) - with open(hookfile_path, 'w') as f: - f.write(hookfile) - os.chmod(hookfile_path, 0o744) - - def populate_paths(manifest, args, create_dockerfiles): # For each release for release_name, release_data in manifest['release_names'].items(): @@ -66,32 +64,10 @@ def populate_paths(manifest, args, create_dockerfiles): os_code_data['release_name'] = release_name os_code_data['os_name'] = os_name os_code_data['os_code_name'] = os_code_name + os_code_data['archs'] = os_code_data['archs'] populate_path(data=os_code_data, path=dockerfolder_dir) if args.auto: # Run the dockerfile generation script create_dockerfiles.main(('dir', '-d' + dockerfolder_dir)) - - # Hacks to add hook scripts for osrf repos - if 'hacks' in manifest: - # For each release - for release_name, release_data in manifest['hacks'].items(): - # For each os supported - for os_name, os_data in release_data['os_names'].items(): - # For each os distro supported - for os_code_name, os_code_data in os_data['os_code_names'].items(): - if os_code_data['tag_names'] is None: - continue - # For each tag supported: - for tag_name, tag_data in os_code_data['tag_names'].items(): - hooksfolder_dir = os.path.join( - release_name, os_name, - os_code_name, - tag_name, - 'hooks') - tag_data['release_name'] = release_name - tag_data['os_name'] = os_name - tag_data['os_code_name'] = os_code_name - tag_data['tag_name'] = tag_name - populate_hooks(tag_data, hooksfolder_dir) diff --git a/docker_templates/packages.py b/docker_templates/packages.py index 7529eaf..1bec5da 100644 --- a/docker_templates/packages.py +++ b/docker_templates/packages.py @@ -16,14 +16,22 @@ import string import re import urllib.request +# import json import rosdistro from docker_templates.eol_distro import isDistroEOL -# TODO: think of a better version pattern like -# r'\d(?!Version\:\s)(.+)(?=(~\w+\n))' but works without a trailing ~ -version_pattern = r'(?<=Version: )\d+\.\d+\.\d+\-\d+' +_cached_package_indexs = {} + +DockerToAptArchLookup = { + 'amd64': 'amd64', + 'arm32v7':'armhf', + 'arm64v8':'arm64', + 'i386':'i386' +} + +version_pattern = r'(?<=Version: ).*\n' sha256_pattern = r'(?<=SHA256: )[0-9a-f]{64}' @@ -42,9 +50,9 @@ } packageVersionTemplateLookup = { - 'gazebo_packages': string.Template('=$package_version*'), - 'ros_packages': string.Template('=$package_version*'), - 'ros2_packages': string.Template('=$package_version*'), + 'gazebo_packages': string.Template('=$package_version'), + 'ros_packages': string.Template('=$package_version'), + 'ros2_packages': string.Template('=$package_version'), } packageNameTemplateLookup = { @@ -56,10 +64,15 @@ def getPackageIndex(data, package_index_url): """Get current online package index""" - # Download package index - req = urllib.request.Request(package_index_url) - with urllib.request.urlopen(req) as response: - package_index = gzip.decompress(response.read()).decode('utf-8') + global _cached_package_indexs + if package_index_url in _cached_package_indexs: + package_index = _cached_package_indexs[package_index_url] + else: + # Download package index + req = urllib.request.Request(package_index_url) + with urllib.request.urlopen(req) as response: + package_index = gzip.decompress(response.read()).decode('utf-8') + _cached_package_indexs[package_index_url] = package_index return package_index @@ -76,6 +89,8 @@ def getPackageInfo(package_pattern, package_index): # Parse for package info matchs = re.search(package_pattern, package_index) + if matchs is None: + return None package_info = matchs.group(0) return package_info @@ -110,6 +125,8 @@ def getPackageVersions(data, package_index, packages, package_type): package_pattern = getPackagePattern(data, package_pattern_template, package) package_name = package_name_template.substitute(data, package=package) package_info = getPackageInfo(package_pattern, package_index) + if package_info is None: + continue package_sha256 = getPackageSHA256(package_info) if data['version'] != False: @@ -123,6 +140,10 @@ def getPackageVersions(data, package_index, packages, package_type): return package_versions def expandPackages(data): + # print("################################################################") + # print(json.dumps(data,sort_keys=True, indent=4)) + # print("################################################################") + data["archs"] = {i: dict() for i in data["archs"]} for package_type in indexUrlTemplateLookup: if package_type in data: # determine if distro is eol and apply the appropriate index URL template @@ -147,7 +168,9 @@ def expandPackages(data): package_index_url_template = indexUrlTemplateLookup[package_type + '_snapshots'] else: package_index_url_template = indexUrlTemplateLookup[package_type] - package_index_url = package_index_url_template.substitute(data) - package_index = getPackageIndex(data, package_index_url) - package_versions = getPackageVersions(data, package_index, data[package_type], package_type) - data[package_type] = package_versions + for arch in data['archs']: + data['arch'] = DockerToAptArchLookup[arch] + package_index_url = package_index_url_template.substitute(data) + package_index = getPackageIndex(data, package_index_url) + package_versions = getPackageVersions(data, package_index, data[package_type], package_type) + data['archs'][arch][package_type] = package_versions diff --git a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em index 1fec499..2ac21ca 100644 --- a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em @@ -52,8 +52,9 @@ ENV LC_ALL C.UTF-8 ENV ROS_DISTRO @rosdistro_name @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', + 'snippet/copy_and_install_package_list.Dockerfile.em', group='ros', + package_type='ros_packages', packages=ros_packages, ))@ @[if 'entrypoint_name' in locals()]@ diff --git a/docker_templates/templates/docker_images/create_ros_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_image.Dockerfile.em index 32d4246..6afc781 100644 --- a/docker_templates/templates/docker_images/create_ros_image.Dockerfile.em +++ b/docker_templates/templates/docker_images/create_ros_image.Dockerfile.em @@ -36,8 +36,9 @@ RUN rosdep init && \ @[if 'ros_packages' in locals()]@ @[ if ros_packages]@ @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', + 'snippet/copy_and_install_package_list.Dockerfile.em', group='ros', + package_type='ros_packages', packages=ros_packages, ))@ @[ end if]@ diff --git a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em index d59182f..89e4ba7 100644 --- a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em @@ -65,8 +65,9 @@ RUN pip3 install -U \ @[end if]@ @ @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', - group='ros2', + 'snippet/copy_and_install_package_list.Dockerfile.em', + group='ros', + package_type='ros2_packages', packages=ros2_packages, ))@ @[if 'entrypoint_name' in locals()]@ diff --git a/docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em index afb121d..6ead5cb 100644 --- a/docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em @@ -53,8 +53,9 @@ RUN pip3 install -U \ @[end if]@ @[if 'ros2_packages' in locals()]@ @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', - group='ros2', + 'snippet/copy_and_install_package_list.Dockerfile.em', + group='ros', + package_type='ros2_packages', packages=ros2_packages, ))@ @[end if]@ diff --git a/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em index b5fe50f..f0ef1d5 100644 --- a/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em @@ -47,15 +47,17 @@ ENV ROS2_DISTRO @ros2distro_name @[if 'ros_packages' in locals()]@ @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', + 'snippet/copy_and_install_package_list.Dockerfile.em', group='ros', + package_type='ros_packages', packages=ros_packages, ))@ @[end if]@ @[if 'ros2_packages' in locals()]@ @(TEMPLATE( - 'snippet/label_and_install_package_list.Dockerfile.em', - group='ros2', + 'snippet/copy_and_install_package_list.Dockerfile.em', + group='ros', + package_type='ros2_packages', packages=ros2_packages, ))@ @[end if]@ diff --git a/docker_templates/templates/snippet/copy_and_install_package_list.Dockerfile.em b/docker_templates/templates/snippet/copy_and_install_package_list.Dockerfile.em new file mode 100644 index 0000000..2b879d6 --- /dev/null +++ b/docker_templates/templates/snippet/copy_and_install_package_list.Dockerfile.em @@ -0,0 +1,11 @@ +@[if packages]@ +# copy @group packages +ARG TARGETARCH +COPY @(package_type)/$TARGETARCH.txt /etc/apt/@(group)/@(package_type).txt + +# install @group packages +RUN apt-get update \ + && xargs -a /etc/apt/@(group)/@(package_type).txt \ + apt-get install -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* +@[end if]