From 93374e068fc6eee2ee457edfc9f702010d08c86c Mon Sep 17 00:00:00 2001 From: ruffsl Date: Tue, 30 Jan 2018 18:32:48 -0800 Subject: [PATCH] Adding multi arch for each dockerfile --- docker_templates/folders.py | 41 ++++++++++++++----- docker_templates/library.py | 14 +++++-- docker_templates/packages.py | 22 +++++----- .../templates/docker_library/ros.em | 6 ++- 4 files changed, 55 insertions(+), 28 deletions(-) diff --git a/docker_templates/folders.py b/docker_templates/folders.py index 6c0d796..fa098a2 100644 --- a/docker_templates/folders.py +++ b/docker_templates/folders.py @@ -2,6 +2,13 @@ import shutil import string +docker_to_deb_arch_mapping = { + 'amd64':'amd64', + 'arm32v7':'armhf', + 'arm64v8':'arm64', + 'i386':'i386', +} + class AltTemplate(string.Template): delimiter = '@' @@ -59,17 +66,30 @@ def populate_paths(manifest, args, create_dockerfiles): for os_name, os_data in release_data['os_names'].items(): # For each os distro supported for os_code_name, os_code_data in os_data['os_code_names'].items(): - dockerfolder_dir = os.path.join(release_name, os_name, os_code_name) - - os_code_data['release_name'] = release_name - os_code_data['os_name'] = os_name - os_code_data['os_code_name'] = os_code_name - - populate_path(data=os_code_data, path=dockerfolder_dir) + # For each arch supported + for arch in os_code_data['archs']: + dockerfolder_dir = os.path.join(release_name, os_name, os_code_name, arch) + + os_code_data['release_name'] = release_name + os_code_data['os_name'] = os_name + os_code_data['os_code_name'] = os_code_name + os_code_data['repo'] = arch + os_code_data['arch'] = docker_to_deb_arch_mapping[arch] + + populate_path(data=os_code_data, path=dockerfolder_dir) + + if args.auto: + # Run the dockerfile generation script + try: + create_dockerfiles.main(('dir', '-d' + dockerfolder_dir)) + except AttributeError as err: + expected = AttributeError("'NoneType' object has no attribute 'groups'") + if str(err) == str(expected): + print("Missing packages, '", dockerfolder_dir, "'") + else: + # pass + raise err - if args.auto: - # Run the dockerfile generation script - create_dockerfiles.main(('dir', '-d' + dockerfolder_dir)) # Hacks to add hook scripts for osrf repos if 'hacks' in manifest: @@ -86,6 +106,7 @@ def populate_paths(manifest, args, create_dockerfiles): hooksfolder_dir = os.path.join( release_name, os_name, os_code_name, + 'amd64', tag_name, 'hooks') tag_data['release_name'] = release_name diff --git a/docker_templates/library.py b/docker_templates/library.py index 2c24178..1d4edc1 100644 --- a/docker_templates/library.py +++ b/docker_templates/library.py @@ -15,6 +15,7 @@ import git import os import string +from collections import OrderedDict def latest_commit_sha(repo, path): @@ -36,8 +37,6 @@ def parse_manifest(manifest, repo, repo_name): # For each os code name supported for os_code_name, os_code_data in list(os_data['os_code_names'].items()): print('os_code_name: ', os_code_name) - commit_path = os.path.join(repo_name, release_name, os_name, os_code_name) - commit_sha = latest_commit_sha(repo, commit_path) if os_code_data['tag_names']: at_leat_one_tag = True for tag_name, tag_data in os_code_data['tag_names'].items(): @@ -53,8 +52,15 @@ def parse_manifest(manifest, repo, repo_name): print('tags: ', tags) tag_data['Tags'] = tags tag_data['Architectures'] = os_code_data['archs'] - tag_data['GitCommit'] = commit_sha - tag_data['Directory'] = os.path.join(commit_path, tag_name) + tag_data['arch_names'] = OrderedDict() + for arch_name in os_code_data['archs']: + tag_data['arch_names'][arch_name] = OrderedDict() + # print("tag_data['arch_names']: \n", tag_data['arch_names']) + for arch_name, arch_data in tag_data['arch_names'].items(): + commit_path = os.path.join(repo_name, release_name, os_name, os_code_name, arch_name, tag_name) + commit_sha = latest_commit_sha(repo, commit_path) + arch_data['GitCommit'] = commit_sha + arch_data['Directory'] = commit_path if not at_leat_one_tag: del manifest['release_names'][release_name] diff --git a/docker_templates/packages.py b/docker_templates/packages.py index c988658..bed80fb 100644 --- a/docker_templates/packages.py +++ b/docker_templates/packages.py @@ -16,14 +16,12 @@ import re import urllib.request -# TODO: think of a better version pattern like -# r'\d(?!Version\:\s)(.+)(?=(~\w+\n))' but works without a trailing ~ -version_pattern = r'(?<= )\d+\.\d+\.\d+\-\d+' +version_pattern = re.compile(r'(\bVersion: )([^\n]*)(\n)') packagePatternTemplateLookup = { - 'gazebo_packages': string.Template(r'(\bPackage: gazebo$gazebo_version\n)(.*\n)'), - 'ros_packages': string.Template(r'(\bPackage: ros-$rosdistro_name-$package\n)(.*\n)'), - 'ros2_packages': string.Template(r'(\bPackage: ros-$ros2distro_name-$package\n)(.*\n)'), + 'gazebo_packages': string.Template(r'((?s)\bPackage: gazebo$gazebo_version\n.*?(?=\n\n))'), + 'ros_packages': string.Template(r'((?s)\bPackage: ros-$rosdistro_name-$package\n.*?(?=\n\n))'), + 'ros2_packages': string.Template(r'((?s)\bPackage: ros-$ros2distro_name-$package\n.*?(?=\n\n))'), } indexUrlTemplateLookup = { @@ -33,9 +31,9 @@ } packageNameVersionTemplateLookup = { - 'gazebo_packages': string.Template('$package=$package_version*'), - 'ros_packages': string.Template('ros-$rosdistro_name-$package=$package_version*'), - 'ros2_packages': string.Template('ros-$ros2distro_name-$package=$package_version*'), + 'gazebo_packages': string.Template('$package=$package_version'), + 'ros_packages': string.Template('ros-$rosdistro_name-$package=$package_version'), + 'ros2_packages': string.Template('ros-$ros2distro_name-$package=$package_version'), } packageNameTemplateLookup = { @@ -66,9 +64,9 @@ def getPackageVersion(data, package_pattern, package, package_index): """Use package index to get package version""" # Parse for version_number - matchs = re.search(package_pattern, package_index) - version_line = matchs.groups(0)[1] # Grab the second line of the first match - package_version = re.search(version_pattern, version_line).group(0) # extract version_number + matchs = re.search(package_pattern, package_index) # Search for the package entry + version_line = re.search(version_pattern, matchs.groups(0)[0]) # Search for the version line + package_version = version_line.group(2) # extract version_number return package_version diff --git a/docker_templates/templates/docker_library/ros.em b/docker_templates/templates/docker_library/ros.em index 108f7eb..affcd97 100644 --- a/docker_templates/templates/docker_library/ros.em +++ b/docker_templates/templates/docker_library/ros.em @@ -14,8 +14,10 @@ GitRepo: @(repo_url) @[ for tag_name, tag_data in os_code_data['tag_names'].items()]@ Tags: @(', '.join(tag_data['Tags'])) Architectures: @(', '.join(tag_data['Architectures'])) -GitCommit: @(tag_data['GitCommit']) -Directory: @(tag_data['Directory']) +@[ for arch_name, arch_data in tag_data['arch_names'].items()]@ +@(arch_name)-GitCommit: @(arch_data['GitCommit']) +@(arch_name)-Directory: @(arch_data['Directory']) +@[ end for]@ @[ end for]@ @[ end if]@