From 0410b55df1513a14d9ae5f936d512bc0e9b75352 Mon Sep 17 00:00:00 2001
From: ruffsl <roxfoxpox@gmail.com>
Date: Fri, 9 Nov 2018 19:55:27 -0800
Subject: [PATCH 1/2] Adding break build section cache for sync

---
 docker_templates/packages.py                  | 24 ++++++++++++++++++-
 .../create_gzserver_image.Dockerfile.em       |  8 +++++++
 .../create_ros2_core_image.Dockerfile.em      |  8 +++++++
 .../create_ros_core_image.Dockerfile.em       |  8 +++++++
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/docker_templates/packages.py b/docker_templates/packages.py
index c988658..22374d9 100644
--- a/docker_templates/packages.py
+++ b/docker_templates/packages.py
@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import string
+import hashlib
 import re
+import string
 import urllib.request
 
 # TODO: think of a better version pattern like
@@ -44,6 +45,12 @@
     'ros2_packages':    string.Template('ros-$ros2distro_name-$package'),
 }
 
+packageReleaseLookup = {
+    'gazebo_packages':  string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/Release'),
+    'ros_packages':     string.Template('http://packages.ros.org/$release/ubuntu/dists/$os_code_name/Release'),
+    'ros2_packages':    string.Template('http://repo.ros2.org/$os_name/main/dists/$os_code_name/Release'),
+}
+
 def getPackageIndex(data, package_index_url):
     """Get current online package index"""
 
@@ -105,6 +112,18 @@ def getPackageVersions(data, package_index, packages, package_type):
 
         return package_versions
 
+def getPackageReleaseToken(data, package_release_url):
+    """Get package release token"""
+
+    # Download package index
+    req = urllib.request.Request(package_release_url)
+    with urllib.request.urlopen(req) as response:
+        package_release = response.read().decode('utf-8')
+    token = {}
+    token['date'] = re.search(r'Date: (.*)', package_release).group(1)
+    token['digest'] = hashlib.sha256(package_release.encode('utf-8')).hexdigest()
+    return token
+
 def expandPackages(data):
     for package_type in indexUrlTemplateLookup:
         if package_type in data:
@@ -113,3 +132,6 @@ def expandPackages(data):
             package_index = getPackageIndex(data, package_index_url)
             package_versions = getPackageVersions(data, package_index, data[package_type], package_type)
             data[package_type] = package_versions
+            package_release_url_template = packageReleaseLookup[package_type]
+            package_release_url = package_release_url_template.substitute(data)
+            data[package_type + '_token'] = getPackageReleaseToken(data, package_release_url)
diff --git a/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em b/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
index fd48303..515570d 100644
--- a/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
@@ -37,6 +37,14 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
 # setup sources.list
 RUN . /etc/os-release \
     && echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list
+@[if 'gazebo_packages_token' in locals()]@
+@[  if gazebo_packages_token]@
+
+# break build cache for sync
+RUN echo "Release: @(gazebo_packages_token['date'])@ " \
+    && echo "Digest: @(gazebo_packages_token['digest'])@ "
+@[  end if]@
+@[end if]@
 
 # install gazebo packages
 RUN apt-get update && apt-get install -q -y \
diff --git a/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
index 3b11760..2279043 100644
--- a/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
@@ -42,6 +42,14 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
 # setup sources.list
 RUN . /etc/os-release \
     && echo "deb http://repo.ros2.org/$ID/main `lsb_release -sc` main" > /etc/apt/sources.list.d/ros2-latest.list
+@[if 'ros2_packages_token' in locals()]@
+@[  if ros2_packages_token]@
+
+# break build cache for sync
+RUN echo "Release: @(ros2_packages_token['date'])@ " \
+    && echo "Digest: @(ros2_packages_token['digest'])@ "
+@[  end if]@
+@[end if]@
 
 # setup environment
 ENV LANG C.UTF-8
diff --git a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
index ebab001..9fd5cfd 100644
--- a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
@@ -38,6 +38,14 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
 
 # setup sources.list
 RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list
+@[if 'ros_packages_token' in locals()]@
+@[  if ros_packages_token]@
+
+# break build cache for sync
+RUN echo "Release: @(ros_packages_token['date'])@ " \
+    && echo "Digest: @(ros_packages_token['digest'])@ "
+@[  end if]@
+@[end if]@
 
 # install bootstrap tools
 RUN apt-get update && apt-get install --no-install-recommends -y \

From ebbe2203ea58b0a90c3772bd16987bf0b3cf21c2 Mon Sep 17 00:00:00 2001
From: ruffsl <roxfoxpox@gmail.com>
Date: Fri, 16 Nov 2018 19:23:02 -0800
Subject: [PATCH 2/2] Use digest to check InRelease

---
 docker_templates/packages.py                       | 14 +++++++-------
 .../create_gzserver_image.Dockerfile.em            |  9 +++++++--
 .../create_ros2_core_image.Dockerfile.em           |  9 +++++++--
 .../create_ros_core_image.Dockerfile.em            |  9 +++++++--
 4 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/docker_templates/packages.py b/docker_templates/packages.py
index 22374d9..fd64c1a 100644
--- a/docker_templates/packages.py
+++ b/docker_templates/packages.py
@@ -45,10 +45,10 @@
     'ros2_packages':    string.Template('ros-$ros2distro_name-$package'),
 }
 
-packageReleaseLookup = {
-    'gazebo_packages':  string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/Release'),
-    'ros_packages':     string.Template('http://packages.ros.org/$release/ubuntu/dists/$os_code_name/Release'),
-    'ros2_packages':    string.Template('http://repo.ros2.org/$os_name/main/dists/$os_code_name/Release'),
+packageInReleaseLookup = {
+    'gazebo_packages':  string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/InRelease'),
+    'ros_packages':     string.Template('http://packages.ros.org/$release/ubuntu/dists/$os_code_name/InRelease'),
+    'ros2_packages':    string.Template('http://repo.ros2.org/$os_name/main/dists/$os_code_name/InRelease'),
 }
 
 def getPackageIndex(data, package_index_url):
@@ -132,6 +132,6 @@ def expandPackages(data):
             package_index = getPackageIndex(data, package_index_url)
             package_versions = getPackageVersions(data, package_index, data[package_type], package_type)
             data[package_type] = package_versions
-            package_release_url_template = packageReleaseLookup[package_type]
-            package_release_url = package_release_url_template.substitute(data)
-            data[package_type + '_token'] = getPackageReleaseToken(data, package_release_url)
+            package_inrelease_url_template = packageInReleaseLookup[package_type]
+            package_inrelease_url = package_inrelease_url_template.substitute(data)
+            data[package_type + '_token'] = getPackageReleaseToken(data, package_inrelease_url)
diff --git a/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em b/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
index 515570d..5a77fd2 100644
--- a/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_gzserver_image.Dockerfile.em
@@ -41,8 +41,13 @@ RUN . /etc/os-release \
 @[  if gazebo_packages_token]@
 
 # break build cache for sync
-RUN echo "Release: @(gazebo_packages_token['date'])@ " \
-    && echo "Digest: @(gazebo_packages_token['digest'])@ "
+RUN apt-get update \
+    && . /etc/os-release \
+    && echo "Release: @(gazebo_packages_token['date'])@ " \
+    && export SUM=@(gazebo_packages_token['digest'])@  \
+    && export FILE=/var/lib/apt/lists/packages.osrfoundation.org_gazebo_$ID-stable_dists_$(lsb_release -sc)_InRelease \
+    && echo "$SUM *$FILE" | sha256sum --check \
+    && rm -rf /var/lib/apt/lists/*
 @[  end if]@
 @[end if]@
 
diff --git a/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
index 2279043..509b711 100644
--- a/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_ros2_core_image.Dockerfile.em
@@ -46,8 +46,13 @@ RUN . /etc/os-release \
 @[  if ros2_packages_token]@
 
 # break build cache for sync
-RUN echo "Release: @(ros2_packages_token['date'])@ " \
-    && echo "Digest: @(ros2_packages_token['digest'])@ "
+RUN apt-get update \
+    && . /etc/os-release \
+    && echo "Release: @(ros2_packages_token['date'])@ " \
+    && export SUM=@(ros2_packages_token['digest'])@  \
+    && export FILE=/var/lib/apt/lists/repo.ros2.org_ubuntu_main_dists_$(lsb_release -sc)_InRelease \
+    && echo "$SUM *$FILE" | sha256sum --check \
+    && rm -rf /var/lib/apt/lists/*
 @[  end if]@
 @[end if]@
 
diff --git a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
index 9fd5cfd..0fe4ccb 100644
--- a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
+++ b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em
@@ -42,8 +42,13 @@ RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/
 @[  if ros_packages_token]@
 
 # break build cache for sync
-RUN echo "Release: @(ros_packages_token['date'])@ " \
-    && echo "Digest: @(ros_packages_token['digest'])@ "
+RUN apt-get update \
+    && . /etc/os-release \
+    && echo "Release: @(ros_packages_token['date'])@ " \
+    && export SUM=@(ros_packages_token['digest'])@  \
+    && export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
+    && echo "$SUM *$FILE" | sha256sum --check \
+    && rm -rf /var/lib/apt/lists/*
 @[  end if]@
 @[end if]@