From 98392b55f860b3d9f60101f124ea3d08e03f5be9 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 27 Apr 2021 13:55:23 +0800 Subject: [PATCH] Add apt/pypi/npm mirror supports --- create_web_container.sh | 13 +++++++++---- emu/cloud_build.py | 2 +- emu/containers/emulator_container.py | 3 ++- emu/emu_docker.py | 22 ++++++++++++++++++++-- emu/templates/Dockerfile | 18 ++++++++++-------- emu/templates/Dockerfile.emulator | 16 +++++++++------- js/docker/nginx.Dockerfile | 4 ++++ js/jwt-provider/Dockerfile | 17 ++++++++++++++--- 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/create_web_container.sh b/create_web_container.sh index bc335acd..975db536 100755 --- a/create_web_container.sh +++ b/create_web_container.sh @@ -15,7 +15,7 @@ DOCKER_YAML=js/docker/docker-compose-build.yaml PASSWDS="$USER,hello" - +COMPOSE_BUILD_ARGS="" # Fancy colors in the terminal if [ -t 1 ]; then RED=$(tput setaf 1) @@ -51,6 +51,9 @@ help() { -s start the container after creation. -p list of username password pairs. Defaults to: [${PASSWDS}] -i install systemd service, with definition in /opt/emulator + -n npm mirror registry. + -P python package mirror index. + -A deb apt source mirror. EOF exit 1 } @@ -73,18 +76,20 @@ generate_keys() { fi } -while getopts 'hasip:' flag; do +while getopts 'hasip:n:P:A:' flag; do case "${flag}" in a) DOCKER_YAML="${DOCKER_YAML} -f js/docker/development.yaml" ;; p) PASSWDS="${OPTARG}" ;; h) help ;; s) START='yes' ;; i) INSTALL='yes' ;; + n) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg npm_mirror=${OPTARG}" ;; + P) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg pip_index=${OPTARG}" ;; + A) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg apt_repo_mirror=${OPTARG}" ;; *) help ;; esac done - # Create the javascript protobufs make -C js deps @@ -105,7 +110,7 @@ cp ~/.android/adbkey js/docker/certs # compose the container pip install docker-compose >/dev/null -docker-compose -f ${DOCKER_YAML} build +docker-compose -f ${DOCKER_YAML} build ${COMPOSE_BUILD_ARGS} rm js/docker/certs/adbkey if [ "${START}" = "yes" ]; then diff --git a/emu/cloud_build.py b/emu/cloud_build.py index 456a7e88..80fce8ef 100644 --- a/emu/cloud_build.py +++ b/emu/cloud_build.py @@ -94,7 +94,7 @@ def cloud_build(args): steps.append(create_build_step(system_container, args.dest)) else: for metrics in [True, False]: - emulator_container = EmulatorContainer(emu, system_container, args.repo, metrics) + emulator_container = EmulatorContainer(emu, system_container, args.repo, metrics, args.aptmirror) emulators.add(emulator_container.props["emu_build_id"]) steps.append(create_build_step(emulator_container, args.dest)) images.append(emulator_container.full_name()) diff --git a/emu/containers/emulator_container.py b/emu/containers/emulator_container.py index 22beb31c..d8335441 100644 --- a/emu/containers/emulator_container.py +++ b/emu/containers/emulator_container.py @@ -30,7 +30,7 @@ class EmulatorContainer(DockerContainer): """ NO_METRICS_MESSAGE = "No metrics are collected when running this container." - def __init__(self, emulator, system_image_container, repository=None, metrics=False, extra=""): + def __init__(self, emulator, system_image_container, repository=None, metrics=False, extra="", apt_mirror=""): self.emulator_zip = AndroidReleaseZip(emulator) self.system_image_container = system_image_container self.metrics = metrics @@ -51,6 +51,7 @@ def __init__(self, emulator, system_image_container, repository=None, metrics=Fa self.props["metrics"] = metrics_msg self.props["emu_build_id"] = self.emulator_zip.build_id() self.props["from_base_img"] = system_image_container.full_name() + self.props["run_apt_repo_replace"] = apt_mirror for expect in [ "ro.build.version.sdk", diff --git a/emu/emu_docker.py b/emu/emu_docker.py index 037806f5..62f4afa2 100644 --- a/emu/emu_docker.py +++ b/emu/emu_docker.py @@ -95,7 +95,7 @@ def create_docker_image(args): if args.sys: continue - emu_docker = EmulatorContainer(emu, sys_docker, args.repo, cfg.collect_metrics(), args.extra) + emu_docker = EmulatorContainer(emu, sys_docker, args.repo, cfg.collect_metrics(), args.extra, args.aptmirror) emu_docker.build(args.dest) if args.start: @@ -129,7 +129,7 @@ def create_docker_image_interactive(args): if not sys_docker.available() and not sys_docker.can_pull(): sys_docker.build(args.dest) - emu_docker = EmulatorContainer(emu_zip, sys_docker, args.repo, metrics) + emu_docker = EmulatorContainer(emu_zip, sys_docker, args.repo, metrics, args.aptmirror) emu_docker.build(args.dest) if args.start: @@ -220,6 +220,12 @@ def main(): "All exposed ports are forwarded, and your private adbkey (if available) is injected but not stored.", ) create_parser.add_argument("--sys", action="store_true", help="Process system image layer only.") + create_parser.add_argument( + "--aptmirror", + default="", + help="Apt repo mirror for apt installing." + 'For example http://ftp2.cn.debian.org/', + ) create_parser.set_defaults(func=create_docker_image) create_inter = subparsers.add_parser( @@ -249,6 +255,12 @@ def main(): action="store_true", help="Display arm images. Note that arm images are not hardware accelerated and are *extremely* slow.", ) + create_inter.add_argument( + "--aptmirror", + default="", + help="Apt repo mirror for apt installing. " + 'For example http://ftp2.cn.debian.org/', + ) create_inter.set_defaults(func=create_docker_image_interactive) dist_parser = subparsers.add_parser( @@ -268,6 +280,12 @@ def main(): dist_parser.add_argument( "--sys", action="store_true", help="Write system image steps, otherwise write emulator steps." ) + dist_parser.add_argument( + "--aptmirror", + default="", + help="Apt repo mirror for apt installing." + 'For example http://ftp2.cn.debian.org/', + ) dist_parser.add_argument( "emuzip", help="Zipfile containing the a publicly released emulator, or (canary|stable|[0-9]+) to use the latest canary, stable, or build id of the emulator to use. " diff --git a/emu/templates/Dockerfile b/emu/templates/Dockerfile index d3313252..d66a7d55 100644 --- a/emu/templates/Dockerfile +++ b/emu/templates/Dockerfile @@ -13,16 +13,18 @@ # limitations under the License. {{from_base_img}} +{{run_apt_repo_replace}} + # Install all the required emulator dependencies. # You can get these by running ./android/scripts/unix/run_tests.sh --verbose --verbose --debs | grep apt | sort -u # pulse audio is needed due to some webrtc dependencies. RUN apt-get update && apt-get install -y --no-install-recommends \ -# Emulator & video bridge dependencies + # Emulator & video bridge dependencies libc6 libdbus-1-3 libfontconfig1 libgcc1 \ libpulse0 libtinfo5 libx11-6 libxcb1 libxdamage1 \ libnss3 libxcomposite1 libxcursor1 libxi6 \ libxext6 libxfixes3 zlib1g libgl1 pulseaudio socat \ -# Enable turncfg through usage of curl + # Enable turncfg through usage of curl curl ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -61,11 +63,11 @@ CMD ["/android/sdk/launch-emulator.sh"] # Note we should use gRPC status endpoint to check for health once the canary release is out. HEALTHCHECK --interval=30s \ - --timeout=30s \ - --start-period=30s \ - --retries=3 \ - CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1" + --timeout=30s \ + --start-period=30s \ + --retries=3 \ + CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1" LABEL maintainer="{{user}}" \ - com.google.android.emulator.description="Pixel 2 Emulator, running API {{api}}" \ - com.google.android.emulator.version="{{tag}}-{{api}}-{{abi}}/{{emu_build_id}}" + com.google.android.emulator.description="Pixel 2 Emulator, running API {{api}}" \ + com.google.android.emulator.version="{{tag}}-{{api}}-{{abi}}/{{emu_build_id}}" diff --git a/emu/templates/Dockerfile.emulator b/emu/templates/Dockerfile.emulator index c9034220..8195e128 100644 --- a/emu/templates/Dockerfile.emulator +++ b/emu/templates/Dockerfile.emulator @@ -13,16 +13,18 @@ # limitations under the License. FROM {{from_base_img}} AS emulator +{{run_apt_repo_replace}} + # Install all the required emulator dependencies. # You can get these by running ./android/scripts/unix/run_tests.sh --verbose --verbose --debs | grep apt | sort -u # pulse audio is needed due to some webrtc dependencies. RUN apt-get update && apt-get install -y --no-install-recommends \ -# Emulator & video bridge dependencies + # Emulator & video bridge dependencies libc6 libdbus-1-3 libfontconfig1 libgcc1 \ libpulse0 libtinfo5 libx11-6 libxcb1 libxdamage1 \ libnss3 libxcomposite1 libxcursor1 libxi6 \ libxext6 libxfixes3 zlib1g libgl1 pulseaudio socat \ -# Enable turncfg through usage of curl + # Enable turncfg through usage of curl curl ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -61,10 +63,10 @@ CMD ["/android/sdk/launch-emulator.sh"] # Note we should use gRPC status endpoint to check for health once the canary release is out. HEALTHCHECK --interval=30s \ - --timeout=30s \ - --start-period=30s \ - --retries=3 \ - CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1" + --timeout=30s \ + --start-period=30s \ + --retries=3 \ + CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1" LABEL maintainer="{{user}}" \ - com.google.android.emulator.version="{{emu_build_id}}" + com.google.android.emulator.version="{{emu_build_id}}" diff --git a/js/docker/nginx.Dockerfile b/js/docker/nginx.Dockerfile index b0270445..4370ab7a 100644 --- a/js/docker/nginx.Dockerfile +++ b/js/docker/nginx.Dockerfile @@ -2,6 +2,10 @@ FROM tiangolo/node-frontend:10 as build-stage WORKDIR /app COPY package*.json /app/ +ARG npm_mirror +RUN if [ x"${npm_mirror}" != "x" ]; then \ + npm config set registry ${npm_mirror}; \ + fi RUN npm install COPY ./ /app/ ARG configuration=production diff --git a/js/jwt-provider/Dockerfile b/js/jwt-provider/Dockerfile index 66d5f181..5282dd6c 100644 --- a/js/jwt-provider/Dockerfile +++ b/js/jwt-provider/Dockerfile @@ -12,10 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM debian:stretch-slim -RUN apt-get update -y -RUN apt-get install -y python-pip python-dev build-essential +ARG apt_repo_mirror +RUN if [ x"$apt_repo_mirror" != "x" ]; then \ + sed -i -E "s#http(s)?://[^\/]+#${apt_repo_mirror}#g" /etc/apt/sources.list; \ + fi +RUN apt-get update -y && \ + apt-get install -y python-pip python-dev build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* COPY . /app WORKDIR /app -RUN pip install -r requirements.txt +ARG pip_index +RUN if [ x"$pip_index" = "x" ]; then \ + pip install -r requirements.txt; \ + else \ + pip install -i ${pip_index} -r requirements.txt; \ + fi ENTRYPOINT ["python"] CMD ["jwt-provider.py"]