From 271cf42cd32af1d77ee4dee9a6482c70c97b66c5 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 15:26:48 +0100 Subject: [PATCH 01/10] feat(justfile): pick latest git release via version="latest-git-tag" --- justfile | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/justfile b/justfile index 94594c0..985d24e 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,14 @@ version := '' image := 'ghcr.io/linkerd/dev' -_tag := if version != '' { "--tag=" + image + ':' + version } else { "" } + +# Auto-detect latest git version if version is 'latest-git-tag' +_version := if version == 'latest-git-tag' { + shell('which git > /dev/null || (echo >&2 "$1error$2: git not available" && exit 1) && git --git-dir="$3/.git" --work-tree="$3" tag -l --sort=-version:refname "v*" | head -n 1', style('error'), NORMAL, justfile_directory()) +} else { + version +} + +_tag := if _version != '' { "--tag=" + image + ':' + _version } else { "" } k3s-image := 'docker.io/rancher/k3s' @@ -26,7 +34,7 @@ build: && _list-if-load for tgt in {{ targets }} ; do just output='{{ output }}' \ image='{{ image }}' \ - version='{{ version }}' \ + version='{{ _version }}' \ _target "$tgt" done @@ -36,22 +44,22 @@ _list-if-load: if [ '{{ load }}' = 'true' ] ; then just image='{{ image }}' \ targets='{{ targets }}' \ - version='{{ version }}' \ + version='{{ _version }}' \ list fi list: #!/usr/bin/env bash set -euo pipefail - if [ -z '{{ version }}' ]; then + if [ -z '{{ _version }}' ]; then echo "Usage: just version= list" >&2 exit 64 fi for tgt in {{ targets }} ; do if [ "$tgt" == "devcontainer" ]; then - docker image ls {{ image }}:{{ version }} | sed 1d + docker image ls {{ image }}:{{ _version }} | sed 1d else - docker image ls {{ image }}:{{ version }}-$tgt | sed 1d + docker image ls {{ image }}:{{ _version }}-$tgt | sed 1d fi done @@ -100,8 +108,9 @@ _target target='': @just \ output='{{ output }}' \ image='{{ image }}' \ + version='{{ _version }}' \ _build --target='{{ target }}' \ - {{ if version == '' { '' } else { '--tag=' + image + ':' + version + if target == 'devcontainer' { '' } else { '-' + target } } }} + {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} # Build the devcontainer image _build *args='': From e570d608fb3eae144a4db4bb066c0e2654630497 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 18:55:25 +0100 Subject: [PATCH 02/10] feat(justfile): allow passing in additional docker args to build --- justfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index 985d24e..816aa75 100644 --- a/justfile +++ b/justfile @@ -28,14 +28,15 @@ export DOCKER_PROGRESS := env_var_or_default('DOCKER_PROGRESS', 'auto') all: sync-k3s-images build -build: && _list-if-load +build *args='': && _list-if-load #!/usr/bin/env bash set -euo pipefail for tgt in {{ targets }} ; do just output='{{ output }}' \ image='{{ image }}' \ version='{{ _version }}' \ - _target "$tgt" + _target "$tgt" \ + {{ args }} done _list-if-load: @@ -104,13 +105,14 @@ _k3s-channels: | {key:.id, value:$tag} ] | from_entries' -_target target='': +_target target='' *args='': @just \ output='{{ output }}' \ image='{{ image }}' \ version='{{ _version }}' \ _build --target='{{ target }}' \ - {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} + {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} \ + {{ args }} # Build the devcontainer image _build *args='': From 05aca3a5218046fca37d7c78f66221a9092fc867 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 15:44:11 +0100 Subject: [PATCH 03/10] feat(Dockerfile): install arch-specific versions of tools --- Dockerfile | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index be1f776..869f679 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,20 +39,22 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update # j5j Turns JSON5 into plain old JSON (i.e. to be processed by jq). FROM apt-base as j5j -ARG J5J_VERSION=v0.2.0 # repo=olix0r/j5j -RUN url="https://github.com/olix0r/j5j/releases/download/${J5J_VERSION}/j5j-${J5J_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ +ARG J5J_VERSION=v0.2.1 # repo=unleashed/j5j +RUN arch=$(uname -m); \ + url="https://github.com/unleashed/j5j/releases/download/${J5J_VERSION}/j5j-${J5J_VERSION}-${arch}-unknown-linux-musl.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin j5j # just runs build/test recipes. Like `make` but a bit more ergonomic. FROM apt-base as just ARG JUST_VERSION=1.43.0 # repo=casey/just -RUN url="https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ +RUN url="https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-$(uname -m)-unknown-linux-musl.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin just # yq is kind of like jq, but for YAML. FROM apt-base as yq ARG YQ_VERSION=v4.47.2 # repo=mikefarah/yq -RUN url="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/' -e 's/x86_/amd/'); \ + url="https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${arch}" ; \ scurl -o /yq "$url" && chmod +x /yq FROM scratch as tools-script @@ -68,20 +70,23 @@ COPY --link bin/scurl /bin/ # helm templates kubernetes manifests. FROM apt-base as helm ARG HELM_VERSION=v3.19.0 # repo=helm/helm -RUN url="https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" ; \ - scurl "$url" | tar xzvf - --strip-components=1 -C /usr/local/bin linux-amd64/helm +RUN arch=$(uname -m | sed -e 's/aarch/arm/' -e 's/x86_/amd/'); \ + url="https://get.helm.sh/helm-${HELM_VERSION}-linux-${arch}.tar.gz" ; \ + scurl "$url" | tar xzvf - --strip-components=1 -C /usr/local/bin linux-${arch}/helm # helm-docs generates documentation from helm charts. FROM apt-base as helm-docs ARG HELM_DOCS_VERSION=v1.14.2 # repo=norwoodj/helm-docs -RUN url="https://github.com/norwoodj/helm-docs/releases/download/$HELM_DOCS_VERSION/helm-docs_${HELM_DOCS_VERSION#v}_Linux_x86_64.tar.gz" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/'); \ + url="https://github.com/norwoodj/helm-docs/releases/download/$HELM_DOCS_VERSION/helm-docs_${HELM_DOCS_VERSION#v}_Linux_${arch}.tar.gz" ; \ scurl "$url" | tar xzvf - -C /usr/local/bin helm-docs # kubectl controls kubernetes clusters. FROM apt-base as kubectl ARG KUBECTL_VERSION=v1.34.1 # repo=kubernetes/kubernetes -RUN url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/'); \ + url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${arch}/kubectl" ; \ scurl -o /usr/local/bin/kubectl "$url" && chmod +x /usr/local/bin/kubectl # k3d runs kubernetes clusters in docker. @@ -116,7 +121,8 @@ COPY --link --from=ghcr.io/anchore/grype:v0.96.1 /grype /bin/ # actionlint lints github actions workflows. FROM apt-base as actionlint ARG ACTIONLINT_VERSION=v1.7.7 # repo=rhysd/actionlint -RUN url="https://github.com/rhysd/actionlint/releases/download/${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION#v}_linux_amd64.tar.gz" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/arm/' -e 's/x86_/amd/'); \ + url="https://github.com/rhysd/actionlint/releases/download/${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION#v}_linux_${arch}.tar.gz" ; \ scurl "$url" | tar xzvf - -C /usr/local/bin actionlint # checksec checks binaries for security issues. @@ -137,7 +143,8 @@ COPY --link bin/action-* bin/just-dev bin/just-sh /bin/ FROM apt-base as protobuf ARG PROTOC_VERSION=v32.1 # repo=protocolbuffers/protobuf -RUN url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/protoc-${PROTOC_VERSION#v}-linux-$(uname -m).zip" ; \ +RUN arch=$(uname -m | sed -e 's/aarch/aarch_/'); \ + url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/protoc-${PROTOC_VERSION#v}-linux-${arch}.zip" ; \ cd $(mktemp -d) && \ scurl -o protoc.zip "$url" && \ unzip protoc.zip bin/protoc include/** && \ @@ -153,30 +160,36 @@ RUN url="https://github.com/google/protobuf/releases/download/$PROTOC_VERSION/pr # cargo-action-fmt formats `cargo build` JSON output to Github Actions annotations. FROM apt-base as cargo-action-fmt ARG CARGO_ACTION_FMT_VERSION=v1.0.4 # ignore -RUN url="https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-${CARGO_ACTION_FMT_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ +RUN arch=$(uname -m); \ + url="https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-${CARGO_ACTION_FMT_VERSION}-${arch}-unknown-linux-musl.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin cargo-action-fmt FROM apt-base as cargo-auditable -ARG CARGO_AUDITABLE_VERSION=v0.6.6 # repo=rust-secure-code/cargo-auditable -RUN url="https://github.com/rust-secure-code/cargo-auditable/releases/download/${CARGO_AUDITABLE_VERSION}/cargo-auditable-x86_64-unknown-linux-gnu.tar.xz" ; \ - scurl "$url" | tar xJvf - --strip-components=1 -C /usr/local/bin cargo-auditable-x86_64-unknown-linux-gnu/cargo-auditable +ARG CARGO_AUDITABLE_VERSION=v0.7.2 # repo=rust-secure-code/cargo-auditable +RUN arch=$(uname -m); \ + libc=$([ "$arch" = "x86_64" ] && echo "musl" || echo "gnu"); \ + url="https://github.com/rust-secure-code/cargo-auditable/releases/download/${CARGO_AUDITABLE_VERSION}/cargo-auditable-${arch}-unknown-linux-${libc}.tar.xz" ; \ + scurl "$url" | tar xJvf - --strip-components=1 -C /usr/local/bin cargo-auditable-${arch}-unknown-linux-${libc}/cargo-auditable # cargo-deny checks cargo dependencies for licensing and RUSTSEC security issues. FROM apt-base as cargo-deny ARG CARGO_DENY_VERSION=0.18.5 # repo=EmbarkStudios/cargo-deny -RUN url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${CARGO_DENY_VERSION}/cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" ; \ - scurl "$url" | tar zvxf - --strip-components=1 -C /usr/local/bin "cargo-deny-${CARGO_DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" +RUN arch=$(uname -m); \ + url="https://github.com/EmbarkStudios/cargo-deny/releases/download/${CARGO_DENY_VERSION}/cargo-deny-${CARGO_DENY_VERSION}-${arch}-unknown-linux-musl.tar.gz" ; \ + scurl "$url" | tar zvxf - --strip-components=1 -C /usr/local/bin "cargo-deny-${CARGO_DENY_VERSION}-${arch}-unknown-linux-musl/cargo-deny" # cargo-nextest is a nicer test runner. FROM apt-base as cargo-nextest ARG NEXTEST_VERSION=0.9.104 # repo=nextest-rs/nextest,prefix=cargo-nextest- -RUN url="https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${NEXTEST_VERSION}/cargo-nextest-${NEXTEST_VERSION}-x86_64-unknown-linux-gnu.tar.gz" ; \ +RUN arch=$(uname -m); \ + url="https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-${NEXTEST_VERSION}/cargo-nextest-${NEXTEST_VERSION}-${arch}-unknown-linux-gnu.tar.gz" ; \ scurl "$url" | tar zvxf - -C /usr/local/bin cargo-nextest # cargo-tarpaulin is a code coverage tool. FROM apt-base as cargo-tarpaulin ARG CARGO_TARPAULIN_VERSION=0.32.8 # repo=xd009642/tarpaulin -RUN url="https://github.com/xd009642/tarpaulin/releases/download/${CARGO_TARPAULIN_VERSION}/cargo-tarpaulin-x86_64-unknown-linux-musl.tar.gz" ;\ +RUN arch=$(uname -m); \ + url="https://github.com/xd009642/tarpaulin/releases/download/${CARGO_TARPAULIN_VERSION}/cargo-tarpaulin-${arch}-unknown-linux-musl.tar.gz" ;\ scurl "$url" | tar xzvf - -C /usr/local/bin cargo-tarpaulin FROM scratch as tools-rust From 0d1b3d635cbde86eddd5a4effc1da1c43a52f738 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 15:48:01 +0100 Subject: [PATCH 04/10] feat(Dockerfile): install docker-compose explicitly Previously the devcontainer build infrastructure would install docker-compose in the container via scripts, but that script ended up requiring global changes to Python code in order to install docker-compose. Now we just install it here to avoid that problem. --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 869f679..c6dc861 100644 --- a/Dockerfile +++ b/Dockerfile @@ -395,6 +395,12 @@ RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ ENV CC=clang-19 \ CXX=clang++-19 +# Install docker-compose since it breaks in the docker-debian script on arm64 +RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/apt \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose + # Use microsoft's Docker setup script to install the Docker CLI. # # A distinct cache is used because the script adds an apt repo that we don't From 311be4ce29c0bb22d481cb1b3ceb997ed0e53411 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 15:59:17 +0100 Subject: [PATCH 05/10] feat(justfile): support specifying architecture via docker_arch --- justfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/justfile b/justfile index 816aa75..fbc1200 100644 --- a/justfile +++ b/justfile @@ -12,6 +12,8 @@ _tag := if _version != '' { "--tag=" + image + ':' + _version } else { "" } k3s-image := 'docker.io/rancher/k3s' +docker_arch := '' + targets := 'go rust rust-musl tools devcontainer' load := 'false' @@ -35,6 +37,7 @@ build *args='': && _list-if-load just output='{{ output }}' \ image='{{ image }}' \ version='{{ _version }}' \ + docker_arch='{{ docker_arch }}' \ _target "$tgt" \ {{ args }} done @@ -110,6 +113,7 @@ _target target='' *args='': output='{{ output }}' \ image='{{ image }}' \ version='{{ _version }}' \ + docker_arch='{{ docker_arch }}' \ _build --target='{{ target }}' \ {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} \ {{ args }} @@ -119,6 +123,7 @@ _build *args='': docker buildx build . {{ _tag }} --pull \ --progress='{{ DOCKER_PROGRESS }}' \ --output='{{ output }}' \ + {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ {{ args }} From 0cb7db24215cd543d6a01ccbf43654741bbbae98 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 16:16:01 +0100 Subject: [PATCH 06/10] feat(justfile): use dry_run=true to print but avoid executing docker commands --- justfile | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index fbc1200..b0aa16b 100644 --- a/justfile +++ b/justfile @@ -12,6 +12,7 @@ _tag := if _version != '' { "--tag=" + image + ':' + _version } else { "" } k3s-image := 'docker.io/rancher/k3s' +dry_run := 'false' docker_arch := '' targets := 'go rust rust-musl tools devcontainer' @@ -38,6 +39,7 @@ build *args='': && _list-if-load image='{{ image }}' \ version='{{ _version }}' \ docker_arch='{{ docker_arch }}' \ + dry_run='{{ dry_run }}' \ _target "$tgt" \ {{ args }} done @@ -61,10 +63,17 @@ list: fi for tgt in {{ targets }} ; do if [ "$tgt" == "devcontainer" ]; then - docker image ls {{ image }}:{{ _version }} | sed 1d + cmd="docker image ls {{ image }}:{{ _version }} | sed 1d" else - docker image ls {{ image }}:{{ _version }}-$tgt | sed 1d + cmd="docker image ls {{ image }}:{{ _version }}-$tgt | sed 1d" fi + + echo "{{ style('error') }}$cmd{{ NORMAL }}" + if [ "{{ dry_run }}" = "true" ]; then + continue + fi + + eval "$cmd" done # Fetch the latest version of k3s images and record their tags and digests. @@ -114,17 +123,28 @@ _target target='' *args='': image='{{ image }}' \ version='{{ _version }}' \ docker_arch='{{ docker_arch }}' \ + dry_run='{{ dry_run }}' \ _build --target='{{ target }}' \ {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} \ {{ args }} # Build the devcontainer image _build *args='': - docker buildx build . {{ _tag }} --pull \ + #!/usr/bin/env bash + set -euo pipefail + + cmd="docker buildx build . {{ _tag }} --pull \ --progress='{{ DOCKER_PROGRESS }}' \ --output='{{ output }}' \ {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ - {{ args }} + {{ args }}" + + echo "{{ style('error') }}$cmd{{ NORMAL }}" + if [ "{{ dry_run }}" = "true" ]; then + exit 0 + fi + + eval "$cmd" md-lint *patterns="'**/*.md' '!repos/**'": From be11c0ce1e53bbf517b3fc0d829f85573f3a7a31 Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 16:25:43 +0100 Subject: [PATCH 07/10] feat(justfile): add podman support --- justfile | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/justfile b/justfile index b0aa16b..047952c 100644 --- a/justfile +++ b/justfile @@ -15,17 +15,32 @@ k3s-image := 'docker.io/rancher/k3s' dry_run := 'false' docker_arch := '' +# Detect docker_bin if not specified: try docker first, then podman +docker_bin := shell('which docker 2> /dev/null || which podman 2> /dev/null || (echo >&2 "$1error$2: neither docker nor podman found" && exit 1)', style('error'), NORMAL) +# Extract basename of docker_bin to identify implementation +_docker_bin_name := file_name(docker_bin) +# Auto-detect if podman is in remote mode (unless explicitly overridden) +podman_remote := if _docker_bin_name == 'podman' { + shell('if $1 info 2>/dev/null | grep -q "remoteSocket"; then echo "true"; else echo "false"; fi', docker_bin) +} else { + 'false' +} + targets := 'go rust rust-musl tools devcontainer' load := 'false' push := 'false' -output := if push == 'true' { - 'type=registry' - } else if load == 'true' { - 'type=docker' - } else { - 'type=image' - } + +# Remote mode cannot use the --output flag +output := if podman_remote == 'true' { + '' +} else if push == 'true' { + '--output=type=registry' +} else if load == 'true' { + '--output=type=docker' +} else { + '--output=type=image' +} export DOCKER_PROGRESS := env_var_or_default('DOCKER_PROGRESS', 'auto') @@ -40,6 +55,8 @@ build *args='': && _list-if-load version='{{ _version }}' \ docker_arch='{{ docker_arch }}' \ dry_run='{{ dry_run }}' \ + docker_bin='{{ docker_bin }}' \ + podman_remote='{{ podman_remote }}' \ _target "$tgt" \ {{ args }} done @@ -63,9 +80,9 @@ list: fi for tgt in {{ targets }} ; do if [ "$tgt" == "devcontainer" ]; then - cmd="docker image ls {{ image }}:{{ _version }} | sed 1d" + cmd="{{ docker_bin }} image ls {{ image }}:{{ _version }} | sed 1d" else - cmd="docker image ls {{ image }}:{{ _version }}-$tgt | sed 1d" + cmd="{{ docker_bin }} image ls {{ image }}:{{ _version }}-$tgt | sed 1d" fi echo "{{ style('error') }}$cmd{{ NORMAL }}" @@ -124,6 +141,8 @@ _target target='' *args='': version='{{ _version }}' \ docker_arch='{{ docker_arch }}' \ dry_run='{{ dry_run }}' \ + docker_bin='{{ docker_bin }}' \ + podman_remote='{{ podman_remote }}' \ _build --target='{{ target }}' \ {{ if _version == '' { '' } else { '--tag=' + image + ':' + _version + if target == 'devcontainer' { '' } else { '-' + target } } }} \ {{ args }} @@ -133,9 +152,9 @@ _build *args='': #!/usr/bin/env bash set -euo pipefail - cmd="docker buildx build . {{ _tag }} --pull \ + cmd="{{ docker_bin }} buildx build . {{ _tag }} --pull \ --progress='{{ DOCKER_PROGRESS }}' \ - --output='{{ output }}' \ + {{ output }} \ {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ {{ args }}" From 5ae7231fb306bd46399857d77d0c03b1b9beb50c Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 16:35:57 +0100 Subject: [PATCH 08/10] feat(justfile): add pull_policy argument so we avoid hitting limits This is set to never with podman since I'm still experimenting. --- justfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 047952c..64a0dec 100644 --- a/justfile +++ b/justfile @@ -26,6 +26,13 @@ podman_remote := if _docker_bin_name == 'podman' { 'false' } +# pull policy +_pull_policy := if _docker_bin_name == 'podman' { + '=never' +} else { + '' +} + targets := 'go rust rust-musl tools devcontainer' load := 'false' @@ -152,7 +159,7 @@ _build *args='': #!/usr/bin/env bash set -euo pipefail - cmd="{{ docker_bin }} buildx build . {{ _tag }} --pull \ + cmd="{{ docker_bin }} buildx build . {{ _tag }} --pull{{ _pull_policy }} \ --progress='{{ DOCKER_PROGRESS }}' \ {{ output }} \ {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ From 4ddb3a5f1a829904ba9613c4a73049dc5186c13e Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 15:44:57 +0100 Subject: [PATCH 09/10] feat(justfile, Dockerfile): make apt cache sharing configurable This is useful to avoid locking the cache, which is desirable if using buildah, since it has had bugs around this feature. The tradeoff is that you could potentially find locking issues if running multiple parallel apt commands (such as in CI). --- Dockerfile | 38 ++++++++++++++++++++------------------ justfile | 8 ++++++++ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6dc861..4b2ab8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -277,11 +277,13 @@ COPY --link --from=tools-script /bin/* /bin/ ## Base images ## +ARG APT_CACHE_SHARING=",sharing=locked" + # A Go build environment. FROM docker.io/library/golang:${GO_TAG} as go RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y file jo jq COPY --link --from=tools-script /bin/* /usr/local/bin/ COPY --link --from=tools-go /bin/* /usr/local/bin/ @@ -294,8 +296,8 @@ ENV PROTOC_NO_VENDOR=1 \ # A Rust build environment. FROM docker.io/library/rust:${RUST_TAG}-slim-bookworm as rust RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ cmake \ curl \ @@ -307,8 +309,8 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ libssl-dev \ pkg-config RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y clang-19 llvm-19 RUN rustup component add clippy rustfmt COPY --link --from=tools-lint /bin/checksec /usr/local/bin/ @@ -332,8 +334,8 @@ RUN rustup target add \ aarch64-unknown-linux-musl \ x86_64-unknown-linux-musl RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ binutils-aarch64-linux-gnu \ g++-aarch64-linux-gnu \ @@ -346,8 +348,8 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ FROM docker.io/library/debian:bookworm as devcontainer RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ cmake \ curl \ @@ -384,13 +386,13 @@ RUN groupadd --gid=1000 code \ # git v2.34+ has new subcommands and supports code signing via SSH. RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y -t bookworm-backports git RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y clang-19 llvm-19 ENV CC=clang-19 \ CXX=clang++-19 @@ -408,16 +410,16 @@ RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/a # # TODO(ver): replace this with a devcontainer feature? RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/apt \ - --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ --mount=type=bind,from=tools,source=/bin/scurl,target=/usr/local/bin/scurl \ scurl https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/docker-debian.sh | bash -s ENV DOCKER_BUILDKIT=1 ARG MARKDOWNLINT_VERSION=0.15.0 RUN --mount=type=cache,from=apt-node,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-node,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,from=apt-node,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,from=apt-node,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-node,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs RUN npm install "markdownlint-cli2@${MARKDOWNLINT_VERSION}" --global diff --git a/justfile b/justfile index 64a0dec..9ee48af 100644 --- a/justfile +++ b/justfile @@ -33,6 +33,13 @@ _pull_policy := if _docker_bin_name == 'podman' { '' } +# apt cache sharing mode hits a bug in buildah +_apt_cache_sharing := if _docker_bin_name == 'podman' { + '' +} else { + ',sharing=locked' +} + targets := 'go rust rust-musl tools devcontainer' load := 'false' @@ -163,6 +170,7 @@ _build *args='': --progress='{{ DOCKER_PROGRESS }}' \ {{ output }} \ {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ + --build-arg APT_CACHE_SHARING={{ _apt_cache_sharing }} \ {{ args }}" echo "{{ style('error') }}$cmd{{ NORMAL }}" From 7614164536d251538357359421892909e8c88f4d Mon Sep 17 00:00:00 2001 From: Alejandro Martinez Ruiz Date: Wed, 19 Nov 2025 19:06:07 +0100 Subject: [PATCH 10/10] Revert "feat(justfile, Dockerfile): make apt cache sharing configurable" This reverts commit 27ab1a5333cc5c5af743c6878f42da23aa5050bf. --- Dockerfile | 38 ++++++++++++++++++-------------------- justfile | 8 -------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b2ab8d..c6dc861 100644 --- a/Dockerfile +++ b/Dockerfile @@ -277,13 +277,11 @@ COPY --link --from=tools-script /bin/* /bin/ ## Base images ## -ARG APT_CACHE_SHARING=",sharing=locked" - # A Go build environment. FROM docker.io/library/golang:${GO_TAG} as go RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y file jo jq COPY --link --from=tools-script /bin/* /usr/local/bin/ COPY --link --from=tools-go /bin/* /usr/local/bin/ @@ -296,8 +294,8 @@ ENV PROTOC_NO_VENDOR=1 \ # A Rust build environment. FROM docker.io/library/rust:${RUST_TAG}-slim-bookworm as rust RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ cmake \ curl \ @@ -309,8 +307,8 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ libssl-dev \ pkg-config RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y clang-19 llvm-19 RUN rustup component add clippy rustfmt COPY --link --from=tools-lint /bin/checksec /usr/local/bin/ @@ -334,8 +332,8 @@ RUN rustup target add \ aarch64-unknown-linux-musl \ x86_64-unknown-linux-musl RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ binutils-aarch64-linux-gnu \ g++-aarch64-linux-gnu \ @@ -348,8 +346,8 @@ RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ FROM docker.io/library/debian:bookworm as devcontainer RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ cmake \ curl \ @@ -386,13 +384,13 @@ RUN groupadd --gid=1000 code \ # git v2.34+ has new subcommands and supports code signing via SSH. RUN --mount=type=cache,from=apt-base,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y -t bookworm-backports git RUN --mount=type=cache,from=apt-llvm,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-llvm,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-llvm,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y clang-19 llvm-19 ENV CC=clang-19 \ CXX=clang++-19 @@ -410,16 +408,16 @@ RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/a # # TODO(ver): replace this with a devcontainer feature? RUN --mount=type=cache,id=apt-docker,from=apt-base,source=/etc/apt,target=/etc/apt \ - --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=apt-docker,from=apt-base,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ --mount=type=bind,from=tools,source=/bin/scurl,target=/usr/local/bin/scurl \ scurl https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/docker-debian.sh | bash -s ENV DOCKER_BUILDKIT=1 ARG MARKDOWNLINT_VERSION=0.15.0 RUN --mount=type=cache,from=apt-node,source=/etc/apt,target=/etc/apt,ro \ - --mount=type=cache,from=apt-node,source=/var/cache/apt,target=/var/cache/apt${APT_CACHE_SHARING} \ - --mount=type=cache,from=apt-node,source=/var/lib/apt/lists,target=/var/lib/apt/lists${APT_CACHE_SHARING} \ + --mount=type=cache,from=apt-node,source=/var/cache/apt,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,from=apt-node,source=/var/lib/apt/lists,target=/var/lib/apt/lists,sharing=locked \ DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs RUN npm install "markdownlint-cli2@${MARKDOWNLINT_VERSION}" --global diff --git a/justfile b/justfile index 9ee48af..64a0dec 100644 --- a/justfile +++ b/justfile @@ -33,13 +33,6 @@ _pull_policy := if _docker_bin_name == 'podman' { '' } -# apt cache sharing mode hits a bug in buildah -_apt_cache_sharing := if _docker_bin_name == 'podman' { - '' -} else { - ',sharing=locked' -} - targets := 'go rust rust-musl tools devcontainer' load := 'false' @@ -170,7 +163,6 @@ _build *args='': --progress='{{ DOCKER_PROGRESS }}' \ {{ output }} \ {{ if docker_arch != '' { '--platform=' + docker_arch } else { '' } }} \ - --build-arg APT_CACHE_SHARING={{ _apt_cache_sharing }} \ {{ args }}" echo "{{ style('error') }}$cmd{{ NORMAL }}"