diff --git a/.github/workflows/bakery-build-native.yml b/.github/workflows/bakery-build-native.yml index 858c1b51..8740d6a2 100644 --- a/.github/workflows/bakery-build-native.yml +++ b/.github/workflows/bakery-build-native.yml @@ -200,7 +200,6 @@ jobs: - name: Build env: GIT_SHA: ${{ github.sha }} - # FIXME: Currently pushes to ghcr.io for caching. Needs to be conditional run: | PLATFORM=${BUILD_PLATFORM#linux/} \ bakery build \ @@ -212,8 +211,9 @@ jobs: --cache-registry "ghcr.io/${{ github.repository_owner }}" \ --temp-registry "ghcr.io/${{ github.repository_owner }}" \ --metadata-file "./${{ matrix.img.image }}-${{ matrix.img.version }}-${{ steps.normalize-platform.outputs.platform }}-metadata.json" \ - --context ${{ inputs.context }} \ - --push + ${{ inputs.push && '--push-cache' || '' }} \ + --push \ + --context ${{ inputs.context }} - name: Test run: | PLATFORM=${BUILD_PLATFORM#linux/} \ diff --git a/.github/workflows/bakery-build.yml b/.github/workflows/bakery-build.yml index 63b6f46f..1adcd665 100644 --- a/.github/workflows/bakery-build.yml +++ b/.github/workflows/bakery-build.yml @@ -173,13 +173,13 @@ jobs: - name: Build env: GIT_SHA: ${{ github.sha }} - # FIXME: Currently pushes to ghcr.io for caching. Needs to be conditional run: | bakery build --load \ --image-name '^${{ matrix.img.image }}$' \ --image-version ${{ matrix.img.version }} \ --dev-versions ${{ inputs.dev-versions }} \ --cache-registry "ghcr.io/${{ github.repository_owner }}" \ + ${{ inputs.push && '--push-cache' || '' }} \ --context ${{ inputs.context }} - name: Test diff --git a/posit-bakery/posit_bakery/cli/build.py b/posit-bakery/posit_bakery/cli/build.py index 0f04da36..117e81d4 100644 --- a/posit-bakery/posit_bakery/cli/build.py +++ b/posit-bakery/posit_bakery/cli/build.py @@ -80,6 +80,13 @@ def build( rich_help_panel=RichHelpPanelEnum.BUILD_CONFIGURATION_AND_OUTPUTS, ), ] = False, + push_cache: Annotated[ + Optional[bool], + typer.Option( + help="Push the build cache to the cache registry without pushing the image.", + rich_help_panel=RichHelpPanelEnum.BUILD_CONFIGURATION_AND_OUTPUTS, + ), + ] = False, clean: Annotated[ Optional[bool], typer.Option( @@ -205,6 +212,7 @@ def build( config.build_targets( load=load, push=push, + push_cache=push_cache, cache=cache, platforms=image_platform, strategy=strategy, diff --git a/posit-bakery/posit_bakery/config/config.py b/posit-bakery/posit_bakery/config/config.py index 6efaa807..d3b30411 100644 --- a/posit-bakery/posit_bakery/config/config.py +++ b/posit-bakery/posit_bakery/config/config.py @@ -734,6 +734,7 @@ def build_targets( self, load: bool = True, push: bool = False, + push_cache: bool = False, cache: bool = True, platforms: list[str] | None = None, strategy: ImageBuildStrategy = ImageBuildStrategy.BAKE, @@ -744,6 +745,7 @@ def build_targets( :param load: If True, load the built images into the local Docker daemon. :param push: If True, push the built images to the configured registries. + :param push_cache: If True, push the build cache to the cache registry without pushing images. :param cache: If True, use the build cache when building images. :param platforms: Optional list of platforms to build for. If None, builds for the configuration specified platform. @@ -752,7 +754,9 @@ def build_targets( :param fail_fast: If True, stop building targets on the first failure. """ if strategy == ImageBuildStrategy.BAKE: - bake_plan = BakePlan.from_image_targets(context=self.base_path, image_targets=self.targets) + bake_plan = BakePlan.from_image_targets( + context=self.base_path, image_targets=self.targets, push_cache=push_cache + ) set_opts = None if self.settings.temp_registry is not None and push: set_opts = { diff --git a/posit-bakery/posit_bakery/image/bake/bake.py b/posit-bakery/posit_bakery/image/bake/bake.py index 2b96870d..be710016 100644 --- a/posit-bakery/posit_bakery/image/bake/bake.py +++ b/posit-bakery/posit_bakery/image/bake/bake.py @@ -90,12 +90,19 @@ def serialize_path(value: Path | str) -> str: return str(value) @classmethod - def from_image_target(cls, image_target: ImageTarget) -> "BakeTarget": + def from_image_target(cls, image_target: ImageTarget, push_cache: bool = False) -> "BakeTarget": """Create a BakeTarget from an ImageTarget.""" kwargs = {"tags": image_target.tags} + platforms = image_target.image_os.platforms if image_target.image_os is not None else DEFAULT_PLATFORMS + if image_target.cache_name is not None: - kwargs["cache_from"] = [{"type": "registry", "ref": image_target.cache_name}] - kwargs["cache_to"] = [{"type": "registry", "ref": image_target.cache_name, "mode": "max"}] + cache_name = image_target.cache_name + # Append platform suffix to cache name + platform_suffix = "-".join(p.removeprefix("linux/").replace("/", "-") for p in platforms) + cache_name = f"{cache_name}-{platform_suffix}" + kwargs["cache_from"] = [{"type": "registry", "ref": cache_name}] + if push_cache: + kwargs["cache_to"] = [{"type": "registry", "ref": cache_name, "mode": "max"}] if image_target.temp_name is not None: kwargs["tags"] = [image_target.temp_name.rsplit(":", 1)[0]] @@ -107,7 +114,7 @@ def from_image_target(cls, image_target: ImageTarget) -> "BakeTarget": image_os=image_target.image_os.name if image_target.image_os else None, dockerfile=image_target.containerfile, labels=image_target.labels, - platforms=image_target.image_os.platforms if image_target.image_os is not None else DEFAULT_PLATFORMS, + platforms=platforms, **kwargs, ) @@ -151,11 +158,14 @@ def update_groups( return groups @classmethod - def from_image_targets(cls, context: Path, image_targets: list[ImageTarget]) -> "BakePlan": + def from_image_targets( + cls, context: Path, image_targets: list[ImageTarget], push_cache: bool = False + ) -> "BakePlan": """Create a BakePlan from a list of ImageTarget objects. :param context: The absolute path to the build context directory. :param image_targets: A list of ImageTarget objects to include in the bake plan. + :param push_cache: Whether to push build cache to the cache registry. :return: A BakePlan object containing the context, groups, and targets. """ @@ -165,7 +175,7 @@ def from_image_targets(cls, context: Path, image_targets: list[ImageTarget]) -> targets: dict[str, BakeTarget] = {} for image_target in image_targets: - bake_target = BakeTarget.from_image_target(image_target=image_target) + bake_target = BakeTarget.from_image_target(image_target=image_target, push_cache=push_cache) groups = cls.update_groups( groups=groups, uid=image_target.uid, diff --git a/posit-bakery/posit_bakery/image/image_target.py b/posit-bakery/posit_bakery/image/image_target.py index eac231d5..24909bc4 100644 --- a/posit-bakery/posit_bakery/image/image_target.py +++ b/posit-bakery/posit_bakery/image/image_target.py @@ -335,7 +335,12 @@ def build( cache_from = None cache_to = None if self.cache_name is not None: - cache_from = f"type=registry,ref={self.cache_name}" + cache_name = self.cache_name + # Append platform suffix to cache name + build_platforms = platforms or self.image_os.platforms + platform_suffix = "-".join(p.removeprefix("linux/").replace("/", "-") for p in build_platforms) + cache_name = f"{cache_name}-{platform_suffix}" + cache_from = f"type=registry,ref={cache_name}" cache_to = f"{cache_from},mode=max" if isinstance(metadata_file, bool) and metadata_file: diff --git a/posit-bakery/test/features/cli/build.feature b/posit-bakery/test/features/cli/build.feature index a41667c0..c95c7045 100644 --- a/posit-bakery/test/features/cli/build.feature +++ b/posit-bakery/test/features/cli/build.feature @@ -10,6 +10,15 @@ Feature: build Then The command succeeds * the bake plan is valid + Scenario: Generating a buildkit bake plan with push-cache flag + Given I call bakery build + * in a temp basic context + * with the arguments: + | --plan | --push-cache | + When I execute the command + Then The command succeeds + * the bake plan is valid + Scenario: Generating a buildkit bake plan with git commit Given I call bakery build * in the basic context diff --git a/posit-bakery/test/image/bake/test_bake.py b/posit-bakery/test/image/bake/test_bake.py index 70fcf0be..1f9bb922 100644 --- a/posit-bakery/test/image/bake/test_bake.py +++ b/posit-bakery/test/image/bake/test_bake.py @@ -144,6 +144,23 @@ def test_from_image_targets_with_cache_registry(self, get_expected_plan, get_con assert plan.bake_file == resource_path / suite / ".bakery-bake.json" assert expected_plan.read_text().strip() == output + @pytest.mark.parametrize("suite", SUCCESS_SUITES) + def test_from_image_targets_with_cache_registry_push_cache(self, get_config_obj, suite): + """Test that bake plans include cache_to when push_cache=True.""" + config_obj = get_config_obj(suite) + + settings = ImageTargetSettings(cache_registry="ghcr.io/posit-dev") + for target in config_obj.targets: + target.settings = settings + + plan = BakePlan.from_image_targets(config_obj.base_path, config_obj.targets, push_cache=True) + + for bake_target in plan.target.values(): + assert bake_target.cache_from is not None + assert bake_target.cache_to is not None + assert bake_target.cache_to[0]["ref"] == bake_target.cache_from[0]["ref"] + assert bake_target.cache_to[0]["mode"] == "max" + @pytest.mark.parametrize("suite", SUCCESS_SUITES) def test_from_image_targets_with_temp_registry(self, get_expected_plan, get_config_obj, suite, resource_path): """Test that bake plans generate as expected with a temp registry.""" diff --git a/posit-bakery/test/image/bake/testdata/cache_registry/barebones_plan.json b/posit-bakery/test/image/bake/testdata/cache_registry/barebones_plan.json index a94e564e..fe7dece2 100644 --- a/posit-bakery/test/image/bake/testdata/cache_registry/barebones_plan.json +++ b/posit-bakery/test/image/bake/testdata/cache_registry/barebones_plan.json @@ -40,14 +40,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/scratch/cache:1.0.0-scratch" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/scratch/cache:1.0.0-scratch", - "mode": "max" + "ref": "ghcr.io/posit-dev/scratch/cache:1.0.0-scratch-amd64" } ] } diff --git a/posit-bakery/test/image/bake/testdata/cache_registry/basic_plan.json b/posit-bakery/test/image/bake/testdata/cache_registry/basic_plan.json index 536740f4..a8748ca7 100644 --- a/posit-bakery/test/image/bake/testdata/cache_registry/basic_plan.json +++ b/posit-bakery/test/image/bake/testdata/cache_registry/basic_plan.json @@ -58,14 +58,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-min" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-min", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-min-amd64" } ] }, @@ -111,14 +104,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-std" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-std", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-image/cache:1.0.0-ubuntu-22.04-std-amd64" } ] } diff --git a/posit-bakery/test/image/bake/testdata/cache_registry/multiplatform_plan.json b/posit-bakery/test/image/bake/testdata/cache_registry/multiplatform_plan.json index df3fac4b..42441f44 100644 --- a/posit-bakery/test/image/bake/testdata/cache_registry/multiplatform_plan.json +++ b/posit-bakery/test/image/bake/testdata/cache_registry/multiplatform_plan.json @@ -58,14 +58,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-min" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-min", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-min-amd64" } ] }, @@ -100,14 +93,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-min" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-min", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-min-amd64-arm64" } ] }, @@ -141,14 +127,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-std" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-std", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-22.04-std-amd64" } ] }, @@ -187,14 +166,7 @@ "cache_from": [ { "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-std" - } - ], - "cache_to": [ - { - "type": "registry", - "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-std", - "mode": "max" + "ref": "ghcr.io/posit-dev/test-multi/cache:1.0.0-ubuntu-24.04-std-amd64-arm64" } ] } diff --git a/posit-bakery/test/image/test_image_target.py b/posit-bakery/test/image/test_image_target.py index b39a9fc4..751aabd8 100644 --- a/posit-bakery/test/image/test_image_target.py +++ b/posit-bakery/test/image/test_image_target.py @@ -398,6 +398,10 @@ def test_build_args(self, basic_standard_image_target): def test_build_args_cache_registry(self, basic_standard_image_target): """Test the build property of an ImageTarget.""" basic_standard_image_target.settings = ImageTargetSettings(cache_registry="ghcr.io/posit-dev") + # Cache name includes platform suffix + platforms = basic_standard_image_target.image_os.platforms + platform_suffix = "-".join(p.removeprefix("linux/").replace("/", "-") for p in platforms) + cache_name_with_platform = f"{basic_standard_image_target.cache_name}-{platform_suffix}" expected_build_args = { "context_path": basic_standard_image_target.context.base_path, "file": basic_standard_image_target.containerfile, @@ -407,10 +411,10 @@ def test_build_args_cache_registry(self, basic_standard_image_target): "push": False, "output": {}, "cache": True, - "cache_from": f"type=registry,ref={basic_standard_image_target.cache_name}", - "cache_to": f"type=registry,ref={basic_standard_image_target.cache_name},mode=max", + "cache_from": f"type=registry,ref={cache_name_with_platform}", + "cache_to": f"type=registry,ref={cache_name_with_platform},mode=max", "metadata_file": None, - "platforms": ["linux/amd64"], + "platforms": platforms, } with patch("python_on_whales.docker.build") as mock_build: