From 24b2bc06c1b09c08eb983b80aef6dbff8aedb334 Mon Sep 17 00:00:00 2001 From: Derek Clair Brown Date: Mon, 17 Feb 2025 15:24:46 -0700 Subject: [PATCH 1/6] build pipeline --- .github/workflows/docker_build.yml | 103 +++++++++++++++++++++++++++++ deployment/dockerfile.appserver | 4 +- deployment/dockerfile.build | 6 +- deployment/dockerfile.tools | 4 +- 4 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/docker_build.yml diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml new file mode 100644 index 0000000..9310eca --- /dev/null +++ b/.github/workflows/docker_build.yml @@ -0,0 +1,103 @@ +name: "Appserver: Build & Push Docker Image" +run-name: Build & Push appserver/${{ github.base_ref || github.ref_name }} +on: + push: + branches: [main] +defaults: + run: + shell: bash +jobs: + build-and-push: + runs-on: ubuntu-latest + env: + REGISTRY: harbor.delivery.iqgeo.cloud/${{ vars.harbor_project_name }} + platform_version: 7.2 + steps: + - + name: Checkout code + uses: actions/checkout@v4 + with: + lfs: true + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - + name: Authenticate Docker to Harbor + uses: docker/login-action@v3 + with: + registry: harbor.delivery.iqgeo.cloud + username: ${{ secrets.container_registry_username }} + password: ${{ secrets.container_registry_password }} + + - + name: "Build: Platform" + id: platform + uses: docker/build-push-action@v6 + with: + push: false + pull: false + context: . + file: deployment/dockerfile.build + build-args: | + CONTAINER_REGISTRY=${{ env.REGISTRY }} + platforms: linux/amd64,linux/arm64 + tags: | + ${{ env.REGISTRY }}/platform-build:${{ env.platform_version }} + iqgeo-myproj-build + cache-to: type=gha,mode=max + cache-from: type=gha + + - + name: Extract Appserver metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/platform-appserver + tags: | + type=raw,value={{branch}}-{{sha}} + type=sha,prefix={{branch}}-,enable={{!is_default_branch}} + type=raw,value=${{ env.platform_version }},enable={{is_default_branch}} + + - + name: "Build: Appserver" + uses: docker/build-push-action@v6 + with: + push: true + pull: true + context: deployment + file: deployment/dockerfile.appserver + build-args: | + CONTAINER_REGISTRY=${{ env.REGISTRY }} + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + + - + name: Extract Appserver Tools metadata (tags, labels) for Docker + id: tools_metadata + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/platform-tools + tags: | + type=raw,value={{branch}}-{{sha}} + type=sha,prefix={{branch}}-,enable={{!is_default_branch}} + type=raw,value=${{ env.platform_version }},enable={{is_default_branch}} + + - + name: "Build: Appserver Tools" + uses: docker/build-push-action@v6 + with: + push: true + pull: true + context: deployment + file: deployment/dockerfile.tools + build-args: | + CONTAINER_REGISTRY=${{ env.REGISTRY }} + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.tools_metadata.outputs.tags }} + cache-from: type=gha + diff --git a/deployment/dockerfile.appserver b/deployment/dockerfile.appserver index 80b6098..99a4673 100644 --- a/deployment/dockerfile.appserver +++ b/deployment/dockerfile.appserver @@ -1,4 +1,4 @@ -ARG CONTAINER_REGISTRY=harbor.delivery.iqgeo.cloud/releases/ +ARG CONTAINER_REGISTRY=${CONTAINER_REGISTRY:-harbor.delivery.iqgeo.cloud/releases} FROM iqgeo-myproj-build AS iqgeo_builder @@ -16,7 +16,7 @@ RUN rm -rf ${MODULES}/*/node_modules \ && rm -rf ${MODULES}/*/native ############################################## project appserver image -FROM ${CONTAINER_REGISTRY}platform-appserver:7.2 +FROM ${CONTAINER_REGISTRY}/platform-appserver:7.2 USER root diff --git a/deployment/dockerfile.build b/deployment/dockerfile.build index 1398acb..ed9573f 100644 --- a/deployment/dockerfile.build +++ b/deployment/dockerfile.build @@ -1,10 +1,10 @@ -ARG CONTAINER_REGISTRY=harbor.delivery.iqgeo.cloud/releases/ +ARG CONTAINER_REGISTRY=${CONTAINER_REGISTRY:-harbor.delivery.iqgeo.cloud/releases} # START SECTION Aliases for Injector images -FROM ${CONTAINER_REGISTRY}comms:3.2 AS comms +FROM ${CONTAINER_REGISTRY}/comms:3.2 AS comms # END SECTION # Create container for building the project -FROM ${CONTAINER_REGISTRY}platform-build:7.2 +FROM ${CONTAINER_REGISTRY}/platform-build:7.2 # START SECTION Copy the modules - if you edit these lines manually note that your change will get lost if you run the IQGeo Project Update tool COPY --link custom ${MODULES}/custom diff --git a/deployment/dockerfile.tools b/deployment/dockerfile.tools index e65488c..1439148 100644 --- a/deployment/dockerfile.tools +++ b/deployment/dockerfile.tools @@ -1,4 +1,4 @@ -ARG CONTAINER_REGISTRY=harbor.delivery.iqgeo.cloud/releases/ +ARG CONTAINER_REGISTRY=${CONTAINER_REGISTRY:-harbor.delivery.iqgeo.cloud/releases} FROM iqgeo-myproj-build AS iqgeo_builder @@ -6,7 +6,7 @@ FROM iqgeo-myproj-build AS iqgeo_builder # END SECTION -FROM ${CONTAINER_REGISTRY}platform-tools:7.2 AS tools_intermediate +FROM ${CONTAINER_REGISTRY}/platform-tools:7.2 AS tools_intermediate USER root From bf3da7c6f229cc26f6bc32ee3490cec3bbd1139c Mon Sep 17 00:00:00 2001 From: Derek Clair Brown Date: Mon, 17 Feb 2025 15:26:43 -0700 Subject: [PATCH 2/6] LFC: method for secure OIDC configuration --- .github/workflows/docker_build.yml | 2 ++ deployment/dockerfile.appserver | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 9310eca..b31a3df 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -69,6 +69,8 @@ jobs: pull: true context: deployment file: deployment/dockerfile.appserver + # secret-files: | + # oidc=${{ secrets.oidc_config }} build-args: | CONTAINER_REGISTRY=${{ env.REGISTRY }} platforms: linux/amd64,linux/arm64 diff --git a/deployment/dockerfile.appserver b/deployment/dockerfile.appserver index 99a4673..8399642 100644 --- a/deployment/dockerfile.appserver +++ b/deployment/dockerfile.appserver @@ -49,3 +49,4 @@ USER www-data COPY --chown=www-data:www-data entrypoint.d/* /entrypoint.d/ COPY --chown=www-data:www-data appserver_config/ /opt/iqgeo/config/ +# RUN --mount=type=secret,id=oidc,target=/opt/iqgeo/config/oidc/conf.json From 59c15a8a16a142d92dce30a50a641da60aa46c87 Mon Sep 17 00:00:00 2001 From: Derek Clair Brown Date: Mon, 17 Feb 2025 16:35:10 -0700 Subject: [PATCH 3/6] Add QEMU setup for multi-arch builds --- .github/workflows/docker_build.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index b31a3df..72021df 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -19,10 +19,6 @@ jobs: with: lfs: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Authenticate Docker to Harbor uses: docker/login-action@v3 @@ -31,6 +27,14 @@ jobs: username: ${{ secrets.container_registry_username }} password: ${{ secrets.container_registry_password }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: "Build: Platform" id: platform From bfcb40150a5d70032871801ebcf16213c3ee1245 Mon Sep 17 00:00:00 2001 From: Derek Clair Brown Date: Thu, 18 Dec 2025 11:11:20 -0700 Subject: [PATCH 4/6] Update docker_build.yml building from https://github.com/IQGeo/project-nmti-trials/blob/dba9e106e092009df2678083b66368de944dad89/.github/workflows/sre-image-build.yml --- .github/workflows/docker_build.yml | 266 ++++++++++++++++++++++------- 1 file changed, 200 insertions(+), 66 deletions(-) diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml index 72021df..82c55ef 100644 --- a/.github/workflows/docker_build.yml +++ b/.github/workflows/docker_build.yml @@ -1,109 +1,243 @@ -name: "Appserver: Build & Push Docker Image" -run-name: Build & Push appserver/${{ github.base_ref || github.ref_name }} +name: Build & Push Docker Images +run-name: Build & Push ${{ github.base_ref || github.ref_name }} + on: push: - branches: [main] -defaults: - run: - shell: bash + branches: [ main, develop ] + pull_request: + branches: [ main ] + workflow_dispatch: + inputs: + + platform_version: + description: 'platform version' + required: true + default: '7.3' + + push_images: + description: 'Y/N push images to container registry' + type: boolean + default: true + +permissions: + id-token: write + contents: read +env: + PLATFORM_VERSION: ${{ github.event.inputs.platform_version || '7.3' }} + DOCKER_BUILD_SUMMARY: true + DOCKER_BUILD_RECORD_UPLOAD: true + jobs: - build-and-push: + + build-build-image: runs-on: ubuntu-latest env: - REGISTRY: harbor.delivery.iqgeo.cloud/${{ vars.harbor_project_name }} - platform_version: 7.2 + image_name: ${{ vars.registry }}/${{ vars.repository }}/${{ vars.image_name }}-build + outputs: + digest: ${{ steps.build.outputs.digest }} + image-id: ${{ steps.build.outputs.imageid }} + metadata: ${{ steps.build.outputs.metadata }} steps: - - name: Checkout code + name: checkout uses: actions/checkout@v4 with: lfs: true + ### `cloud-addon-modules` has been officially deprecated. + # - name: checkout `cloud-addon-modules` + # uses: actions/checkout@v4 + # with: + # repository: IQGeo/cloud-addon-modules + # path: cloud-addons + # token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT (personal access token) + # lfs: true + - - name: Authenticate Docker to Harbor + name: setup Docker BuildX + uses: docker/setup-buildx-action@v3 + - + name: login to Harbor Registry (for pulling base images) uses: docker/login-action@v3 with: - registry: harbor.delivery.iqgeo.cloud - username: ${{ secrets.container_registry_username }} - password: ${{ secrets.container_registry_password }} - - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + registry: ${{ vars.registry }} + username: ${{ vars.registry_username }} + password: ${{ secrets.registry_password }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + name: extract metadata for `build` image + id: meta-build + uses: docker/metadata-action@v5 + with: + images: ${{ env.image_name }} + tags: | + type=raw,value=${{ env.PLATFORM_VERSION }} + type=raw,value=${{ github.run_number }} + type=raw,value=${{ github.base_ref || github.ref_name }} + type=sha,prefix={{branch}}- - - name: "Build: Platform" - id: platform + name: build and push `build` image + id: build uses: docker/build-push-action@v6 with: - push: false - pull: false + platforms: linux/amd64,linux/arm64 context: . - file: deployment/dockerfile.build + file: ./deployment/dockerfile.build + pull: true + push: false + tags: ${{ steps.meta-build.outputs.tags }} + labels: ${{ steps.meta-build.outputs.labels }} + annotations: ${{ steps.meta-build.outputs.annotations }} build-args: | - CONTAINER_REGISTRY=${{ env.REGISTRY }} - platforms: linux/amd64,linux/arm64 - tags: | - ${{ env.REGISTRY }}/platform-build:${{ env.platform_version }} - iqgeo-myproj-build - cache-to: type=gha,mode=max - cache-from: type=gha + PRODUCT_REGISTRY=harbor.delivery.iqgeo.cloud/releases_ + cache-from: | + type=gha,scope=build-latest + type=gha,scope=build-${{ github.sha }} + type=registry,ref=${{ env.image_name }}:buildcache + cache-to: | + type=gha,scope=build-${{ github.sha }},mode=min + type=registry,ref=${{ env.image_name }}:buildcache,mode=min + provenance: mode=max + sbom: true + build-appserver-image: + needs: build-build-image + runs-on: ubuntu-latest + env: + image_name: ${{ vars.registry }}/${{ vars.repository }}/${{ vars.image_name }}-appserver + outputs: + digest: ${{ steps.build.outputs.digest }} + image-id: ${{ steps.build.outputs.imageid }} + metadata: ${{ steps.build.outputs.metadata }} + steps: + - + name: checkout + uses: actions/checkout@v4 + with: + lfs: true - - name: Extract Appserver metadata (tags, labels) for Docker - id: meta + name: setup Docker BuildX + uses: docker/setup-buildx-action@v3 + - + name: login to Harbor Registry (for pulling base images) + uses: docker/login-action@v3 + with: + registry: ${{ vars.registry }} + username: ${{ vars.registry_username }} + password: ${{ secrets.registry_password }} + - + name: extract metadata for `appserver` image + id: meta-appserver uses: docker/metadata-action@v5 with: - images: | - ${{ env.REGISTRY }}/platform-appserver + images: ${{ env.image_name }} tags: | - type=raw,value={{branch}}-{{sha}} - type=sha,prefix={{branch}}-,enable={{!is_default_branch}} - type=raw,value=${{ env.platform_version }},enable={{is_default_branch}} - + type=raw,value=${{ env.PLATFORM_VERSION }} + type=raw,value=${{ github.run_number }} + type=raw,value=${{ github.base_ref || github.ref_name }} + type=sha,prefix={{branch}}- - - name: "Build: Appserver" + name: build and push `appserver` image + id: build uses: docker/build-push-action@v6 with: - push: true + platforms: linux/amd64,linux/arm64 + context: ./deployment + file: ./deployment/dockerfile.appserver pull: true - context: deployment - file: deployment/dockerfile.appserver - # secret-files: | - # oidc=${{ secrets.oidc_config }} + push: ${{ github.event.inputs.push_images != 'false' }} + tags: ${{ steps.meta-appserver.outputs.tags }} + labels: ${{ steps.meta-appserver.outputs.labels }} + annotations: ${{ steps.meta-appserver.outputs.annotations }} build-args: | - CONTAINER_REGISTRY=${{ env.REGISTRY }} - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha + PRODUCT_REGISTRY=harbor.delivery.iqgeo.cloud/releases_ + cache-from: | + type=gha,scope=appserver-latest + type=gha,scope=appserver-${{ github.sha }} + type=registry,ref=${{ env.image_name }}:buildcache + cache-to: | + type=gha,scope=appserver-${{ github.sha }},mode=min + type=registry,ref=${{ env.image_name }}:buildcache,mode=min + provenance: mode=max + sbom: true + build-tools-image: + needs: build-build-image + runs-on: ubuntu-latest + env: + image_name: ${{ vars.registry }}/${{ vars.repository }}/${{ vars.image_name }}-tools + outputs: + digest: ${{ steps.build.outputs.digest }} + image-id: ${{ steps.build.outputs.imageid }} + metadata: ${{ steps.build.outputs.metadata }} + steps: - - name: Extract Appserver Tools metadata (tags, labels) for Docker - id: tools_metadata + name: checkout + uses: actions/checkout@v4 + with: + lfs: true + - + name: setup Docker BuildX + uses: docker/setup-buildx-action@v3 + - + name: login to Harbor Registry (for pulling base images) + uses: docker/login-action@v3 + with: + registry: ${{ vars.registry }} + username: ${{ vars.registry_username }} + password: ${{ secrets.registry_password }} + - + name: extract metadata for `tools` image + id: meta-tools uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index with: - images: | - ${{ env.REGISTRY }}/platform-tools + images: ${{ env.image_name }} tags: | - type=raw,value={{branch}}-{{sha}} - type=sha,prefix={{branch}}-,enable={{!is_default_branch}} - type=raw,value=${{ env.platform_version }},enable={{is_default_branch}} - + type=raw,value=${{ env.PLATFORM_VERSION }} + type=raw,value=${{ github.run_number }} + type=raw,value=${{ github.base_ref || github.ref_name }} + type=sha,prefix={{branch}}- - - name: "Build: Appserver Tools" + name: build and push `tools` image + id: build uses: docker/build-push-action@v6 with: - push: true + platforms: linux/amd64,linux/arm64 + context: . + file: ./deployment/dockerfile.tools pull: true - context: deployment - file: deployment/dockerfile.tools + push: ${{ github.event.inputs.push_images != 'false' }} + tags: ${{ steps.meta-tools.outputs.tags }} + labels: ${{ steps.meta-tools.outputs.labels }} + annotations: ${{ steps.meta-tools.outputs.annotations }} build-args: | - CONTAINER_REGISTRY=${{ env.REGISTRY }} - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.tools_metadata.outputs.tags }} - cache-from: type=gha + PRODUCT_REGISTRY=harbor.delivery.iqgeo.cloud/releases_ + cache-from: | + type=gha,scope=tools-latest + type=gha,scope=tools-${{ github.sha }} + type=registry,ref=${{ env.image_name }}:buildcache + cache-to: | + type=gha,scope=tools-${{ github.sha }},mode=min + type=registry,ref=${{ env.image_name }}:buildcache,mode=min + provenance: mode=max + sbom: true + cleanup: + if: always() + needs: + - build-build-image + - build-appserver-image + - build-tools-image + runs-on: ubuntu-latest + steps: + - name: clean-up temporary artifacts + uses: geekyeggo/delete-artifact@v5 + with: + name: | + build-image-metadata + appserver-image-metadata + tools-image-metadata + if: always() From 692cc97cca84efa7be5c2aea30f543c88952c1d5 Mon Sep 17 00:00:00 2001 From: Derek Clair Date: Thu, 18 Dec 2025 11:48:53 -0700 Subject: [PATCH 5/6] Update workflow: multi-job Docker builds with metadata, caching, provenance From 1cef9600327141eec362581ff5d7bd3ab04811c7 Mon Sep 17 00:00:00 2001 From: Derek Clair Brown Date: Thu, 18 Dec 2025 12:51:34 -0700 Subject: [PATCH 6/6] Update PROJECT_README.md --- PROJECT_README.md | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/PROJECT_README.md b/PROJECT_README.md index f63fb72..86f5e56 100644 --- a/PROJECT_README.md +++ b/PROJECT_README.md @@ -6,7 +6,10 @@ Project description - [Project Name](#project-name) - [Development](#development) - - [Running a dev environment on windows.](#running-a-dev-environment-on-windows) + - [Running a dev environment on Windows](#running-a-dev-environment-on-windows) + - [GitHub Actions Configuration](#github-actions-configuration) + - [Required Variables](#required-variables) + - [Required Secrets](#required-secrets) - [Deployment](#deployment) - [Container images hierarchy](#container-images-hierarchy) @@ -14,12 +17,35 @@ Project description Check out the [development README](.devcontainer/README.md) for instructions on how to build and run the development environment. -### Running a dev environment on windows. +### Running a dev environment on Windows Using host-bound volumes when running linux containers on a windows host comes with considerable overhead. Using **myw_product build** and **myw_product watch** within a container becomes impractical. By following these steps, you will be able to checkout and access your source code within WSL2, and cut on the need to access the windows host at all. [Developing with Containers on Windows](https://github.com/IQGeo/utils-project-template/wiki/Developing-with-containers-on-Windows) +## GitHub Actions Configuration + +The project uses GitHub Actions workflows for building and pushing Docker images. The following variables and secrets must be configured in your repository settings to enable the Docker build workflow (`.github/workflows/docker_build.yml`). + +### Required Variables + +Configure these in **Settings > Secrets and variables > Actions > Variables**: + +| Variable | Description | Example | +| ------------------- | ------------------------------------ | ---------------------- | +| `registry` | Container registry URL | `harbor.example.com` | +| `repository` | Repository path within the registry | `myproject/containers` | +| `image_name` | Base name for the Docker images | `myproject` | +| `registry_username` | Username for registry authentication | `robot-account` | + +### Required Secrets + +Configure these in **Settings > Secrets and variables > Actions > Secrets**: + +| Secret | Description | +| ------------------- | --------------------------------------------- | +| `registry_password` | Password or token for registry authentication | + ## Deployment Check out the [deployment README](deployment/README.md) for instructions on how to build and run the deployment environment. @@ -64,4 +90,4 @@ flowchart TD style K fill:#D50000,color:#FFFFFF style L fill:#D50000,color:#FFFFFF -``` \ No newline at end of file +```