From 6644fffdb8efbe61870e6014779c5df2b219b324 Mon Sep 17 00:00:00 2001 From: Jakob Jensen Date: Sun, 18 Jan 2026 15:20:48 +0100 Subject: [PATCH 1/5] process configuration in chunks --- .github/workflows/scheduled-jobs.yaml | 127 +++++++++++++++++++------- .github/workflows/tests.yaml | 4 - make.d/config-chunks.mk | 9 ++ make.d/help.txt | 3 +- make.d/update.mk | 4 +- 5 files changed, 110 insertions(+), 37 deletions(-) create mode 100644 make.d/config-chunks.mk diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 62181398c..5b0169db5 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -10,57 +10,122 @@ on: - schema/** - definitions/** - docs/COMPARISON.md - workflow_dispatch: - inputs: - reset: - description: Space-separated paths to remove from schema before running an update - default: "" - type: string + workflow_dispatch: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - update-schemas: - name: Update schemas - runs-on: ubuntu-latest + configuration-chunks: + name: Split configuration if: github.repository == 'CustomResourceDefinition/catalog' || github.event_name == 'workflow_dispatch' - permissions: - contents: write - packages: write + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@v6 with: - ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }} + sparse-checkout: | + build + configuration.yaml + Makefile + make.d - - name: Optionally remove paths - if: github.event_name == 'workflow_dispatch' && github.event.inputs.reset != '' - run: | - cd schema - rm -rf ${{ github.event.inputs.reset }} || true - cd - + - run: make config-chunks + env: + CHUNK_SIZE: 100 + + - run: mv build/configuration_*.yml . - cd definitions - rm -rf ${{ github.event.inputs.reset }} || true - cd - + - run: | + { + printf "matrix=" + echo configuration*.yml | tr ' ' "\n" | jq -Rsc 'split("\n")[:-1]' + } | tee -a $GITHUB_OUTPUT + id: matrix + + - uses: actions/upload-artifact@v5 + with: + name: configurations + path: configuration_*.yml + retention-days: 1 + + update: + name: Create patch files + needs: configuration-chunks + runs-on: ubuntu-latest + strategy: + matrix: + file: ${{ fromJSON(needs.configuration-chunks.outputs.matrix) }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Setup Golang cache + uses: actions/cache@v5 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - run: make update env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} + CONFIGURATION: ${{ matrix.file }} + + - run: git diff schema definitions | tee ${{ matrix.file }}.patch + + - uses: actions/upload-artifact@v5 + with: + path: ${{ matrix.file }}.patch + retention-days: 1 + + commit: + name: Update files + needs: update + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }} + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Setup Golang cache + uses: actions/cache@v5 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - uses: actions/checkout@v6 with: repository: datreeio/CRDs-catalog path: build/remote/datreeio + - uses: actions/download-artifact@v6 + with: + merge-multiple: true + + - run: git apply --allow-empty *.patch + - run: make comparison - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} + + - if: github.ref != 'refs/heads/main' + run: git diff --stat - uses: EndBug/add-and-commit@v9 name: Publish changes @@ -76,7 +141,7 @@ jobs: name: Report failures runs-on: ubuntu-latest needs: - - update-schemas + - commit - sync-tags if: failure() && github.event_name != 'workflow_dispatch' permissions: @@ -96,7 +161,7 @@ jobs: sync-tags: name: Synchronize tags with kubernetes runs-on: ubuntu-latest - needs: update-schemas + needs: commit permissions: contents: write steps: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a97d08435..483b8615e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,3 @@ jobs: steps: - uses: actions/checkout@v6 - run: make test - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} diff --git a/make.d/config-chunks.mk b/make.d/config-chunks.mk new file mode 100644 index 000000000..b29d2fbd3 --- /dev/null +++ b/make.d/config-chunks.mk @@ -0,0 +1,9 @@ +CHUNK_SIZE ?= 100 + +config-chunks: + cd build && \ + yq -o=json '.' ../configuration.yaml \ + | jq -c ". as \$$arr \ + | [range(0; \$$arr | length; $(CHUNK_SIZE)) | \$$arr[. : .+$(CHUNK_SIZE)]]" \ + | yq -P \ + | yq '.[]' -s '"configuration_" + $$index' diff --git a/make.d/help.txt b/make.d/help.txt index f0fca66d8..17eb03005 100644 --- a/make.d/help.txt +++ b/make.d/help.txt @@ -8,8 +8,9 @@ Available targets: check Runs update using the `build/configuration.yaml` file and outputting to `build/ephemeral/schema` - this is meant as a tool to test a new configuration item locally before adding it clean Removes temporarily generated files and builds and removes local container images comparison Updates COMPARISON.md based on the /schema contents + config-chunks Generates configuration chunk files help Displays this help - schema-check Verifies all schema files against jsonschema schema file - this takes a very long time + schema-check Verifies all schema files against jsonschema schema file - this takes a very long time tags Synchronize semantic version tags from kubernetes into this repository test Runs the entire test suite update Updates the /schema contents diff --git a/make.d/update.mk b/make.d/update.mk index e7cb73096..ade667856 100644 --- a/make.d/update.mk +++ b/make.d/update.mk @@ -1,3 +1,5 @@ +CONFIGURATION ?= configuration.yaml + update: build/bin/catalog df -h - build/bin/catalog update --configuration configuration.yaml --output schema --definitions definitions + build/bin/catalog update --configuration $(CONFIGURATION) --output schema --definitions definitions From 9183a7894555415cd841e19f4a80b69657eb1b40 Mon Sep 17 00:00:00 2001 From: Jakob Jensen Date: Sun, 18 Jan 2026 15:38:05 +0100 Subject: [PATCH 2/5] update workflow limitations --- .github/workflows/scheduled-jobs.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 5b0169db5..04656bc72 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -143,7 +143,7 @@ jobs: needs: - commit - sync-tags - if: failure() && github.event_name != 'workflow_dispatch' + if: failure() && github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' permissions: contents: read issues: write @@ -162,6 +162,7 @@ jobs: name: Synchronize tags with kubernetes runs-on: ubuntu-latest needs: commit + if: github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main' permissions: contents: write steps: From 26777c4ae80d03b2777f25352b8d2f8a7701409d Mon Sep 17 00:00:00 2001 From: Jakob Jensen Date: Sun, 18 Jan 2026 15:41:13 +0100 Subject: [PATCH 3/5] download configuration chunk to use it --- .github/workflows/scheduled-jobs.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 04656bc72..10ba85a41 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -75,6 +75,10 @@ jobs: restore-keys: | ${{ runner.os }}-go- + - uses: actions/download-artifact@v6 + with: + name: ${{ matrix.file }} + - run: make update env: CONFIGURATION: ${{ matrix.file }} From d736cadfa6eeaba47df7a4b621b9f6cef7b294f5 Mon Sep 17 00:00:00 2001 From: Jakob Jensen Date: Sun, 18 Jan 2026 15:43:00 +0100 Subject: [PATCH 4/5] download configuration chunk to use it --- .github/workflows/scheduled-jobs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 10ba85a41..9d81fbe94 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -77,7 +77,7 @@ jobs: - uses: actions/download-artifact@v6 with: - name: ${{ matrix.file }} + name: configurations - run: make update env: From 7581e930fd45537aedd43ca74ffae2b3583e5604 Mon Sep 17 00:00:00 2001 From: Jakob Jensen Date: Sun, 18 Jan 2026 15:51:55 +0100 Subject: [PATCH 5/5] name the patch files uniquely --- .github/workflows/scheduled-jobs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 9d81fbe94..2c5a52496 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -87,6 +87,7 @@ jobs: - uses: actions/upload-artifact@v5 with: + name: ${{ matrix.file }}.patch path: ${{ matrix.file }}.patch retention-days: 1