diff --git a/.github/workflows/scheduled-jobs.yaml b/.github/workflows/scheduled-jobs.yaml index 62181398c..2c5a52496 100644 --- a/.github/workflows/scheduled-jobs.yaml +++ b/.github/workflows/scheduled-jobs.yaml @@ -10,57 +10,127 @@ on: - schema/** - definitions/** - docs/COMPARISON.md - workflow_dispatch: - inputs: - reset: - description: Space-separated paths to remove from schema before running an update - default: "" - type: string + workflow_dispatch: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - update-schemas: - name: Update schemas - runs-on: ubuntu-latest + configuration-chunks: + name: Split configuration if: github.repository == 'CustomResourceDefinition/catalog' || github.event_name == 'workflow_dispatch' - permissions: - contents: write - packages: write + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@v6 with: - ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }} + sparse-checkout: | + build + configuration.yaml + Makefile + make.d + + - run: make config-chunks + env: + CHUNK_SIZE: 100 + + - run: mv build/configuration_*.yml . + + - run: | + { + printf "matrix=" + echo configuration*.yml | tr ' ' "\n" | jq -Rsc 'split("\n")[:-1]' + } | tee -a $GITHUB_OUTPUT + id: matrix + + - uses: actions/upload-artifact@v5 + with: + name: configurations + path: configuration_*.yml + retention-days: 1 + + update: + name: Create patch files + needs: configuration-chunks + runs-on: ubuntu-latest + strategy: + matrix: + file: ${{ fromJSON(needs.configuration-chunks.outputs.matrix) }} + steps: + - uses: actions/checkout@v6 - - name: Optionally remove paths - if: github.event_name == 'workflow_dispatch' && github.event.inputs.reset != '' - run: | - cd schema - rm -rf ${{ github.event.inputs.reset }} || true - cd - + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod - cd definitions - rm -rf ${{ github.event.inputs.reset }} || true - cd - + - name: Setup Golang cache + uses: actions/cache@v5 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - uses: actions/download-artifact@v6 + with: + name: configurations - run: make update env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} + CONFIGURATION: ${{ matrix.file }} + + - run: git diff schema definitions | tee ${{ matrix.file }}.patch + + - uses: actions/upload-artifact@v5 + with: + name: ${{ matrix.file }}.patch + path: ${{ matrix.file }}.patch + retention-days: 1 + + commit: + name: Update files + needs: update + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }} + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Setup Golang cache + uses: actions/cache@v5 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - uses: actions/checkout@v6 with: repository: datreeio/CRDs-catalog path: build/remote/datreeio + - uses: actions/download-artifact@v6 + with: + merge-multiple: true + + - run: git apply --allow-empty *.patch + - run: make comparison - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} + + - if: github.ref != 'refs/heads/main' + run: git diff --stat - uses: EndBug/add-and-commit@v9 name: Publish changes @@ -76,9 +146,9 @@ jobs: name: Report failures runs-on: ubuntu-latest needs: - - update-schemas + - commit - sync-tags - if: failure() && github.event_name != 'workflow_dispatch' + if: failure() && github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' permissions: contents: read issues: write @@ -96,7 +166,8 @@ jobs: sync-tags: name: Synchronize tags with kubernetes runs-on: ubuntu-latest - needs: update-schemas + needs: commit + if: github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main' permissions: contents: write steps: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a97d08435..483b8615e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,3 @@ jobs: steps: - uses: actions/checkout@v6 - run: make test - env: - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REF: ${{ github.ref }} diff --git a/make.d/config-chunks.mk b/make.d/config-chunks.mk new file mode 100644 index 000000000..b29d2fbd3 --- /dev/null +++ b/make.d/config-chunks.mk @@ -0,0 +1,9 @@ +CHUNK_SIZE ?= 100 + +config-chunks: + cd build && \ + yq -o=json '.' ../configuration.yaml \ + | jq -c ". as \$$arr \ + | [range(0; \$$arr | length; $(CHUNK_SIZE)) | \$$arr[. : .+$(CHUNK_SIZE)]]" \ + | yq -P \ + | yq '.[]' -s '"configuration_" + $$index' diff --git a/make.d/help.txt b/make.d/help.txt index f0fca66d8..17eb03005 100644 --- a/make.d/help.txt +++ b/make.d/help.txt @@ -8,8 +8,9 @@ Available targets: check Runs update using the `build/configuration.yaml` file and outputting to `build/ephemeral/schema` - this is meant as a tool to test a new configuration item locally before adding it clean Removes temporarily generated files and builds and removes local container images comparison Updates COMPARISON.md based on the /schema contents + config-chunks Generates configuration chunk files help Displays this help - schema-check Verifies all schema files against jsonschema schema file - this takes a very long time + schema-check Verifies all schema files against jsonschema schema file - this takes a very long time tags Synchronize semantic version tags from kubernetes into this repository test Runs the entire test suite update Updates the /schema contents diff --git a/make.d/update.mk b/make.d/update.mk index e7cb73096..ade667856 100644 --- a/make.d/update.mk +++ b/make.d/update.mk @@ -1,3 +1,5 @@ +CONFIGURATION ?= configuration.yaml + update: build/bin/catalog df -h - build/bin/catalog update --configuration configuration.yaml --output schema --definitions definitions + build/bin/catalog update --configuration $(CONFIGURATION) --output schema --definitions definitions