From ac228f3ac00305441bf5198440b4a6078432ad0a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 14 Aug 2025 01:29:46 +0100 Subject: [PATCH 1/2] CI: fix & optimise build matrix --- .github/workflows/build.yml | 90 ++++++++++++++++++--------- recipe/conda_build_config.yaml | 24 ------- recipe/meta.yaml | 7 ++- scripts/requirements-test-windows.yml | 3 +- scripts/requirements-test.yml | 3 +- 5 files changed, 66 insertions(+), 61 deletions(-) delete mode 100644 recipe/conda_build_config.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 182ca8ac04..d8e1cb1990 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,18 +103,17 @@ jobs: echo "os=['ubuntu']" >> $GITHUB_OUTPUT fi fi - conda: + conda-build: defaults: {run: {shell: 'bash -el {0}'}} runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }} needs: conda-matrix strategy: matrix: python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} - numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }} os: ${{ fromJson(needs.conda-matrix.outputs.os) }} include: - - {python-version: 3.12, numpy-version: 1.26, os: ubuntu} - - {python-version: 3.12, numpy-version: 1.26, os: windows} + - {python-version: 3.12, os: ubuntu} + - {python-version: 3.12, os: windows} steps: - uses: actions/checkout@v4 with: @@ -129,38 +128,69 @@ jobs: mamba-version: "*" channels: conda-forge conda-remove-defaults: "true" - - run: conda install boa conda-verify anaconda-client - - name: conda build & test - working-directory: recipe - run: | - conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder . + - run: conda install boa conda-verify + - name: conda build + run: > + conda mambabuild -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} + --no-test --output-folder dist recipe - uses: actions/upload-artifact@v4 with: - name: cil-package-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version }} - path: recipe/*/cil-*.tar.bz2 - - name: anaconda upload -c ccpi - if: > - (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')) && matrix.numpy-version == '1.26' - run: | - anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} recipe/*/cil-*.tar.bz2 - - name: conda upload -c tomography.stfc.ac.uk/conda - if: startsWith(github.ref, 'refs/tags') - run: | - echo '${{ secrets.STFC_SSH_KEY }}' > ./key - chmod 600 ./key - for plat in linux-64 win-64; do - rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/$plat/cil-*.tar.bz2 \ - '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/$plat/' - done - conda-reindex: - if: startsWith(github.ref, 'refs/tags') - needs: conda + name: cil-py${{ matrix.python-version }}-${{ matrix.os }} + path: dist/*/cil-*.tar.bz2 + conda-test: + defaults: {run: {shell: 'bash -el {0}'}} + runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }} + needs: [conda-matrix, conda-build] + strategy: + matrix: + python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} + numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }} + os: ${{ fromJson(needs.conda-matrix.outputs.os) }} + include: + - {python-version: 3.12, numpy-version: 1.26, os: ubuntu} + - {python-version: 3.12, numpy-version: 1.26, os: windows} + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + mamba-version: "*" + channels: conda-forge + conda-remove-defaults: "true" + - run: conda install boa + - uses: actions/download-artifact@v4 + with: + name: cil-py${{ matrix.python-version }}-${{ matrix.os }} + path: dist + - name: conda test + run: > + conda mambabuild -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} + --test dist/*/cil-*.tar.bz2 --extra-deps numpy=${{ matrix.numpy-version }} + conda-upload: + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') + defaults: {run: {shell: 'bash -el {0}', working-directory: dist}} runs-on: ubuntu-22.04 + needs: conda-test steps: - - name: conda index tomography.stfc.ac.uk/conda + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + mamba-version: "*" + channels: conda-forge + conda-remove-defaults: "true" + - run: conda install anaconda-client + - uses: actions/download-artifact@v4 + with: {pattern: cil-py*-*, path: dist, merge-multiple: true} + - name: anaconda upload -c ccpi + run: > + anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force + --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} */cil-*.tar.bz2 + - if: startsWith(github.ref, 'refs/tags') + name: conda upload -c tomography.stfc.ac.uk/conda run: | echo '${{ secrets.STFC_SSH_KEY }}' > ./key chmod 600 ./key + rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */cil-*.tar.bz2 '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}' ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \ 'bash -lic "conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}"' docs: @@ -266,6 +296,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} pass: - needs: [test-cuda, test, conda, docs, docker] + needs: [test-cuda, test, conda-test, docs, docker] runs-on: ubuntu-22.04 steps: [{run: echo success}] diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml deleted file mode 100644 index 44de043847..0000000000 --- a/recipe/conda_build_config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2018 United Kingdom Research and Innovation -# Copyright 2018 The University of Manchester - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt - -ipp_version: - - '2021.12' -python: - - 3.10 - - 3.11 - - 3.12 diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 6a3e9de307..deffc64d2f 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -11,7 +11,6 @@ build: test: requires: - - python - pip - python-wget - cvxpy # [linux] @@ -31,6 +30,8 @@ test: - pip install unittest-parametrize - python -m unittest discover -v -s Wrappers/Python/test +{% set ipp_version = '2021.12' %} + requirements: build: - cmake >=3.16 @@ -42,14 +43,14 @@ requirements: - llvm-openmp # [osx] - ipp-include {{ ipp_version }} - ipp-devel {{ ipp_version }} - - python >=3.10,<3.13 + - python x.x - pip - setuptools >=64 - setuptools_scm >=8 - scikit-build-core >=0.10 run: - - python + - {{ pin_compatible('python', min_pin='x.x') }} - ipp {{ ipp_version }} - numpy >=1.23,<2 - scipy >=1.4 diff --git a/scripts/requirements-test-windows.yml b/scripts/requirements-test-windows.yml index c317da5368..616db62ee9 100644 --- a/scripts/requirements-test-windows.yml +++ b/scripts/requirements-test-windows.yml @@ -17,8 +17,7 @@ channels: - nvidia - https://software.repos.intel.com/python/conda dependencies: - # base (vis. recipe/conda_build_config.yaml) - - python >=3.10,<3.13 + - python >=3.10 - numpy >=1.23,<2 - ccpi::cil-data >=22 - ccpi::tigre 2.6 diff --git a/scripts/requirements-test.yml b/scripts/requirements-test.yml index 03adbe1972..496c19b3ad 100644 --- a/scripts/requirements-test.yml +++ b/scripts/requirements-test.yml @@ -17,8 +17,7 @@ channels: - nvidia - https://software.repos.intel.com/python/conda dependencies: - # base (vis. recipe/conda_build_config.yaml) - - python >=3.10,<3.13 + - python >=3.10 - numpy >=1.23,<2 - libgomp # [linux] - ccpi::cil-data >=22 From f41627bc409089e6bcfdd95be070d2031fd754c4 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 14 Aug 2025 02:40:46 +0100 Subject: [PATCH 2/2] reduce PR matrix --- .github/workflows/build.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8e1cb1990..13a23e5525 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,6 +87,8 @@ jobs: python-version: ${{ steps.matrix.outputs.python-version }} numpy-version: ${{ steps.matrix.outputs.numpy-version }} os: ${{ steps.matrix.outputs.os }} + include: ${{ steps.matrix.outputs.include }} + include-numpy: ${{ steps.matrix.outputs.include-numpy }} steps: - id: matrix run: | @@ -94,15 +96,25 @@ jobs: echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT echo "os=['ubuntu', 'windows']" >> $GITHUB_OUTPUT + include_max=true else echo "python-version=['3.10']" >> $GITHUB_OUTPUT echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT if ${{ github.ref == 'refs/heads/master' }}; then echo "os=['ubuntu', 'windows']" >> $GITHUB_OUTPUT + include_max=true else echo "os=['ubuntu']" >> $GITHUB_OUTPUT + include_max=false fi fi + if $include_max; then + echo "include=[{'python-version': 3.12, 'os': 'ubuntu'}, {'python-version': 3.12, 'os': 'windows'}]" >> $GITHUB_OUTPUT + echo "include-numpy=[{'python-version': 3.12, 'numpy-version': 1.26, 'os': 'ubuntu'}, {'python-version': 3.12, 'numpy-version': 1.26, 'os': 'windows'}]" >> $GITHUB_OUTPUT + else + echo "include=[]" >> $GITHUB_OUTPUT + echo "include-numpy=[]" >> $GITHUB_OUTPUT + fi conda-build: defaults: {run: {shell: 'bash -el {0}'}} runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }} @@ -111,9 +123,7 @@ jobs: matrix: python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} os: ${{ fromJson(needs.conda-matrix.outputs.os) }} - include: - - {python-version: 3.12, os: ubuntu} - - {python-version: 3.12, os: windows} + include: ${{ fromJson(needs.conda-matrix.outputs.include) }} steps: - uses: actions/checkout@v4 with: @@ -146,9 +156,7 @@ jobs: python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }} os: ${{ fromJson(needs.conda-matrix.outputs.os) }} - include: - - {python-version: 3.12, numpy-version: 1.26, os: ubuntu} - - {python-version: 3.12, numpy-version: 1.26, os: windows} + include: ${{ fromJson(needs.conda-matrix.outputs.include-numpy) }} steps: - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3