From bf900d0acb116d8615132422691c6bc3effff25a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 29 Aug 2024 10:32:13 +0100 Subject: [PATCH 1/4] CI: GHA conda build & upload - part of #1870 - related to #1875 - related to #1899 <- #1842 <- #1835 - part of #1598 - related to #1430 - related to #1416 - related to #1205 --- .github/workflows/build.yml | 51 +++++++++++++++++++++------------- recipe/conda_build_config.yaml | 43 ---------------------------- recipe/meta.yaml | 30 ++++---------------- 3 files changed, 38 insertions(+), 86 deletions(-) delete mode 100644 recipe/conda_build_config.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4915cdb127..0816a9cd59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,16 +1,3 @@ -# Copyright 2021 United Kingdom Research and Innovation -# Copyright 2021 The University of Manchester -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt name: build on: release: @@ -105,8 +92,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.11] - numpy-version: [1.25] + python-version: ['3.10', 3.11] + numpy-version: [1.23, 1.24, 1.25, 1.26] + include: + - python-version: 3.12 + numpy-version: 1.26 steps: - uses: actions/checkout@v4 with: @@ -122,12 +112,35 @@ jobs: working-directory: recipe run: | conda install boa - conda mambabuild . -c conda-forge -c https://software.repos.intel.com/python/conda -c ccpi --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder . - - name: Upload artifact of the conda package - uses: actions/upload-artifact@v4 + conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder . + - uses: actions/upload-artifact@v4 with: - name: cil-package + name: cil-package-py${{ matrix.python-version }}-np${{ matrix.numpy-version }} path: recipe/linux-64/cil* + - name: anaconda upload -c ccpi + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') + run: > + anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force + --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} + recipe/linux-64/cil* + - name: conda upload -c tomography.stfc.ac.uk/conda + if: startsWith(github.ref, 'refs/tags') + run: | + echo '${{ secrets.STFC_SSH_KEY }}' > ./key + chmod 600 ./key + rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/linux-64/cil* \ + '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/linux-64/' + conda-reindex: + if: startsWith(github.ref, 'refs/tags') + needs: conda + runs-on: ubuntu-latest + steps: + - name: conda index tomography.stfc.ac.uk/conda + run: | + echo '${{ secrets.STFC_SSH_KEY }}' > ./key + chmod 600 ./key + ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \ + 'conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}' docs: defaults: {run: {shell: 'bash -el {0}', working-directory: docs}} runs-on: ubuntu-latest diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml deleted file mode 100644 index 28e16d0af2..0000000000 --- a/recipe/conda_build_config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2018 United Kingdom Research and Innovation -# Copyright 2018 The University of Manchester - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt -# Tomas Kulhanek (STFC) - -#creates pairs of versions using zip_keys, lists must be the same length -python: - - 3.10 - - 3.10 - - 3.10 - - 3.10 - - 3.11 - - 3.11 - - 3.11 - - 3.11 - - 3.12 -numpy: - - 1.23 - - 1.24 - - 1.25 - - 1.26 - - 1.23 - - 1.24 - - 1.25 - - 1.26 - - 1.26 -zip_keys: - - python - - numpy diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5e19c411a8..3355fee710 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,21 +1,5 @@ -# Copyright 2018 United Kingdom Research and Innovation -# Copyright 2018 The University of Manchester -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt -# Kyle Pidgeon (UKRI-STFC) - package: name: cil - # use the latest stable release version version: {{ environ.get('GIT_DESCRIBE_TAG','v')[1:].split('-')[0] }} source: @@ -33,20 +17,18 @@ build: test: requires: - python-wget - - cvxpy # [ linux ] + - cvxpy # [linux] - scikit-image - - tomophantom=2.0.0 # [ linux ] + - tomophantom=2.0.0 # [linux] - tigre=2.6 - packaging - - ccpi-regulariser=24.0.1 # [ not osx ] - - astra-toolbox=2.1=cuda* # [ not osx ] + - ccpi-regulariser=24.0.1 # [not osx] + - astra-toolbox=2.1=cuda* # [not osx] source_files: - - ./Wrappers/Python/test # [win] - - ./Wrappers/Python/test # [not win] + - ./Wrappers/Python/test commands: - - python -c "import os; print ('TESTING IN THIS DIRECTORY' , os.getcwd())" - python -m unittest discover -v -s Wrappers/Python/test # [win] - python -m unittest discover -s Wrappers/Python/test -v # [not win] @@ -98,6 +80,6 @@ requirements: - ipywidgets about: - home: https://ccpi.ac.uk/cil + home: https://TomographicImaging.github.io/CIL license: Apache 2.0 License summary: 'CCPi Core Imaging Library' From 8ac069dbdce6e3ef29eb1fd7bf244a7b9352d82e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 29 Aug 2024 11:26:45 +0100 Subject: [PATCH 2/4] skip most of conda matrix on PR --- .github/workflows/build.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0816a9cd59..4c30ef6e4e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,13 +87,29 @@ jobs: cmake --build ./build --target install - name: test run: python -m unittest discover -v ./Wrappers/Python/test + conda-matrix: + runs-on: ubuntu-latest + outputs: + python-version: ${{ steps.matrix.outputs.python-version }} + numpy-version: ${{ steps.matrix.outputs.numpy-version }} + steps: + - id: matrix + run: | + if ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') }}; then + echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT + echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT + else + echo "python-version=['3.10']" >> $GITHUB_OUTPUT + echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT + fi conda: defaults: {run: {shell: 'bash -el {0}'}} runs-on: ubuntu-latest + needs: conda-matrix strategy: matrix: - python-version: ['3.10', 3.11] - numpy-version: [1.23, 1.24, 1.25, 1.26] + python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }} + numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }} include: - python-version: 3.12 numpy-version: 1.26 From 79923de15b981132a29a7daec32a539f52c8fc50 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 30 Aug 2024 11:40:05 +0100 Subject: [PATCH 3/4] restore conda_build_config.yaml --- recipe/conda_build_config.yaml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 recipe/conda_build_config.yaml diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml new file mode 100644 index 0000000000..28e16d0af2 --- /dev/null +++ b/recipe/conda_build_config.yaml @@ -0,0 +1,43 @@ +# Copyright 2018 United Kingdom Research and Innovation +# Copyright 2018 The University of Manchester + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Authors: +# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt +# Tomas Kulhanek (STFC) + +#creates pairs of versions using zip_keys, lists must be the same length +python: + - 3.10 + - 3.10 + - 3.10 + - 3.10 + - 3.11 + - 3.11 + - 3.11 + - 3.11 + - 3.12 +numpy: + - 1.23 + - 1.24 + - 1.25 + - 1.26 + - 1.23 + - 1.24 + - 1.25 + - 1.26 + - 1.26 +zip_keys: + - python + - numpy From 13eaa66a1e7ee9544743b7cae11ed4be6d2230a9 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 30 Aug 2024 13:39:26 +0100 Subject: [PATCH 4/4] docs: drop jenkins --- .github/workflows/README.md | 8 ++++++-- CHANGELOG.md | 1 + README.md | 4 +--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index cf395ad959..5acd379cbb 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -23,12 +23,16 @@ Details on some of these jobs are given below. ## conda -When opening or modifying a pull request to `master`, a single variant is built and tested. This variant is for linux with `python=3.11` and `numpy=1.25`. +When opening or modifying a pull request to `master`, two variants are built and tested (for linux with minimum & maximum supported `python` & `numpy` versions). > [!NOTE] -> The action does not publish to conda, instead this is done by jenkins. We will eventually move from jenkins to conda-forge instead. > When pushing to `master` or creating an [annotated tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging), *all* variants are built and tested. + + +> [!NOTE] +> The action publishes `ccpi` as well as `https://tomography.stfc.ac.uk/conda/` conda channels. We will eventually move to conda-forge instead. + It looks for conda-build dependencies in the channels listed [here](./build.yml#L118). If you add any new dependencies, the appropriate channels need to be added to this line. > [!TIP] diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0885792e..bb2d98df2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Testing: - New unit tests for operators and functions to check for in place errors and the behaviour of `out` (#1805) - Updates in SPDHG vs PDHG unit test to reduce test time and adjustments to parameters (#1898) + - Drop Jenkins in favour of GHA for conda builds (#1914) - Bug fixes: - `ImageData` removes dimensions of size 1 from the input array. This fixes an issue where single slice reconstructions from 3D data would fail due to shape mismatches (#1885) - Make Binner accept accelerated=False (#1887) diff --git a/README.md b/README.md index 315cdff911..0c4937c318 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # CIL - Core Imaging Library -Master | Development | Conda binaries --|-|- -[![CI-master](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework) | [![CI-dev](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework-dev)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework-dev) | ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil) +[![CI-master](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml/badge.svg)](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml) ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TomographicImaging/CIL-Demos/HEAD?urlpath=lab/tree/binder%2Findex.ipynb)