Build Python wheels #212
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Python wheels | |
| on: | |
| push: | |
| tags: ["*"] | |
| pull_request: | |
| paths: | |
| # build wheels in PR if this file changed | |
| - '.github/workflows/build-wheels.yml' | |
| # build wheels in PR if any of the build system files changed | |
| - '**/VERSION' | |
| - '**/setup.py' | |
| - '**/pyproject.toml' | |
| - '**/MANIFEST.in' | |
| - '**/CMakeLists.txt' | |
| schedule: | |
| # check the build once a week on mondays | |
| - cron: '0 10 * * 1' | |
| concurrency: | |
| group: python-wheels-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| build-torch-wheels: | |
| runs-on: ${{ matrix.os }} | |
| name: ${{ matrix.name }} (torch v${{ matrix.torch-version }}) | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-24.04', 'ubuntu-24.04-arm', 'macos-15-intel', 'macos-15', 'windows-2022'] | |
| arch: ['arm64', 'x86_64'] | |
| torch-version: ['2.1', '2.2', '2.3', '2.4', '2.5', '2.6', '2.7', '2.8', '2.9'] | |
| exclude: | |
| # remove mismatched arch-os combinations | |
| - {os: macos-15-intel, arch: arm64} | |
| - {os: windows-2022, arch: arm64} | |
| - {os: ubuntu-24.04, arch: arm64} | |
| - {os: macos-15, arch: x86_64} | |
| - {os: ubuntu-24.04-arm, arch: x86_64} | |
| # arch x86_64 on macos is only supported for torch <2.3 | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.3'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.4'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.5'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.6'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.7'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.8'} | |
| - {os: macos-15-intel, arch: x86_64, torch-version: '2.9'} | |
| include: | |
| # add `cibw-arch` and `rust-target` to the different configurations | |
| - name: x86_64 Linux | |
| os: ubuntu-24.04 | |
| arch: x86_64 | |
| rust-target: x86_64-unknown-linux-gnu | |
| cibw-arch: x86_64 | |
| - name: arm64 Linux | |
| os: ubuntu-24.04-arm | |
| arch: arm64 | |
| rust-target: aarch64-unknown-linux-gnu | |
| cibw-arch: aarch64 | |
| - name: x86_64 macOS | |
| os: macos-15-intel | |
| arch: x86_64 | |
| rust-target: x86_64-apple-darwin | |
| cibw-arch: x86_64 | |
| - name: arm64 macOS | |
| os: macos-15 | |
| arch: arm64 | |
| rust-target: aarch64-apple-darwin | |
| cibw-arch: arm64 | |
| - name: x86_64 Windows | |
| os: windows-2022 | |
| arch: x86_64 | |
| rust-target: x86_64-pc-windows-msvc | |
| cibw-arch: AMD64 | |
| # add the right python version image for each torch version | |
| - {torch-version: '2.1', cibw-python: 'cp311-*'} | |
| - {torch-version: '2.2', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.3', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.4', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.5', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.6', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.7', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.8', cibw-python: 'cp312-*'} | |
| - {torch-version: '2.9', cibw-python: 'cp312-*'} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - name: install dependencies | |
| run: python -m pip install cibuildwheel | |
| - name: build custom manylinux docker images | |
| if: matrix.os == 'ubuntu-24.04' || matrix.os == 'ubuntu-24.04-arm' | |
| run: | | |
| docker buildx build \ | |
| -t gcc11-manylinux_2_28_${{ matrix.cibw-arch }} \ | |
| python/scripts/gcc11-manylinux_2_28_${{ matrix.cibw-arch }} | |
| - name: build metatomic-torch wheel | |
| run: python -m cibuildwheel python/metatomic_torch | |
| env: | |
| CIBW_BUILD: ${{ matrix.cibw-python}} | |
| CIBW_SKIP: "*musllinux*" | |
| CIBW_ARCHS: ${{ matrix.cibw-arch }} | |
| CIBW_BUILD_VERBOSITY: 1 | |
| CIBW_MANYLINUX_X86_64_IMAGE: gcc11-manylinux_2_28_x86_64 | |
| CIBW_MANYLINUX_AARCH64_IMAGE: gcc11-manylinux_2_28_aarch64 | |
| CIBW_ENVIRONMENT: > | |
| METATOMIC_TORCH_BUILD_WITH_TORCH_VERSION=${{ matrix.torch-version }}.* | |
| PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cpu | |
| MACOSX_DEPLOYMENT_TARGET=11 | |
| # do not complain for missing shared libraries provided by other packages | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: | | |
| delocate-wheel --ignore-missing-dependencies --require-archs {delocate_archs} -w {dest_dir} -v {wheel} | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: | | |
| auditwheel repair --exclude libmetatensor.so --exclude libmetatensor_torch.so --exclude libtorch.so --exclude libtorch_cpu.so --exclude libc10.so -w {dest_dir} {wheel} | |
| - uses: actions/upload-artifact@v5 | |
| with: | |
| name: torch-single-version-wheel-${{ matrix.torch-version }}-${{ matrix.os }}-${{ matrix.arch }} | |
| path: ./wheelhouse/*.whl | |
| merge-torch-wheels: | |
| needs: build-torch-wheels | |
| runs-on: ubuntu-24.04 | |
| name: merge metatomic-torch ${{ matrix.name }} | |
| strategy: | |
| matrix: | |
| include: | |
| - name: x86_64 Linux | |
| os: ubuntu-24.04 | |
| arch: x86_64 | |
| - name: arm64 Linux | |
| os: ubuntu-24.04-arm | |
| arch: arm64 | |
| - name: x86_64 macOS | |
| os: macos-15-intel | |
| arch: x86_64 | |
| - name: arm64 macOS | |
| os: macos-15 | |
| arch: arm64 | |
| - name: x86_64 Windows | |
| os: windows-2022 | |
| arch: x86_64 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download wheels | |
| uses: actions/download-artifact@v6 | |
| with: | |
| pattern: torch-single-version-wheel-*-${{ matrix.os }}-${{ matrix.arch }} | |
| merge-multiple: false | |
| path: dist | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - name: install dependencies | |
| run: python -m pip install twine wheel | |
| - name: merge wheels | |
| run: | | |
| # collect all torch versions used for the build | |
| REQUIRES_TORCH=$(find dist -name "*.whl" -exec unzip -p {} "metatomic_torch-*.dist-info/METADATA" \; | grep "Requires-Dist: torch") | |
| MERGED_TORCH_REQUIRE=$(python scripts/create-torch-versions-range.py "$REQUIRES_TORCH") | |
| echo MERGED_TORCH_REQUIRE=$MERGED_TORCH_REQUIRE | |
| # unpack all single torch versions wheels in the same directory | |
| mkdir dist/unpacked | |
| find dist -name "*.whl" -print -exec python -m wheel unpack --dest dist/unpacked/ {} ';' | |
| sed -i "s/Requires-Dist: torch.*/$MERGED_TORCH_REQUIRE/" dist/unpacked/metatomic_torch-*/metatomic_torch-*.dist-info/METADATA | |
| echo "\n\n METADATA = \n\n" | |
| cat dist/unpacked/metatomic_torch-*/metatomic_torch-*.dist-info/METADATA | |
| # check the right metadata was added to the file. grep will exit with | |
| # code `1` if the line is not found, which will stop CI | |
| grep "$MERGED_TORCH_REQUIRE" dist/unpacked/metatomic_torch-*/metatomic_torch-*.dist-info/METADATA | |
| # repack the directory as a new wheel | |
| mkdir wheelhouse | |
| python -m wheel pack --dest wheelhouse/ dist/unpacked/* | |
| - name: check wheels with twine | |
| run: twine check wheelhouse/* | |
| - uses: actions/upload-artifact@v5 | |
| with: | |
| name: torch-wheel-${{ matrix.os }}-${{ matrix.arch }} | |
| path: ./wheelhouse/*.whl | |
| build-others: | |
| name: Build other wheels/sdists | |
| runs-on: ubuntu-24.04 | |
| env: | |
| METATOMIC_NO_LOCAL_DEPS: "1" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - name: install dependencies | |
| run: python -m pip install wheel build twine | |
| - name: build metatomic-torch sdist | |
| run: python -m build python/metatomic_torch --sdist --outdir=dist/ | |
| - name: build metatomic sdist and wheel | |
| run: python -m build . --outdir=dist/ | |
| - name: check sdist and wheels with twine | |
| run: twine check dist/*.tar.gz dist/*.whl | |
| - name: create C++ tarballs | |
| run: | | |
| ./scripts/package-torch.sh dist/cxx/ | |
| - uses: actions/upload-artifact@v5 | |
| with: | |
| name: others | |
| path: | | |
| dist/*.tar.gz | |
| dist/cxx/*.tar.gz | |
| dist/*.whl | |
| merge-and-release: | |
| name: Merge and release wheels/sdists | |
| needs: [merge-torch-wheels, build-others] | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - name: Download metatomic-torch wheels | |
| uses: actions/download-artifact@v6 | |
| with: | |
| path: wheels | |
| pattern: torch-wheel-* | |
| merge-multiple: true | |
| - name: Download other wheels and sdists | |
| uses: actions/download-artifact@v6 | |
| with: | |
| path: wheels | |
| name: others | |
| - name: Re-upload a single wheels artifact | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: wheels | |
| path: | | |
| wheels/* | |
| wheels/cxx/* | |
| - name: Comment with download link | |
| uses: PicoCentauri/comment-artifact@v1 | |
| if: github.event.pull_request.head.repo.fork == false | |
| with: | |
| name: wheels | |
| description: ⚙️ Download Python wheels for this pull-request (you can install these with pip) | |
| - name: upload to GitHub release (metatomic) | |
| if: startsWith(github.ref, 'refs/tags/metatomic-v') | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: | | |
| wheels/metatomic-* | |
| prerelease: ${{ contains(github.ref, '-rc') }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: upload to GitHub release (metatomic-torch) | |
| if: startsWith(github.ref, 'refs/tags/metatomic-torch-v') | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: | | |
| wheels/cxx/metatomic-torch-cxx-*.tar.gz | |
| wheels/metatomic_torch-* | |
| prerelease: ${{ contains(github.ref, '-rc') }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| test-build-external: | |
| # This checks building the wheels with external libraries. This setup is | |
| # mainly used for the conda packages metatensor-*-python, which use the | |
| # libmetatensor-* conda packages to provide the native code. | |
| runs-on: ${{ matrix.os }} | |
| name: External libraries / ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-24.04 | |
| rust-target: x86_64-unknown-linux-gnu | |
| torch-version: "2.9" | |
| libtorch-url: https://download.pytorch.org/libtorch/cpu/libtorch-shared-with-deps-2.9.0%2Bcpu.zip | |
| - os: macos-15 | |
| rust-target: aarch64-apple-darwin | |
| torch-version: "2.9" | |
| libtorch-url: https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-2.9.0.zip | |
| - os: windows-2022 | |
| rust-target: x86_64-pc-windows-msvc | |
| torch-version: "2.9" | |
| libtorch-url: https://download.pytorch.org/libtorch/cpu/libtorch-win-shared-with-deps-2.9.0%2Bcpu.zip | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - name: setup environment | |
| run: | | |
| pip install build | |
| echo "CMAKE_PREFIX_PATH=${RUNNER_TEMP//\\//}/usr" >> "$GITHUB_ENV" | |
| - name: setup libtorch | |
| run: | | |
| curl --output libtorch.zip ${{ matrix.libtorch-url }} | |
| unzip -q libtorch.zip -d $CMAKE_PREFIX_PATH/ | |
| mv $CMAKE_PREFIX_PATH/libtorch/* $CMAKE_PREFIX_PATH/ | |
| - name: setup libmetatensor | |
| run: | | |
| curl -L -O https://github.com/metatensor/metatensor/releases/download/metatensor-core-v0.1.17/metatensor-core-cxx-0.1.17.tar.gz | |
| tar xf metatensor-core-cxx-0.1.17.tar.gz | |
| cmake -B build-metatensor -S metatensor-core-cxx-0.1.17 \ | |
| -DMETATENSOR_INSTALL_BOTH_STATIC_SHARED=OFF \ | |
| -DCMAKE_INSTALL_PREFIX=$CMAKE_PREFIX_PATH \ | |
| -DCMAKE_BUILD_TYPE=Debug | |
| cmake --build build-metatensor --config Debug | |
| cmake --install build-metatensor --config Debug | |
| - name: setup libmetatensor-torch | |
| run: | | |
| curl -L -O https://github.com/metatensor/metatensor/releases/download/metatensor-torch-v0.8.2/metatensor-torch-cxx-0.8.2.tar.gz | |
| tar xf metatensor-torch-cxx-0.8.2.tar.gz | |
| cmake -B build-metatensor-torch -S metatensor-torch-cxx-0.8.2 \ | |
| -DCMAKE_INSTALL_PREFIX=$CMAKE_PREFIX_PATH \ | |
| -DCMAKE_BUILD_TYPE=Debug | |
| cmake --build build-metatensor-torch --config Debug | |
| cmake --install build-metatensor-torch --config Debug | |
| - name: build libmetatomic-torch | |
| run: | | |
| cmake -B build-metatomic-torch -S metatomic-torch \ | |
| -DCMAKE_INSTALL_PREFIX=$CMAKE_PREFIX_PATH \ | |
| -DCMAKE_BUILD_TYPE=Debug | |
| cmake --build build-metatomic-torch --config Debug | |
| cmake --install build-metatomic-torch --config Debug | |
| - name: build metatomic-torch wheels | |
| run: | | |
| pip install build | |
| python -m build python/metatomic_torch --wheel --outdir=dist/ | |
| unzip -l dist/metatomic_torch*.whl | grep "_external.py" | |
| env: | |
| METATOMIC_TORCH_PYTHON_USE_EXTERNAL_LIB: "ON" | |
| METATOMIC_TORCH_BUILD_WITH_TORCH_VERSION: ${{ matrix.torch-version }}.* | |
| PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu |