diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml deleted file mode 100644 index 44703825..00000000 --- a/.azure_pipeline.yml +++ /dev/null @@ -1,214 +0,0 @@ -# Adapted from https://github.com/pandas-dev/pandas/blob/master/azure-pipelines.yml - -# Global variables for all jobs -variables: - VIRTUALENV: 'testvenv' - JUNITXML: 'test-data.xml' - CODECOV_TOKEN: 'cee0e505-c12e-4139-aa43-621fb16a2347' - -schedules: -- cron: "0 1 * * *" # 1am UTC - displayName: Run nightly build - branches: - include: - - master - always: true - -stages: -- stage: - jobs: - - - job: 'linting' - displayName: Linting - pool: - vmImage: ubuntu-latest - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - script: | - pip install black==25.1.0 - displayName: install black - - script: | - black --check --diff . - displayName: Run black - - - template: continuous_integration/windows.yml - parameters: - name: Windows - vmImage: windows-latest - matrix: - pylatest_conda_forge_mkl: - PYTHON_VERSION: '*' - PACKAGER: 'conda-forge' - BLAS: 'mkl' - py310_conda_forge_openblas: - PYTHON_VERSION: '3.10' - PACKAGER: 'conda-forge' - BLAS: 'openblas' - py39_conda: - PYTHON_VERSION: '3.9' - PACKAGER: 'conda' - py38_pip: - PYTHON_VERSION: '3.8' - PACKAGER: 'pip' - - - template: continuous_integration/posix.yml - parameters: - name: Linux - vmImage: ubuntu-20.04 - matrix: - # Linux environment with development versions of numpy and scipy - pylatest_pip_dev: - PACKAGER: 'pip-dev' - PYTHON_VERSION: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux environment to test that packages that comes with Ubuntu 20.04 - # are correctly handled. - py38_ubuntu_atlas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libatlas3-base libatlas-base-dev' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - py38_ubuntu_openblas_gcc_gcc: - PACKAGER: 'ubuntu' - APT_BLAS: 'libopenblas-base libopenblas-dev' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux + Python 3.9 and homogeneous runtime nesting. - py39_conda_openblas_clang_clang: - PACKAGER: 'conda' - PYTHON_VERSION: '3.9' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'clang-10' - # Linux environment with MKL and Clang (known to be unsafe for - # threadpoolctl) to only test the warning from multiple OpenMP. - pylatest_conda_mkl_clang_gcc: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'gcc' - TESTS: 'libomp_libiomp_warning' - # Linux environment with MKL, safe for threadpoolctl. - pylatest_conda_mkl_gcc_gcc: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - MKL_THREADING_LAYER: 'INTEL' - # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI - # and heterogeneous OpenMP runtimes. - py38_pip_openblas_gcc_clang: - PACKAGER: 'pip' - PYTHON_VERSION: '3.8' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-10' - # Linux environment with numpy from conda-forge channel and openblas-openmp - pylatest_conda_forge: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'openblas' - OPENBLAS_THREADING_LAYER: 'openmp' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - # Linux environment with no numpy and heterogeneous OpenMP runtimes. - pylatest_conda_nonumpy_gcc_clang: - PACKAGER: 'conda' - NO_NUMPY: 'true' - PYTHON_VERSION: '*' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'clang-10' - # Linux environment with numpy linked to BLIS - pylatest_blis_gcc_clang_openmp: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'clang-10' - BLIS_ENABLE_THREADING: 'openmp' - pylatest_blis_clang_gcc_pthreads: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '4' - CC_OUTER_LOOP: 'clang-10' - CC_INNER_LOOP: 'clang-10' - BLIS_CC: 'gcc-8' - BLIS_ENABLE_THREADING: 'pthreads' - pylatest_blis_no_threading: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'blis' - BLIS_NUM_THREADS: '1' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - BLIS_CC: 'gcc-8' - BLIS_ENABLE_THREADING: 'no' - pylatest_flexiblas: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'flexiblas' - PLATFORM_SPECIFIC_PACKAGES: 'mkl' - CC_OUTER_LOOP: 'gcc' - CC_INNER_LOOP: 'gcc' - - - - template: continuous_integration/posix.yml - parameters: - name: macOS - vmImage: macOS-latest - matrix: - # MacOS environment with OpenMP installed through homebrew - py38_conda_homebrew_libomp: - PACKAGER: 'conda' - PYTHON_VERSION: '3.8' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'homebrew' - # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers - py39_conda_forge_clang_openblas: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'openblas' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'conda-forge' - # MacOS environment with OpenMP installed through conda-forge compilers - pylatest_conda_forge_clang: - PACKAGER: 'conda-forge' - PYTHON_VERSION: '*' - BLAS: 'mkl' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - INSTALL_LIBOMP: 'conda-forge' - pylatest_flexiblas: - PACKAGER: 'conda' - PYTHON_VERSION: '*' - INSTALL_BLAS: 'flexiblas' - PLATFORM_SPECIFIC_PACKAGES: 'mkl' - CC_OUTER_LOOP: 'clang' - CC_INNER_LOOP: 'clang' - -- stage: - jobs: - # Meta-test to ensure that at least of the above CI configurations had - # the necessary platform settings to execute each test without raising - # skipping. - - job: 'no_test_always_skipped' - displayName: 'No test always skipped' - pool: - vmImage: ubuntu-latest - steps: - - download: current - - script: | - python continuous_integration/check_no_test_skipped.py $(Pipeline.Workspace) - displayName: 'No test always skipped' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc9eb44c..1520b10b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,15 @@ on: pull_request: branches: - master + schedule: + # Daily build at 1:00 AM UTC + - cron: "0 1 * * *" + +# Cancel in-progress workflows when pushing +# a new commit on the same branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: @@ -27,3 +36,239 @@ jobs: - name: Run black run: | black --check --diff . + + testing: + needs: linting + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + + # Windows env with numpy, scipy,MKL installed through conda-forge + - name: pylatest_conda_forge_mkl + os: windows-latest + PYTHON_VERSION: "*" + PACKAGER: "conda-forge" + BLAS: "mkl" + # Windows env with numpy, scipy, OpenBLAS installed through conda-forge + - name: py310_conda_forge_openblas + os: windows-latest + PYTHON_VERSION: "3.10" + PACKAGER: "conda-forge" + BLAS: "openblas" + # Windows env with numpy, scipy installed through conda + - name: py39_conda + os: windows-latest + PYTHON_VERSION: "3.9" + PACKAGER: "conda" + # Windows env with numpy, scipy installed through pip + - name: py38_pip + os: windows-latest + PYTHON_VERSION: "3.8" + PACKAGER: "pip" + + # MacOS env with OpenMP installed through homebrew + - name: py38_conda_homebrew_libomp + os: macos-latest + PYTHON_VERSION: "3.8" + PACKAGER: "conda" + BLAS: "openblas" + CC_OUTER_LOOP: "clang" + CC_INNER_LOOP: "clang" + INSTALL_LIBOMP: "homebrew" + # MacOS env with OpenBLAS and OpenMP installed through conda-forge compilers + - name: pylatest_conda_forge_clang_openblas + os: macos-latest + PYTHON_VERSION: "*" + PACKAGER: "conda-forge" + BLAS: "openblas" + INSTALL_LIBOMP: "conda-forge" + # MacOS env with FlexiBLAS + - name: pylatest_flexiblas + os: macos-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "llvm-openmp" + + # Linux environments to test that packages that comes with Ubuntu 20.04 + # are correctly handled. + - name: py38_ubuntu_atlas_gcc_gcc + os: ubuntu-20.04 + PYTHON_VERSION: "3.8" + PACKAGER: "ubuntu" + APT_BLAS: "libatlas3-base libatlas-base-dev" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + - name: py38_ubuntu_openblas_gcc_gcc + os: ubuntu-20.04 + PYTHON_VERSION: "3.8" + PACKAGER: "ubuntu" + APT_BLAS: "libopenblas-base libopenblas-dev" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + + # Linux environment with development versions of numpy and scipy + - name: pylatest_pip_dev + os : ubuntu-latest + PACKAGER: "pip-dev" + PYTHON_VERSION: "*" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + # Linux + Python 3.9 and homogeneous runtime nesting. + - name: py39_conda_openblas_clang_clang + os: ubuntu-latest + PACKAGER: "conda" + PYTHON_VERSION: "3.9" + BLAS: "openblas" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "clang-18" + # Linux environment with MKL and Clang (known to be unsafe for + # threadpoolctl) to only test the warning from multiple OpenMP. + - name: pylatest_conda_mkl_clang_gcc + os: ubuntu-latest + PYTHON_VERSION: "*" + PACKAGER: "conda" + BLAS: "mkl" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "gcc" + TESTS: "libomp_libiomp_warning" + # Linux environment with MKL, safe for threadpoolctl. + - name: pylatest_conda_mkl_gcc_gcc + os: ubuntu-latest + PYTHON_VERSION: "*" + PACKAGER: "conda" + BLAS: "mkl" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + MKL_THREADING_LAYER: "INTEL" + # Linux + Python 3.8 with numpy / scipy installed with pip from PyPI + # and heterogeneous OpenMP runtimes. + - name: py38_pip_openblas_gcc_clang + os: ubuntu-latest + PACKAGER: "pip" + PYTHON_VERSION: "3.8" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "clang-18" + # Linux environment with numpy from conda-forge channel and openblas-openmp + - name: pylatest_conda_forge + os: ubuntu-latest + PACKAGER: "conda-forge" + PYTHON_VERSION: "*" + BLAS: "openblas" + OPENBLAS_THREADING_LAYER: "openmp" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + # Linux environment with no numpy and heterogeneous OpenMP runtimes. + - name: pylatest_conda_nonumpy_gcc_clang + os: ubuntu-latest + PACKAGER: "conda" + PYTHON_VERSION: "*" + NO_NUMPY: "true" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "clang-18" + + # Linux environments with numpy linked to BLIS + - name: pylatest_blis_gcc_clang_openmp + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREAEDS: "4" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + BLIS_CC: "clang-18" + BLIS_ENABLE_THREADING: "openmp" + - name: pylatest_blis_clang_gcc_pthreads + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREADS: "4" + CC_OUTER_LOOP: "clang-18" + CC_INNER_LOOP: "clang-18" + BLIS_CC: "gcc-12" + BLIS_ENABLE_THREADING: "pthreads" + - name: pylatest_blis_no_threading + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "blis" + BLIS_NUM_THREADS: "1" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + BLIS_CC: "gcc-12" + BLIS_ENABLE_THREADING: "no" + + # Linux env with FlexiBLAS + - name: pylatest_flexiblas + os: ubuntu-latest + PYTHON_VERSION: "*" + INSTALL_BLAS: "flexiblas" + PLATFORM_SPECIFIC_PACKAGES: "mkl" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + + env: ${{ matrix }} + + runs-on: ${{ matrix.os }} + + defaults: + run: + # Need to use this shell to get conda working properly. + # See https://github.com/marketplace/actions/setup-miniconda#important + shell: ${{ matrix.os == 'windows-latest' && 'cmd /C CALL {0}' || 'bash -el {0}' }} + + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: true + auto-update-conda: true + miniforge-version: latest + + - name: Install dependencies + run: | + bash -el continuous_integration/install.sh + + - name: Test library + run: | + bash -el continuous_integration/run_tests.sh + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + # Requires a unique name for each job in the matrix of this run + name: test_result_${{github.run_id}}_${{ matrix.os }}_${{ matrix.name }} + path: test_result.xml + retention-days: 1 + + - name: Upload to Codecov + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + + # Meta-test to ensure that at least one of the above CI configurations had + # the necessary platform settings to execute each test without raising + # skipping. + meta_test: + needs: testing + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download tests results + uses: actions/download-artifact@v4 + with: + path: test_results + + - name: Check no test always skipped + run: | + python continuous_integration/check_no_test_skipped.py test_results diff --git a/continuous_integration/build_test_ext.sh b/continuous_integration/build_test_ext.sh index bd0e1ae7..4bc27696 100755 --- a/continuous_integration/build_test_ext.sh +++ b/continuous_integration/build_test_ext.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe if [[ "$OSTYPE" == "linux-gnu"* ]]; then pushd tests/_pyMylib diff --git a/continuous_integration/check_no_test_skipped.py b/continuous_integration/check_no_test_skipped.py index f831d29a..18fd6bc2 100644 --- a/continuous_integration/check_no_test_skipped.py +++ b/continuous_integration/check_no_test_skipped.py @@ -13,11 +13,11 @@ always_skipped = {} for name in os.listdir(base_dir): - # all test result files are in /base_dir/jobs.*/ dirs - if name.startswith("stage1."): - print("> processing test result from job", name.replace("stage1", "")) + # all test result files are in base_dir/test_result_*/ dirs + if name.startswith("test_result_"): + print(f"> processing test result from job {name.replace('test_result_', '')}") print(" > tests skipped:") - result_file = os.path.join(base_dir, name, "test-data.xml") + result_file = os.path.join(base_dir, name, "test_result.xml") root = ET.parse(result_file).getroot() # All tests are identified by the xml tag testcase. diff --git a/continuous_integration/install.cmd b/continuous_integration/install.cmd deleted file mode 100644 index 40731e05..00000000 --- a/continuous_integration/install.cmd +++ /dev/null @@ -1,38 +0,0 @@ -@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd -@rem The cmd /C hack circumvents a regression where conda installs a conda.bat -@rem script in non-root environments. -set CONDA_INSTALL=cmd /C conda install -q -y -set PIP_INSTALL=pip install -q - -@echo on - -@rem Deactivate any environment -call deactivate -@rem Clean up any left-over from a previous build and install version of python -conda update -n base conda conda-libmamba-solver -q -y -conda config --set solver libmamba -conda remove --all -q -y -n %VIRTUALENV% -conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% - -call activate %VIRTUALENV% -python -m pip install -U pip -python --version -pip --version - -@rem Install dependencies with either conda or pip. -set TO_INSTALL=numpy scipy cython pytest - -if "%PACKAGER%" == "conda" (%CONDA_INSTALL% %TO_INSTALL%) -if "%PACKAGER%" == "conda-forge" (%CONDA_INSTALL% -c conda-forge %TO_INSTALL% blas[build=%BLAS%]) -if "%PACKAGER%" == "pip" (%PIP_INSTALL% %TO_INSTALL%) - -@rem Install extra developer dependencies -pip install -q -r dev-requirements.txt - -@rem Install package -flit install --symlink - -@rem Build the cython test helper for openmp -bash ./continuous_integration/build_test_ext.sh - -if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh old mode 100755 new mode 100644 index d0e8402f..7a958980 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -1,28 +1,34 @@ #!/bin/bash -set -e +# License: BSD 3-Clause + +set -xe UNAMESTR=`uname` -if [[ "$CC_OUTER_LOOP" == "clang-10" || "$CC_INNER_LOOP" == "clang-10" ]]; then - # Assume Ubuntu: install a recent version of clang and libomp + +# Install a recent version of clang and libomp if needed +# Only applicable to linux jobs +if [[ "$CC_OUTER_LOOP" == "clang-18" ]] || \ + [[ "$CC_INNER_LOOP" == "clang-18" ]] || \ + [[ "$BLIS_CC" == "clang-18" ]] +then wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 10 + sudo ./llvm.sh 18 sudo apt-get install libomp-dev fi + make_conda() { - TO_INSTALL="$@" + CHANNEL="$1" + TO_INSTALL="$2" if [[ "$UNAMESTR" == "Darwin" ]]; then if [[ "$INSTALL_LIBOMP" == "conda-forge" ]]; then # Install an OpenMP-enabled clang/llvm from conda-forge # assumes conda-forge is set on priority channel TO_INSTALL="$TO_INSTALL compilers llvm-openmp" - export CFLAGS="$CFLAGS -I$CONDA/envs/$VIRTUALENV/include" - export LDFLAGS="$LDFLAGS -Wl,-rpath,$CONDA/envs/$VIRTUALENV/lib -L$CONDA/envs/$VIRTUALENV/lib" - elif [[ "$INSTALL_LIBOMP" == "homebrew" ]]; then # Install a compiler with a working openmp HOMEBREW_NO_AUTO_UPDATE=1 brew install libomp @@ -30,37 +36,49 @@ make_conda() { # enable OpenMP support for Apple-clang export CC=/usr/bin/clang export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp" - export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include" - export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" + export CFLAGS="$CFLAGS -I/opt/homebrew/opt/libomp/include" + export LDFLAGS="$LDFLAGS -Wl,-rpath,/opt/homebrew/opt/libomp/lib -L/opt/homebrew/opt/libomp/lib -lomp" fi fi + + if [[ "$PYTHON_VERSION" == "*" ]]; then + # Avoid installing free-threaded python + TO_INSTALL="$TO_INSTALL python-gil" + fi + + # prevent mixing conda channels + conda config --set channel_priority strict + conda config --add channels $CHANNEL + conda update -n base conda conda-libmamba-solver -q --yes conda config --set solver libmamba - conda create -n $VIRTUALENV -q --yes $TO_INSTALL - source activate $VIRTUALENV + + conda create -n testenv -q --yes python=$PYTHON_VERSION $TO_INSTALL + conda activate testenv } + if [[ "$PACKAGER" == "conda" ]]; then - TO_INSTALL="python=$PYTHON_VERSION pip" + TO_INSTALL="" if [[ "$NO_NUMPY" != "true" ]]; then - TO_INSTALL="$TO_INSTALL numpy scipy blas[build=$BLAS]" + TO_INSTALL="$TO_INSTALL numpy scipy" + if [[ -n "$BLAS" ]]; then + TO_INSTALL="$TO_INSTALL blas=*=$BLAS" + fi fi - make_conda $TO_INSTALL + make_conda "defaults" "$TO_INSTALL" elif [[ "$PACKAGER" == "conda-forge" ]]; then - conda config --prepend channels conda-forge - conda config --set channel_priority strict - TO_INSTALL="python=$PYTHON_VERSION numpy scipy blas[build=$BLAS]" + TO_INSTALL="numpy scipy blas=*=$BLAS" if [[ "$BLAS" == "openblas" && "$OPENBLAS_THREADING_LAYER" == "openmp" ]]; then TO_INSTALL="$TO_INSTALL libopenblas=*=*openmp*" fi - make_conda $TO_INSTALL + make_conda "conda-forge" "$TO_INSTALL" elif [[ "$PACKAGER" == "pip" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy - TO_INSTALL="python=$PYTHON_VERSION pip" - make_conda $TO_INSTALL + make_conda "conda-forge" "" if [[ "$NO_NUMPY" != "true" ]]; then pip install numpy scipy fi @@ -68,8 +86,7 @@ elif [[ "$PACKAGER" == "pip" ]]; then elif [[ "$PACKAGER" == "pip-dev" ]]; then # Use conda to build an empty python env and then use pip to install # numpy and scipy dev versions - TO_INSTALL="python=$PYTHON_VERSION pip" - make_conda $TO_INSTALL + make_conda "conda-forge" "" dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy @@ -80,14 +97,29 @@ elif [[ "$PACKAGER" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update sudo apt-get install python3-scipy python3-virtualenv $APT_BLAS - python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV - source $VIRTUALENV/bin/activate -fi + python3 -m virtualenv --system-site-packages --python=python3 testenv + source testenv/bin/activate + +elif [[ "$INSTALL_BLAS" == "blis" ]]; then + TO_INSTALL="cython meson-python pkg-config" + make_conda "conda-forge" "$TO_INSTALL" + source ./continuous_integration/install_blis.sh +elif [[ "$INSTALL_BLAS" == "flexiblas" ]]; then + TO_INSTALL="cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers" + make_conda "conda-forge" "$TO_INSTALL" + source ./continuous_integration/install_flexiblas.sh -python -m pip install -q -r dev-requirements.txt +fi + +python -m pip install -v -q -r dev-requirements.txt bash ./continuous_integration/build_test_ext.sh +# Check which BLAS is linked (only available on linux) +if [[ "$UNAMESTR" == "Linux" && "$NO_NUMPY" != "true" ]]; then + ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so +fi + python --version python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" diff --git a/continuous_integration/install_blis.sh b/continuous_integration/install_blis.sh new file mode 100644 index 00000000..e8fa8074 --- /dev/null +++ b/continuous_integration/install_blis.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -xe + + +# Install gcc 12 to build BLIS +if [[ "$BLIS_CC" == "gcc-12" ]]; then + sudo apt install gcc-12 +fi + +# step outside of threadpoolctl directory +pushd .. +ABS_PATH=$(pwd) + +# build & install blis +mkdir BLIS_install +git clone https://github.com/flame/blis.git +pushd blis + +./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$BLIS_ENABLE_THREADING CC=$BLIS_CC auto +make -j4 +make install +popd + +# build & install numpy +git clone https://github.com/numpy/numpy.git +pushd numpy +git submodule update --init + +echo "libdir=$ABS_PATH/BLIS_install/lib/ +includedir=$ABS_PATH/BLIS_install/include/blis/ +version=latest +extralib=-lm -lpthread -lgfortran +Name: blis +Description: BLIS +Version: \${version} +Libs: -L\${libdir} -lblis +Libs.private: \${extralib} +Cflags: -I\${includedir}" > blis.pc + +PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=blis + +export CFLAGS=-I$ABS_PATH/BLIS_install/include/blis +export LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" + +popd + +# back to threadpoolctl directory +popd diff --git a/continuous_integration/install_with_flexiblas.sh b/continuous_integration/install_flexiblas.sh old mode 100755 new mode 100644 similarity index 63% rename from continuous_integration/install_with_flexiblas.sh rename to continuous_integration/install_flexiblas.sh index 59eec612..3d45ab54 --- a/continuous_integration/install_with_flexiblas.sh +++ b/continuous_integration/install_flexiblas.sh @@ -1,19 +1,10 @@ #!/bin/bash -set -e +set -xe +# step outside of threadpoolctl directory pushd .. ABS_PATH=$(pwd) -popd - -# create conda env -conda update -n base conda conda-libmamba-solver -q --yes -conda config --set solver libmamba -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ - pip cython openblas $PLATFORM_SPECIFIC_PACKAGES meson-python pkg-config compilers -source activate $VIRTUALENV - -pushd .. # build & install FlexiBLAS mkdir flexiblas_install @@ -66,22 +57,11 @@ Libs.private: \${extralib} Cflags: -I\${includedir}" > flexiblas.pc PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=flexiblas -Csetup-args=-Dlapack=flexiblas -popd -popd +export CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \ +export LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \ -python -m pip install -q -r dev-requirements.txt -CFLAGS=-I$ABS_PATH/flexiblas_install/include/flexiblas \ - LDFLAGS="-L$ABS_PATH/flexiblas_install/lib -Wl,-rpath,$ABS_PATH/flexiblas_install/lib" \ - bash ./continuous_integration/build_test_ext.sh - -# Check that FlexiBLAS is linked -if [[ $(uname) != "Darwin" ]]; then - ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so -fi - -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" -python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" +popd -python -m flit install --symlink +# back to threadpoolctl directory +popd diff --git a/continuous_integration/install_with_blis.sh b/continuous_integration/install_with_blis.sh deleted file mode 100755 index 15ef3a5e..00000000 --- a/continuous_integration/install_with_blis.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -pushd .. -ABS_PATH=$(pwd) -popd - -# Install a recent version of clang and libomp -wget https://apt.llvm.org/llvm.sh -chmod +x llvm.sh -sudo ./llvm.sh 10 -sudo apt-get install libomp-dev - -# create conda env -conda update -n base conda conda-libmamba-solver -q --yes -conda config --set solver libmamba -conda create -n $VIRTUALENV -q --yes -c conda-forge python=$PYTHON_VERSION \ - pip cython meson-python pkg-config -source activate $VIRTUALENV - -if [[ "$BLIS_CC" == "gcc-8" ]]; then - sudo apt install gcc-8 -fi - -pushd .. - -# build & install blis -mkdir BLIS_install -git clone https://github.com/flame/blis.git -pushd blis - -./configure --prefix=$ABS_PATH/BLIS_install --enable-cblas --enable-threading=$BLIS_ENABLE_THREADING CC=$BLIS_CC auto -make -j4 -make install -popd - -# build & install numpy -git clone https://github.com/numpy/numpy.git -pushd numpy -git submodule update --init - -echo "libdir=$ABS_PATH/BLIS_install/lib/ -includedir=$ABS_PATH/BLIS_install/include/blis/ -version=latest -extralib=-lm -lpthread -lgfortran -Name: blis -Description: BLIS -Version: \${version} -Libs: -L\${libdir} -lblis -Libs.private: \${extralib} -Cflags: -I\${includedir}" > blis.pc - -PKG_CONFIG_PATH=$ABS_PATH/numpy/ pip install . -v --no-build-isolation -Csetup-args=-Dblas=blis -popd - -popd - -python -m pip install -q -r dev-requirements.txt -CFLAGS=-I$ABS_PATH/BLIS_install/include/blis \ - LDFLAGS="-L$ABS_PATH/BLIS_install/lib -Wl,-rpath,$ABS_PATH/BLIS_install/lib" \ - bash ./continuous_integration/build_test_ext.sh - -# Check that BLIS is linked -ldd tests/_openmp_test_helper/nested_prange_blas.cpython*.so - -python --version -python -c "import numpy; print(f'numpy {numpy.__version__}')" || echo "no numpy" -python -c "import scipy; print(f'scipy {scipy.__version__}')" || echo "no scipy" - -python -m flit install --symlink diff --git a/continuous_integration/posix.yml b/continuous_integration/posix.yml deleted file mode 100644 index a1598d37..00000000 --- a/continuous_integration/posix.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - name: '' - vmImage: '' - matrix: [] - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - condition: or(startsWith(variables['PACKAGER'], 'conda'), startsWith(variables['PACKAGER'], 'pip')) - - bash: sudo chown -R $USER $CONDA - # On Hosted macOS, the agent user doesn't have ownership of Miniconda's installation directory/ - # We need to take ownership if we want to update conda or install packages globally - displayName: Take ownership of conda installation - condition: eq('${{ parameters.name }}', 'macOS') - - script: | - continuous_integration/install.sh - displayName: 'Install without custom BLAS' - condition: eq(variables['INSTALL_BLAS'], '') - - script: | - continuous_integration/install_with_blis.sh - displayName: 'Install with BLIS' - condition: eq(variables['INSTALL_BLAS'], 'blis') - - script: | - continuous_integration/install_with_flexiblas.sh - displayName: 'Install with FlexiBLAS' - condition: eq(variables['INSTALL_BLAS'], 'flexiblas') - - script: | - continuous_integration/test_script.sh - displayName: 'Test Library' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(JUNITXML)' - testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} - displayName: 'Publish Test Results' - condition: succeededOrFailed() - - publish: $(JUNITXML) - - script: | - bash continuous_integration/upload_codecov.sh - displayName: 'Upload to codecov' - condition: succeeded() diff --git a/continuous_integration/test_script.sh b/continuous_integration/run_tests.sh similarity index 60% rename from continuous_integration/test_script.sh rename to continuous_integration/run_tests.sh index 8eeaba19..f26f0fdc 100755 --- a/continuous_integration/test_script.sh +++ b/continuous_integration/run_tests.sh @@ -1,23 +1,21 @@ #!/bin/bash -set -e +set -xe -if [[ "$PACKAGER" == conda* ]]; then - source activate $VIRTUALENV +if [[ "$PACKAGER" == conda* ]] || [[ -z "$PACKAGER" ]]; then + conda activate testenv conda list elif [[ "$PACKAGER" == pip* ]]; then # we actually use conda to install the base environment: - source activate $VIRTUALENV + conda activate testenv pip list elif [[ "$PACKAGER" == "ubuntu" ]]; then - source $VIRTUALENV/bin/activate + source testenv/bin/activate pip list fi -set -x - # Use the CLI to display the effective runtime environment prior to # launching the tests: python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner -pytest -vlrxXs -W error -k "$TESTS" --junitxml=$JUNITXML --cov=threadpoolctl +pytest -vlrxXs -W error -k "$TESTS" --junitxml=test_result.xml --cov=threadpoolctl --cov-report xml diff --git a/continuous_integration/test_script.cmd b/continuous_integration/test_script.cmd deleted file mode 100644 index 4da20228..00000000 --- a/continuous_integration/test_script.cmd +++ /dev/null @@ -1,10 +0,0 @@ -call activate %VIRTUALENV% - -# Display version information -python -m pip list - -# Use the CLI to display the effective runtime environment prior to -# launching the tests: -python -m threadpoolctl -i numpy scipy.linalg tests._openmp_test_helper.openmp_helpers_inner - -pytest -vlrxXs --junitxml=%JUNITXML% --cov=threadpoolctl diff --git a/continuous_integration/upload_codecov.sh b/continuous_integration/upload_codecov.sh deleted file mode 100755 index 1fb312c0..00000000 --- a/continuous_integration/upload_codecov.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$PACKAGER" == "conda" ]]; then - source activate $VIRTUALENV -elif [[ "$PACKAGER" == "pip" ]]; then - source activate $VIRTUALENV -elif [[ "$PACKAGER" == "ubuntu" ]]; then - source $VIRTUALENV/bin/activate -fi - -pip install codecov - -codecov || echo "codecov upload failed" diff --git a/continuous_integration/windows.yml b/continuous_integration/windows.yml deleted file mode 100644 index 7d6e53da..00000000 --- a/continuous_integration/windows.yml +++ /dev/null @@ -1,35 +0,0 @@ - -parameters: - name: '' - vmImage: '' - matrix: [] - -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - - strategy: - matrix: - ${{ insert }}: ${{ parameters.matrix }} - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - script: | - continuous_integration\\install.cmd - displayName: 'Install' - - script: | - continuous_integration\\test_script.cmd - displayName: 'Test threadpoolctl' - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(JUNITXML)' - testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} - displayName: 'Publish Test Results' - condition: succeededOrFailed() - - publish: $(JUNITXML) - - script: | - bash continuous_integration\\upload_codecov.sh - displayName: 'Upload to codecov' - condition: succeeded() diff --git a/dev-requirements.txt b/dev-requirements.txt index d118f1e7..04901b66 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,3 +3,4 @@ coverage pytest pytest-cov cython +setuptools diff --git a/tests/_openmp_test_helper/setup_inner.py b/tests/_openmp_test_helper/setup_inner.py index 4e045c02..547e257a 100644 --- a/tests/_openmp_test_helper/setup_inner.py +++ b/tests/_openmp_test_helper/setup_inner.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/_openmp_test_helper/setup_nested_prange_blas.py b/tests/_openmp_test_helper/setup_nested_prange_blas.py index 809f981c..dcb39ca6 100644 --- a/tests/_openmp_test_helper/setup_nested_prange_blas.py +++ b/tests/_openmp_test_helper/setup_nested_prange_blas.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/_openmp_test_helper/setup_outer.py b/tests/_openmp_test_helper/setup_outer.py index 98a84d58..dee39b1a 100644 --- a/tests/_openmp_test_helper/setup_outer.py +++ b/tests/_openmp_test_helper/setup_outer.py @@ -1,7 +1,6 @@ import os -from distutils.core import setup +from setuptools import Extension, setup from Cython.Build import cythonize -from distutils.extension import Extension from build_utils import set_cc_variables from build_utils import get_openmp_flag diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index d6e95363..7f6e090e 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -624,6 +624,7 @@ def test_architecture(): # XXX: add more as needed by CI or developer laptops "skx", "haswell", + "zen3", ) for lib_info in threadpool_info(): if lib_info["internal_api"] == "openblas": @@ -657,7 +658,7 @@ def test_openblas_threading_layer(): # skip test if not run in a azure pipelines job since it relies on a specific flexiblas # installation. @pytest.mark.skipif( - "TF_BUILD" not in os.environ, reason="not running in azure pipelines" + "GITHUB_ACTIONS" not in os.environ, reason="not running in azure pipelines" ) def test_flexiblas(): # Check that threadpool_info correctly recovers the FlexiBLAS backends. @@ -695,7 +696,7 @@ def test_flexiblas_switch_error(): # skip test if not run in a azure pipelines job since it relies on a specific flexiblas # installation. @pytest.mark.skipif( - "TF_BUILD" not in os.environ, reason="not running in azure pipelines" + "GITHUB_ACTIONS" not in os.environ, reason="not running in azure pipelines" ) def test_flexiblas_switch(): # Check that the backend can be switched. @@ -717,13 +718,13 @@ def test_flexiblas_switch(): assert fb_controller.current_backend == "NETLIB" assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB"] - ext = ".so" if sys.platform == "linux" else ".dylib" - mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt{ext}" - fb_controller.switch_backend(mkl_path) - assert fb_controller.current_backend == mkl_path - assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB", mkl_path] - # switching the backend triggered a new search for loaded shared libs - assert len(controller.select(internal_api="mkl").lib_controllers) == 1 + if sys.platform == "linux": + mkl_path = f"{os.getenv('CONDA_PREFIX')}/lib/libmkl_rt.so" + fb_controller.switch_backend(mkl_path) + assert fb_controller.current_backend == mkl_path + assert fb_controller.loaded_backends == ["OPENBLAS_CONDA", "NETLIB", mkl_path] + # switching the backend triggered a new search for loaded shared libs + assert len(controller.select(internal_api="mkl").lib_controllers) == 1 # switch back to default to avoid side effects fb_controller.switch_backend("OPENBLAS_CONDA")