Upgrade python and tensorflow and add unittests for main tools

.github/workflows/python-package-conda.yml

@@ -1,4 +1,3 @@
-
 name: CI
 
 on:
@@ -15,39 +14,61 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04]
-        pyv: [ '3.10','3.11', '3.12', '3.13']
-      max-parallel: 5
+        python-version: ['3.12', '3.13', '3.14']
+        dl1dh-version: ['latest', 'nightly']
+      max-parallel: 6
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.dl1dh-version == 'nightly' || matrix.python-version == '3.14' }}
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.pyv }}
-      run: |
-        # Install Miniconda
-        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-        bash miniconda.sh -b -p $HOME/miniconda
-        echo "$HOME/miniconda/bin" >> $GITHUB_PATH
-        source $HOME/miniconda/bin/activate
-        # Install Mamba via conda (since we don't have mamba yet)
-        $HOME/miniconda/bin/conda config --add channels conda-forge
-        $HOME/miniconda/bin/conda install -y mamba=2.0.8
-        mamba install -y python=${{ matrix.pyv }}
-    - name: Add MKL_THREADING_LAYER variable
-      run: echo "MKL_THREADING_LAYER=GNU" >> $GITHUB_ENV
-    - name: Install dependencies with Mamba
-      run: |
-        source $HOME/miniconda/bin/activate
-        mamba env update --file environment.yml --name base
-    - name: Lint with flake8
-      run: |
-        source $HOME/miniconda/bin/activate
-        mamba install flake8
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Install with pip
-      run: |
-        pip install -e .
-    - name: Test with pytest
-      run: |
-        source $HOME/miniconda/bin/activate
-        mamba install pytest
-        pytest
+      - uses: actions/checkout@v4
+
+      - name: Set up Miniconda
+        run: |
+          wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+          bash miniconda.sh -b -p $HOME/miniconda
+          echo "$HOME/miniconda/bin" >> $GITHUB_PATH
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          conda config --add channels conda-forge
+          conda install -y mamba
+
+      - name: Create Python environment
+        run: |
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          mamba create -y -n ctlearn python==${{ matrix.python-version }} -c conda-forge
+          conda activate ctlearn
+
+      - name: Install dependencies
+        run: |
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          conda activate ctlearn
+          sudo apt-get update
+          sudo apt-get install -y git
+          pip install --upgrade pip
+          pip install pylint pylint-exit anybadge eventio pytest flake8
+          if [ "${{ matrix.dl1dh-version }}" = "nightly" ]; then
+            pip install git+https://github.com/cta-observatory/dl1-data-handler.git
+          else
+            pip install dl1-data-handler
+          fi
+
+      - name: Add MKL_THREADING_LAYER variable
+        run: echo "MKL_THREADING_LAYER=GNU" >> $GITHUB_ENV
+
+      - name: Lint with flake8
+        run: |
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          conda activate ctlearn
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Install package with pip
+        run: |
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          conda activate ctlearn
+          pip install -e .
+
+      - name: Run pytest
+        run: |
+          source $HOME/miniconda/etc/profile.d/conda.sh
+          conda activate ctlearn
+          pytest

.github/workflows/release.yml

@@ -2,13 +2,17 @@ name: Release CD
 
 on:
   release:
-   types: [published]
+    types: [published]
   workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+
 jobs:
   pypi-publish:
     name: Publish release to PyPI
     environment:
-      name: pypi 
+      name: pypi
       url: https://pypi.org/project/ctlearn/
     permissions:
       id-token: write
@@ -20,64 +24,29 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04]
-        pyv: ['3.10']
+        pyv: ['3.12']
       max-parallel: 5
     runs-on: ${{ matrix.os }}
+
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.pyv }}
-      run: |
-        conda install -y python=${{ matrix.pyv }}
-
-    - name: Add conda to system path
-      run: |
-        #$CONDA is an environment variable pointing to the root of the miniconda directory
-        echo $CONDA/bin >> $GITHUB_PATH 
-
-    - name: Install dependencies
-      run: |
-        conda env update --file environment.yml --name base 
-
-    - name: Build package
-      run: |
-        python --version
-        pip install -U build
-        python -m build
-    - name: Publish package distributions to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      - uses: actions/checkout@v4
 
+      - name: Set up micromamba (Python ${{ matrix.pyv }})
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-name: ctlearn
+          create-args: >-
+            python=${{ matrix.pyv }}
+            pip
+          cache-environment: true
 
-  condapublish:
-    strategy:
-      matrix:
-        os: [ubuntu-22.04]
-        pyv: ["3.10"]
-      max-parallel: 5
-    runs-on: ${{ matrix.os }}
-    permissions:
-      id-token: write
-      contents: write
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - name: Set up Python ${{ matrix.pyv }}
-      run: |
-        conda install -y python=${{ matrix.pyv }}
-
-    - name: Add conda to system path
-      run: |
-        # $CONDA is an environment variable pointing to the root of the miniconda directory
-        echo $CONDA/bin >> $GITHUB_PATH
-
-    - name: Install dependencies
-      run: |
-        conda env update --file environment.yml --name base
-        sudo apt-get install python3-numpy
-
-    - name: publish-to-conda
-      uses: fcakyon/conda-publish-action@v1.3
-      with:
-        subdir: '.github/conda'
-        anacondatoken: ${{ secrets.ANACONDA_TOKEN }}
+      - name: Build package
+        shell: micromamba-shell {0}
+        run: |
+          python --version
+          pip install -U build
+          python -m build
+
+      - name: Publish package distributions to PyPI
+        if: github.event_name == 'release'
+        uses: pypa/gh-action-pypi-publish@release/v1

.readthedocs.yaml

@@ -6,7 +6,7 @@ sphinx:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.12"
 
 python:
   install:

Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Build the ctlearn wheel using a standard Python image
-FROM python:3.11 AS builder
+FROM python:3.12 AS builder
 
 # Install git (needed for setuptools_scm during build) and build tool
 RUN apt-get update \
@@ -18,9 +18,7 @@ COPY ./.git ./.git/
 RUN python -m build --wheel
 
 # Stage 2: Create the final runtime image BASED ON NVIDIA's TF image
-# TODO what version to use ? after 24.?? TF 2.14 is not found in the container
-FROM nvcr.io/nvidia/tensorflow:24.01-tf2-py3 
-
+FROM nvcr.io/nvidia/tensorflow:25.02-tf2-py3 
 # Copy only the built wheel from the builder stage's dist directory
 COPY --from=builder /repo/dist /tmp/dist
 

README.rst

@@ -28,24 +28,24 @@ CTLearn is a package under active development to run deep learning models to ana
 Installation for users
 ----------------------
 
-Download and install `Anaconda <https://www.anaconda.com/download/>`_\ , or, for a minimal installation, `Miniconda <https://conda.io/miniconda.html>`_.
 
-The following command will set up a conda virtual environment, add the
-necessary package channels, and install CTLearn specified version and its dependencies:
+Installation
+------------
+
+First, create and activate a fresh conda environment:
 
 .. code-block:: bash
 
-   CTLEARN_VER=0.10.3
-   wget https://raw.githubusercontent.com/ctlearn-project/ctlearn/v$CTLEARN_VER/environment.yml
-   conda env create -n [ENVIRONMENT_NAME] -f environment.yml
-   conda activate [ENVIRONMENT_NAME]
-   pip install ctlearn==$CTLEARN_VER
-   ctlearn -h
+   mamba create -n ctlearn -c conda-forge python==3.12 llvmlite
+   conda activate ctlearn
+
+The lastest version fo this package can be installed as a pip package:
 
+.. code-block:: bash
 
-This should automatically install all dependencies (NOTE: this may take some time, as by default MKL is included as a dependency of NumPy and it is very large).
+   pip install ctlearn
 
-See the documentation for further information like `installation instructions for developers <https://ctlearn.readthedocs.io/en/latest/installation.html#installing-with-pip-setuptools-from-source-for-development>`_, `package usage <https://ctlearn.readthedocs.io/en/stable/usage.html>`_, and `dependencies <https://ctlearn.readthedocs.io/en/stable/installation.html#dependencies>`_ among other topics.
+See the documentation for further information like `installation instructions for the IT-cluster <https://ctlearn.readthedocs.io/en/latest/installation.html#install-a-released-version>`_, `installation instructions for developers <https://ctlearn.readthedocs.io/en/latest/installation.html#installing-with-pip-setuptools-from-source-for-development>`_, `package usage <https://ctlearn.readthedocs.io/en/stable/usage.html>`_, and `dependencies <https://ctlearn.readthedocs.io/en/stable/installation.html#dependencies>`_ among other topics.
 
 Citing this software
 --------------------