Merge pull request #109 from easyscience/develop

Release: merge develop into master

Author: AndrewSazonov

.badgery.yaml

@@ -52,6 +52,7 @@ cards:
   - group: Coverage
     type: codecov
     title: Unit test coverage (Codecov)
+    flag: unittests
     enabled: true
 
   - group: Coverage

.github/workflows/backmerge-pr.yaml

@@ -1,12 +1,16 @@
-# This workflow creates a backmerge PR from DEFAULT_BRANCH into `develop`
-# whenever a new version tag is pushed (v*).
+# This workflow creates a backmerge PR into `develop` whenever a new version tag is pushed (v*).
 #
-# Usage:
-#   - Triggered automatically on tag push (v*).
-#   - Creates a PR titled "Backmerge: DEFAULT_BRANCH into develop".
-#   - Adds the label "[maintainer] auto-pull-request" so it is excluded from changelogs.
+# Key points:
+# - The PR head is a temporary branch that points at the *tag commit* (refs/tags/vX.Y.Z).
+#   After merging, `develop` can `git describe` as vX.Y.Z-... because the tag commit becomes an ancestor.
+# - The PR is auto-merged using a MERGE COMMIT (not squash) via a GitHub App token.
+#   The GitHub App must be added to the develop ruleset bypass list.
+#
+# Required repo config:
+# - Actions secret:   ES_BACKMERGE_PRIVATE_KEY  (GitHub App private key PEM)
+# - Actions variable: ES_BACKMERGE_APP_ID       (GitHub App ID)
 
-name: Backmerge PR creation
+name: Backmerge PR (tag -> develop)
 
 on:
   push:
@@ -17,28 +21,76 @@ permissions:
   contents: write
   pull-requests: write
 
-# Set the environment variables to be used in all jobs defined in this workflow
-env:
-  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-
 jobs:
   create-backmerge-pr:
     runs-on: ubuntu-latest
+
     steps:
-      - name: Checkout ${{ env.DEFAULT_BRANCH }} branch
+      - name: Create GitHub App installation token
+        id: app-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.ES_BACKMERGE_APP_ID }}
+          private-key: ${{ secrets.ES_BACKMERGE_PRIVATE_KEY }}
+
+      - name: Checkout tag commit
         uses: actions/checkout@v5
         with:
-          ref: ${{ env.DEFAULT_BRANCH }}
+          ref: ${{ github.ref }} # refs/tags/vX.Y.Z
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+
+      - name: Create and push backmerge branch at tag
+        id: vars
+        run: |
+          set -euo pipefail
+
+          TAG='${{ github.ref_name }}'
+          BRANCH="backmerge/${TAG}"
+
+          echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
+          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
+
+          git config user.name  "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create/move branch to point exactly at the tag commit
+          git checkout -B "$BRANCH"
+
+          # Push (force makes re-runs idempotent for the same tag)
+          git push --force --set-upstream origin "$BRANCH"
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+
+      - name:
+          Create PR from tag backmerge branch to develop (or reuse if exists)
+        run: |
+          set -euo pipefail
+
+          TITLE="Backmerge: ${{ steps.vars.outputs.tag }} into develop"
+
+          BODY="⚠️ This PR is created automatically for backmerge of a new release tag commit \`${{ steps.vars.outputs.tag }}\` into \`develop\`.
+
+          It is labeled '[maintainer] auto-pull-request' and is excluded from release notes and version bump logic."
+
+          if gh pr view --repo "${{ github.repository }}" --head "${{ steps.vars.outputs.branch }}" >/dev/null 2>&1; then
+            echo "PR already exists for head=${{ steps.vars.outputs.branch }}"
+          else
+            gh pr create \
+              --repo "${{ github.repository }}" \
+              --base develop \
+              --head "${{ steps.vars.outputs.branch }}" \
+              --title "$TITLE" \
+              --label "[maintainer] auto-pull-request" \
+              --body "$BODY"
+          fi
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
 
-      - name: Create PR from ${{ env.DEFAULT_BRANCH }} to develop
+      - name: Enable auto-merge using MERGE COMMIT
         run: |
-          gh pr create \
-            --base develop \
-            --head ${{ env.DEFAULT_BRANCH }} \
-            --title "Backmerge: ${{ env.DEFAULT_BRANCH }} into develop" \
-            --label "[maintainer] auto-pull-request" \
-            --body "⚠️ This PR is created automatically for backmerges changes from \`${{ env.DEFAULT_BRANCH }}\` into \`develop\`, following a new release tag push.
-
-            It is labeled \`[maintainer] auto-pull-request\` and is excluded from release notes and version bump logic."
+          set -euo pipefail
+          # Merge the PR identified by its head branch.
+          gh pr merge --repo "${{ github.repository }}" --merge --auto "${{ steps.vars.outputs.branch }}"
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_API_PERSONAL_ACCESS_TOKEN }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}

.github/workflows/coverage.yaml

@@ -86,8 +86,8 @@ jobs:
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  # Job 2: Run functional tests with coverage and upload to Codecov
-  func-tests-coverage:
+  # Job 2: Run integration tests with coverage and upload to Codecov
+  integration-tests-coverage:
     runs-on: ubuntu-latest
 
     steps:
@@ -108,23 +108,25 @@ jobs:
         shell: bash
         run: pixi run dev
 
-      - name: Run functional tests with coverage
-        run: pixi run func-tests-coverage --cov-report=xml:coverage-func.xml
+      - name: Run integration tests with coverage
+        run:
+          pixi run integration-tests-coverage
+          --cov-report=xml:coverage-integration.xml
 
-      - name: Upload functional tests coverage to Codecov
+      - name: Upload integration tests coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v5
         with:
-          name: func-tests-job
-          flags: functional
-          files: ./coverage-func.xml
+          name: integration-tests-job
+          flags: integration
+          files: ./coverage-integration.xml
           fail_ci_if_error: true
           verbose: true
           token: ${{ secrets.CODECOV_TOKEN }}
 
   # Job 4: Trigger dashboard build
   dashboard-build-trigger:
-    needs: [docstring-coverage, unit-tests-coverage, func-tests-coverage] # depend on the previous jobs
+    needs: [docstring-coverage, unit-tests-coverage, integration-tests-coverage] # depend on the previous jobs
 
     runs-on: ubuntu-latest
 

.github/workflows/dashboard.yaml

@@ -41,33 +41,34 @@ jobs:
 
       - name: Install and setup development dependencies
         shell: bash
-        run: pixi run dev
+        run: |
+          pixi run dev
+          pixi add --pypi --git https://github.com/enhantica/badgery badgery
 
       - name: Run docstring coverage and code complexity/maintainability checks
         run: |
           for BRANCH in ${{ env.DEFAULT_BRANCH }} ${{ env.DEVELOP_BRANCH }} ${{ env.CI_BRANCH }}; do
             echo "=== Processing branch $BRANCH ==="
-            if [ -d "../worktree-$BRANCH" ]; then
+            if [ -d "../$BRANCH" ]; then
               echo "Branch $BRANCH already processed, skipping"
               continue
             fi
-            git worktree add ../worktree-$BRANCH origin/$BRANCH
+            git worktree add ../$BRANCH origin/$BRANCH
             mkdir -p reports/$BRANCH
             echo "Docstring coverage for branch $BRANCH"
-            pixi run interrogate -c pyproject.toml --fail-under=0 ../worktree-$BRANCH/src > reports/$BRANCH/coverage-docstring.txt
+            pixi run interrogate -c pyproject.toml --fail-under=0 ../$BRANCH/src > reports/$BRANCH/coverage-docstring.txt
             echo "Cyclomatic complexity for branch $BRANCH"
-            pixi run radon cc -s -j ../worktree-$BRANCH/src --exclude ../worktree-$BRANCH/src/easydiffraction/crystallography/space_group_lookup_table.py > reports/$BRANCH/cyclomatic-complexity.json
+            pixi run radon cc -s -j ../$BRANCH/src > reports/$BRANCH/cyclomatic-complexity.json
             echo "Maintainability index for branch $BRANCH"
-            pixi run radon mi -j ../worktree-$BRANCH/src --exclude ../worktree-$BRANCH/src/easydiffraction/crystallography/space_group_lookup_table.py > reports/$BRANCH/maintainability-index.json
+            pixi run radon mi -j ../$BRANCH/src > reports/$BRANCH/maintainability-index.json
             echo "Raw metrics for branch $BRANCH"
-            pixi run radon raw -s -j ../worktree-$BRANCH/src --exclude ../worktree-$BRANCH/src/easydiffraction/crystallography/space_group_lookup_table.py > reports/$BRANCH/raw-metrics.json
+            pixi run radon raw -s -j ../$BRANCH/src > reports/$BRANCH/raw-metrics.json
           done
 
       - name: Generate dashboard HTML
-        run: |
-          pixi add --pypi --git https://github.com/enhantica/badgery badgery
-          pixi run pip show badgery
-          pixi run python -m badgery --config .badgery.yaml --repo ${{ github.repository }} --branch ${{ env.CI_BRANCH }} --output index.html
+        run: >
+          pixi run python -m badgery --config .badgery.yaml --repo ${{
+          github.repository }} --branch ${{ env.CI_BRANCH }} --output index.html
 
       - name: Prepare publish directory
         run: |

.github/workflows/docs.yaml

@@ -137,7 +137,14 @@ jobs:
       # Input: docs/ directory containing the Markdown files
       # Output: site/ directory containing the generated HTML files
       - name: Build site with MkDocs
-        run: pixi run docs-build
+        run: |
+          if [[ "${CI_BRANCH}" == "develop" || "${CI_BRANCH}" == "master" || "${CI_BRANCH}" == "main" ]]; then
+            echo "📘 Building production docs for ${CI_BRANCH}"
+            pixi run docs-build
+          else
+            echo "📗 Building local docs for ${CI_BRANCH}"
+            pixi run docs-local
+          fi
 
       # Set up the Pages action to configure the static files to be deployed
       # NOTE: The repository must have GitHub Pages enabled and configured to build using GitHub Actions

.github/workflows/pypi-test.yaml

@@ -20,6 +20,7 @@ env:
   DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
 
 jobs:
+  # Job 1: Test installation from PyPI on multiple OS
   pypi-package-tests:
     strategy:
       matrix:
@@ -54,24 +55,45 @@ jobs:
       - name: Create the environment and install dependencies
         run: pixi install
 
-      - name: Install package from PyPI with dev and visualization extras
-        run: |
-          pixi add --pypi "easydiffraction[dev,visualization]"
-          pixi run easydiffraction --version
+      - name: Install package from PyPI with all extras
+        run: pixi add --pypi "easydiffraction[all]"
 
       - name: Run unit tests to verify the installation
         run: pixi run unit-tests
 
-      - name: Run functional tests to verify the installation
-        run: pixi run func-tests
+      - name: Run integration tests to verify the installation
+        run: pixi run integration-tests
 
       # Github token to avoid hitting the unauthenticated API rate limit
       - name: List and fetch the EasyDiffraction tutorials
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          pixi run tutorials-list
-          pixi run tutorials-fetch
+          pixi run easydiffraction --version
+          pixi run easydiffraction list-tutorials
+          pixi run easydiffraction fetch-tutorials
 
       - name: Test tutorials as notebooks
         run: pixi run notebook-tests
+
+  # Job 2: Trigger dashboard build
+  dashboard-build-trigger:
+    needs: pypi-package-tests
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check-out repository
+        uses: actions/checkout@v5
+
+      - name: Trigger dashboard build
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "dashboard.yaml",
+              ref: "${{ env.CI_BRANCH }}"
+            });

.github/workflows/test.yaml

@@ -56,15 +56,15 @@ jobs:
       pytest-marks: ${{ steps.set-mark.outputs.pytest_marks }}
 
     steps:
-      # Determine if functional tests should be run fully or only the fast ones
+      # Determine if integration tests should be run fully or only the fast ones
       # (to save time on branches other than master and develop)
-      - name: Set mark for functional tests
+      - name: Set mark for integration tests
         id: set-mark
         run: |
           if [[ "${{ env.CI_BRANCH }}" == "master" || "${{ env.CI_BRANCH }}" == "develop" ]]; then
             echo "pytest_marks=" >> $GITHUB_OUTPUT
           else
-            echo "pytest_marks=-m 'fast'" >> $GITHUB_OUTPUT
+            echo "pytest_marks=-m fast" >> $GITHUB_OUTPUT
           fi
 
   # Job 2: Test code
@@ -115,12 +115,13 @@ jobs:
             pixi run --environment $env unit-tests
           done
 
-      - name: Run functional tests ${{ needs.env-prepare.outputs.pytest-marks }}
+      - name:
+          Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
         shell: bash
         run: |
           for env in ${{ env.PIXI_ENVS }}; do
             echo "🔹🔸🔹🔸🔹   Current env: $env   🔹🔸🔹🔸🔹"
-            pixi run --environment $env func-tests "${{ needs.env-prepare.outputs.pytest-marks }}"
+            pixi run --environment $env integration-tests ${{ needs.env-prepare.outputs.pytest-marks }}
           done
 
       # Delete all local tags when not on a tagged commit to force versioningit
@@ -147,6 +148,10 @@ jobs:
         shell: bash
         run: pixi remove --pypi easydiffraction
 
+      - name: Remove Python cache files before uploading
+        shell: bash
+        run: pixi run clean-pycache
+
       # More than one file/dir need to be specified in 'path', to preserve the
       # structure of the dist/ directory, not only its contents.
       - name: Upload Python package for the next job
@@ -156,6 +161,7 @@ jobs:
           path: |
             dist/
             tests/
+            pytest.ini
             pixi.toml
             pixi.lock
           if-no-files-found: 'error'
@@ -221,12 +227,13 @@ jobs:
             pixi run --environment $env unit-tests
           done
 
-      - name: Run functional tests ${{ needs.env-prepare.outputs.pytest-marks }}
+      - name:
+          Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
         shell: bash
         run: |
           for env in ${{ env.PIXI_ENVS }}; do
             echo "🔹🔸🔹🔸🔹   Current env: $env   🔹🔸🔹🔸🔹"
-            pixi run --environment $env func-tests "${{ needs.env-prepare.outputs.pytest-marks }}"
+            pixi run --environment $env integration-tests ${{ needs.env-prepare.outputs.pytest-marks }}
           done
 
   # Job 4: Trigger dashboard build