From 6ce88a0e80be64c4baea4d679cf6a28b5157dc65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 25 Nov 2025 12:17:19 +0100 Subject: [PATCH 01/38] perf: bump to dna-seq-benchmark 1.14.1 --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 7c07716..62ef004 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -18,7 +18,7 @@ module benchmark: github( "snakemake-workflows/dna-seq-benchmark", path="workflow/Snakefile", - tag="v1.13.0", + tag="v1.14.1", ) config: config From 45f3fbf1636a0b694c72b701bc197b82541f6fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bianca=20St=C3=B6cker?= Date: Thu, 18 Dec 2025 12:23:33 +0100 Subject: [PATCH 02/38] chore: Update snakemake-github-action to version 2.0.3 --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6397d07..60e937b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,7 @@ jobs: uses: actions/checkout@v4 - name: Download reference genome - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -68,7 +68,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Download truthsets - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -82,7 +82,7 @@ jobs: # because their modification dates are too new or too old. # (as git does not preserve modification dates) - name: Fix modification dates - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -92,7 +92,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Run analysis - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -103,7 +103,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Create report - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" From 5c11a721812653daa68efb1d5805a5710cec661a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 14:16:37 +0100 Subject: [PATCH 03/38] try: dividing into subcategories --- .github/workflows/main.yml | 115 +++++++++++++++++++++++++++---------- workflow/rules/common.smk | 10 ++++ 2 files changed, 95 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60e937b..dc44492 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,46 +7,22 @@ on: pull_request: workflow_dispatch: - concurrency: # Cancel concurrent flows group: ci-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: - evaluate: + # First job: Download shared resources once to avoid multiple Zenodo requests + prepare: runs-on: ubuntu-latest - permissions: - actions: write env: - FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} - ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} - BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} - BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} - CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} - BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} - BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} steps: - - uses: 8BitJonny/gh-get-current-pr@3.0.0 - id: pr - with: - sha: ${{ github.event.pull_request.head.sha }} - filterOutClosed: true - - - run: | - echo "is PR: ${{ steps.pr.outputs.pr_found }}" - echo "current branch: ${{ github.ref }}" - - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow android: true dotnet: true haskell: true @@ -77,6 +53,51 @@ jobs: pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http + - name: Upload shared resources + uses: actions/upload-artifact@v4 + with: + name: shared-resources + path: | + resources/reference/ + resources/truthsets/ + retention-days: 1 + + # Matrix job: Run benchmarks in parallel per subcategory + evaluate: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + subcategory: + - NA12878-agilent + - NA12878-twist + - CHM-eval + permissions: + actions: write + env: + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + docker-images: false + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Download shared resources + uses: actions/download-artifact@v4 + with: + name: shared-resources + path: resources/ + # This step is necessary (after downloading the truthsets above) to ensure # that the files coming from the git repo are not triggering reruns # because their modification dates are too new or too old. @@ -91,17 +112,52 @@ jobs: pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http - - name: Run analysis - uses: snakemake/snakemake-github-action@v2.0.3 + - name: Run analysis for ${{ matrix.subcategory }} + uses: snakemake/snakemake-github-action@v2 with: directory: "." snakefile: "workflow/Snakefile" args: > - --cores 4 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp + --cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp + --config subcategory=${{ matrix.subcategory }} stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.subcategory }} + path: results/ + retention-days: 1 + + # Final job: Combine results and create report + report: + needs: evaluate + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - uses: 8BitJonny/gh-get-current-pr@3.0.0 + id: pr + with: + sha: ${{ github.event.pull_request.head.sha }} + filterOutClosed: true + + - run: | + echo "is PR: ${{ steps.pr.outputs.pr_found }}" + echo "current branch: ${{ github.ref }}" + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + pattern: results-* + path: results/ + merge-multiple: true + - name: Create report uses: snakemake/snakemake-github-action@v2.0.3 with: @@ -113,7 +169,6 @@ jobs: pip install snakemake-storage-plugin-http - name: Upload report as artifact - uses: actions/upload-artifact@v4 with: name: report diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 8ebc811..3764dce 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -5,6 +5,16 @@ def get_zenodo_tag(entry): return "zenodo" +# Filter callsets by subcategory if specified via --config subcategory=... +# This allows running benchmarks in parallel across multiple CI runners +if "subcategory" in config: + config["variant-calls"] = { + key: callset + for key, callset in config["variant-calls"].items() + if callset.get("subcategory") == config["subcategory"] + } + + # add path to callsets for key, callset in config["variant-calls"].items(): if "zenodo" in callset: From 6c50241aaf0752057dc1caa22fbc5d8981d88441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 14:26:26 +0100 Subject: [PATCH 04/38] test: add back secrets --- .github/workflows/main.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc44492..9b25b59 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,14 @@ jobs: env: SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} steps: + - name: Debug - Check if token is set + run: | + if [ -n "$SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN" ]; then + echo "Token is set (length: ${#SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN})" + else + echo "WARNING: Token is NOT set!" + fi + - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 with: @@ -38,7 +46,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache resources/reference/genome.fasta" + args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache --retries 3 --resources downloads=1 resources/reference/genome.fasta" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http @@ -48,7 +56,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --until benchmark_get_truth" + args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --retries 3 --resources downloads=1 --until benchmark_get_truth" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http @@ -76,6 +84,13 @@ jobs: permissions: actions: write env: + FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} + BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} + CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} + BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} + BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} steps: - name: Free Disk Space (Ubuntu) @@ -119,6 +134,7 @@ jobs: snakefile: "workflow/Snakefile" args: > --cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp + --retries 3 --resources downloads=1 --config subcategory=${{ matrix.subcategory }} stagein: | pip install snakemake-storage-plugin-zenodo From 307f3a16f9fc8d2161d0156ede77fb30bf40aaae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 14:31:13 +0100 Subject: [PATCH 05/38] fix: args syntax --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b25b59..ab4df5e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,10 +132,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: > - --cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp - --retries 3 --resources downloads=1 - --config subcategory=${{ matrix.subcategory }} + args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config subcategory=${{ matrix.subcategory }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 66b6c3b7097f7e493a7e8678d9867a6af7757e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 14:39:31 +0100 Subject: [PATCH 06/38] fix: args syntax --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ab4df5e..edcfc6f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,7 +132,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config subcategory=${{ matrix.subcategory }}" + args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --config subcategory=${{ matrix.subcategory }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 939d4e64640e964047eabeaca14426464f5fb0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 14:58:18 +0100 Subject: [PATCH 07/38] fix: action version --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index edcfc6f..71edb1f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,7 +128,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Run analysis for ${{ matrix.subcategory }} - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" From de17e289792fc00edfb5cb34e92b5983a78bd58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 15:10:23 +0100 Subject: [PATCH 08/38] fix: dl reference --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71edb1f..24b62cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache --retries 3 --resources downloads=1 resources/reference/genome.fasta" + args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache resources/reference/genome.fasta" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 6e73b80aaa0ac1a0aae4426c51e77bd6cf53f7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 23 Jan 2026 16:14:08 +0100 Subject: [PATCH 09/38] test: split matrix by benchmark --- .github/workflows/main.yml | 115 ++++++++++++------------------------- workflow/rules/common.smk | 10 ++++ 2 files changed, 47 insertions(+), 78 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24b62cd..f0bdf14 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,74 +13,18 @@ concurrency: cancel-in-progress: true jobs: - # First job: Download shared resources once to avoid multiple Zenodo requests - prepare: - runs-on: ubuntu-latest - env: - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - steps: - - name: Debug - Check if token is set - run: | - if [ -n "$SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN" ]; then - echo "Token is set (length: ${#SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN})" - else - echo "WARNING: Token is NOT set!" - fi - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true - docker-images: false - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: Download reference genome - uses: snakemake/snakemake-github-action@v2.0.3 - with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache resources/reference/genome.fasta" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http - - - name: Download truthsets - uses: snakemake/snakemake-github-action@v2.0.3 - with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --retries 3 --resources downloads=1 --until benchmark_get_truth" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http - - - name: Upload shared resources - uses: actions/upload-artifact@v4 - with: - name: shared-resources - path: | - resources/reference/ - resources/truthsets/ - retention-days: 1 - - # Matrix job: Run benchmarks in parallel per subcategory + # Matrix job: Run benchmarks in parallel per benchmark evaluate: - needs: prepare runs-on: ubuntu-latest strategy: fail-fast: false + max-parallel: 2 matrix: - subcategory: - - NA12878-agilent - - NA12878-twist - - CHM-eval + benchmark: + - giab-NA12878-agilent-75M + - giab-NA12878-agilent-200M + - giab-NA12878-twist + - chm-eval permissions: actions: write env: @@ -107,40 +51,41 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - - name: Download shared resources - uses: actions/download-artifact@v4 + - name: Restore caches (conda + snakemake) + uses: actions/cache@v4 with: - name: shared-resources - path: resources/ + path: | + /home/runner/miniconda3/pkgs + /home/runner/miniconda3/envs/snakemake + /home/runner/.cache/pip + /home/runner/.cache/snakemake + .snakemake + key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} + restore-keys: | + ${{ runner.os }}-snakemake- # This step is necessary (after downloading the truthsets above) to ensure # that the files coming from the git repo are not triggering reruns # because their modification dates are too new or too old. # (as git does not preserve modification dates) - name: Fix modification dates - uses: snakemake/snakemake-github-action@v2.0.3 - with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--cores 1 --sdm conda --touch resources/regions/*/test-regions.cov-*.bed" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http + run: touch resources/regions/*/test-regions.cov-*.bed - - name: Run analysis for ${{ matrix.subcategory }} + - name: Run analysis for ${{ matrix.benchmark }} uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --config subcategory=${{ matrix.subcategory }}" + args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http + show-disk-usage-on-error: true - name: Upload results uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.subcategory }} + name: results-${{ matrix.benchmark }} path: results/ retention-days: 1 @@ -164,6 +109,19 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 + - name: Restore caches (conda + snakemake) + uses: actions/cache@v4 + with: + path: | + /home/runner/miniconda3/pkgs + /home/runner/miniconda3/envs/snakemake + /home/runner/.cache/pip + /home/runner/.cache/snakemake + .snakemake + key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} + restore-keys: | + ${{ runner.os }}-snakemake- + - name: Download all results uses: actions/download-artifact@v4 with: @@ -180,6 +138,7 @@ jobs: stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http + show-disk-usage-on-error: true - name: Upload report as artifact uses: actions/upload-artifact@v4 diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 3764dce..2795bf3 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -15,6 +15,16 @@ if "subcategory" in config: } +# Filter callsets by benchmark if specified via --config benchmark=... +# This allows splitting CI work further (e.g. giab-NA12878-agilent-75M vs 200M). +if "benchmark" in config: + config["variant-calls"] = { + key: callset + for key, callset in config["variant-calls"].items() + if callset.get("benchmark") == config["benchmark"] + } + + # add path to callsets for key, callset in config["variant-calls"].items(): if "zenodo" in callset: From d314bf1edce1e8500d0e44d33d8e006df3548993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 26 Jan 2026 15:17:30 +0100 Subject: [PATCH 10/38] test: change matrix --- .github/workflows/main.yml | 93 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f0bdf14..3e3db75 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,8 +13,89 @@ concurrency: cancel-in-progress: true jobs: - # Matrix job: Run benchmarks in parallel per benchmark + # PR job: Run a lightweight benchmark to keep CI fast and within disk limits + evaluate-pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 1 + matrix: + benchmark: + - chm-eval + permissions: + actions: write + env: + FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} + BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} + CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} + BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} + BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} + TMPDIR: /mnt/tmp + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + docker-images: false + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Prepare /mnt workspace + run: | + sudo mkdir -p /mnt/conda /mnt/tmp + sudo chown -R $USER:$USER /mnt/conda /mnt/tmp + + - name: Restore caches (conda + snakemake) + uses: actions/cache@v4 + with: + path: | + /home/runner/miniconda3/pkgs + /home/runner/miniconda3/envs/snakemake + /home/runner/.cache/pip + /home/runner/.cache/snakemake + .snakemake + key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} + restore-keys: | + ${{ runner.os }}-snakemake- + + # This step is necessary (after downloading the truthsets above) to ensure + # that the files coming from the git repo are not triggering reruns + # because their modification dates are too new or too old. + # (as git does not preserve modification dates) + - name: Fix modification dates + run: touch resources/regions/*/test-regions.cov-*.bed + + - name: Run analysis for ${{ matrix.benchmark }} + uses: snakemake/snakemake-github-action@v2.0.3 + with: + directory: "." + snakefile: "workflow/Snakefile" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http + show-disk-usage-on-error: true + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.benchmark }} + path: results/ + retention-days: 1 + + # Main/dispatch job: Run full benchmark suite in parallel evaluate: + if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,6 +117,7 @@ jobs: BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} + TMPDIR: /mnt/tmp steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 @@ -51,6 +133,11 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 + - name: Prepare /mnt workspace + run: | + sudo mkdir -p /mnt/conda /mnt/tmp + sudo chown -R $USER:$USER /mnt/conda /mnt/tmp + - name: Restore caches (conda + snakemake) uses: actions/cache@v4 with: @@ -76,7 +163,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http @@ -91,7 +178,7 @@ jobs: # Final job: Combine results and create report report: - needs: evaluate + needs: [evaluate-pr, evaluate] runs-on: ubuntu-latest permissions: actions: write From 53ef4cf494dbea86e8e53c35cb963f9913add80b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 26 Jan 2026 15:23:05 +0100 Subject: [PATCH 11/38] fix: full matrix --- .github/workflows/main.yml | 85 +------------------------------------- 1 file changed, 2 insertions(+), 83 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e3db75..b55b105 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,89 +13,8 @@ concurrency: cancel-in-progress: true jobs: - # PR job: Run a lightweight benchmark to keep CI fast and within disk limits - evaluate-pr: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 1 - matrix: - benchmark: - - chm-eval - permissions: - actions: write - env: - FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} - ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} - BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} - BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} - CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} - BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} - BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - TMPDIR: /mnt/tmp - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true - docker-images: false - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: Prepare /mnt workspace - run: | - sudo mkdir -p /mnt/conda /mnt/tmp - sudo chown -R $USER:$USER /mnt/conda /mnt/tmp - - - name: Restore caches (conda + snakemake) - uses: actions/cache@v4 - with: - path: | - /home/runner/miniconda3/pkgs - /home/runner/miniconda3/envs/snakemake - /home/runner/.cache/pip - /home/runner/.cache/snakemake - .snakemake - key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} - restore-keys: | - ${{ runner.os }}-snakemake- - - # This step is necessary (after downloading the truthsets above) to ensure - # that the files coming from the git repo are not triggering reruns - # because their modification dates are too new or too old. - # (as git does not preserve modification dates) - - name: Fix modification dates - run: touch resources/regions/*/test-regions.cov-*.bed - - - name: Run analysis for ${{ matrix.benchmark }} - uses: snakemake/snakemake-github-action@v2.0.3 - with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http - show-disk-usage-on-error: true - - - name: Upload results - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.benchmark }} - path: results/ - retention-days: 1 - - # Main/dispatch job: Run full benchmark suite in parallel + # Full job: Run full benchmark suite in parallel evaluate: - if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -178,7 +97,7 @@ jobs: # Final job: Combine results and create report report: - needs: [evaluate-pr, evaluate] + needs: evaluate runs-on: ubuntu-latest permissions: actions: write From 3e95dd41493cb3c514bb06099d3df8341564e607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 26 Jan 2026 15:48:00 +0100 Subject: [PATCH 12/38] fix: add token --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b55b105..976c284 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,6 +101,8 @@ jobs: runs-on: ubuntu-latest permissions: actions: write + env: + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} steps: - uses: 8BitJonny/gh-get-current-pr@3.0.0 id: pr From 2e41623c60fa8802ae9a6e0b5a9871d87a4cad32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 26 Jan 2026 16:10:40 +0100 Subject: [PATCH 13/38] fix: tmp --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 976c284..96858d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --all-temp --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From b19d15bd2f415fa4b54238582495c9a597a32d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 08:02:39 +0100 Subject: [PATCH 14/38] test: add rule all --- workflow/Snakefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index 62ef004..12d1dc8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,6 +27,11 @@ module benchmark: use rule * from benchmark as benchmark_* +rule all: + input: + rules.benchmark_all.input, + + rule download_zenodo: input: get_zenodo_input, From 89e1adc749d76c747a7f3ffd7b18ab28377025bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 08:29:56 +0100 Subject: [PATCH 15/38] test: modify snakemake command --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 96858d7..273b70a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} benchmark_all" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 2a31f63ab02e20393bd01875a6159c31bd60a150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 08:43:09 +0100 Subject: [PATCH 16/38] fix: move target --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 273b70a..2bfb761 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} benchmark_all" + args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From d2a64380849c57c7192e7c1fbd1dac978ab34f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 09:26:58 +0100 Subject: [PATCH 17/38] fix: move target --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2bfb761..96858d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 801c558df0801dc5e665c5cc9ef184c3c5c9742b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 09:37:54 +0100 Subject: [PATCH 18/38] fix: rule all --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 12d1dc8..86ec994 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -29,7 +29,7 @@ use rule * from benchmark as benchmark_* rule all: input: - rules.benchmark_all.input, + rules.benchmark_benchmark_all.input, rule download_zenodo: From 2e74f12ae07eac7c2e9b78b0705a882ce07b5baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 09:59:16 +0100 Subject: [PATCH 19/38] fix: rule all --- .github/workflows/main.yml | 4 ++-- workflow/Snakefile | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 96858d7..5c01d30 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} benchmark_all" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http @@ -142,7 +142,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--report report.zip" + args: "--report report.zip benchmark_all" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http diff --git a/workflow/Snakefile b/workflow/Snakefile index 86ec994..62ef004 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,11 +27,6 @@ module benchmark: use rule * from benchmark as benchmark_* -rule all: - input: - rules.benchmark_benchmark_all.input, - - rule download_zenodo: input: get_zenodo_input, From 843c5b656f9e57c9bdb1156d1fa9bba57b9fa936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 10:52:41 +0100 Subject: [PATCH 20/38] fix: move target --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c01d30..8e3f720 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} benchmark_all" + args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From e26c47621985631b9113cb7ead49727f7cd87204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 13:04:32 +0100 Subject: [PATCH 21/38] test: other rerun trigger --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e3f720..c3f4b0a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,7 +82,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --rerun-triggers mtime --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http From 6e915d3953e541fb5cd012ed878a0525ec32f458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 13:13:53 +0100 Subject: [PATCH 22/38] test: change directory --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3f4b0a..b555389 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,7 +65,7 @@ jobs: /home/runner/miniconda3/envs/snakemake /home/runner/.cache/pip /home/runner/.cache/snakemake - .snakemake + /mnt/conda key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} restore-keys: | ${{ runner.os }}-snakemake- From 4442b4b2c1878d263d3c485caba0b8005bbd3021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 13:58:34 +0100 Subject: [PATCH 23/38] test: add debug --- .github/workflows/main.yml | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b555389..49ffa5f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,9 +66,9 @@ jobs: /home/runner/.cache/pip /home/runner/.cache/snakemake /mnt/conda - key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} + key: ${{ runner.os }}-conda-v2-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} restore-keys: | - ${{ runner.os }}-snakemake- + ${{ runner.os }}-conda-v2- # This step is necessary (after downloading the truthsets above) to ensure # that the files coming from the git repo are not triggering reruns @@ -77,17 +77,31 @@ jobs: - name: Fix modification dates run: touch resources/regions/*/test-regions.cov-*.bed + - name: Debug - List results directory before run + run: | + echo "=== Contents of results/ before run ===" + ls -la results/ 2>/dev/null || echo "results/ does not exist" + echo "=== Contents of .snakemake/ before run ===" + ls -la .snakemake/ 2>/dev/null || echo ".snakemake/ does not exist" + - name: Run analysis for ${{ matrix.benchmark }} uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" - args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }}" + args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} --forceall" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http show-disk-usage-on-error: true + - name: Debug - List results directory after run + if: always() + run: | + echo "=== Contents of results/ after run ===" + ls -la results/ 2>/dev/null || echo "results/ does not exist" + find results/ -type f 2>/dev/null | head -50 || true + - name: Upload results uses: actions/upload-artifact@v4 with: @@ -117,18 +131,16 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - - name: Restore caches (conda + snakemake) + - name: Restore caches (conda) uses: actions/cache@v4 with: path: | /home/runner/miniconda3/pkgs /home/runner/miniconda3/envs/snakemake /home/runner/.cache/pip - /home/runner/.cache/snakemake - .snakemake - key: ${{ runner.os }}-snakemake-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} + key: ${{ runner.os }}-conda-v2-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} restore-keys: | - ${{ runner.os }}-snakemake- + ${{ runner.os }}-conda-v2- - name: Download all results uses: actions/download-artifact@v4 From e1aab9fbca4e7f5b3bb8a97e29117a7130aec52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:06:29 +0100 Subject: [PATCH 24/38] test: add dry-run --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49ffa5f..c12ce72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,10 +89,14 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} --forceall" + args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} --forceall --verbose --printshellcmds" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http + echo "=== Dry-run to see what jobs would be executed ===" + snakemake --directory . --snakefile workflow/Snakefile benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --config benchmark=${{ matrix.benchmark }} --dry-run --reason || echo "Dry-run failed or no jobs" + echo "=== Checking rule inputs with --summary ===" + snakemake --directory . --snakefile workflow/Snakefile benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --config benchmark=${{ matrix.benchmark }} --summary 2>&1 | head -100 || echo "Summary failed" show-disk-usage-on-error: true - name: Debug - List results directory after run From 51d40946d67fbb40e0b72c1d5383289a509705bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:22:14 +0100 Subject: [PATCH 25/38] test: different structure of gha --- .github/workflows/main.yml | 83 ++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c12ce72..6f15755 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,20 +84,50 @@ jobs: echo "=== Contents of .snakemake/ before run ===" ls -la .snakemake/ 2>/dev/null || echo ".snakemake/ does not exist" - - name: Run analysis for ${{ matrix.benchmark }} - uses: snakemake/snakemake-github-action@v2.0.3 + - name: Setup Snakemake environment + uses: conda-incubator/setup-miniconda@v3 with: - directory: "." - snakefile: "workflow/Snakefile" - args: "benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --conda-cleanup-pkgs cache --conda-cleanup-envs --retries 3 --resources downloads=1 --config benchmark=${{ matrix.benchmark }} --forceall --verbose --printshellcmds" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http - echo "=== Dry-run to see what jobs would be executed ===" - snakemake --directory . --snakefile workflow/Snakefile benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --config benchmark=${{ matrix.benchmark }} --dry-run --reason || echo "Dry-run failed or no jobs" - echo "=== Checking rule inputs with --summary ===" - snakemake --directory . --snakefile workflow/Snakefile benchmark_all --cores 2 --sdm conda --conda-prefix /mnt/conda --config benchmark=${{ matrix.benchmark }} --summary 2>&1 | head -100 || echo "Summary failed" - show-disk-usage-on-error: true + miniforge-version: latest + channels: conda-forge,bioconda + activate-environment: snakemake + python-version: "3.12" + + - name: Install Snakemake and plugins + shell: bash -l {0} + run: | + conda install -n snakemake -c conda-forge -c bioconda snakemake -y + pip install snakemake-storage-plugin-zenodo snakemake-storage-plugin-http + snakemake --version + + - name: Debug - Dry-run Snakemake + shell: bash -l {0} + run: | + echo "=== Dry-run to show what jobs would run ===" + snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ + --cores 2 --sdm conda --conda-prefix /mnt/conda \ + --config benchmark=${{ matrix.benchmark }} \ + --dry-run -p 2>&1 || echo "Dry-run exit code: $?" + + - name: Run Snakemake directly + shell: bash -l {0} + run: | + set -x + echo "=== Starting Snakemake run ===" + snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ + --cores 2 --sdm conda --conda-prefix /mnt/conda \ + --conda-cleanup-pkgs cache --conda-cleanup-envs \ + --retries 3 --resources downloads=1 \ + --config benchmark=${{ matrix.benchmark }} \ + --forceall --printshellcmds --verbose \ + 2>&1 || { + EXIT_CODE=$? + echo "=== Snakemake failed with exit code: $EXIT_CODE ===" + echo "=== Checking .snakemake/log/ ===" + ls -la .snakemake/log/ 2>/dev/null || echo "No log directory" + cat .snakemake/log/*.log 2>/dev/null | tail -100 || echo "No logs found" + exit $EXIT_CODE + } + echo "=== Snakemake completed successfully ===" - name: Debug - List results directory after run if: always() @@ -153,16 +183,25 @@ jobs: path: results/ merge-multiple: true - - name: Create report - uses: snakemake/snakemake-github-action@v2.0.3 + - name: Setup Snakemake environment + uses: conda-incubator/setup-miniconda@v3 with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--report report.zip benchmark_all" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http - show-disk-usage-on-error: true + miniforge-version: latest + channels: conda-forge,bioconda + activate-environment: snakemake + python-version: "3.12" + + - name: Install Snakemake and plugins + shell: bash -l {0} + run: | + conda install -n snakemake -c conda-forge -c bioconda snakemake -y + pip install snakemake-storage-plugin-zenodo snakemake-storage-plugin-http + + - name: Create report + shell: bash -l {0} + run: | + snakemake --directory . --snakefile workflow/Snakefile \ + --report report.zip benchmark_all - name: Upload report as artifact uses: actions/upload-artifact@v4 From 503cf5c625eec4c9ce34e64e26e5455a9e4be585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:40:45 +0100 Subject: [PATCH 26/38] test: more debug --- .github/workflows/main.yml | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f15755..c88f815 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,32 +99,49 @@ jobs: pip install snakemake-storage-plugin-zenodo snakemake-storage-plugin-http snakemake --version - - name: Debug - Dry-run Snakemake + - name: Debug - Show config and DAG info + shell: bash -l {0} + run: | + echo "=== Benchmark: ${{ matrix.benchmark }} ===" + echo "=== Python version ===" + python --version + echo "=== Snakemake version ===" + snakemake --version + echo "=== Config filtering check ===" + python -c " + import yaml + with open('config/config.yaml') as f: + config = yaml.safe_load(f) + benchmark = '${{ matrix.benchmark }}' + filtered = {k: v for k, v in config.get('variant-calls', {}).items() if v.get('benchmark') == benchmark} + print(f'Callsets for {benchmark}: {list(filtered.keys())}') + print(f'Number of callsets: {len(filtered)}') + " + + - name: Debug - Dry-run Snakemake with reasons shell: bash -l {0} run: | echo "=== Dry-run to show what jobs would run ===" snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ --cores 2 --sdm conda --conda-prefix /mnt/conda \ --config benchmark=${{ matrix.benchmark }} \ - --dry-run -p 2>&1 || echo "Dry-run exit code: $?" + --dry-run --printshellcmds --reason 2>&1 | head -200 || echo "Dry-run exit code: $?" - - name: Run Snakemake directly + - name: Run Snakemake shell: bash -l {0} run: | - set -x echo "=== Starting Snakemake run ===" snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ --cores 2 --sdm conda --conda-prefix /mnt/conda \ --conda-cleanup-pkgs cache --conda-cleanup-envs \ --retries 3 --resources downloads=1 \ --config benchmark=${{ matrix.benchmark }} \ - --forceall --printshellcmds --verbose \ - 2>&1 || { + --printshellcmds 2>&1 || { EXIT_CODE=$? echo "=== Snakemake failed with exit code: $EXIT_CODE ===" echo "=== Checking .snakemake/log/ ===" ls -la .snakemake/log/ 2>/dev/null || echo "No log directory" - cat .snakemake/log/*.log 2>/dev/null | tail -100 || echo "No logs found" + cat .snakemake/log/*.log 2>/dev/null | tail -200 || echo "No logs found" exit $EXIT_CODE } echo "=== Snakemake completed successfully ===" @@ -183,6 +200,11 @@ jobs: path: results/ merge-multiple: true + - name: Debug - Show downloaded results + run: | + echo "=== Downloaded results structure ===" + find results/ -type f 2>/dev/null | head -100 || echo "No files found" + - name: Setup Snakemake environment uses: conda-incubator/setup-miniconda@v3 with: From 6d81500e6f4b6f890721d9399030d4843b696c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:48:53 +0100 Subject: [PATCH 27/38] test: create conda envs separately --- .github/workflows/main.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c88f815..75c4bff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -127,6 +127,15 @@ jobs: --config benchmark=${{ matrix.benchmark }} \ --dry-run --printshellcmds --reason 2>&1 | head -200 || echo "Dry-run exit code: $?" + - name: Create conda environments + shell: bash -l {0} + run: | + echo "=== Creating conda environments first ===" + snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ + --cores 2 --sdm conda --conda-prefix /mnt/conda \ + --config benchmark=${{ matrix.benchmark }} \ + --conda-create-envs-only + - name: Run Snakemake shell: bash -l {0} run: | @@ -136,15 +145,8 @@ jobs: --conda-cleanup-pkgs cache --conda-cleanup-envs \ --retries 3 --resources downloads=1 \ --config benchmark=${{ matrix.benchmark }} \ - --printshellcmds 2>&1 || { - EXIT_CODE=$? - echo "=== Snakemake failed with exit code: $EXIT_CODE ===" - echo "=== Checking .snakemake/log/ ===" - ls -la .snakemake/log/ 2>/dev/null || echo "No log directory" - cat .snakemake/log/*.log 2>/dev/null | tail -200 || echo "No logs found" - exit $EXIT_CODE - } - echo "=== Snakemake completed successfully ===" + --printshellcmds --show-failed-logs --nolock + echo "=== Snakemake completed with exit code: $? ===" - name: Debug - List results directory after run if: always() From 89d77f29fe0e5778f7188ba49b9b0c249d2bc7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:54:26 +0100 Subject: [PATCH 28/38] test: new gha structure --- .github/workflows/main.yml | 268 ++++++------------------------------- 1 file changed, 42 insertions(+), 226 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75c4bff..8bc28ff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,241 +2,57 @@ name: benchmark on: push: - branches: - - main + branches: [main] pull_request: workflow_dispatch: -concurrency: - # Cancel concurrent flows - group: ci-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - jobs: - # Full job: Run full benchmark suite in parallel + # 1. NEW JOB: Detect samples + setup: + runs-on: ubuntu-latest + outputs: + samples: ${{ steps.set-matrix.outputs.samples }} + steps: + - uses: actions/checkout@v4 + - id: set-matrix + run: | + # Option A: Get sample names from directory names + # This looks into resources/regions/, filters out non-directories, and formats as JSON + SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') + + # Option B: If your samples are defined in a Snakemake config.yaml + # You could use: SAMPLES=$(python3 -c "import yaml; print(list(yaml.safe_load(open('config/config.yaml'))['samples'].keys()))" | jq -c .) + + echo "Detected samples: $SAMPLES" + echo "samples=$SAMPLES" >> $GITHUB_OUTPUT + + # 2. EVALUATE JOB: Now uses the output from 'setup' evaluate: + needs: setup runs-on: ubuntu-latest strategy: fail-fast: false - max-parallel: 2 matrix: - benchmark: - - giab-NA12878-agilent-75M - - giab-NA12878-agilent-200M - - giab-NA12878-twist - - chm-eval - permissions: - actions: write - env: - FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} - ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} - BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} - BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} - CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} - BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} - BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - TMPDIR: /mnt/tmp + # This dynamically injects the list from the setup job + sample: ${{ fromJson(needs.setup.outputs.samples) }} + + # ... rest of your evaluate job ... steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - swap-storage: true - docker-images: false - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: Prepare /mnt workspace - run: | - sudo mkdir -p /mnt/conda /mnt/tmp - sudo chown -R $USER:$USER /mnt/conda /mnt/tmp - - - name: Restore caches (conda + snakemake) - uses: actions/cache@v4 - with: - path: | - /home/runner/miniconda3/pkgs - /home/runner/miniconda3/envs/snakemake - /home/runner/.cache/pip - /home/runner/.cache/snakemake - /mnt/conda - key: ${{ runner.os }}-conda-v2-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-v2- - - # This step is necessary (after downloading the truthsets above) to ensure - # that the files coming from the git repo are not triggering reruns - # because their modification dates are too new or too old. - # (as git does not preserve modification dates) - - name: Fix modification dates - run: touch resources/regions/*/test-regions.cov-*.bed - - - name: Debug - List results directory before run - run: | - echo "=== Contents of results/ before run ===" - ls -la results/ 2>/dev/null || echo "results/ does not exist" - echo "=== Contents of .snakemake/ before run ===" - ls -la .snakemake/ 2>/dev/null || echo ".snakemake/ does not exist" - - - name: Setup Snakemake environment - uses: conda-incubator/setup-miniconda@v3 - with: - miniforge-version: latest - channels: conda-forge,bioconda - activate-environment: snakemake - python-version: "3.12" - - - name: Install Snakemake and plugins - shell: bash -l {0} - run: | - conda install -n snakemake -c conda-forge -c bioconda snakemake -y - pip install snakemake-storage-plugin-zenodo snakemake-storage-plugin-http - snakemake --version - - - name: Debug - Show config and DAG info - shell: bash -l {0} - run: | - echo "=== Benchmark: ${{ matrix.benchmark }} ===" - echo "=== Python version ===" - python --version - echo "=== Snakemake version ===" - snakemake --version - echo "=== Config filtering check ===" - python -c " - import yaml - with open('config/config.yaml') as f: - config = yaml.safe_load(f) - benchmark = '${{ matrix.benchmark }}' - filtered = {k: v for k, v in config.get('variant-calls', {}).items() if v.get('benchmark') == benchmark} - print(f'Callsets for {benchmark}: {list(filtered.keys())}') - print(f'Number of callsets: {len(filtered)}') - " - - - name: Debug - Dry-run Snakemake with reasons - shell: bash -l {0} - run: | - echo "=== Dry-run to show what jobs would run ===" - snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ - --cores 2 --sdm conda --conda-prefix /mnt/conda \ - --config benchmark=${{ matrix.benchmark }} \ - --dry-run --printshellcmds --reason 2>&1 | head -200 || echo "Dry-run exit code: $?" - - - name: Create conda environments - shell: bash -l {0} - run: | - echo "=== Creating conda environments first ===" - snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ - --cores 2 --sdm conda --conda-prefix /mnt/conda \ - --config benchmark=${{ matrix.benchmark }} \ - --conda-create-envs-only - - - name: Run Snakemake - shell: bash -l {0} - run: | - echo "=== Starting Snakemake run ===" - snakemake --directory . --snakefile workflow/Snakefile benchmark_all \ - --cores 2 --sdm conda --conda-prefix /mnt/conda \ - --conda-cleanup-pkgs cache --conda-cleanup-envs \ - --retries 3 --resources downloads=1 \ - --config benchmark=${{ matrix.benchmark }} \ - --printshellcmds --show-failed-logs --nolock - echo "=== Snakemake completed with exit code: $? ===" - - - name: Debug - List results directory after run - if: always() - run: | - echo "=== Contents of results/ after run ===" - ls -la results/ 2>/dev/null || echo "results/ does not exist" - find results/ -type f 2>/dev/null | head -50 || true - - - name: Upload results - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.benchmark }} - path: results/ - retention-days: 1 - - # Final job: Combine results and create report + - uses: actions/checkout@v4 + # ... (disk space, etc.) + + - name: Run analysis + uses: snakemake/snakemake-github-action@v2 + with: + directory: "." + snakefile: "workflow/Snakefile" + args: > + --cores 4 --sdm conda + results/benchmarks/${{ matrix.sample }}/results.zip + # ... stagein ... + + # 3. REPORT JOB: Stays the same, but needs both setup and evaluate report: - needs: evaluate + needs: [setup, evaluate] runs-on: ubuntu-latest - permissions: - actions: write - env: - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - steps: - - uses: 8BitJonny/gh-get-current-pr@3.0.0 - id: pr - with: - sha: ${{ github.event.pull_request.head.sha }} - filterOutClosed: true - - - run: | - echo "is PR: ${{ steps.pr.outputs.pr_found }}" - echo "current branch: ${{ github.ref }}" - - - name: Check out repository code - uses: actions/checkout@v4 - - - name: Restore caches (conda) - uses: actions/cache@v4 - with: - path: | - /home/runner/miniconda3/pkgs - /home/runner/miniconda3/envs/snakemake - /home/runner/.cache/pip - key: ${{ runner.os }}-conda-v2-${{ hashFiles('workflow/Snakefile', 'workflow/rules/**/*.smk', 'config/config.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-v2- - - - name: Download all results - uses: actions/download-artifact@v4 - with: - pattern: results-* - path: results/ - merge-multiple: true - - - name: Debug - Show downloaded results - run: | - echo "=== Downloaded results structure ===" - find results/ -type f 2>/dev/null | head -100 || echo "No files found" - - - name: Setup Snakemake environment - uses: conda-incubator/setup-miniconda@v3 - with: - miniforge-version: latest - channels: conda-forge,bioconda - activate-environment: snakemake - python-version: "3.12" - - - name: Install Snakemake and plugins - shell: bash -l {0} - run: | - conda install -n snakemake -c conda-forge -c bioconda snakemake -y - pip install snakemake-storage-plugin-zenodo snakemake-storage-plugin-http - - - name: Create report - shell: bash -l {0} - run: | - snakemake --directory . --snakefile workflow/Snakefile \ - --report report.zip benchmark_all - - - name: Upload report as artifact - uses: actions/upload-artifact@v4 - with: - name: report - path: report.zip - - - name: Trigger homepage build - if: ${{ (steps.pr.outputs.pr_found != 'true') && (github.ref == 'refs/heads/main') }} - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: deploy-page - repo: ncbench/ncbench.github.io - token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} + # ... (same as previous response) \ No newline at end of file From 894f40b2395007d214916897e23ac6cb10cf3bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 14:55:49 +0100 Subject: [PATCH 29/38] fix: workflow --- .github/workflows/main.yml | 104 ++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8bc28ff..c4b1940 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,12 +2,17 @@ name: benchmark on: push: - branches: [main] + branches: + - main pull_request: workflow_dispatch: +concurrency: + group: ci-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: - # 1. NEW JOB: Detect samples + # 1. SETUP: Detect samples based on directory names setup: runs-on: ubuntu-latest outputs: @@ -16,43 +21,104 @@ jobs: - uses: actions/checkout@v4 - id: set-matrix run: | - # Option A: Get sample names from directory names - # This looks into resources/regions/, filters out non-directories, and formats as JSON + # Finds directories in resources/regions/, gets folder names, and formats as JSON array SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') - - # Option B: If your samples are defined in a Snakemake config.yaml - # You could use: SAMPLES=$(python3 -c "import yaml; print(list(yaml.safe_load(open('config/config.yaml'))['samples'].keys()))" | jq -c .) - - echo "Detected samples: $SAMPLES" echo "samples=$SAMPLES" >> $GITHUB_OUTPUT - # 2. EVALUATE JOB: Now uses the output from 'setup' + # 2. EVALUATE: Run benchmarks in parallel evaluate: needs: setup runs-on: ubuntu-latest strategy: fail-fast: false matrix: - # This dynamically injects the list from the setup job sample: ${{ fromJson(needs.setup.outputs.samples) }} - # ... rest of your evaluate job ... + permissions: + actions: write + + env: + FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} + BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} + CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} + BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} + BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} + steps: - uses: actions/checkout@v4 - # ... (disk space, etc.) - - - name: Run analysis + + - name: Free Disk Space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Run benchmark analysis uses: snakemake/snakemake-github-action@v2 with: directory: "." snakefile: "workflow/Snakefile" + # Target only the results for this specific sample args: > - --cores 4 --sdm conda + --cores 4 --sdm conda --conda-cleanup-pkgs cache + --rerun-triggers mtime results/benchmarks/${{ matrix.sample }}/results.zip - # ... stagein ... + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http - # 3. REPORT JOB: Stays the same, but needs both setup and evaluate + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.sample }} + path: results/benchmarks/${{ matrix.sample }}/ + retention-days: 1 + + # 3. REPORT: Collect all artifacts and generate the final report report: needs: [setup, evaluate] runs-on: ubuntu-latest - # ... (same as previous response) \ No newline at end of file + steps: + - uses: actions/checkout@v4 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + path: results/benchmarks/ + # This downloads every artifact from the previous job into its own folder + + - name: Create report + uses: snakemake/snakemake-github-action@v2 + with: + directory: "." + snakefile: "workflow/Snakefile" + args: "--report report.zip" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http + + - name: Upload report artifact + uses: actions/upload-artifact@v4 + with: + name: report + path: report.zip + + # 4. DEPLOY: Trigger homepage update + deploy: + needs: report + if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - name: Trigger homepage build + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: deploy-page + repo: ncbench/ncbench.github.io + token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} \ No newline at end of file From fe4cf199bc500dd44e6444a3542b1c3e69cca351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 15:03:49 +0100 Subject: [PATCH 30/38] fix: bm analysis --- .github/workflows/main.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4b1940..ae8035b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,14 +65,13 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - # Target only the results for this specific sample + # We move the target path to the front to avoid the argument confusion args: > - --cores 4 --sdm conda --conda-cleanup-pkgs cache - --rerun-triggers mtime results/benchmarks/${{ matrix.sample }}/results.zip - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http + --cores 4 + --sdm conda + --conda-cleanup-pkgs cache + --rerun-triggers mtime - name: Upload results uses: actions/upload-artifact@v4 From 9b98961518f255a4eb9e4a2ac7adafd074a84883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 15:14:21 +0100 Subject: [PATCH 31/38] fix: envs --- .github/workflows/main.yml | 80 ++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ae8035b..74c0504 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,17 +2,12 @@ name: benchmark on: push: - branches: - - main + branches: [main] pull_request: workflow_dispatch: -concurrency: - group: ci-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - jobs: - # 1. SETUP: Detect samples based on directory names + # 1. SETUP: Detect samples dynamically setup: runs-on: ubuntu-latest outputs: @@ -21,11 +16,11 @@ jobs: - uses: actions/checkout@v4 - id: set-matrix run: | - # Finds directories in resources/regions/, gets folder names, and formats as JSON array + # Detect folder names in resources/regions/ SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') echo "samples=$SAMPLES" >> $GITHUB_OUTPUT - # 2. EVALUATE: Run benchmarks in parallel + # 2. EVALUATE: Parallel processing with Micromamba evaluate: needs: setup runs-on: ubuntu-latest @@ -34,9 +29,6 @@ jobs: matrix: sample: ${{ fromJson(needs.setup.outputs.samples) }} - permissions: - actions: write - env: FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} @@ -60,17 +52,25 @@ jobs: large-packages: true swap-storage: true - - name: Run benchmark analysis - uses: snakemake/snakemake-github-action@v2 + - name: Setup Snakemake + Plugins + uses: mamba-org/setup-micromamba@v1 with: - directory: "." - snakefile: "workflow/Snakefile" - # We move the target path to the front to avoid the argument confusion - args: > - results/benchmarks/${{ matrix.sample }}/results.zip - --cores 4 - --sdm conda - --conda-cleanup-pkgs cache + environment-name: snakemake-env + create-args: >- + python=3.11 + snakemake + snakemake-storage-plugin-zenodo + snakemake-storage-plugin-http + cache-environment: true + + - name: Run Snakemake + shell: bash -el {0} + run: | + snakemake \ + results/benchmarks/${{ matrix.sample }}/results.zip \ + --cores 4 \ + --sdm conda \ + --conda-cleanup-pkgs cache \ --rerun-triggers mtime - name: Upload results @@ -80,39 +80,43 @@ jobs: path: results/benchmarks/${{ matrix.sample }}/ retention-days: 1 - # 3. REPORT: Collect all artifacts and generate the final report + # 3. REPORT: Combine results report: needs: [setup, evaluate] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Download all results + - name: Setup Snakemake + Plugins + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: snakemake-env + create-args: >- + python=3.11 + snakemake + snakemake-storage-plugin-zenodo + snakemake-storage-plugin-http + cache-environment: true + + - name: Download all artifacts uses: actions/download-artifact@v4 with: path: results/benchmarks/ - # This downloads every artifact from the previous job into its own folder - - name: Create report - uses: snakemake/snakemake-github-action@v2 - with: - directory: "." - snakefile: "workflow/Snakefile" - args: "--report report.zip" - stagein: | - pip install snakemake-storage-plugin-zenodo - pip install snakemake-storage-plugin-http + - name: Generate Report + shell: bash -el {0} + run: snakemake --report report.zip - - name: Upload report artifact + - name: Upload report uses: actions/upload-artifact@v4 with: - name: report + name: final-report path: report.zip - # 4. DEPLOY: Trigger homepage update + # 4. DEPLOY: Trigger external update deploy: needs: report - if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' }} + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: - name: Trigger homepage build From a112fc75d37cb65399ad595c2fc7bee0575e05ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 15:20:38 +0100 Subject: [PATCH 32/38] fix: envs --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74c0504..98cb1a7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,6 +56,7 @@ jobs: uses: mamba-org/setup-micromamba@v1 with: environment-name: snakemake-env + channels: conda-forge,bioconda create-args: >- python=3.11 snakemake From 6d8060e52407246ada2f6dc658d9dd5f1a07be40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 15:29:01 +0100 Subject: [PATCH 33/38] fix: envs --- .github/workflows/main.yml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98cb1a7..fe5025c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,11 +16,11 @@ jobs: - uses: actions/checkout@v4 - id: set-matrix run: | - # Detect folder names in resources/regions/ + # Detect sample folders and format as a JSON array for the matrix SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') echo "samples=$SAMPLES" >> $GITHUB_OUTPUT - # 2. EVALUATE: Parallel processing with Micromamba + # 2. EVALUATE: Run parallel benchmarks evaluate: needs: setup runs-on: ubuntu-latest @@ -29,16 +29,6 @@ jobs: matrix: sample: ${{ fromJson(needs.setup.outputs.samples) }} - env: - FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} - ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} - BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} - BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} - CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} - BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} - BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - steps: - uses: actions/checkout@v4 @@ -56,7 +46,10 @@ jobs: uses: mamba-org/setup-micromamba@v1 with: environment-name: snakemake-env - channels: conda-forge,bioconda + condarc: | + channels: + - conda-forge + - bioconda create-args: >- python=3.11 snakemake @@ -66,6 +59,15 @@ jobs: - name: Run Snakemake shell: bash -el {0} + env: + FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} + BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} + BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} + CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} + BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} + BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} run: | snakemake \ results/benchmarks/${{ matrix.sample }}/results.zip \ @@ -81,7 +83,7 @@ jobs: path: results/benchmarks/${{ matrix.sample }}/ retention-days: 1 - # 3. REPORT: Combine results + # 3. REPORT: Combine parallel results into a single report report: needs: [setup, evaluate] runs-on: ubuntu-latest @@ -92,6 +94,10 @@ jobs: uses: mamba-org/setup-micromamba@v1 with: environment-name: snakemake-env + condarc: | + channels: + - conda-forge + - bioconda create-args: >- python=3.11 snakemake From 9837a89f57bb0d298eb312cfc29f03b779d3d05a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 15:55:21 +0100 Subject: [PATCH 34/38] test: new gha --- .github/workflows/main.yml | 139 +++++++++++++------------------------ 1 file changed, 50 insertions(+), 89 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe5025c..2ae0c1d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,8 +6,12 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ci-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: - # 1. SETUP: Detect samples dynamically + # 1. SETUP: Dynamically find the samples setup: runs-on: ubuntu-latest outputs: @@ -16,11 +20,10 @@ jobs: - uses: actions/checkout@v4 - id: set-matrix run: | - # Detect sample folders and format as a JSON array for the matrix SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') echo "samples=$SAMPLES" >> $GITHUB_OUTPUT - # 2. EVALUATE: Run parallel benchmarks + # 2. EVALUATE: The matrix-based analysis evaluate: needs: setup runs-on: ubuntu-latest @@ -28,11 +31,18 @@ jobs: fail-fast: false matrix: sample: ${{ fromJson(needs.setup.outputs.samples) }} - - steps: - - uses: actions/checkout@v4 + env: + FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} + ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} + BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} + BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} + CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} + BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} + BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} + SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - - name: Free Disk Space + steps: + - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 with: tool-cache: false @@ -41,94 +51,45 @@ jobs: haskell: true large-packages: true swap-storage: true + docker-images: false - - name: Setup Snakemake + Plugins - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: snakemake-env - condarc: | - channels: - - conda-forge - - bioconda - create-args: >- - python=3.11 - snakemake - snakemake-storage-plugin-zenodo - snakemake-storage-plugin-http - cache-environment: true - - - name: Run Snakemake - shell: bash -el {0} - env: - FTP_PASSWORD: ${{ secrets.FTP_PASSWORD }} - ZENODO_TOKEN: ${{ secrets.ZENODO_TOKEN }} - SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN: ${{ secrets.SNAKEMAKE_STORAGE_ZENODO_ACCESS_TOKEN }} - BENCHMARK_GIAB_NA12878_AGILENT_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_AGILENT_TOKEN }} - BO_AGILENT_TOKEN: ${{ secrets.BO_AGILENT_TOKEN }} - CO_AGILENT_TOKEN: ${{ secrets.CO_AGILENT_TOKEN }} - BO_CORE_UNIT_TOKEN: ${{ secrets.BO_CORE_UNIT_TOKEN }} - BENCHMARK_GIAB_NA12878_TWIST_TOKEN: ${{ secrets.BENCHMARK_GIAB_NA12878_TWIST_TOKEN }} - run: | - snakemake \ - results/benchmarks/${{ matrix.sample }}/results.zip \ - --cores 4 \ - --sdm conda \ - --conda-cleanup-pkgs cache \ - --rerun-triggers mtime - - - name: Upload results - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sample }} - path: results/benchmarks/${{ matrix.sample }}/ - retention-days: 1 - - # 3. REPORT: Combine parallel results into a single report - report: - needs: [setup, evaluate] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + - name: Check out repository code + uses: actions/checkout@v4 - - name: Setup Snakemake + Plugins - uses: mamba-org/setup-micromamba@v1 + - name: Download reference genome + uses: snakemake/snakemake-github-action@v2 with: - environment-name: snakemake-env - condarc: | - channels: - - conda-forge - - bioconda - create-args: >- - python=3.11 - snakemake - snakemake-storage-plugin-zenodo - snakemake-storage-plugin-http - cache-environment: true + directory: "." + snakefile: "workflow/Snakefile" + args: "--cores 1 --sdm conda --conda-cleanup-pkgs cache resources/reference/genome.fasta" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http - - name: Download all artifacts - uses: actions/download-artifact@v4 + - name: Download truthsets + uses: snakemake/snakemake-github-action@v2 with: - path: results/benchmarks/ - - - name: Generate Report - shell: bash -el {0} - run: snakemake --report report.zip + directory: "." + snakefile: "workflow/Snakefile" + args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --until benchmark_get_truth" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http - - name: Upload report - uses: actions/upload-artifact@v4 + - name: Fix modification dates + uses: snakemake/snakemake-github-action@v2 with: - name: final-report - path: report.zip + directory: "." + snakefile: "workflow/Snakefile" + # Only touch the files for THIS matrix sample + args: "--cores 1 --sdm conda --touch resources/regions/${{ matrix.sample }}/test-regions.cov-*.bed" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http - # 4. DEPLOY: Trigger external update - deploy: - needs: report - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - steps: - - name: Trigger homepage build - uses: benc-uk/workflow-dispatch@v1 + - name: Run analysis + uses: snakemake/snakemake-github-action@v2 with: - workflow: deploy-page - repo: ncbench/ncbench.github.io - token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} \ No newline at end of file + directory: "." + snakefile: "workflow/Snakefile" + # We specify the specific target for \ No newline at end of file From c95a2f45ae28adaddc22d9ad85b4d949b20648fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 16:40:28 +0100 Subject: [PATCH 35/38] fix: upgrade version --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ae0c1d..f0c91bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v4 - name: Download reference genome - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -67,7 +67,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Download truthsets - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -77,7 +77,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Fix modification dates - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" @@ -88,7 +88,7 @@ jobs: pip install snakemake-storage-plugin-http - name: Run analysis - uses: snakemake/snakemake-github-action@v2 + uses: snakemake/snakemake-github-action@v2.0.3 with: directory: "." snakefile: "workflow/Snakefile" From eeabfbb0d3f50631acb21b5156d25f520f0233b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 27 Jan 2026 16:51:13 +0100 Subject: [PATCH 36/38] fix: add analysis and downstream --- .github/workflows/main.yml | 53 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f0c91bd..90d26ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,8 +88,57 @@ jobs: pip install snakemake-storage-plugin-http - name: Run analysis - uses: snakemake/snakemake-github-action@v2.0.3 + uses: snakemake/snakemake-github-action@v2 with: directory: "." snakefile: "workflow/Snakefile" - # We specify the specific target for \ No newline at end of file + # We specify the specific target for this matrix run + args: > + results/benchmarks/${{ matrix.sample }}/results.zip + --cores 4 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.sample }} + path: results/benchmarks/${{ matrix.sample }}/ + retention-days: 1 + + # 3. REPORT: Combine and Deploy + report: + needs: evaluate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + path: results/benchmarks/ + + - name: Create report + uses: snakemake/snakemake-github-action@v2 + with: + directory: "." + snakefile: "workflow/Snakefile" + args: "--report report.zip" + stagein: | + pip install snakemake-storage-plugin-zenodo + pip install snakemake-storage-plugin-http + + - name: Upload final report + uses: actions/upload-artifact@v4 + with: + name: report + path: report.zip + + - name: Trigger homepage build + if: github.ref == 'refs/heads/main' + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: deploy-page + repo: ncbench/ncbench.github.io + token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} \ No newline at end of file From 4363cacd74b22f95ed4d7fb1bb2063b83e83c907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 30 Jan 2026 11:16:18 +0100 Subject: [PATCH 37/38] fix: target --- .github/workflows/main.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90d26ef..1f7725c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,9 +92,9 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - # We specify the specific target for this matrix run args: > - results/benchmarks/${{ matrix.sample }}/results.zip + benchmark_all + --config benchmark=${{ matrix.sample }} --cores 4 --sdm conda --conda-cleanup-pkgs cache --rerun-triggers mtime --all-temp stagein: | pip install snakemake-storage-plugin-zenodo @@ -104,7 +104,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ matrix.sample }} - path: results/benchmarks/${{ matrix.sample }}/ + # Upload the workflow outputs (keeps the on-disk paths Snakemake expects). + path: results/ retention-days: 1 # 3. REPORT: Combine and Deploy @@ -117,7 +118,9 @@ jobs: - name: Download all results uses: actions/download-artifact@v4 with: - path: results/benchmarks/ + # Merge all benchmark artifacts back into the workspace so Snakemake sees `results/`. + path: . + merge-multiple: true - name: Create report uses: snakemake/snakemake-github-action@v2 @@ -141,4 +144,4 @@ jobs: with: workflow: deploy-page repo: ncbench/ncbench.github.io - token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} \ No newline at end of file + token: ${{ secrets.WORKFLOW_DISPATCH_TOKEN }} From cee58a814cd6516220eb59c954bcc9ccfc86b728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Fri, 30 Jan 2026 13:22:18 +0100 Subject: [PATCH 38/38] fix: change target --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f7725c..a0f4e74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,9 @@ jobs: - uses: actions/checkout@v4 - id: set-matrix run: | - SAMPLES=$(ls -d resources/regions/*/ | xargs -n 1 basename | jq -R -s -c 'split("\n")[:-1]') + # Auto-discover benchmarks from resources/regions/, but only keep + # those that include a target regions BED (required by the workflow). + SAMPLES=$(for d in resources/regions/*/; do [ -f "$d/target-regions.bed" ] && basename "$d"; done | jq -R -s -c 'split("\n")[:-1]') echo "samples=$SAMPLES" >> $GITHUB_OUTPUT # 2. EVALUATE: The matrix-based analysis @@ -71,7 +73,7 @@ jobs: with: directory: "." snakefile: "workflow/Snakefile" - args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --until benchmark_get_truth" + args: "--sdm conda --cores 1 --conda-cleanup-pkgs cache --until benchmark_get_truth --config benchmark=${{ matrix.sample }}" stagein: | pip install snakemake-storage-plugin-zenodo pip install snakemake-storage-plugin-http