Add script to run pipeline locally

.github/workflows/pipeline.yml

@@ -2,71 +2,26 @@ name: Implementation process pipeline
 run-name: ${{ github.actor }} is running the pipeline
 on: [push]
 
-env:
-  ap_url:
-  shacl_url:
-
 jobs:
   Set-Up-Repo:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository code
         uses: actions/checkout@v5
-      - name: Create temp file if AP and SHACL url exist
-        run: ./scripts/create-env-temp-file.sh $ap_url $shacl_url
-      - name: Check if tmp file exists
-        id: check_env_temp
-        uses: andstor/file-existence-action@v3
+      - uses: actions/setup-node@v6
         with:
-          files: "env.txt"
-      - name: Download SHACL if needed
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/download-shacl.sh $shacl_url
+          node-version: 22
+      - name: Set up repo
+        run: ./scripts/set-up-repo.sh
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: in-shacl/shacl.ttl
           message: "Add SHACL"
-      - name: Add AP and SHACL links to README
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/update-readme.sh $ap_url $shacl_url
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: README.md original-README.md
           message: "Add AP and SHACL links to README"
-      - name: Check if template Excel exists
-        if: steps.check_env_temp.outputs.files_exists == 'true'
-        id: check_template
-        uses: andstor/file-existence-action@v3
-        with:
-          files: "template.xlsx"
-      - name: Clone ap-data-to-dashboard to generate template Excel
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        uses: actions/checkout@v5
-        with:
-          repository: RMLio/ap-data-to-dashboard
-          path: ap-data-to-dashboard
-      - uses: actions/setup-node@v6
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        with:
-          node-version: 22
-      - name: "Copy shacl file in-shacl directory"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: cp in-shacl/* ap-data-to-dashboard/in-shacl
-      - name: "Install ap-data-in-dashboard"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: ./scripts/install-ap-data-to-dashboard.sh
-      - name: "Generate template Excel, template schema, and example data"
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
-        run: | 
-          cd ap-data-to-dashboard && node src/shacl-to-template.js
-          cp in-shacl/template.xlsx ../template.xlsx
-          cp in-shacl/template.schema.json ../in-shacl/template.schema.json
-          rm -rf ../example-data && mkdir ../example-data
-          cp in-shacl/dummy* ../example-data
       - uses: EndBug/add-and-commit@v9
-        if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true'
         with:
           add: template.xlsx in-shacl/template.schema.json example-data
           message: "Add template Excel, template schema, and example data"
@@ -77,59 +32,34 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref.name }}
       - name: Check if there is any data available
         id: check_data
         uses: andstor/file-existence-action@v3
         with:
           files: data/*.xlsx
-      - name: Clone ap-data-to-dashboard
-        if: steps.check_data.outputs.files_exists == 'true'
-        uses: actions/checkout@v5
-        with:
-          repository: RMLio/ap-data-to-dashboard
-          path: ap-data-to-dashboard
       - uses: actions/setup-node@v6
         if: steps.check_data.outputs.files_exists == 'true'
         with:
           node-version: 22
-      - name: "Copy input data to in directory"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: | 
-          rm -rf ap-data-to-dashboard/in
-          mkdir ap-data-to-dashboard/in
-          cp data/* ap-data-to-dashboard/in
-      - name: "Copy in-shacl directory content to ap-data-to-dashboard"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: cp in-shacl/* ap-data-to-dashboard/in-shacl
-      - name: "Install ap-data-in-dashboard"
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: ./scripts/install-ap-data-to-dashboard.sh
-      - name: Copy dashboard config
-        if: steps.check_data.outputs.files_exists == 'true'
-        run: ./scripts/copy-dashboard-config.sh
       - name: "Generate RDF and build Miravi"
         if: steps.check_data.outputs.files_exists == 'true'
-        run: |
-          cd ap-data-to-dashboard
-          ./run.sh -u https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/
+        run: ./scripts/generate-rdf-build-miravi.sh https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/
       - name: Clone gh-pages branch of this repo
         uses: actions/checkout@v5
         with:
           ref: gh-pages
           path: gh-pages
-      - uses: actions/setup-node@v6
       - name: Move Miravi dist directory to correct folder in gh-pages
-        run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }}
-      - name: Move generated RDF to output directory
         if: steps.check_data.outputs.files_exists == 'true'
+        run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }}
+      - name: Move output and mappings to root of repo
         run: |
-          rm -rf output
-          mkdir output
-          rm -rf mappings
-          mkdir mappings
-          mv ap-data-to-dashboard/out/serve-me/* output/
-          mv ap-data-to-dashboard/out/*.rml.ttl mappings/
-          mv ap-data-to-dashboard/out/*.yml mappings/
+          rm -rf output mappings
+          mv dist/output .
+          mv dist/mappings .
       - uses: EndBug/add-and-commit@v9
         if: steps.check_data.outputs.files_exists == 'true'
         with:

.gitignore

@@ -1 +1,3 @@
 node_modules
+dist
+tmp

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Script to run pipeline locally
+
+### Changed
+
+- GitHub actions uses scripts that are introduced to run pipeline locally
+
 ## [v2.0.0] - 2025-11-25
 
 ### Added

README.md

@@ -34,8 +34,8 @@ You only have to do these steps once.
    make sure that "Read and write permissions" is checked.
 4. [Set up GitHub pages](https://docs.github.com/en/pages/getting-started-with-github-pages/creating-a-github-pages-site#creating-your-site)
    and let it point to the root directory on the `gh-pages` branch.
-5. Set the link to the application profile at `env.ap_url` in the file `.github/workflows/pipeline.yml`.
-6. Set the link to the SHACL at `env.shacl_url` in the file `.github/workflows/pipeline.yml`.
+5. Set the link to the application profile as value for `AP_URL` in the file `urls.env`.
+6. Set the link to the SHACL as value for `SHACL_URL` in the file `urls.env`.
 7. Read and, if necessary, update [the license](LICENSE).
 8. Commit and push your changes to GitHub.
    This will trigger the workflows, which will also add example data to the [example-data](./example-data) directory.
@@ -168,10 +168,15 @@ Note that the dashboard will still rely on the RDF in this repository.
   ./scripts/reset-repo.sh
   ```
 
-- Some example env values are
+- Some example values for `urls.env` are
 
    ```yaml
-   env:
-     ap_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/
-     shacl_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl
+   AP_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/
+   SHACL_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl
+   ```
+
+- You can run the pipeline locally via
+
+   ```bash
+   ./scripts/run-pipeline-locally.sh
    ```

scripts/README.md

@@ -0,0 +1,7 @@
+# Scripts
+
+- `run-pipeline-locally.sh`: this script runs the pipeline locally.
+- `set-up-repo.sh`: this script sets up the repo, including downloading the SHACL file, generating a template Excel file,
+   and generating example data.
+- `generate-rdf-build-miravi-locally.sh`: this script generates the RDF and builds Miravi locally.
+   It relies on `generate-rdf-build-miravi.sh`.

scripts/clone-ap-data-to-dashboard.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+WORK_DIR="$SCRIPTS_DIR/../tmp"
+DIRECTORY="$WORK_DIR/ap-data-to-dashboard"
+
+if [ -d "$DIRECTORY" ]; then
+  echo "Folder $DIRECTORY already exists, so no cloning"
+else
+  echo "Cloning ap-data-to-dashboard"
+  git clone --depth 1 https://github.com/RMLio/ap-data-to-dashboard.git $DIRECTORY &> /dev/null
+fi
+

scripts/copy-dashboard-config.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
-DIRECTORY=dashboard-config
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+WORK_DIR="$SCRIPTS_DIR/../tmp"
+DIRECTORY="$SCRIPTS_DIR/../dashboard-config"
 
 if [ -d "$DIRECTORY" ]; then
   echo "$DIRECTORY exists."
-  rm -rf ap-data-to-dashboard/miravi-initial-config
-  mv $DIRECTORY ap-data-to-dashboard/miravi-initial-config
+  rm -rf $WORK_DIR/ap-data-to-dashboard/miravi-initial-config
+  mv $DIRECTORY $WORK_DIR/ap-data-to-dashboard/miravi-initial-config
 else
   echo "$DIRECTORY doesn't exist. Using initial config."
 fi

scripts/download-shacl.sh

@@ -4,7 +4,7 @@ DIR=in-shacl
 FILE="$DIR/shacl.ttl"
 
 if [ -d "$DIR" ]; then
-  echo "DIR already exists."
+  echo "$DIR already exists."
 else
   mkdir $DIR
 fi

scripts/generate-rdf-build-miravi-locally.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+OUTPUT_DIR=$SCRIPTS_DIR"/../dist"
+WORK_DIR=$SCRIPTS_DIR"/../tmp"
+BASE_URL="http://localhost:5500"
+
+if compgen -G "data/*.xlsx" > /dev/null; then
+  $SCRIPTS_DIR/generate-rdf-build-miravi.sh $BASE_URL
+
+  echo "Moving Miravi dist to docs"
+  rm -rf $OUTPUT_DIR/docs && mkdir $OUTPUT_DIR/docs
+  mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* $OUTPUT_DIR/docs
+
+  echo ""
+  echo ""
+  echo "1. Host RDF files via \"npx http-server $OUTPUT_DIR/output -p 5500 --cors true\""
+  echo "2. Host Miravi via \"npx http-server $OUTPUT_DIR/docs -p 8080\""
+  echo "3. Browse to http://localhost:8080"
+else
+  echo "No Excel files found in the data folder."
+  exit 1
+fi

scripts/generate-rdf-build-miravi.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR=`pwd`
+WORK_DIR=$ROOT_DIR"/tmp"
+OUTPUT_DIR=$ROOT_DIR/"dist"
+BASE_URL=$1
+
+mkdir -p $WORK_DIR
+rm -rf $OUTPUT_DIR
+mkdir $OUTPUT_DIR
+
+cd $WORK_DIR
+$SCRIPTS_DIR/clone-ap-data-to-dashboard.sh || exit 1
+rm -rf ap-data-to-dashboard/in
+mkdir ap-data-to-dashboard/in
+
+echo "Copying ../data/* to ap-data-to-dashboard/in"
+cp $ROOT_DIR/data/* ap-data-to-dashboard/in
+
+echo "Copying ../in-shacl/* to ap-data-to-dashboard/in-shacl"
+cp $ROOT_DIR/in-shacl/* ap-data-to-dashboard/in-shacl
+
+echo "Installing ap-data-to-dashboard"
+$SCRIPTS_DIR/install-ap-data-to-dashboard.sh
+
+echo "Copying dashboard config"
+$SCRIPTS_DIR/copy-dashboard-config.sh
+
+echo "Running ap-data-to-dashboard"
+cd $WORK_DIR/ap-data-to-dashboard
+./run.sh -u $BASE_URL
+cd $OUTPUT_DIR
+
+echo "Moving RDF and mappings"
+rm -rf output
+mkdir output
+rm -rf mappings
+mkdir mappings
+mv $WORK_DIR/ap-data-to-dashboard/out/serve-me/* output/
+mv $WORK_DIR/ap-data-to-dashboard/out/*.rml.ttl mappings/
+mv $WORK_DIR/ap-data-to-dashboard/out/*.yml mappings/

scripts/install-ap-data-to-dashboard.sh

@@ -1,5 +1,18 @@
 #!/usr/bin/env bash
 
-cd ap-data-to-dashboard
-npm i
-npm run setup
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+WORK_DIR="$SCRIPTS_DIR/../tmp"
+TEST_FILE="install-setup-done"
+ORIGINAL_DIR=$PWD
+
+cd $WORK_DIR/ap-data-to-dashboard
+
+if [ -f "$TEST_FILE" ]; then
+  echo "Installation and setup already done"
+else
+  npm i &> /dev/null
+  npm run setup &> /dev/null
+  touch $TEST_FILE
+fi
+
+cd $ORIGINAL_DIR

scripts/move-miravi-dist.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 FORBIDDEN_BRANCH_NAMES=("assets" "queries" "images")
 CURRENT_BRANCH=$1
 USE_DIST=$2
+WORK_DIR=$SCRIPTS_DIR"/../tmp"
 
 # Sanitize branch
 CURRENT_BRANCH=${CURRENT_BRANCH//\//-}
@@ -17,18 +19,18 @@ if [[ "$CURRENT_BRANCH" == "main" ]]; then
   rm -rf gh-pages/assets gh-pages/images gh-pages/queries
 
   if [[ "$USE_DIST" == "true" ]]; then
-    mv ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages
+    mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages
   else
-    cp scripts/dashboard-placeholder.html gh-pages/index.html
+    cp $SCRIPTS_DIR/dashboard-placeholder.html gh-pages/index.html
   fi
 else
   echo "Using subdirectory because branch is not main."
   rm -rf gh-pages/$CURRENT_BRANCH
   mkdir -p gh-pages/$CURRENT_BRANCH
 
   if [[ "$USE_DIST" == "true" ]]; then
-    mv ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages/$CURRENT_BRANCH
+    mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages/$CURRENT_BRANCH
   else
-    cp scripts/dashboard-placeholder.html gh-pages/$CURRENT_BRANCH/index.html
+    cp $SCRIPTS_DIR/dashboard-placeholder.html gh-pages/$CURRENT_BRANCH/index.html
   fi
 fi

scripts/reset-readme.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-FILE=README.md
-BACKUP_FILE=original-README.md
+SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+FILE="$SCRIPTS_DIR/../README.md"
+BACKUP_FILE="$SCRIPTS_DIR/../original-README.md"
 
 mv $BACKUP_FILE $FILE

scripts/reset-repo.sh

@@ -9,7 +9,10 @@ rm -rf example-data \
   data \
   output \
   template.xlsx \
-  env.txt
+  dist \
+  tmp
 
 mkdir data
 touch data/.gitkeep
+echo "AP_URL=" > urls.env
+echo "SHACL_URL=" >> urls.env