diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index bed5791..ab310fb 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -2,71 +2,26 @@ name: Implementation process pipeline run-name: ${{ github.actor }} is running the pipeline on: [push] -env: - ap_url: - shacl_url: - jobs: Set-Up-Repo: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v5 - - name: Create temp file if AP and SHACL url exist - run: ./scripts/create-env-temp-file.sh $ap_url $shacl_url - - name: Check if tmp file exists - id: check_env_temp - uses: andstor/file-existence-action@v3 + - uses: actions/setup-node@v6 with: - files: "env.txt" - - name: Download SHACL if needed - if: steps.check_env_temp.outputs.files_exists == 'true' - run: ./scripts/download-shacl.sh $shacl_url + node-version: 22 + - name: Set up repo + run: ./scripts/set-up-repo.sh - uses: EndBug/add-and-commit@v9 - if: steps.check_env_temp.outputs.files_exists == 'true' with: add: in-shacl/shacl.ttl message: "Add SHACL" - - name: Add AP and SHACL links to README - if: steps.check_env_temp.outputs.files_exists == 'true' - run: ./scripts/update-readme.sh $ap_url $shacl_url - uses: EndBug/add-and-commit@v9 - if: steps.check_env_temp.outputs.files_exists == 'true' with: add: README.md original-README.md message: "Add AP and SHACL links to README" - - name: Check if template Excel exists - if: steps.check_env_temp.outputs.files_exists == 'true' - id: check_template - uses: andstor/file-existence-action@v3 - with: - files: "template.xlsx" - - name: Clone ap-data-to-dashboard to generate template Excel - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' - uses: actions/checkout@v5 - with: - repository: RMLio/ap-data-to-dashboard - path: ap-data-to-dashboard - - uses: actions/setup-node@v6 - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' - with: - node-version: 22 - - name: "Copy shacl file in-shacl directory" - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' - run: cp in-shacl/* ap-data-to-dashboard/in-shacl - - name: "Install ap-data-in-dashboard" - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' - run: ./scripts/install-ap-data-to-dashboard.sh - - name: "Generate template Excel, template schema, and example data" - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' - run: | - cd ap-data-to-dashboard && node src/shacl-to-template.js - cp in-shacl/template.xlsx ../template.xlsx - cp in-shacl/template.schema.json ../in-shacl/template.schema.json - rm -rf ../example-data && mkdir ../example-data - cp in-shacl/dummy* ../example-data - uses: EndBug/add-and-commit@v9 - if: steps.check_template.outputs.files_exists == 'false' && steps.check_env_temp.outputs.files_exists == 'true' with: add: template.xlsx in-shacl/template.schema.json example-data message: "Add template Excel, template schema, and example data" @@ -77,59 +32,34 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ github.ref.name }} - name: Check if there is any data available id: check_data uses: andstor/file-existence-action@v3 with: files: data/*.xlsx - - name: Clone ap-data-to-dashboard - if: steps.check_data.outputs.files_exists == 'true' - uses: actions/checkout@v5 - with: - repository: RMLio/ap-data-to-dashboard - path: ap-data-to-dashboard - uses: actions/setup-node@v6 if: steps.check_data.outputs.files_exists == 'true' with: node-version: 22 - - name: "Copy input data to in directory" - if: steps.check_data.outputs.files_exists == 'true' - run: | - rm -rf ap-data-to-dashboard/in - mkdir ap-data-to-dashboard/in - cp data/* ap-data-to-dashboard/in - - name: "Copy in-shacl directory content to ap-data-to-dashboard" - if: steps.check_data.outputs.files_exists == 'true' - run: cp in-shacl/* ap-data-to-dashboard/in-shacl - - name: "Install ap-data-in-dashboard" - if: steps.check_data.outputs.files_exists == 'true' - run: ./scripts/install-ap-data-to-dashboard.sh - - name: Copy dashboard config - if: steps.check_data.outputs.files_exists == 'true' - run: ./scripts/copy-dashboard-config.sh - name: "Generate RDF and build Miravi" if: steps.check_data.outputs.files_exists == 'true' - run: | - cd ap-data-to-dashboard - ./run.sh -u https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/ + run: ./scripts/generate-rdf-build-miravi.sh https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/main/output/ - name: Clone gh-pages branch of this repo uses: actions/checkout@v5 with: ref: gh-pages path: gh-pages - - uses: actions/setup-node@v6 - name: Move Miravi dist directory to correct folder in gh-pages - run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }} - - name: Move generated RDF to output directory if: steps.check_data.outputs.files_exists == 'true' + run: ./scripts/move-miravi-dist.sh ${{ github.ref_name }} ${{ steps.check_data.outputs.files_exists }} + - name: Move output and mappings to root of repo run: | - rm -rf output - mkdir output - rm -rf mappings - mkdir mappings - mv ap-data-to-dashboard/out/serve-me/* output/ - mv ap-data-to-dashboard/out/*.rml.ttl mappings/ - mv ap-data-to-dashboard/out/*.yml mappings/ + rm -rf output mappings + mv dist/output . + mv dist/mappings . - uses: EndBug/add-and-commit@v9 if: steps.check_data.outputs.files_exists == 'true' with: diff --git a/.gitignore b/.gitignore index 3c3629e..142ab74 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules +dist +tmp diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f2f44..84ac026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Script to run pipeline locally + +### Changed + +- GitHub actions uses scripts that are introduced to run pipeline locally + ## [v2.0.0] - 2025-11-25 ### Added diff --git a/README.md b/README.md index 348bc00..95ed338 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ You only have to do these steps once. make sure that "Read and write permissions" is checked. 4. [Set up GitHub pages](https://docs.github.com/en/pages/getting-started-with-github-pages/creating-a-github-pages-site#creating-your-site) and let it point to the root directory on the `gh-pages` branch. -5. Set the link to the application profile at `env.ap_url` in the file `.github/workflows/pipeline.yml`. -6. Set the link to the SHACL at `env.shacl_url` in the file `.github/workflows/pipeline.yml`. +5. Set the link to the application profile as value for `AP_URL` in the file `urls.env`. +6. Set the link to the SHACL as value for `SHACL_URL` in the file `urls.env`. 7. Read and, if necessary, update [the license](LICENSE). 8. Commit and push your changes to GitHub. This will trigger the workflows, which will also add example data to the [example-data](./example-data) directory. @@ -168,10 +168,15 @@ Note that the dashboard will still rely on the RDF in this repository. ./scripts/reset-repo.sh ``` -- Some example env values are +- Some example values for `urls.env` are ```yaml - env: - ap_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/ - shacl_url: https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl + AP_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/ontwerpstandaard/2025-03-21/ + SHACL_URL=https://data.vlaanderen.be/doc/applicatieprofiel/leermiddelen/kandidaatstandaard/2025-08-01/shacl/leermiddelen-SHACL.ttl + ``` + +- You can run the pipeline locally via + + ```bash + ./scripts/run-pipeline-locally.sh ``` diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..4ab3e7f --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,7 @@ +# Scripts + +- `run-pipeline-locally.sh`: this script runs the pipeline locally. +- `set-up-repo.sh`: this script sets up the repo, including downloading the SHACL file, generating a template Excel file, + and generating example data. +- `generate-rdf-build-miravi-locally.sh`: this script generates the RDF and builds Miravi locally. + It relies on `generate-rdf-build-miravi.sh`. diff --git a/scripts/clone-ap-data-to-dashboard.sh b/scripts/clone-ap-data-to-dashboard.sh new file mode 100755 index 0000000..88dd8d2 --- /dev/null +++ b/scripts/clone-ap-data-to-dashboard.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +WORK_DIR="$SCRIPTS_DIR/../tmp" +DIRECTORY="$WORK_DIR/ap-data-to-dashboard" + +if [ -d "$DIRECTORY" ]; then + echo "Folder $DIRECTORY already exists, so no cloning" +else + echo "Cloning ap-data-to-dashboard" + git clone --depth 1 https://github.com/RMLio/ap-data-to-dashboard.git $DIRECTORY &> /dev/null +fi + diff --git a/scripts/copy-dashboard-config.sh b/scripts/copy-dashboard-config.sh index 0f75200..9276654 100755 --- a/scripts/copy-dashboard-config.sh +++ b/scripts/copy-dashboard-config.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -DIRECTORY=dashboard-config +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +WORK_DIR="$SCRIPTS_DIR/../tmp" +DIRECTORY="$SCRIPTS_DIR/../dashboard-config" if [ -d "$DIRECTORY" ]; then echo "$DIRECTORY exists." - rm -rf ap-data-to-dashboard/miravi-initial-config - mv $DIRECTORY ap-data-to-dashboard/miravi-initial-config + rm -rf $WORK_DIR/ap-data-to-dashboard/miravi-initial-config + mv $DIRECTORY $WORK_DIR/ap-data-to-dashboard/miravi-initial-config else echo "$DIRECTORY doesn't exist. Using initial config." fi diff --git a/scripts/create-env-temp-file.sh b/scripts/create-env-temp-file.sh deleted file mode 100755 index 76210fb..0000000 --- a/scripts/create-env-temp-file.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -FILE=env.txt - -if [ $# -eq 2 ] - then - touch $FILE -fi diff --git a/scripts/download-shacl.sh b/scripts/download-shacl.sh index c3cfa4c..5a4a2c2 100755 --- a/scripts/download-shacl.sh +++ b/scripts/download-shacl.sh @@ -4,7 +4,7 @@ DIR=in-shacl FILE="$DIR/shacl.ttl" if [ -d "$DIR" ]; then - echo "DIR already exists." + echo "$DIR already exists." else mkdir $DIR fi diff --git a/scripts/generate-rdf-build-miravi-locally.sh b/scripts/generate-rdf-build-miravi-locally.sh new file mode 100755 index 0000000..29fb938 --- /dev/null +++ b/scripts/generate-rdf-build-miravi-locally.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +OUTPUT_DIR=$SCRIPTS_DIR"/../dist" +WORK_DIR=$SCRIPTS_DIR"/../tmp" +BASE_URL="http://localhost:5500" + +if compgen -G "data/*.xlsx" > /dev/null; then + $SCRIPTS_DIR/generate-rdf-build-miravi.sh $BASE_URL + + echo "Moving Miravi dist to docs" + rm -rf $OUTPUT_DIR/docs && mkdir $OUTPUT_DIR/docs + mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* $OUTPUT_DIR/docs + + echo "" + echo "" + echo "1. Host RDF files via \"npx http-server $OUTPUT_DIR/output -p 5500 --cors true\"" + echo "2. Host Miravi via \"npx http-server $OUTPUT_DIR/docs -p 8080\"" + echo "3. Browse to http://localhost:8080" +else + echo "No Excel files found in the data folder." + exit 1 +fi diff --git a/scripts/generate-rdf-build-miravi.sh b/scripts/generate-rdf-build-miravi.sh new file mode 100755 index 0000000..8dcbe4e --- /dev/null +++ b/scripts/generate-rdf-build-miravi.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT_DIR=`pwd` +WORK_DIR=$ROOT_DIR"/tmp" +OUTPUT_DIR=$ROOT_DIR/"dist" +BASE_URL=$1 + +mkdir -p $WORK_DIR +rm -rf $OUTPUT_DIR +mkdir $OUTPUT_DIR + +cd $WORK_DIR +$SCRIPTS_DIR/clone-ap-data-to-dashboard.sh || exit 1 +rm -rf ap-data-to-dashboard/in +mkdir ap-data-to-dashboard/in + +echo "Copying ../data/* to ap-data-to-dashboard/in" +cp $ROOT_DIR/data/* ap-data-to-dashboard/in + +echo "Copying ../in-shacl/* to ap-data-to-dashboard/in-shacl" +cp $ROOT_DIR/in-shacl/* ap-data-to-dashboard/in-shacl + +echo "Installing ap-data-to-dashboard" +$SCRIPTS_DIR/install-ap-data-to-dashboard.sh + +echo "Copying dashboard config" +$SCRIPTS_DIR/copy-dashboard-config.sh + +echo "Running ap-data-to-dashboard" +cd $WORK_DIR/ap-data-to-dashboard +./run.sh -u $BASE_URL +cd $OUTPUT_DIR + +echo "Moving RDF and mappings" +rm -rf output +mkdir output +rm -rf mappings +mkdir mappings +mv $WORK_DIR/ap-data-to-dashboard/out/serve-me/* output/ +mv $WORK_DIR/ap-data-to-dashboard/out/*.rml.ttl mappings/ +mv $WORK_DIR/ap-data-to-dashboard/out/*.yml mappings/ diff --git a/scripts/install-ap-data-to-dashboard.sh b/scripts/install-ap-data-to-dashboard.sh index 4b78fd2..c9c577b 100755 --- a/scripts/install-ap-data-to-dashboard.sh +++ b/scripts/install-ap-data-to-dashboard.sh @@ -1,5 +1,18 @@ #!/usr/bin/env bash -cd ap-data-to-dashboard -npm i -npm run setup +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +WORK_DIR="$SCRIPTS_DIR/../tmp" +TEST_FILE="install-setup-done" +ORIGINAL_DIR=$PWD + +cd $WORK_DIR/ap-data-to-dashboard + +if [ -f "$TEST_FILE" ]; then + echo "Installation and setup already done" +else + npm i &> /dev/null + npm run setup &> /dev/null + touch $TEST_FILE +fi + +cd $ORIGINAL_DIR diff --git a/scripts/move-miravi-dist.sh b/scripts/move-miravi-dist.sh index 568abaa..c337b3d 100755 --- a/scripts/move-miravi-dist.sh +++ b/scripts/move-miravi-dist.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) FORBIDDEN_BRANCH_NAMES=("assets" "queries" "images") CURRENT_BRANCH=$1 USE_DIST=$2 +WORK_DIR=$SCRIPTS_DIR"/../tmp" # Sanitize branch CURRENT_BRANCH=${CURRENT_BRANCH//\//-} @@ -17,9 +19,9 @@ if [[ "$CURRENT_BRANCH" == "main" ]]; then rm -rf gh-pages/assets gh-pages/images gh-pages/queries if [[ "$USE_DIST" == "true" ]]; then - mv ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages + mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages else - cp scripts/dashboard-placeholder.html gh-pages/index.html + cp $SCRIPTS_DIR/dashboard-placeholder.html gh-pages/index.html fi else echo "Using subdirectory because branch is not main." @@ -27,8 +29,8 @@ else mkdir -p gh-pages/$CURRENT_BRANCH if [[ "$USE_DIST" == "true" ]]; then - mv ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages/$CURRENT_BRANCH + mv $WORK_DIR/ap-data-to-dashboard/node_modules/miravi/main/dist/* gh-pages/$CURRENT_BRANCH else - cp scripts/dashboard-placeholder.html gh-pages/$CURRENT_BRANCH/index.html + cp $SCRIPTS_DIR/dashboard-placeholder.html gh-pages/$CURRENT_BRANCH/index.html fi fi diff --git a/scripts/reset-readme.sh b/scripts/reset-readme.sh index 5bd8f8c..b8d16a9 100755 --- a/scripts/reset-readme.sh +++ b/scripts/reset-readme.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -FILE=README.md -BACKUP_FILE=original-README.md +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +FILE="$SCRIPTS_DIR/../README.md" +BACKUP_FILE="$SCRIPTS_DIR/../original-README.md" mv $BACKUP_FILE $FILE diff --git a/scripts/reset-repo.sh b/scripts/reset-repo.sh index 63b106b..5deef9a 100755 --- a/scripts/reset-repo.sh +++ b/scripts/reset-repo.sh @@ -9,7 +9,10 @@ rm -rf example-data \ data \ output \ template.xlsx \ - env.txt + dist \ + tmp mkdir data touch data/.gitkeep +echo "AP_URL=" > urls.env +echo "SHACL_URL=" >> urls.env diff --git a/scripts/run-pipeline-locally.sh b/scripts/run-pipeline-locally.sh new file mode 100644 index 0000000..8830d0e --- /dev/null +++ b/scripts/run-pipeline-locally.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +echo "Setting up repo" +$SCRIPTS_DIR/set-up-repo.sh || exit 1 + +echo "Generating RDF and building Miravi" +$SCRIPTS_DIR/generate-rdf-build-miravi-locally.sh || exit 1 + diff --git a/scripts/set-up-repo.sh b/scripts/set-up-repo.sh new file mode 100755 index 0000000..353294f --- /dev/null +++ b/scripts/set-up-repo.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -a # automatically export all variables +source urls.env +set +a + +SCRIPTS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT_DIR=`pwd` +WORK_DIR=$ROOT_DIR"/tmp" + +mkdir -p $WORK_DIR + +if [ "$SHACL_URL" != "" -a "$AP_URL" != "" ]; then + echo "AP url is $AP_URL" + echo "SHACL url is $SHACL_URL" + + echo "Downloading SHACL" + $SCRIPTS_DIR/download-shacl.sh $SHACL_URL || exit 1 + + echo "Adding AP and SHACL links to README" + $SCRIPTS_DIR/update-readme.sh $ap_url $shacl_url + + if [ -f "$ROOT_DIR/template.xlsx" ]; then + echo "Template and example data already exist" + else + cd $WORK_DIR + $SCRIPTS_DIR/clone-ap-data-to-dashboard.sh || exit 1 + + echo "Copying shacl file in-shacl directory" + cp $ROOT_DIR/in-shacl/* $WORK_DIR/ap-data-to-dashboard/in-shacl || exit 1 + + cd $WORK_DIR + echo "Installing ap-data-to-dashboard" + $SCRIPTS_DIR/install-ap-data-to-dashboard.sh &> /dev/null || exit 1 + + echo "Generating template and example data" + cd ap-data-to-dashboard + node src/shacl-to-template.js || exit 1 + + echo "Moving template and example data" + cp in-shacl/template.xlsx $ROOT_DIR/template.xlsx + cp in-shacl/template.schema.json $ROOT_DIR/in-shacl/template.schema.json + rm -rf $ROOT_DIR/example-data && mkdir $ROOT_DIR/example-data + cp in-shacl/dummy* $ROOT_DIR/example-data + fi +else + echo "ERROR: AP url or SHACL url is not defined. Please update the file \"urls.env\"." + exit 1 +fi diff --git a/urls.env b/urls.env new file mode 100644 index 0000000..3d1ff27 --- /dev/null +++ b/urls.env @@ -0,0 +1,2 @@ +AP_URL= +SHACL_URL=