From 7949656c8228fbc1b61fc960f827c1679d0fbb50 Mon Sep 17 00:00:00 2001 From: Yurely Date: Thu, 30 Mar 2023 17:49:43 -0400 Subject: [PATCH 01/24] Create file for implementation of containers-sugar --- .containers-sugar.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .containers-sugar.yaml diff --git a/.containers-sugar.yaml b/.containers-sugar.yaml new file mode 100644 index 00000000..dc15a9d7 --- /dev/null +++ b/.containers-sugar.yaml @@ -0,0 +1,31 @@ +version: 1.0.0 +compose-app: docker-compose +service-groups: + - name: default + project-name: egh-prod + compose-path: + - containers/compose-base.yaml + - containers/compose-prod.yaml + env-file: .env + services: + default: superset,airflow + list: + - name: redis + - name: flower + - name: minio + - name: superset + - name: airflow + - name: postgres + + - name: dev + project-name: egh-dev + compose-path: + - containers/compose-base.yaml + - containers/compose-dev.yaml + env-file: .env + services: superset,airflow + default: + list: + - name: superset + - name: airflow + - name: postgres \ No newline at end of file From b5422420e7a92e218970b4c836a0e64d9b30a552 Mon Sep 17 00:00:00 2001 From: Yurely Date: Thu, 30 Mar 2023 17:49:53 -0400 Subject: [PATCH 02/24] Create file for implementation of makim --- .makim.yaml | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .makim.yaml diff --git a/.makim.yaml b/.makim.yaml new file mode 100644 index 00000000..21c2d185 --- /dev/null +++ b/.makim.yaml @@ -0,0 +1,82 @@ +version: 1.0.0 +env-file: .env +groups: + + host: + targets: + prepare: + help: Preparing host + run: | + bash scripts/prepare-host.sh + + aws: + targets: + dev-create-s3-credential: + help: Create AWS S3 Credentials + run: | + set -e + ./scripts/dev/create-s3-credential.sh + + + conda: + targets: + lock: + help: Configure a conda-lock + run: | + cd conda + rm -f conda-*.lock + conda-lock --conda which mamba \ + -f prod.yaml \ + -p osx-64 \ + -p linux-64 \ + --kind explicit + + containers: + targets: + wait: + help: Wait for a service + args: + timeout: + help: Define the timeout for the healthcheck + type: integer + default: 30 + service: + help: Service name that would be awaited + type: string + required: True + + run: | + timeout {{ args.timeout }} ./containers/scripts/healthcheck.sh {{ args.service }} + + wait-all: + run: | + if $ENV == "dev": + makim containers.wait --service "postgres" + + for service in ["postgres","airflow","superset","redis","flower","minio"]: + makim containers.wait --service @(service) + + start-services: + help: bla + shell: bash + dependencies: + - targets: host.prepare + args: + services: Services names separated by comma + help: + type: string + default: "airflow,superset" + + run: | + set -e + if [ "${ENV}" = "dev" ]; then \ + containers-sugar start --services "postgres"; \ + makim containers.wait --service "postgres"; \ + fi + containers-sugar start --services {{ args.services }} + makim containers.wait --service "airflow" + + dev-prepare-db: + help: Prepare a database. Used for development + run: | + containers-sugar exec --extras="-T" --services superset --cmd "bash /opt/EpiGraphHub/containers/postgresql/scripts/dev/prepare-db.sh" \ No newline at end of file From a3e45181a8b85a1597eb0f147c2396ace713f877 Mon Sep 17 00:00:00 2001 From: Yurely Date: Thu, 30 Mar 2023 17:50:29 -0400 Subject: [PATCH 03/24] Adding dependencies --- conda/base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/base.yaml b/conda/base.yaml index e9eb3e12..f942ad8b 100644 --- a/conda/base.yaml +++ b/conda/base.yaml @@ -12,3 +12,5 @@ dependencies: - pip - pip: - epigraphhub + - containers-sugar==1.4.3 + - makim==1.6.4 From c64e6c80e8301ccb57c9ddb8395f126cd532e8ec Mon Sep 17 00:00:00 2001 From: Yurely Date: Thu, 30 Mar 2023 17:51:07 -0400 Subject: [PATCH 04/24] Update commands for makim and containers-sugar --- .github/workflows/main.yaml | 26 +++++++++++----------- README.md | 44 ++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b5dcc8be..d06c8033 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@v2 - name: setup - run: make prepare-host + run: makim host.prepare - name: Create environment variables file run: | @@ -97,26 +97,26 @@ jobs: --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}" - name: configure aws s3 credentials to minio - run: make dev-create-s3-credential + run: makim aws.dev-create-s3-credential - - name: build docker containers - run: make containers-build + - name: build containers-sugar + run: containers-sugar build - name: start services run: | - make containers-start-services SERVICES="superset airflow" + makim containers.start-services --services "superset,airflow" - name: wait for the services are properly working run: | docker ps - make containers-wait-all + makim containers.wait-all - name: Trigger all DAGs run: | - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test foph 2022-10-25' + containers-sugar exec --args="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25' + containers-sugar exec --extra="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25' - name: Test MinIO access run: aws --endpoint-url http://localhost:9000/ s3 ls @@ -124,11 +124,11 @@ jobs: - name: Generate logs if: ${{ failure() }} run: | - make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log + makim containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log - - name: Archive log artifacts +- name: Archive log artifacts uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: containers-services - path: /tmp/containers-services.log + path: /tmp/containers-services.log \ No newline at end of file diff --git a/README.md b/README.md index d2d2977b..9301edb2 100644 --- a/README.md +++ b/README.md @@ -166,26 +166,26 @@ $ conda activate epigraphhub ![fig2](https://user-images.githubusercontent.com/140123/165057193-c5a2b2a2-8f30-426d-9bac-8f559c01265d.png) -The project provides three **compose** files, where one is the base -definition (`containers/compose-base.yaml`) and the others are one for -development (`containers/compose-dev.yaml`) and other for production -(`containers/compose-prod.yaml`). Additionally, there is a **Makefile** -file that provides `make` targets that helps to manage the docker services: - -- `containers-build`: builds docker images -- `containers-start`: starts docker services -- `containers-stop`: stops docker services -- `containers-restart`: restarts docker services -- `containers-logs`: shows docker -- `containers-dev-prepare-db`: prepares the development version of the database -- `containers-run-cron`: run the cron tasks -- `containers-bash`: opens the docker service bash - -This `make` targets runs by default docker for development. For production, +The project provides three **compose** files, where one is the base definition +(`containers/compose-base.yaml`) and the others are one for development +(`containers/compose-dev.yaml`) and other for production +(`containers/compose-prod.yaml`). Additionally, there is a **.makim.yaml** file +that provides `makim` targets that helps to define targets and dependencies and a +**.containers.sugar.yaml** file that provides `containers-sugar` commands to +manage the usage of containers: + +- `containers-sugar build`: builds docker images +- `containers-sugar start`: starts docker services +- `containers-sugar stop`: stops docker services +- `containers-sugar restart`: restarts docker services +- `containers-sugar logs`: shows docker +- `makim containers.dev-prepare-db`: prepares the development version of the database + +This `makim` targets runs by default docker for development. For production, it is necessary to pass the argument `ENV=prod` , for example: ```bash -make containers-build +makim containers-sugar build ``` A common workflow to prepare the system would be: @@ -193,13 +193,13 @@ A common workflow to prepare the system would be: 1. Build the docker image: ```bash -$ make containers-build +$ containers-sugar build ``` 2. Start the services: ```bash -$ make containers-start +$ containers-sugar start ``` Before moving forward, check if the services are working properly: @@ -211,7 +211,7 @@ $ ./containers/healthcheck.sh epigraphhub 3. Now, prepare the development database (skip it for production): ```bash -$ make containers-prepare-db +$ makim containers.dev-prepare-db ``` ## AIRFLOW @@ -231,7 +231,7 @@ For development, it is already created by the containers stack. In order to configure it locally, run: ```bash -make dev-create-s3-credentials +makim aws.dev-create-s3-credentials ``` Note: it needs the AWS env variable on the environment (or from .env file). @@ -291,4 +291,4 @@ Linked-in : https://www.linkedin.com/company/the-graph-network [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555 [linkedin-url]: https://www.linkedin.com/company/the-graph-network [discord-shield]: https://img.shields.io/discord/865572405229518878?color=%235460e6&label=DISCORD&style=for-the-badge -[discord-url]: https://discord.gg/56thARPrnJ +[discord-url]: https://discord.gg/56thARPrnJ \ No newline at end of file From 3a8cfd4c481212159f031f4b39a52c9b95f2b829 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 3 Apr 2023 17:06:03 -0400 Subject: [PATCH 05/24] Adding an empty line at the end of the files --- .containers-sugar.yaml | 2 +- .github/workflows/main.yaml | 5 +++-- .makim.yaml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.containers-sugar.yaml b/.containers-sugar.yaml index dc15a9d7..c0d62676 100644 --- a/.containers-sugar.yaml +++ b/.containers-sugar.yaml @@ -28,4 +28,4 @@ service-groups: list: - name: superset - name: airflow - - name: postgres \ No newline at end of file + - name: postgres diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d06c8033..280c3a6f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -126,9 +126,10 @@ jobs: run: | makim containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log -- name: Archive log artifacts + - name: Archive log artifacts uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: containers-services - path: /tmp/containers-services.log \ No newline at end of file + path: /tmp/containers-services.log + \ No newline at end of file diff --git a/.makim.yaml b/.makim.yaml index 21c2d185..e896f021 100644 --- a/.makim.yaml +++ b/.makim.yaml @@ -79,4 +79,4 @@ groups: dev-prepare-db: help: Prepare a database. Used for development run: | - containers-sugar exec --extras="-T" --services superset --cmd "bash /opt/EpiGraphHub/containers/postgresql/scripts/dev/prepare-db.sh" \ No newline at end of file + containers-sugar exec --extras="-T" --services superset --cmd "bash /opt/EpiGraphHub/containers/postgresql/scripts/dev/prepare-db.sh" From e93b6a346d29de51bb71972d89ab9fce44239ec3 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 3 Apr 2023 17:20:00 -0400 Subject: [PATCH 06/24] Delete command --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 280c3a6f..dd2affec 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -124,7 +124,7 @@ jobs: - name: Generate logs if: ${{ failure() }} run: | - makim containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log + containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log - name: Archive log artifacts uses: actions/upload-artifact@v3 From 4fe8405d6eda70a6a02b71358fea160cd2c0383a Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 13:33:23 -0400 Subject: [PATCH 07/24] Changes in main.yaml --- .github/workflows/main.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dd2affec..b54aca7d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -72,15 +72,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: setup - run: makim host.prepare - - - name: Create environment variables file - run: | - export HOST_UID=$(id -u) - export HOST_GID=$(id -g) - envsubst < .env.tpl > .env - - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" @@ -91,6 +82,16 @@ jobs: use-mamba: true miniforge-variant: Mambaforge + - name: setup + run: makim host.prepare + + - name: Create environment variables file + run: | + export HOST_UID=$(id -u) + export HOST_GID=$(id -g) + envsubst < .env.tpl > .env + + - name: configure epigraphhub credentials run: | epigraphhub-config --name ci-epigraphhub \ From 8c3b620283115b8b242e86edd217905ea77f01d4 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 13:49:04 -0400 Subject: [PATCH 08/24] main.yaml --- .github/workflows/main.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b54aca7d..0d08c2ac 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 55 defaults: - run: + run: shell: bash -l {0} concurrency: group: ci-${{ github.ref }} @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 - with: + with: miniconda-version: "latest" mamba-version: "*" environment-file: conda/base.yaml @@ -91,7 +91,6 @@ jobs: export HOST_GID=$(id -g) envsubst < .env.tpl > .env - - name: configure epigraphhub credentials run: | epigraphhub-config --name ci-epigraphhub \ From e7ed016b97b7bbd5b2602b2e61355f28ce742aef Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 13:55:11 -0400 Subject: [PATCH 09/24] main.yaml --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0d08c2ac..83b58bd4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,8 +63,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 55 defaults: - run: - shell: bash -l {0} + run: + shell: bash -l {0} concurrency: group: ci-${{ github.ref }} cancel-in-progress: true From 3f99bcd074486a9e46013bd83aeb4f5320dcf22b Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 13:58:44 -0400 Subject: [PATCH 10/24] Revert main.yaml --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 83b58bd4..0d08c2ac 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,8 +63,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 55 defaults: - run: - shell: bash -l {0} + run: + shell: bash -l {0} concurrency: group: ci-${{ github.ref }} cancel-in-progress: true From e4f21568be121a58ea49411c9d7f837539f9d218 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:02:42 -0400 Subject: [PATCH 11/24] Update main.yaml --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0d08c2ac..01979291 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -64,7 +64,7 @@ jobs: timeout-minutes: 55 defaults: run: - shell: bash -l {0} + shell: bash -l {0} concurrency: group: ci-${{ github.ref }} cancel-in-progress: true From 841faefcfbe9b850faf586e4a044b59976f24fdb Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:05:40 -0400 Subject: [PATCH 12/24] revert --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 01979291..cbbc5e0a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,8 +63,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 55 defaults: - run: - shell: bash -l {0} + run: + shell: bash -l {0} concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 - with: + with: miniconda-version: "latest" mamba-version: "*" environment-file: conda/base.yaml From abddf77fe047e9a42e91ceb34fd5b963887d487d Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:12:03 -0400 Subject: [PATCH 13/24] update containers-sugar.yaml --- .containers-sugar.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.containers-sugar.yaml b/.containers-sugar.yaml index c0d62676..fc3fa551 100644 --- a/.containers-sugar.yaml +++ b/.containers-sugar.yaml @@ -23,8 +23,8 @@ service-groups: - containers/compose-base.yaml - containers/compose-dev.yaml env-file: .env - services: superset,airflow - default: + services: + default: superset,airflow list: - name: superset - name: airflow From ff77e333ff8e93f28c0ae7b5956f23e4678bed72 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:31:16 -0400 Subject: [PATCH 14/24] Update makim.yaml --- .makim.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.makim.yaml b/.makim.yaml index e896f021..fffe6921 100644 --- a/.makim.yaml +++ b/.makim.yaml @@ -63,7 +63,7 @@ groups: - targets: host.prepare args: services: Services names separated by comma - help: + help: Name of service type: string default: "airflow,superset" From bdfa2bf9c271c6b4f574f0fa9e1e6c2e9381277c Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:42:03 -0400 Subject: [PATCH 15/24] Update makim.yaml --- .makim.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.makim.yaml b/.makim.yaml index fffe6921..5656624b 100644 --- a/.makim.yaml +++ b/.makim.yaml @@ -62,8 +62,8 @@ groups: dependencies: - targets: host.prepare args: - services: Services names separated by comma - help: Name of service + services: + help: Services names separated by comma type: string default: "airflow,superset" From 913aa27fcbc9e420633cbd62b850ea1b73fdbbb0 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:51:41 -0400 Subject: [PATCH 16/24] makim.yaml test --- .makim.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.makim.yaml b/.makim.yaml index 5656624b..db256a2b 100644 --- a/.makim.yaml +++ b/.makim.yaml @@ -7,7 +7,7 @@ groups: prepare: help: Preparing host run: | - bash scripts/prepare-host.sh + bash ./scripts/prepare-host.sh aws: targets: From a3704acabec15d9710278cf00335637caa38ca25 Mon Sep 17 00:00:00 2001 From: Yurely Date: Mon, 10 Apr 2023 14:59:28 -0400 Subject: [PATCH 17/24] Changes in makim.yaml --- .makim.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.makim.yaml b/.makim.yaml index db256a2b..47a004b5 100644 --- a/.makim.yaml +++ b/.makim.yaml @@ -6,8 +6,9 @@ groups: targets: prepare: help: Preparing host + shell: bash run: | - bash ./scripts/prepare-host.sh + ./scripts/prepare-host.sh aws: targets: From 8e9635979698a8b61f0646884a0760f3ca7bd709 Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Mon, 10 Apr 2023 19:04:46 -0400 Subject: [PATCH 18/24] Add temporarily debugging session --- .github/workflows/main.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cbbc5e0a..430d2ea7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -113,7 +113,7 @@ jobs: - name: Trigger all DAGs run: | - containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25' containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test foph 2022-10-25' containers-sugar exec --args="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25' containers-sugar exec --extra="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25' @@ -132,4 +132,7 @@ jobs: with: name: containers-services path: /tmp/containers-services.log - \ No newline at end of file + + - name: Setup tmate session + if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'status: ci:enable-debugging')) }}" + uses: mxschmitt/action-tmate@v3 From 5b72295e4f811318044d16d4f9ff59911f84abad Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Mon, 10 Apr 2023 19:11:15 -0400 Subject: [PATCH 19/24] Update label trigger --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 430d2ea7..c09db79f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -134,5 +134,5 @@ jobs: path: /tmp/containers-services.log - name: Setup tmate session - if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'status: ci:enable-debugging')) }}" + if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'ci:enable-debugging')) }}" uses: mxschmitt/action-tmate@v3 From 6fd127d02993df3d2a90d235fd4a63b3af81bd44 Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Mon, 10 Apr 2023 19:18:22 -0400 Subject: [PATCH 20/24] Fix ci --- .github/workflows/main.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c09db79f..4938f387 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -82,15 +82,15 @@ jobs: use-mamba: true miniforge-variant: Mambaforge - - name: setup - run: makim host.prepare - - name: Create environment variables file run: | export HOST_UID=$(id -u) export HOST_GID=$(id -g) envsubst < .env.tpl > .env + - name: setup + run: makim host.prepare + - name: configure epigraphhub credentials run: | epigraphhub-config --name ci-epigraphhub \ @@ -132,7 +132,3 @@ jobs: with: name: containers-services path: /tmp/containers-services.log - - - name: Setup tmate session - if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'ci:enable-debugging')) }}" - uses: mxschmitt/action-tmate@v3 From 48082110727856bdb1384e9e7b1e7da47b555d86 Mon Sep 17 00:00:00 2001 From: Yurely Date: Tue, 11 Apr 2023 17:20:34 -0400 Subject: [PATCH 21/24] Change in path --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4938f387..77e4ebc6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,7 +14,7 @@ env: _AIRFLOW_WWW_USER_LAST_NAME: Admin _AIRFLOW_WWW_USER_PASSWORD: airflow _AIRFLOW_WWW_USER_USERNAME: airflow - AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/docker/airflow + AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/airflow AIRFLOW_HOME: /opt/airflow AIRFLOW_PORT: 8099 AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth From 1f86a4605ceccbaf94702bb1ebfdb483f586b0a7 Mon Sep 17 00:00:00 2001 From: Yurely Date: Tue, 11 Apr 2023 17:31:19 -0400 Subject: [PATCH 22/24] update --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 77e4ebc6..3bd32f19 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,7 +14,7 @@ env: _AIRFLOW_WWW_USER_LAST_NAME: Admin _AIRFLOW_WWW_USER_PASSWORD: airflow _AIRFLOW_WWW_USER_USERNAME: airflow - AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/airflow + AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/containers/airflow AIRFLOW_HOME: /opt/airflow AIRFLOW_PORT: 8099 AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth From b36f862a54b592916aa5d92c22ac10608d039396 Mon Sep 17 00:00:00 2001 From: Yurely Date: Tue, 11 Apr 2023 18:18:05 -0400 Subject: [PATCH 23/24] update --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3bd32f19..c17595b9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -115,8 +115,8 @@ jobs: run: | containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25' containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test foph 2022-10-25' - containers-sugar exec --args="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25' - containers-sugar exec --extra="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25' - name: Test MinIO access run: aws --endpoint-url http://localhost:9000/ s3 ls From 2416acb63ca7247a2fdcaecb40c8560eb63539e0 Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Tue, 11 Apr 2023 18:57:54 -0400 Subject: [PATCH 24/24] Add debugging step --- .github/workflows/main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c17595b9..fd5928f6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -132,3 +132,7 @@ jobs: with: name: containers-services path: /tmp/containers-services.log + + - name: Setup tmate session + if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'ci:enable-debugging')) }}" + uses: mxschmitt/action-tmate@v3