diff --git a/.containers-sugar.yaml b/.containers-sugar.yaml new file mode 100644 index 00000000..fc3fa551 --- /dev/null +++ b/.containers-sugar.yaml @@ -0,0 +1,31 @@ +version: 1.0.0 +compose-app: docker-compose +service-groups: + - name: default + project-name: egh-prod + compose-path: + - containers/compose-base.yaml + - containers/compose-prod.yaml + env-file: .env + services: + default: superset,airflow + list: + - name: redis + - name: flower + - name: minio + - name: superset + - name: airflow + - name: postgres + + - name: dev + project-name: egh-dev + compose-path: + - containers/compose-base.yaml + - containers/compose-dev.yaml + env-file: .env + services: + default: superset,airflow + list: + - name: superset + - name: airflow + - name: postgres diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b5dcc8be..fd5928f6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,7 +14,7 @@ env: _AIRFLOW_WWW_USER_LAST_NAME: Admin _AIRFLOW_WWW_USER_PASSWORD: airflow _AIRFLOW_WWW_USER_USERNAME: airflow - AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/docker/airflow + AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/containers/airflow AIRFLOW_HOME: /opt/airflow AIRFLOW_PORT: 8099 AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth @@ -72,15 +72,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: setup - run: make prepare-host - - - name: Create environment variables file - run: | - export HOST_UID=$(id -u) - export HOST_GID=$(id -g) - envsubst < .env.tpl > .env - - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" @@ -91,32 +82,41 @@ jobs: use-mamba: true miniforge-variant: Mambaforge + - name: Create environment variables file + run: | + export HOST_UID=$(id -u) + export HOST_GID=$(id -g) + envsubst < .env.tpl > .env + + - name: setup + run: makim host.prepare + - name: configure epigraphhub credentials run: | epigraphhub-config --name ci-epigraphhub \ --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}" - name: configure aws s3 credentials to minio - run: make dev-create-s3-credential + run: makim aws.dev-create-s3-credential - - name: build docker containers - run: make containers-build + - name: build containers-sugar + run: containers-sugar build - name: start services run: | - make containers-start-services SERVICES="superset airflow" + makim containers.start-services --services "superset,airflow" - name: wait for the services are properly working run: | docker ps - make containers-wait-all + makim containers.wait-all - name: Trigger all DAGs run: | - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25' - make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test owid 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test foph 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test colombia 2022-10-25' + containers-sugar exec --extras="-T" --services airflow --cmd 'airflow dags test web_status_test 2022-10-25' - name: Test MinIO access run: aws --endpoint-url http://localhost:9000/ s3 ls @@ -124,7 +124,7 @@ jobs: - name: Generate logs if: ${{ failure() }} run: | - make containers-logs ARGS="--tail 1000" SERVICES="" > /tmp/containers-services.log + containers-sugar logs --extras "--tail 1000" --all > /tmp/containers-services.log - name: Archive log artifacts uses: actions/upload-artifact@v3 @@ -132,3 +132,7 @@ jobs: with: name: containers-services path: /tmp/containers-services.log + + - name: Setup tmate session + if: "${{ failure() && (contains(github.event.pull_request.labels.*.name, 'ci:enable-debugging')) }}" + uses: mxschmitt/action-tmate@v3 diff --git a/.makim.yaml b/.makim.yaml new file mode 100644 index 00000000..47a004b5 --- /dev/null +++ b/.makim.yaml @@ -0,0 +1,83 @@ +version: 1.0.0 +env-file: .env +groups: + + host: + targets: + prepare: + help: Preparing host + shell: bash + run: | + ./scripts/prepare-host.sh + + aws: + targets: + dev-create-s3-credential: + help: Create AWS S3 Credentials + run: | + set -e + ./scripts/dev/create-s3-credential.sh + + + conda: + targets: + lock: + help: Configure a conda-lock + run: | + cd conda + rm -f conda-*.lock + conda-lock --conda which mamba \ + -f prod.yaml \ + -p osx-64 \ + -p linux-64 \ + --kind explicit + + containers: + targets: + wait: + help: Wait for a service + args: + timeout: + help: Define the timeout for the healthcheck + type: integer + default: 30 + service: + help: Service name that would be awaited + type: string + required: True + + run: | + timeout {{ args.timeout }} ./containers/scripts/healthcheck.sh {{ args.service }} + + wait-all: + run: | + if $ENV == "dev": + makim containers.wait --service "postgres" + + for service in ["postgres","airflow","superset","redis","flower","minio"]: + makim containers.wait --service @(service) + + start-services: + help: bla + shell: bash + dependencies: + - targets: host.prepare + args: + services: + help: Services names separated by comma + type: string + default: "airflow,superset" + + run: | + set -e + if [ "${ENV}" = "dev" ]; then \ + containers-sugar start --services "postgres"; \ + makim containers.wait --service "postgres"; \ + fi + containers-sugar start --services {{ args.services }} + makim containers.wait --service "airflow" + + dev-prepare-db: + help: Prepare a database. Used for development + run: | + containers-sugar exec --extras="-T" --services superset --cmd "bash /opt/EpiGraphHub/containers/postgresql/scripts/dev/prepare-db.sh" diff --git a/README.md b/README.md index d2d2977b..9301edb2 100644 --- a/README.md +++ b/README.md @@ -166,26 +166,26 @@ $ conda activate epigraphhub ![fig2](https://user-images.githubusercontent.com/140123/165057193-c5a2b2a2-8f30-426d-9bac-8f559c01265d.png) -The project provides three **compose** files, where one is the base -definition (`containers/compose-base.yaml`) and the others are one for -development (`containers/compose-dev.yaml`) and other for production -(`containers/compose-prod.yaml`). Additionally, there is a **Makefile** -file that provides `make` targets that helps to manage the docker services: - -- `containers-build`: builds docker images -- `containers-start`: starts docker services -- `containers-stop`: stops docker services -- `containers-restart`: restarts docker services -- `containers-logs`: shows docker -- `containers-dev-prepare-db`: prepares the development version of the database -- `containers-run-cron`: run the cron tasks -- `containers-bash`: opens the docker service bash - -This `make` targets runs by default docker for development. For production, +The project provides three **compose** files, where one is the base definition +(`containers/compose-base.yaml`) and the others are one for development +(`containers/compose-dev.yaml`) and other for production +(`containers/compose-prod.yaml`). Additionally, there is a **.makim.yaml** file +that provides `makim` targets that helps to define targets and dependencies and a +**.containers.sugar.yaml** file that provides `containers-sugar` commands to +manage the usage of containers: + +- `containers-sugar build`: builds docker images +- `containers-sugar start`: starts docker services +- `containers-sugar stop`: stops docker services +- `containers-sugar restart`: restarts docker services +- `containers-sugar logs`: shows docker +- `makim containers.dev-prepare-db`: prepares the development version of the database + +This `makim` targets runs by default docker for development. For production, it is necessary to pass the argument `ENV=prod` , for example: ```bash -make containers-build +makim containers-sugar build ``` A common workflow to prepare the system would be: @@ -193,13 +193,13 @@ A common workflow to prepare the system would be: 1. Build the docker image: ```bash -$ make containers-build +$ containers-sugar build ``` 2. Start the services: ```bash -$ make containers-start +$ containers-sugar start ``` Before moving forward, check if the services are working properly: @@ -211,7 +211,7 @@ $ ./containers/healthcheck.sh epigraphhub 3. Now, prepare the development database (skip it for production): ```bash -$ make containers-prepare-db +$ makim containers.dev-prepare-db ``` ## AIRFLOW @@ -231,7 +231,7 @@ For development, it is already created by the containers stack. In order to configure it locally, run: ```bash -make dev-create-s3-credentials +makim aws.dev-create-s3-credentials ``` Note: it needs the AWS env variable on the environment (or from .env file). @@ -291,4 +291,4 @@ Linked-in : https://www.linkedin.com/company/the-graph-network [linkedin-shield]: https://img.shields.io/badge/-LinkedIn-black.svg?style=for-the-badge&logo=linkedin&colorB=555 [linkedin-url]: https://www.linkedin.com/company/the-graph-network [discord-shield]: https://img.shields.io/discord/865572405229518878?color=%235460e6&label=DISCORD&style=for-the-badge -[discord-url]: https://discord.gg/56thARPrnJ +[discord-url]: https://discord.gg/56thARPrnJ \ No newline at end of file diff --git a/conda/base.yaml b/conda/base.yaml index e9eb3e12..f942ad8b 100644 --- a/conda/base.yaml +++ b/conda/base.yaml @@ -12,3 +12,5 @@ dependencies: - pip - pip: - epigraphhub + - containers-sugar==1.4.3 + - makim==1.6.4