From 27895a9f29154afa04396fec16fe75bb4f796574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 14:40:24 -0300 Subject: [PATCH 1/6] refactor(airflow): airflow system load balancer using microsservices arch --- containers/airflow/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/containers/airflow/Dockerfile b/containers/airflow/Dockerfile index 96068ff..6ef36a2 100644 --- a/containers/airflow/Dockerfile +++ b/containers/airflow/Dockerfile @@ -111,4 +111,3 @@ RUN sudo mkdir -p /opt/scripts /sources /opt/airflow \ && sudo chown -R airflow /var/log/* ENTRYPOINT [ "/opt/entrypoint.sh" ] -CMD /opt/scripts/startup.sh From 758933ab6cdf27aa049c9253c268e59cb413a7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 14:51:45 -0300 Subject: [PATCH 2/6] docker-compose: Restructuring docker-compose, Configuring airflow-common config & Configuring airflow_webserver service --- containers/compose-base.yaml | 149 +++++++++++++++++------------------ containers/compose-dev.yaml | 6 -- 2 files changed, 73 insertions(+), 82 deletions(-) diff --git a/containers/compose-base.yaml b/containers/compose-base.yaml index 975f0b7..f8a3d80 100644 --- a/containers/compose-base.yaml +++ b/containers/compose-base.yaml @@ -1,5 +1,46 @@ version: '3.4' +x-airflow-common: + &airflow-common + build: + context: .. + dockerfile: containers/airflow/Dockerfile + args: + UID: ${HOST_UID} + GID: ${HOST_GID} + environment: + &airflow-common-env + AIRFLOW_HOME: /opt/airflow + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + HOST_UID: ${HOST_UID} + HOST_GID: ${HOST_GID} + _AIRFLOW_DB_UPGRADE: ${_AIRFLOW_DB_UPGRADE} + _AIRFLOW_WWW_USER_CREATE: ${_AIRFLOW_WWW_USER_CREATE} + _AIRFLOW_WWW_USER_EMAIL: ${_AIRFLOW_WWW_USER_EMAIL} + _AIRFLOW_WWW_USER_FIRST_NAME: ${_AIRFLOW_WWW_USER_FIRST_NAME} + _AIRFLOW_WWW_USER_LAST_NAME: ${_AIRFLOW_WWW_USER_LAST_NAME} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + AIRFLOW_FILES_PATH_DIR_HOST: ${AIRFLOW_FILES_PATH_DIR_HOST} + AIRFLOW_PORT: ${AIRFLOW_PORT} + AIRFLOW__API__AUTH_BACKENDS: ${AIRFLOW__API__AUTH_BACKENDS} + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: ${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION} + AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN} + AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} + volumes: + - ${AIRFLOW_FILES_PATH_DIR_HOST}/dags:${AIRFLOW_HOME}/dags + - ${AIRFLOW_FILES_PATH_DIR_HOST}/logs:${AIRFLOW_HOME}/logs + - ${AIRFLOW_FILES_PATH_DIR_HOST}/plugins:${AIRFLOW_HOME}/plugins + user: "${HOST_GID}:0" + depends_on: + &airflow-common-depends-on + redis: + condition: service_healthy + postgres: + condition: service_healthy + + services: superset: platform: linux/amd64 @@ -30,63 +71,6 @@ services: - ${SUPERSET_PORT}:8088 depends_on: - redis - - flower - - airflow: - platform: linux/amd64 - hostname: airflow - restart: unless-stopped - env_file: - - ../.env - build: - context: .. - dockerfile: containers/airflow/Dockerfile - args: - # note: if you want to use a specific UID and GID, ensure to add it to - # the .env file - - UID=${HOST_UID:-1000} - - GID=${HOST_GID:-1000} - environment: - AIRFLOW_UID: "${HOST_UID:-1000}" - user: "epigraphhub:epigraphhub" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/health"] - interval: 30s - timeout: 10s - retries: 10 - start_period: 40s # requires docker-compose 3.4 - volumes: - - ..:/opt/EpiGraphHub - - $HOME/.config:/home/epigraphhub/.config - - ${SUPERSET_DB_PATH_DIR_HOST}:/opt/data/superset/ - - ${AIRFLOW_FILES_PATH_DIR_HOST}/logs:/opt/airflow/logs - - ${AIRFLOW_FILES_PATH_DIR_HOST}/plugins:/opt/airflow/plugins - - ./airflow/dags:/opt/airflow/dags - - ~/.config:/home/epigraphhub/.config - ports: - - ${AIRFLOW_PORT}:8080 - depends_on: - - redis - - flower - - minio - - redis: - platform: linux/amd64 - image: redis:alpine - hostname: redis - restart: unless-stopped - # environment: - # DISABLE_COMMANDS: "FLUSHDB,FLUSHALL,CONFIG" - # ALLOW_EMPTY_PASSWORD: "no" - # REDIS_PASSWORD: "${REDIS_PASSWORD}" - healthcheck: - test: ["CMD", "redis-cli","ping"] - interval: 60s - timeout: 5s - retries: 5 - start_period: 30s # requires docker-compose 3.4 - volumes: - - redis:/data postgres: platform: linux/amd64 @@ -120,28 +104,23 @@ services: retries: 10 command: -p ${POSTGRES_PORT} - flower: + redis: platform: linux/amd64 - image: mher/flower:0.9.7 + image: redis:alpine + hostname: redis restart: unless-stopped - env_file: - - ../.env - depends_on: - - redis - command: [ - "flower", - "--broker=redis://redis:6379/0", - "--port=8888", - "--basic_auth=admin:${FLOWER_PASSWORD}" - ] - ports: - - ${FLOWER_PORT}:8888 + # environment: + # DISABLE_COMMANDS: "FLUSHDB,FLUSHALL,CONFIG" + # ALLOW_EMPTY_PASSWORD: "no" + # REDIS_PASSWORD: "${REDIS_PASSWORD}" healthcheck: - test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:8888/healthcheck"] + test: ["CMD", "redis-cli","ping"] interval: 60s - timeout: 10s + timeout: 5s retries: 5 - start_period: 40s # requires docker-compose 3.4 + start_period: 30s # requires docker-compose 3.4 + volumes: + - redis:/data minio: platform: linux/amd64 @@ -153,8 +132,7 @@ services: "server", "/data", "--console-address", - ":9090" - ] + ":9090"] ports: - ${MINIO_PORT_1}:9000 - ${MINIO_PORT_2}:9090 @@ -167,6 +145,25 @@ services: retries: 5 start_period: 40s # requires docker-compose 3.4 + airflow_webserver: + <<: *airflow-common + command: webserver + image: eph_airflow:latest + hostname: webserver + container_name: airflow_webserver + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${AIRFLOW_PORT}/health"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 40s # requires docker-compose 3.4 + ports: + - ${AIRFLOW_PORT}:8080 + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully volumes: redis: diff --git a/containers/compose-dev.yaml b/containers/compose-dev.yaml index e02314f..66abf24 100644 --- a/containers/compose-dev.yaml +++ b/containers/compose-dev.yaml @@ -6,9 +6,3 @@ services: - postgres - redis - flower - - airflow: - depends_on: - - postgres - - redis - - flower From 6a1d834dffe664679ab9d96c354d015469e4219b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 14:55:55 -0300 Subject: [PATCH 3/6] docker-compose: scheduler service --- containers/compose-base.yaml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/containers/compose-base.yaml b/containers/compose-base.yaml index f8a3d80..42d7636 100644 --- a/containers/compose-base.yaml +++ b/containers/compose-base.yaml @@ -63,7 +63,7 @@ services: interval: 60s timeout: 10s retries: 5 - start_period: 40s # requires docker-compose 3.4 + start_period: 40s volumes: - ..:/opt/EpiGraphHub - ${SUPERSET_DB_PATH_DIR_HOST}:/opt/data/superset/ @@ -118,7 +118,7 @@ services: interval: 60s timeout: 5s retries: 5 - start_period: 30s # requires docker-compose 3.4 + start_period: 30s volumes: - redis:/data @@ -143,12 +143,12 @@ services: interval: 60s timeout: 10s retries: 5 - start_period: 40s # requires docker-compose 3.4 + start_period: 40s airflow_webserver: <<: *airflow-common command: webserver - image: eph_airflow:latest + image: eph_airflow hostname: webserver container_name: airflow_webserver restart: unless-stopped @@ -157,7 +157,7 @@ services: interval: 30s timeout: 10s retries: 10 - start_period: 40s # requires docker-compose 3.4 + start_period: 40s ports: - ${AIRFLOW_PORT}:8080 depends_on: @@ -165,6 +165,24 @@ services: startup: condition: service_completed_successfully + scheduler: + <<: *airflow-common + command: scheduler + image: eph_airflow + hostname: scheduler + container_name: eph_airflow_scheduler + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + volumes: redis: minio: From d8f5d35fb49ba72b54aef61e2b5042ce8e9e39dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 15:32:28 -0300 Subject: [PATCH 4/6] docker-compose: airflow_worker service --- containers/compose-base.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/containers/compose-base.yaml b/containers/compose-base.yaml index 42d7636..0f732c2 100644 --- a/containers/compose-base.yaml +++ b/containers/compose-base.yaml @@ -165,7 +165,7 @@ services: startup: condition: service_completed_successfully - scheduler: + airflow_scheduler: <<: *airflow-common command: scheduler image: eph_airflow @@ -183,6 +183,31 @@ services: startup: condition: service_completed_successfully + airflow_worker: + <<: *airflow-common + command: celery worker + image: eph_airflow + hostname: worker + container_name: eph_airflow_worker + healthcheck: + test: + - "CMD-SHELL" + - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + environment: + <<: *airflow-common-env + # Required to handle warm shutdown of the celery workers properly + # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + volumes: redis: minio: From 16219391e9c54a53c7e008d5f265f51d74c61c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 16:23:14 -0300 Subject: [PATCH 5/6] dockerfile & compose: add triggerer and startup --- containers/airflow/Dockerfile | 2 +- containers/compose-base.yaml | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/containers/airflow/Dockerfile b/containers/airflow/Dockerfile index 6ef36a2..cde8156 100644 --- a/containers/airflow/Dockerfile +++ b/containers/airflow/Dockerfile @@ -110,4 +110,4 @@ RUN sudo mkdir -p /opt/scripts /sources /opt/airflow \ && sudo touch /var/log/colombia_fetch.log \ && sudo chown -R airflow /var/log/* -ENTRYPOINT [ "/opt/entrypoint.sh" ] +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/containers/compose-base.yaml b/containers/compose-base.yaml index 0f732c2..f1aac55 100644 --- a/containers/compose-base.yaml +++ b/containers/compose-base.yaml @@ -208,6 +208,47 @@ services: startup: condition: service_completed_successfully + airflow_triggerer: + <<: *airflow-common + command: triggerer + image: eph_airflow + hostname: triggerer + container_name: eph_airflow_triggerer + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + startup: + condition: service_completed_successfully + + airflow_startup: + <<: *airflow-common + entrypoint: /bin/bash + image: eph_airflow + container_name: eph_airflow_startup + # yamllint disable rule:line-length + command: + - -c + - | + mkdir -p /sources/logs /sources/dags /sources/plugins + chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + exec /entrypoint airflow version + # yamllint enable rule:line-length + environment: + <<: *airflow-common-env + _AIRFLOW_DB_UPGRADE: 'true' + _AIRFLOW_WWW_USER_CREATE: 'true' + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} + user: "0:0" + volumes: + - ${AIRFLOW_FILES_PATH_DIR_HOST}:/sources + volumes: redis: minio: From 99ed4fddab1fa8e6093c29ccb6bae40edbed1a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 22 Jun 2023 17:17:55 -0300 Subject: [PATCH 6/6] Additional config --- .env.tpl | 5 +++-- containers/compose-base.yaml | 37 +++++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.env.tpl b/.env.tpl index 23c6945..c7654a1 100644 --- a/.env.tpl +++ b/.env.tpl @@ -3,8 +3,9 @@ _AIRFLOW_WWW_USER_CREATE=${_AIRFLOW_WWW_USER_CREATE:-True} _AIRFLOW_WWW_USER_USERNAME=${_AIRFLOW_WWW_USER_USERNAME} _AIRFLOW_WWW_USER_PASSWORD=${_AIRFLOW_WWW_USER_PASSWORD} _AIRFLOW_WWW_USER_EMAIL=${_AIRFLOW_WWW_USER_EMAIL} -_AIRFLOW_WWW_USER_FIRST_NAME=${_AIRFLOW_WWW_USER_FIRST_NAME} -_AIRFLOW_WWW_USER_LAST_NAME=${_AIRFLOW_WWW_USER_LAST_NAME} +_AIRFLOW_WWW_USER_FIRSTNAME=${_AIRFLOW_WWW_USER_FIRSTNAME} +_AIRFLOW_WWW_USER_LASTNAME=${_AIRFLOW_WWW_USER_LASTNAME} +_AIRFLOW_WWW_USER_ROLE=${_AIRFLOW_WWW_USER_ROLE} _PIP_ADDITIONAL_REQUIREMENTS=${_PIP_ADDITIONAL_REQUIREMENTS} AIRFLOW__API__AUTH_BACKENDS=${AIRFLOW__API__AUTH_BACKENDS:-"airflow.api.auth.backend.basic_auth"} AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION} diff --git a/containers/compose-base.yaml b/containers/compose-base.yaml index f1aac55..b04412d 100644 --- a/containers/compose-base.yaml +++ b/containers/compose-base.yaml @@ -17,10 +17,11 @@ x-airflow-common: _AIRFLOW_DB_UPGRADE: ${_AIRFLOW_DB_UPGRADE} _AIRFLOW_WWW_USER_CREATE: ${_AIRFLOW_WWW_USER_CREATE} _AIRFLOW_WWW_USER_EMAIL: ${_AIRFLOW_WWW_USER_EMAIL} - _AIRFLOW_WWW_USER_FIRST_NAME: ${_AIRFLOW_WWW_USER_FIRST_NAME} - _AIRFLOW_WWW_USER_LAST_NAME: ${_AIRFLOW_WWW_USER_LAST_NAME} + _AIRFLOW_WWW_USER_FIRSTNAME: ${_AIRFLOW_WWW_USER_FIRSTNAME} + _AIRFLOW_WWW_USER_LASTNAME: ${_AIRFLOW_WWW_USER_LASTNAME} _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD} _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME} + _AIRFLOW_WWW_USER_ROLE: ${_AIRFLOW_WWW_USER_ROLE} AIRFLOW_FILES_PATH_DIR_HOST: ${AIRFLOW_FILES_PATH_DIR_HOST} AIRFLOW_PORT: ${AIRFLOW_PORT} AIRFLOW__API__AUTH_BACKENDS: ${AIRFLOW__API__AUTH_BACKENDS} @@ -122,6 +123,28 @@ services: volumes: - redis:/data + flower: + platform: linux/amd64 + image: mher/flower:0.9.7 + restart: unless-stopped + env_file: + - ../.env + depends_on: + - redis + command: [ + "flower", + "--broker=redis://redis:6379/0", + "--port=8888", + "--basic_auth=admin:${FLOWER_PASSWORD}"] + ports: + - ${FLOWER_PORT}:8888 + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "http://localhost:8888/healthcheck"] + interval: 60s + timeout: 10s + retries: 5 + start_period: 40s # requires docker-compose 3.4 + minio: platform: linux/amd64 image: quay.io/minio/minio @@ -162,7 +185,7 @@ services: - ${AIRFLOW_PORT}:8080 depends_on: <<: *airflow-common-depends-on - startup: + airflow_startup: condition: service_completed_successfully airflow_scheduler: @@ -180,7 +203,7 @@ services: restart: always depends_on: <<: *airflow-common-depends-on - startup: + airflow_startup: condition: service_completed_successfully airflow_worker: @@ -205,7 +228,7 @@ services: restart: always depends_on: <<: *airflow-common-depends-on - startup: + airflow_startup: condition: service_completed_successfully airflow_triggerer: @@ -223,7 +246,7 @@ services: restart: always depends_on: <<: *airflow-common-depends-on - startup: + airflow_startup: condition: service_completed_successfully airflow_startup: @@ -236,7 +259,7 @@ services: - -c - | mkdir -p /sources/logs /sources/dags /sources/plugins - chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + chown -R "${HOST_UID}:0" /sources/{logs,dags,plugins} exec /entrypoint airflow version # yamllint enable rule:line-length environment: