From 53c6c2e28ae22163ab520998b210ae1be61f33cf Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 17:57:19 -0700 Subject: [PATCH 01/17] fix(docker-compose.sh): load the env file, fix shellcheck docker-compose, somewhat confusingly, has two overloaded meanings for "env file": 1. Expanded inside the container, by passing environment variables to the started process inside the container, which is services.SVC.env_file 2. Expanded inside docker-compose files, which is --env-file We were missing both of them in different places, and the latter is fixed by this commit. https://docs.docker.com/compose/environment-variables/set-environment-variables/ Fixes these warnings: WARN[0000] The "NOMOS_DB_USER" variable is not set. Defaulting to a blank string. WARN[0000] The "NOMOS_DB_PASSWORD" variable is not set. Defaulting to a blank string. WARN[0000] The "NOMOS_RABBITMQ_USER" variable is not set. Defaulting to a blank string. WARN[0000] The "NOMOS_RABBITMQ_PASSWORD" variable is not set. Defaulting to a blank string. WARN[0000] The "NOMOS_RABBITMQ_VHOST" variable is not set. Defaulting to a blank string. --- docker-compose.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker-compose.sh b/docker-compose.sh index 0322447a..3eca6580 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -1,5 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -export COMPOSE_FILE=$(cat docker-compose.conf | egrep -v '^(#|;|$)' | xargs | tr ' ' ':') +SCRIPT_DIR=$(realpath "$(dirname "$0")") -/usr/bin/env docker-compose $@ +export COMPOSE_FILE=$(< docker-compose.conf grep -E -v '^(#|;|$)' | xargs | tr ' ' ':') + +# --env-file is passed in here to make the variables inside available for +# expansion *inside rules definitions*. This is separate from the +# service.SVC.env_file directive, which defines the environment *inside* the +# container. +exec docker-compose --env-file "$SCRIPT_DIR"/docker/nomos.env "$@" From 8f572cba2d3bebbeeca8d9eeadea6549398f6543 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:06:41 -0700 Subject: [PATCH 02/17] feat(docker-compose): add self-contained dev configuration This seems to be the most useful configuration to use for development so we should ship it for ease of use. --- docker-compose.dev.conf | 150 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 docker-compose.dev.conf diff --git a/docker-compose.dev.conf b/docker-compose.dev.conf new file mode 100644 index 00000000..c705c025 --- /dev/null +++ b/docker-compose.dev.conf @@ -0,0 +1,150 @@ +################################################################################################### +## ## +## Self-contained development configuration using docker-compose. ## +## This configuration provides an internal mysql and RabbitMQ server. ## +## ## +################################################################################################### + +## Always use the core +docker-compose/core.yml + +## +## Expose port +## +# docker-compose/core.ports.yml + +## +## Lift environments from nomos.env file +## + +docker-compose/files-local-nomos-env.yml + +## +## Build core +## + +docker-compose/build-backend.yml +docker-compose/build-frontend.yml + +## +## Enable bridge network_mode for core services +## +#docker-compose/core.network-bridge.yml + +## +## Enable proxy network for core services +## +docker-compose/core.network-proxy.yml + +## +## MySQL +## + +## +## Local MySQL instance +## + +docker-compose/mysql-local.yml +#docker-compose/mysql-local-network-bridge.yml +docker-compose/mysql-local-network-mysql.yml + +## +## External MySQL instance +## + +# docker-compose/mysql-external-mysqld.yml + +## +## External MySQL network (mysql) +## + +# docker-compose/mysql-external-network-mysql.yml + +## +## Inject the MySQL container through EXTERNAL_MYSQL_HOST +## + +# docker-compose/mysql-external-variable.yml + +## +## Webhooker +## +## Webhooker depends on RabbitMQ, so always enable a RabbitMQ +## + +docker-compose/webhooker.yml + +## +## Webhooker logs +## + +docker-compose/webhooker-logs-local.yml + +## +## Build webhooker +## + +docker-compose/webhooker-build.yml + +## +## Enable bridge network_mode for webhooker +## + +# docker-compose/webhooker-network-bridge.yml + +## +## Enable proxy network for webhooker +## + +docker-compose/webhooker-network-proxy.yml + +## +## Enable rabbitmq network for webhooker +## + +# docker-compose/webhooker-network-rabbitmq.yml + +## +## Local RabbitMQ instance +## + +docker-compose/rabbitmq-local.yml +docker-compose/rabbitmq-local-management.yml +# docker-compose/rabbitmq-local-network-bridge.yml +docker-compose/rabbitmq-local-network-rabbitmq.yml + +## +## External RabbitMQ instance +## + +# docker-compose/rabbitmq-external.yml + +## +## Vhosts +## +## TODO: Example Traefik config +## + +## +## Enable membership.vanhack.ca for nginx-proxy +## + +# docker-compose/vhost-membership-vanhack-ca.yml + +## +## Enable membership.test.vanhack.ca for nginx-proxy +## + +# docker-compose/vhost-membership-test-vanhack-ca.yml + +## +## Override backend files from local filesystem +## + +docker-compose/files-local-backend.yml + +## +## Override frontend files from local filesystem +## + +docker-compose/files-local-frontend.yml From 0de2ad2c313e1a01801ff085346e1555c7b53010 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:07:43 -0700 Subject: [PATCH 03/17] fix(docker-compose): load the environment into the webhooker service Previously, the webhooker service was always pulling defaults and was not actually using any of its variables in the environment file. --- docker-compose/webhooker-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose/webhooker-build.yml b/docker-compose/webhooker-build.yml index 4685dbd6..ba9741a1 100644 --- a/docker-compose/webhooker-build.yml +++ b/docker-compose/webhooker-build.yml @@ -5,3 +5,5 @@ services: build: context: .. dockerfile: docker-compose/Dockerfile.webhooker + env_file: + ../docker/nomos.env From d32c0282ffc5188dbfa2a944d472bfbb91338d50 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:10:41 -0700 Subject: [PATCH 04/17] docs: update repo URL in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 08f6a9c2..acbf5086 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This system in a way acts as the rule set for how things are governed, via membe ## Development See here for complete setup, API, and philosophy: -https://github.com/vhs/membership-manager-pro/wiki +https://github.com/vhs/nomos/wiki First steps should be installing your test environment: -https://github.com/vhs/membership-manager-pro/wiki/Contributing \ No newline at end of file +https://github.com/vhs/nomos/wiki/Contributing From 2c1cc26c2dd7d507df2aa2e5dadcae0d81b1f9da Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:10:57 -0700 Subject: [PATCH 05/17] fix(docker-compose): make the mysql service actually work I found several bugs: 1. The service was taking 30 seconds (!!!?!) to start on my machine (Arch Linux x86_64) and taking 16 GB of memory (...). This seems to be some kind of strange bug in the container, which is worked around by setting some ulimit values. 2. MYSQL_PASS is the wrong variable for the password; the correct one is MYSQL_PASSWORD. 3. A root password must be set for the container to start. 4. We forgot to set the database to create, which is important to giving the app access to it. --- docker-compose/mysql-local.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docker-compose/mysql-local.yml b/docker-compose/mysql-local.yml index 17c37fdc..429b7afe 100644 --- a/docker-compose/mysql-local.yml +++ b/docker-compose/mysql-local.yml @@ -1,11 +1,24 @@ version: "3.5" services: + # backend needs the db started first + nomos-backend: + depends_on: + nomos-mysql: + condition: service_started nomos-mysql: image: mysql:5.7 container_name: nomos-mysql environment: - MYSQL_USER=${NOMOS_DB_USER} - - MYSQL_PASS=${NOMOS_DB_PASSWORD} + - MYSQL_PASSWORD=${NOMOS_DB_PASSWORD} + - MYSQL_DATABASE=${NOMOS_DB_DATABASE} + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} volumes: - ../data/mysql:/var/lib/mysql + # Workaround this bug in the mysql image: https://github.com/docker-library/mysql/issues/579 + ulimits: + nproc: 65535 + nofile: + soft: 20000 + hard: 40000 From ef2ed4eb1fe14a9e38b741677182b906c810cda3 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:14:21 -0700 Subject: [PATCH 06/17] chore(webhooker): npm added name to the lockfile --- webhooker/package-lock.json | 1 + 1 file changed, 1 insertion(+) diff --git a/webhooker/package-lock.json b/webhooker/package-lock.json index 495457d0..f7cf5c13 100644 --- a/webhooker/package-lock.json +++ b/webhooker/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "webhooker", "version": "1.0.0", "license": "MIT", "dependencies": { From 93abaefb22b2a958fce71d74e2f3cc01ba2f05d2 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:14:30 -0700 Subject: [PATCH 07/17] feat(setup): add a script to create an API key for the webhook service Previously this was done in the tools/vagrant_provision.sh script, but I want to deprecate Vagrant, so we need a second way. --- tools/make-webhook-key.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 tools/make-webhook-key.sh diff --git a/tools/make-webhook-key.sh b/tools/make-webhook-key.sh new file mode 100755 index 00000000..3c18dbf8 --- /dev/null +++ b/tools/make-webhook-key.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +container_name=nomos-frontend +# evil villainy to get the IP from the host +frontend_addr="$(docker inspect "$container_name" | jq -r '.[0].NetworkSettings.Networks | to_entries | .[0].value.IPAddress')" +frontend_base_uri="http://vhs:password@${frontend_addr}/services/web" + +key_resp=$(curl -s "${frontend_base_uri}/ApiKeyService1.svc/GenerateSystemApiKey?notes=webhooker") +key_id=$(echo "$key_resp" | jq -r .id) +key_value=$(echo "$key_resp" | jq -r .key) + +echo "API key ID: $key_id; key: $key_value" >&2 + +function is_webhook_privilege_present() { + local found_webhook + found_webhook=$(curl -s "${frontend_base_uri}/PrivilegeService1.svc/GetAllPrivileges" | jq '.[] | select(.code == "webhook")') + + if [[ -n ${found_webhook} ]]; then + return 0 + else + return 1 + fi +} + +if ! is_webhook_privilege_present; then + echo 'Creating webhook privilege' >&2 + curl -s "${frontend_base_uri}/PrivilegeService1.svc/CreatePrivilege?name=webhook&code=webhook&description=webhook&icon=webhook&enabled=true" \ + >/dev/null || echo "Failed to create privilege" >&2 +fi + +curl -s "${frontend_base_uri}/ApiKeyService1.svc/PutApiKeyPrivileges?keyid=${key_id}&privileges=webhook" \ + >/dev/null || echo "Failed to add API key privilege" >&2 + +echo -e "Add the following to docker/nomos.env:\n\nNOMOS_RABBITMQ_NOMOS_TOKEN=${key_value}" >&2 From 34346de818ec63a81db26bd12994724f3c785230 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:22:20 -0700 Subject: [PATCH 08/17] chore: populate the env template with plausible dev values We were also missing some environment variables related to Stripe which the backend was angry about, so I fixed those too. --- docker/nomos.env.template | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docker/nomos.env.template b/docker/nomos.env.template index 4558e476..b71993ed 100644 --- a/docker/nomos.env.template +++ b/docker/nomos.env.template @@ -1,7 +1,9 @@ -NOMOS_DB_SERVER= -NOMOS_DB_USER= -NOMOS_DB_PASSWORD= -NOMOS_DB_DATABASE= +NOMOS_DB_SERVER=nomos-mysql +NOMOS_DB_USER=nomos +NOMOS_DB_PASSWORD=Password1 +NOMOS_DB_DATABASE=nomos +MYSQL_ROOT_PASSWORD=Password1 + NOMOS_AWS_SES_CLIENT_ID= NOMOS_AWS_SES_SECRET= NOMOS_OAUTH_GITHUB_CLIENT= @@ -11,10 +13,15 @@ NOMOS_OAUTH_SLACK_SECRET= NOMOS_OAUTH_SLACK_TEAM= NOMOS_OAUTH_GOOGLE_CLIENT= NOMOS_OAUTH_GOOGLE_SECRET= -NOMOS_RABBITMQ_HOST= -NOMOS_RABBITMQ_PORT= -NOMOS_RABBITMQ_USER= -NOMOS_RABBITMQ_PASSWORD= -NOMOS_RABBITMQ_VHOST= + +NOMOS_STRIPE_API_KEY= +NOMOS_STRIPE_WEBHOOK_SECRET= +NOMOS_STRIPE_PRODUCTS= + +NOMOS_RABBITMQ_HOST=nomos-rabbitmq +NOMOS_RABBITMQ_PORT=5672 +NOMOS_RABBITMQ_USER=nomos +NOMOS_RABBITMQ_PASSWORD=Password1 +NOMOS_RABBITMQ_VHOST=nomos NOMOS_RABBITMQ_NOMOS_HOST= -NOMOS_RABBITMQ_NOMOS_TOKEN= \ No newline at end of file +NOMOS_RABBITMQ_NOMOS_TOKEN=please-run-tools-slash-make-webhook-key-dot-sh From 13e8460647ffa297279e4b09e2e57f68c3491e4a Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 18:38:39 -0700 Subject: [PATCH 09/17] docs: move docker-compose.md where it will be seen, write up new devenv --- README.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++- docker-compose.md | 7 ------ 2 files changed, 57 insertions(+), 8 deletions(-) delete mode 100644 docker-compose.md diff --git a/README.md b/README.md index acbf5086..0986b697 100644 --- a/README.md +++ b/README.md @@ -10,5 +10,61 @@ This system in a way acts as the rule set for how things are governed, via membe See here for complete setup, API, and philosophy: https://github.com/vhs/nomos/wiki -First steps should be installing your test environment: +For the old development guide, see: https://github.com/vhs/nomos/wiki/Contributing + +### docker-compose + +#### Site configuration files + +Two configuration files are used, which will vary based on the installation: +`docker/nomos.env`, and `docker-compose.conf`. These should be set up before +running the `docker-compose` setup. + +There is a development configuration at `docker-compose.dev.conf`, which can be +used for development. + +#### Usage + +- Copy `docker-compose.template.conf`, `docker-compose.sample.conf`, or `docker-compose.dev.conf` to `docker-compose.conf` +- Edit/uncomment the relevant lines in `docker-compose.conf` to enable - or even add - specific functionality +- Run `./docker-compose.sh` as a 1:1 wrapper for docker-compose, or generate a + local `docker-compose.yml` file for direct usage with `docker-compose` with + `./docker-compose.sh config > docker-compose.yml` + +#### Development setup guide + +On Linux, first install `docker` and `docker-compose` from your distribution +package manager. On Mac/Windows, you probably want [Docker Desktop], which is a +fancy app that makes and manages a Linux virtual machine that it runs Docker in. + +[Docker Desktop]: https://docs.docker.com/get-docker/ + +Copy `docker/nomos.env.template` to `docker/nomos.env`. + +Copy `docker-compose.dev.conf` to `docker-compose.conf`. + +Grant write permission to all users on the log directory: `chmod a+w logs`. The +reason this is needed is because the back-end PHP code runs as a non-root user +inside the container, and by default permissions don't grant write access to +non-owners of directories. + +Start the service with `./docker-compose.sh up`. This should bring everything up, +but the webhook service will still be failing, which is expected. + +To get the webhook service working, run `tools/make-webhook-key.sh`, which will +provide the correct value of `NOMOS_RABBITMQ_NOMOS_TOKEN`. Then, edit that into +`docker/nomos.env`. + +Once you have done this, press Ctrl-C in the terminal with `./docker-compose.sh up`, +then run `./docker-compose.sh up` again. + +You're all set! You can get the address to access the Nomos service locally by +running the following in a separate terminal as `docker-compose.sh`: + +``` +$ docker inspect nomos-frontend | jq -r '.[0].NetworkSettings.Networks | to_entries | .[0].value.IPAddress' +``` + +The username is `vhs` and the password is `password`. + diff --git a/docker-compose.md b/docker-compose.md deleted file mode 100644 index 363d6834..00000000 --- a/docker-compose.md +++ /dev/null @@ -1,7 +0,0 @@ -# docker-compose - -## Usage - -- Copy `docker-compose.template.conf` or `docker-compose.sample.conf` to `docker-compose.conf` -- Edit/uncomment the relevant lines in `docker-compose.conf` to enable - or even add - specific functionality -- Run `./docker-compose.sh` as a 1:1 wrapper for docker-compose, or generate a local `docker-compose.yml` file for direct usage with `docker-compose` From 37d99d6d685aee4ed657249a51bc44b96956138b Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 28 Apr 2023 17:54:05 -0700 Subject: [PATCH 10/17] feat(docker-compose): use a separate config file for internal nets This was causing errors such as the following in docker-compose: ``` network rabbitmq declared as external, but could not be found ``` This is because `external` means that the network should be managed outside the lifecycle of the application and must be created first. This seems like the incorrect move in this case. --- docker-compose.dev.conf | 16 +++++++++++++--- docker-compose.template.conf | 10 ++++++++++ docker-compose/core.network-proxy-internal.yml | 17 +++++++++++++++++ .../rabbitmq-local-network-internal.yml | 17 +++++++++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 docker-compose/core.network-proxy-internal.yml create mode 100644 docker-compose/rabbitmq-local-network-internal.yml diff --git a/docker-compose.dev.conf b/docker-compose.dev.conf index c705c025..9e34f749 100644 --- a/docker-compose.dev.conf +++ b/docker-compose.dev.conf @@ -34,7 +34,12 @@ docker-compose/build-frontend.yml ## ## Enable proxy network for core services ## -docker-compose/core.network-proxy.yml +#docker-compose/core.network-proxy.yml + +## +## Enable proxy network for core services, managed by docker-compose +## +docker-compose/core.network-proxy-internal.yml ## ## MySQL @@ -96,7 +101,7 @@ docker-compose/webhooker-build.yml ## Enable proxy network for webhooker ## -docker-compose/webhooker-network-proxy.yml +# docker-compose/webhooker-network-proxy.yml ## ## Enable rabbitmq network for webhooker @@ -111,7 +116,12 @@ docker-compose/webhooker-network-proxy.yml docker-compose/rabbitmq-local.yml docker-compose/rabbitmq-local-management.yml # docker-compose/rabbitmq-local-network-bridge.yml -docker-compose/rabbitmq-local-network-rabbitmq.yml + +# Network not managed with docker-compose +#docker-compose/rabbitmq-local-network-rabbitmq.yml + +# Network managed with docker-compose +docker-compose/rabbitmq-local-network-internal.yml ## ## External RabbitMQ instance diff --git a/docker-compose.template.conf b/docker-compose.template.conf index 1336446a..abc67977 100644 --- a/docker-compose.template.conf +++ b/docker-compose.template.conf @@ -29,6 +29,11 @@ docker-compose/core.yml ## #docker-compose/core.network-proxy.yml +## +## Enable proxy network for core services, managed by docker-compose +## +#docker-compose/core.network-proxy-internal.yml + ## ## MySQL ## @@ -104,8 +109,13 @@ docker-compose/core.yml #docker-compose/rabbitmq-local.yml #docker-compose/rabbitmq-local-management.yml # docker-compose/rabbitmq-local-network-bridge.yml + +# Network not managed with docker-compose #docker-compose/rabbitmq-local-network-rabbitmq.yml +# Network managed with docker-compose +# docker-compose/rabbitmq-local-network-internal.yml + ## ## External RabbitMQ instance ## diff --git a/docker-compose/core.network-proxy-internal.yml b/docker-compose/core.network-proxy-internal.yml new file mode 100644 index 00000000..57ad9d96 --- /dev/null +++ b/docker-compose/core.network-proxy-internal.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + nomos-frontend: + networks: + - proxy + + nomos-backend: + networks: + - proxy + + nomos-webhooker: + networks: + - proxy + +networks: + proxy: diff --git a/docker-compose/rabbitmq-local-network-internal.yml b/docker-compose/rabbitmq-local-network-internal.yml new file mode 100644 index 00000000..230ad3ee --- /dev/null +++ b/docker-compose/rabbitmq-local-network-internal.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + nomos-rabbitmq: + networks: + - rabbitmq + + nomos-webhooker: + networks: + - rabbitmq + + nomos-backend: + networks: + - rabbitmq + +networks: + rabbitmq: From 980738a7bd59af7665e43be1e0d0593032252a5e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 4 May 2023 21:28:14 -0700 Subject: [PATCH 11/17] feat(otel): implement root span --- app/app.php | 2 + composer.json | 18 ++++- .../PsrToVhsLoggerAdapterTest.php | 48 +++++++++++++ vhs/observability/OpenTelemetry.php | 70 +++++++++++++++++++ vhs/observability/PsrToVhsLoggerAdapter.php | 51 ++++++++++++++ vhs/web/HttpServer.php | 47 +++++++++++++ 6 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 tests/observability/PsrToVhsLoggerAdapterTest.php create mode 100644 vhs/observability/OpenTelemetry.php create mode 100644 vhs/observability/PsrToVhsLoggerAdapter.php diff --git a/app/app.php b/app/app.php index be720a95..4e62822d 100644 --- a/app/app.php +++ b/app/app.php @@ -18,6 +18,8 @@ $serverLog = new \vhs\loggers\FileLogger(dirname(__FILE__) . "/../logs/server.log"); +\vhs\observability\OpenTelemetry::Init($serverLog); + \vhs\web\HttpContext::Init(new \vhs\web\HttpServer(new \vhs\web\modules\HttpServerInfoModule("Nomos"), $serverLog)); \vhs\web\HttpContext::Server()->register(new \app\security\HttpApiAuthModule(\app\security\Authenticate::getInstance())); diff --git a/composer.json b/composer.json index b833a839..ae6189cc 100644 --- a/composer.json +++ b/composer.json @@ -3,9 +3,25 @@ "aws/aws-sdk-php": "2.*", "nicmart/string-template": "~0.1", "league/oauth2-client": "0.10.*", - "php-amqplib/php-amqplib": "2.5.*" + "php-amqplib/php-amqplib": "2.5.*", + "open-telemetry/api": "^0.0.17", + "open-telemetry/sdk": "^0.0.17", + "symfony/http-client": "^5.4", + "nyholm/psr7": "^1.8", + "guzzlehttp/promises": "^1.5", + "php-http/message-factory": "^1.1", + "open-telemetry/exporter-otlp": "^0.0.17", + "psr/http-client": "^1.0" }, "require-dev": { "phpunit/phpunit": "^6" + }, + "config": { + "platform": { + "php": "7.4.15" + }, + "allow-plugins": { + "php-http/discovery": true + } } } diff --git a/tests/observability/PsrToVhsLoggerAdapterTest.php b/tests/observability/PsrToVhsLoggerAdapterTest.php new file mode 100644 index 00000000..c5861246 --- /dev/null +++ b/tests/observability/PsrToVhsLoggerAdapterTest.php @@ -0,0 +1,48 @@ +logger = new StringLogger(); + } + + public function test_NoContext(): void + { + $adapter = new PsrToVhsLoggerAdapter($this->logger); + + $adapter->debug('nya nya'); + + $this->assertEquals(['nya nya'], $this->logger->history); + } + + public function test_Levels(): void + { + $adapter = new PsrToVhsLoggerAdapter($this->logger, LogLevel::INFO); + + $adapter->debug('nya nya'); + + $this->assertEquals([], $this->logger->history); + } + + public function test_ContextPrinting(): void + { + $adapter = new PsrToVhsLoggerAdapter($this->logger); + + $adapter->debug('nya nya', ['hmm', 'var' => 'yeah']); + + $this->assertEquals(["nya nya\nContext:\n0 = hmm\nvar = yeah"], $this->logger->history); + } +} diff --git a/vhs/observability/OpenTelemetry.php b/vhs/observability/OpenTelemetry.php new file mode 100644 index 00000000..4afbac95 --- /dev/null +++ b/vhs/observability/OpenTelemetry.php @@ -0,0 +1,70 @@ +create(); + $sampler = (new SamplerFactory())->create(); + $spanProcessor = (new SpanProcessorFactory())->create($exporter); + + // A "resource" is an attribute that goes on the root span, + // representing the environment that the service instance is running in. + $resource = ResourceInfoFactory::merge( + ResourceInfo::create(Attributes::create([ResourceAttributes::SERVICE_NAME => 'nomos'])), + ResourceInfoFactory::defaultResource() + ); + + $tracerProvider = TracerProvider::builder() + ->setSampler($sampler) + ->addSpanProcessor($spanProcessor) + ->setResource($resource) + ->build(); + + Sdk::builder() + ->setTracerProvider($tracerProvider) + ->setPropagator(TraceContextPropagator::getInstance()) + ->setAutoShutdown(true) + ->buildAndRegisterGlobal(); + } +} diff --git a/vhs/observability/PsrToVhsLoggerAdapter.php b/vhs/observability/PsrToVhsLoggerAdapter.php new file mode 100644 index 00000000..fbf90ee7 --- /dev/null +++ b/vhs/observability/PsrToVhsLoggerAdapter.php @@ -0,0 +1,51 @@ +level = $this->levelToInt($level); + $this->vhsLogger = $vhsLogger; + } + + private function levelToInt(string $level): int + { + $levels = [ + LogLevel::DEBUG => 0, + LogLevel::INFO => 1, + LogLevel::NOTICE => 2, + LogLevel::WARNING => 3, + LogLevel::ERROR => 4, + LogLevel::CRITICAL => 5, + LogLevel::ALERT => 6, + LogLevel::EMERGENCY => 7 + ]; + if (!isset($levels[$level])) { + throw new InvalidArgumentException('Unknown log level'); + } + return $levels[$level]; + } + + public function log($level, $message, array $context = []): void + { + if ($level >= $this->level) { + $listOfKvs = array_map(function ($v, $k) { + return "$k = $v"; + }, array_values($context), array_keys($context)); + $contextS = sizeof($context) === 0 ? '' : "\nContext:\n" . implode("\n", $listOfKvs); + $this->vhsLogger->log($message . $contextS); + } + } +} diff --git a/vhs/web/HttpServer.php b/vhs/web/HttpServer.php index 34c70cda..a52e95e3 100644 --- a/vhs/web/HttpServer.php +++ b/vhs/web/HttpServer.php @@ -9,6 +9,12 @@ namespace vhs\web; +use OpenTelemetry\API\Common\Instrumentation\CachedInstrumentation; +use OpenTelemetry\API\Trace\SpanInterface; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\Context\ScopeInterface; +use OpenTelemetry\SemConv\TraceAttributes; use vhs\Logger; use vhs\loggers\SilentLogger; use vhs\web\modules\HttpServerInfoModule; @@ -23,6 +29,9 @@ class HttpServer { private $http_response_code; private $endset = false; + private ?SpanInterface $root_span = null; + private ?ScopeInterface $root_scope = null; + /** @var HttpRequest */ public $request; @@ -51,6 +60,37 @@ public function register(IHttpModule $module) { array_push($this->modules, $module); } + /** + * Starts the root span for a request, which will appear as one individual + * trace in OpenTelemetry. + */ + private function beginRootSpan(): void { + // NOTE: OpenTelemetry root spans cannot be implemented as a module + // since it needs to create the root span *before* starting other + // modules, and end it *after* all the other modules complete. + $tracer = (new CachedInstrumentation('vhs\\web\\HttpServer'))->tracer(); + $this->root_span = $tracer + ->spanBuilder($this->request->url) + ->setSpanKind(SpanKind::KIND_SERVER) + ->setAttribute(TraceAttributes::HTTP_METHOD, $this->request->method) + ->startSpan(); + $this->root_scope = $this->root_span->activate(); + } + + /** + * Ends the root span for a request. Expected to be called unconditionally + * at the end of a request's lifecycle (if not, the trace will most likely + * be shown as missing a root span). + */ + private function endRootSpan(): void { + if ($this->root_span) { + $this->root_span->end(); + } + if ($this->root_scope) { + $this->root_scope->detach(); + } + } + public function handle() { $this->handling = true; $this->clear(); @@ -59,6 +99,8 @@ public function handle() { session_start(); + $this->beginRootSpan(); + $exception = null; /** @var IHttpModule $module */ $index = 0; @@ -69,6 +111,8 @@ public function handle() { $module->handle($this); } catch(\Exception $ex) { $exception = $ex; + $this->root_span->recordException($ex); + $this->root_span->setStatus(StatusCode::STATUS_ERROR); break; } $index += 1; @@ -109,6 +153,8 @@ private function endResponse() { $module->endResponse($this); } catch(\Exception $ex) { $exception = $ex; + $this->root_span->recordException($ex); + $this->root_span->setStatus(StatusCode::STATUS_ERROR); break; } $index += 1; @@ -121,6 +167,7 @@ private function endResponse() { } } + $this->endRootSpan(); exit(); } From 18c3075f341937f300df0f4edf74235d6ab42605 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 4 May 2023 21:57:13 -0700 Subject: [PATCH 12/17] feat(otel): instrument queries --- vhs/database/engines/mysql/MySqlEngine.php | 10 +----- .../engines/mysql/MySqlOtelWrapper.php | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 vhs/database/engines/mysql/MySqlOtelWrapper.php diff --git a/vhs/database/engines/mysql/MySqlEngine.php b/vhs/database/engines/mysql/MySqlEngine.php index 8f3bc500..15b5a7f2 100644 --- a/vhs/database/engines/mysql/MySqlEngine.php +++ b/vhs/database/engines/mysql/MySqlEngine.php @@ -9,22 +9,14 @@ namespace vhs\database\engines\mysql; use vhs\database\Column; -use vhs\database\Columns; use vhs\database\Engine; use vhs\database\exceptions\DatabaseConnectionException; use vhs\database\exceptions\DatabaseException; -use vhs\database\limits\Limit; -use vhs\database\offsets\Offset; -use vhs\database\orders\OrderBy; -use vhs\database\queries\Query; use vhs\database\queries\QueryDelete; use vhs\database\queries\QueryInsert; use vhs\database\queries\QuerySelect; use vhs\database\queries\QueryUpdate; use vhs\database\queries\QueryCount; -use vhs\database\Table; -use vhs\database\types\Type; -use vhs\database\wheres\Where; use vhs\Logger; use vhs\loggers\SilentLogger; @@ -60,7 +52,7 @@ public function setLogger(Logger $logger) { public function connect() { if(isset($this->conn) && !is_null($this->conn)) return true; - $this->conn = new \mysqli( + $this->conn = new MySqlOtelWrapper( $this->info->getServer(), $this->info->getUsername(), $this->info->getPassword() diff --git a/vhs/database/engines/mysql/MySqlOtelWrapper.php b/vhs/database/engines/mysql/MySqlOtelWrapper.php new file mode 100644 index 00000000..92db7095 --- /dev/null +++ b/vhs/database/engines/mysql/MySqlOtelWrapper.php @@ -0,0 +1,36 @@ +tracer(); + } + + /** + * @return \mysqli_result|bool + */ + public function query($query, $resultmode = null) + { + // FIXME(jade): replace this whole thing with the auto instrumentation + // extension when we upgrade to PHP 8 where that is available. Benefit: + // caller details and it's not as much of a dirty hack. + $span = self::getTracer() + ->spanBuilder($query) + ->setSpanKind(SpanKind::KIND_CLIENT) + ->startSpan(); + $scope = $span->activate(); + try { + return parent::query($query, $resultmode); + } finally { + $span->end(); + $scope->detach(); + } + } +} From 660dee5ebbe070e41c1679e7cb56b56859a39787 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 4 May 2023 23:39:27 -0700 Subject: [PATCH 13/17] fix(docker): generate env.php correctly in presence of = in env-vars Previously, it would try to do: define('NOMOS_TRACE_URL_FORMAT=https://blah/?a=b', '') This fixes that to now only eat up to the first = sign using the ${parameter/pattern/string} parameter expansion form in Bash. --- docker/docker_env_config.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/docker_env_config.sh b/docker/docker_env_config.sh index c1a25d66..880360aa 100755 --- a/docker/docker_env_config.sh +++ b/docker/docker_env_config.sh @@ -1,9 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash echo " Date: Thu, 4 May 2023 23:55:02 -0700 Subject: [PATCH 14/17] feat(otel): trace links in headers --- app/app.php | 1 + conf/config.ini.php.docker | 8 ++++ conf/config.ini.php.template | 8 ++++ vhs/web/modules/HttpTraceLinkModule.php | 64 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 vhs/web/modules/HttpTraceLinkModule.php diff --git a/app/app.php b/app/app.php index 4e62822d..aa73f06e 100644 --- a/app/app.php +++ b/app/app.php @@ -29,6 +29,7 @@ \vhs\web\HttpContext::Server()->register(\app\modules\HttpPaymentGatewayHandlerModule::getInstance()); \vhs\web\HttpContext::Server()->register(\app\security\oauth\modules\OAuthHandlerModule::getInstance()); \vhs\web\HttpContext::Server()->register(new \vhs\web\modules\HttpJsonServiceHandlerModule("web")); +\vhs\web\HttpContext::Server()->register(new \vhs\web\modules\HttpTraceLinkModule()); \app\modules\HttpPaymentGatewayHandlerModule::register(new \app\gateways\PaypalGateway()); diff --git a/conf/config.ini.php.docker b/conf/config.ini.php.docker index 6a54de2c..4421090f 100644 --- a/conf/config.ini.php.docker +++ b/conf/config.ini.php.docker @@ -39,6 +39,14 @@ require_once('env.php'); define('STRIPE_WEBHOOK_SECRET', NOMOS_STRIPE_WEBHOOK_SECRET); define('STRIPE_PRODUCTS', json_decode(NOMOS_STRIPE_PRODUCTS, true)); + // Format for URLs to link to traces (see HttpTraceLinkModule) + // For Honeycomb, see: https://docs.honeycomb.io/api/direct-trace-links/ + // https://ui.honeycomb.io//environments//datasets/nomos/trace?trace_id=%TRACE_ID%&trace_start_ts=%TRACE_START_TS% + // The following parameters will be replaced: + // %TRACE_ID% - trace ID in use + // %TRACE_START_TS% - timestamp of the start of the root span + define('TRACE_URL_FORMAT', defined('NOMOS_TRACE_URL_FORMAT') ? NOMOS_TRACE_URL_FORMAT : '(trace url format not configured)'); + /** * Show MySql Errors. * Not recomended for live site. true/false diff --git a/conf/config.ini.php.template b/conf/config.ini.php.template index 7465fec2..d1630784 100644 --- a/conf/config.ini.php.template +++ b/conf/config.ini.php.template @@ -35,6 +35,14 @@ define('RABBITMQ_PASSWORD', 'password'); define('RABBITMQ_VHOST', 'nomos'); + // Format for URLs to link to traces (see HttpTraceLinkModule) + // For Honeycomb, see: https://docs.honeycomb.io/api/direct-trace-links/ + // https://ui.honeycomb.io//environments//datasets/nomos/trace?trace_id=%TRACE_ID%&trace_start_ts=%TRACE_START_TS% + // The following parameters will be replaced: + // %TRACE_ID% - trace ID in use + // %TRACE_START_TS% - timestamp of the start of the root span + define('TRACE_URL_FORMAT', '(trace url format not configured)'); + /** * Show MySql Errors. * Not recomended for live site. true/false diff --git a/vhs/web/modules/HttpTraceLinkModule.php b/vhs/web/modules/HttpTraceLinkModule.php new file mode 100644 index 00000000..a9877ab1 --- /dev/null +++ b/vhs/web/modules/HttpTraceLinkModule.php @@ -0,0 +1,64 @@ +getContext(); + // If the trace is not sampled, it's *definitely* not going to be + // sent out, so don't provide a trace id. + if ($currContext->isSampled()) { + return $currContext->getTraceId(); + } else { + return null; + } + } + + private static function makeTraceLink(string $template, string $traceId, int $startTs): string + { + return str_replace(['%TRACE_ID%', '%TRACE_START_TS%'], [$traceId, (string)$startTs], $template); + } + + private static function generateTraceLink(): string + { + $traceId = self::getTraceId(); + if (is_null($traceId)) { + return '(not sampled)'; + } + + // It is Non-obvious(tm) how to get the concrete instance of the root + // Span to find the actual value, so just use the current time minus a + // minute. This retains the property of links containing approximately + // the time range to look around. + $startTs = (new DateTimeImmutable())->sub(DateInterval::createFromDateString('1 minute'))->getTimestamp(); + return self::makeTraceLink(TRACE_URL_FORMAT, $traceId, $startTs); + } + + public function handle(HttpServer $server): void + { + $link = self::generateTraceLink(); + $server->header("vhs-trace-link: $link"); + } + + public function endResponse(HttpServer $server): void + { + } + + public function handleException(HttpServer $server, Exception $ex): void + { + } + +} From cbd96cf7f084d7ac9cb258c638640ee7fc200e77 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 5 May 2023 16:12:28 -0700 Subject: [PATCH 15/17] feat(otel): implement otel collector container This fixes an issue where the service is blocking on a network round-trip off-machine to export all the traces to finish request processing. Noted upstream as a potential documentation/feature issue: https://github.com/open-telemetry/opentelemetry-php/issues/993 --- conf/otel-collector-config.yaml | 25 +++++++++++++ docker-compose.dev.conf | 7 ++++ docker-compose.template.conf | 7 ++++ docker-compose/opentelemetry-collector.yml | 42 ++++++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 conf/otel-collector-config.yaml create mode 100644 docker-compose/opentelemetry-collector.yml diff --git a/conf/otel-collector-config.yaml b/conf/otel-collector-config.yaml new file mode 100644 index 00000000..8f7897de --- /dev/null +++ b/conf/otel-collector-config.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + protocols: + grpc: # port 4317 + http: # port 4318 + +processors: + batch: + +exporters: + otlp: + endpoint: "${env:OTLP_UPSTREAM}" + headers: + "x-honeycomb-team": "${env:HONEYCOMB_API_KEY}" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp] diff --git a/docker-compose.dev.conf b/docker-compose.dev.conf index 9e34f749..7c2d45c2 100644 --- a/docker-compose.dev.conf +++ b/docker-compose.dev.conf @@ -129,6 +129,13 @@ docker-compose/rabbitmq-local-network-internal.yml # docker-compose/rabbitmq-external.yml +## +## OpenTelemetry collector process (realistically required for using +## OpenTelemetry) +## + +# docker-compose/opentelemetry-collector.yml + ## ## Vhosts ## diff --git a/docker-compose.template.conf b/docker-compose.template.conf index abc67977..d3a1599c 100644 --- a/docker-compose.template.conf +++ b/docker-compose.template.conf @@ -122,6 +122,13 @@ docker-compose/core.yml # docker-compose/rabbitmq-external.yml +## +## OpenTelemetry collector process (realistically required for using +## OpenTelemetry) +## + +# docker-compose/opentelemetry-collector.yml + ## ## Vhosts ## diff --git a/docker-compose/opentelemetry-collector.yml b/docker-compose/opentelemetry-collector.yml new file mode 100644 index 00000000..ba8d05d3 --- /dev/null +++ b/docker-compose/opentelemetry-collector.yml @@ -0,0 +1,42 @@ +version: "3.5" + +services: + # This is the OpenTelemetry Collector: https://opentelemetry.io/docs/collector/ + # + # In this case, we use it for batching and queueing traces for multiple + # requests from PHP, since OpenTelemetry for PHP is forced to send out all + # traces for a request immediately at the end of a request, and it blocks the + # completion of the request processing (due to how PHP does init/teardown on + # every request). + # + # It is thus basically mandatory unless the telemetry receiver process is on + # the same box and has no rate limits. + nomos-otel-collector: + image: otel/opentelemetry-collector:0.76.1 + command: [--config=/etc/otel-collector-config.yaml] + volumes: + - ../conf/otel-collector-config.yaml:/etc/otel-collector-config.yaml + container_name: nomos-otel-collector + restart: always + + environment: + # just set this to garbage/empty if you aren't using Honeycomb; it will be + # ignored + - "HONEYCOMB_API_KEY=${OT_COLLECTOR_HONEYCOMB_API_KEY}" + - "OTLP_UPSTREAM=${OT_COLLECTOR_OTLP_UPSTREAM}" + networks: + - otel-collector + nomos-backend: + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://nomos-otel-collector:4318 + - "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" + - OTEL_TRACES_EXPORTER=otlp + - OTEL_PHP_TRACES_PROCESSOR=batch + - OTEL_TRACES_SAMPLER=parentbased_always_on + depends_on: + - nomos-otel-collector + networks: + - otel-collector + +networks: + otel-collector: From c3d8001997d59fc9b9c61475b2202d1272ef4e5d Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 5 May 2023 16:34:44 -0700 Subject: [PATCH 16/17] fix(otel): do not wait for exit handlers to send responses --- app/app.php | 19 ++++++++++++++++++- vhs/RequestFinished.php | 10 ++++++++++ vhs/web/HttpServer.php | 9 +++++---- 3 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 vhs/RequestFinished.php diff --git a/app/app.php b/app/app.php index aa73f06e..65ffbae8 100644 --- a/app/app.php +++ b/app/app.php @@ -43,4 +43,21 @@ \app\security\oauth\modules\OAuthHandlerModule::register(new \app\security\oauth\modules\GoogleOAuthHandler()); \app\security\oauth\modules\OAuthHandlerModule::register(new \app\security\oauth\modules\SlackOAuthHandler()); -\vhs\web\HttpContext::Server()->handle(); +try { + \vhs\web\HttpContext::Server()->handle(); +} catch (\vhs\RequestFinished $e) { +} finally { + /* + * This is here to fix a problem where exit-handlers need to finish dealing + * with a request e.g. by sending traces to an OpenTelemetry service, and + * that may take between a few ms and a while, which we should never make + * the client wait for. + * + * https://stackoverflow.com/questions/15273570/continue-processing-php-after-sending-http-response + */ + session_write_close(); + fastcgi_finish_request(); + // Ensure that the root span of the server is always closed cleanly. + \vhs\web\HttpContext::Server()->endRootSpan(); + exit(); +} diff --git a/vhs/RequestFinished.php b/vhs/RequestFinished.php new file mode 100644 index 00000000..2fc4bff3 --- /dev/null +++ b/vhs/RequestFinished.php @@ -0,0 +1,10 @@ +root_span) { $this->root_span->end(); } @@ -167,8 +169,7 @@ private function endResponse() { } } - $this->endRootSpan(); - exit(); + throw new \vhs\RequestFinished(); } public function clear() { @@ -217,7 +218,7 @@ public function sendOnlyHeaders() { $self = $this; array_push($this->headerBuffer, function() use ($self) { - exit(); + throw new \vhs\RequestFinished(); }); } From 32a3827e5424d708799f59d921fd812219e5790d Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 5 May 2023 17:11:25 -0700 Subject: [PATCH 17/17] doc(otel): write README and example env vars --- README.md | 40 +++++++++++++++++++++++++++++++++++++++ docker/nomos.env.template | 5 +++++ 2 files changed, 45 insertions(+) diff --git a/README.md b/README.md index 0986b697..b4af96a1 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,43 @@ $ docker inspect nomos-frontend | jq -r '.[0].NetworkSettings.Networks | to_entr The username is `vhs` and the password is `password`. +### OpenTelemetry + +Nomos optionally supports [OpenTelemetry](https://opentelemetry.io/docs/), a +system for collecting traces, logs, and metrics (we have not yet implemented +metrics and logs) from production and dev systems to greatly ease debugging and +performance work. It supports distributed tracing (we have not implemented this +yet), allowing tracing operations through their whole lifecycle through multiple +systems. + +One feature we've implemented is the `vhs-trace-link` HTTP header, which allows +you to look at dev tools in your browser, then click the link to open the trace +for any problematic request and immediately see all the database queries +performed by that request. + +To set it up, you need to choose some kind of aggregation service: +- [Jaeger](https://jaegertracing.io/) is an open source trace aggregation + service that you can self host. +- [Honeycomb](https://honeycomb.io/) is a proprietary cloud tracing service + which does metrics, traces, and logs, as well as dashboards and alerting. + They have a generous free tier. + +Once you've chosen and set up one of these services, enable +[opentelemetry-collector.yml](./docker-compose/opentelemetry-collector.yml) in +`docker-compose.conf`. Next, configure `docker/nomos.env`: set +`OT_COLLECTOR_OTLP_UPSTREAM` to point to your aggregation service's ingest +endpoint and optionally set `OT_COLLECTOR_HONEYCOMB_API_KEY` if using Honeycomb +(make it blank otherwise). + +Then, optionally set up the `vhs-trace-link` system by setting +`NOMOS_TRACE_URL_FORMAT` to the appropriate value for your trace aggregator. +There is an example given for Honeycomb. + +You should start seeing traces in your trace aggregator of choice when you +start the service and make some backend requests. + +#### Caveats + +Currently the database traces show full statements because the Nomos database +library escapes strings instead of using placeholders. This may expose PII if +used in production. Fixing this is planned. diff --git a/docker/nomos.env.template b/docker/nomos.env.template index b71993ed..7cc86e84 100644 --- a/docker/nomos.env.template +++ b/docker/nomos.env.template @@ -4,6 +4,11 @@ NOMOS_DB_PASSWORD=Password1 NOMOS_DB_DATABASE=nomos MYSQL_ROOT_PASSWORD=Password1 +# Example for Honeycomb +# OT_COLLECTOR_OTLP_UPSTREAM=api.honeycomb.io:443 +# OT_COLLECTOR_HONEYCOMB_API_KEY=YOUR_API_KEY +# NOMOS_TRACE_URL_FORMAT=https://ui.honeycomb.io/YOUR_ACCOUNT/environments/YOUR_ENV/datasets/nomos/trace?trace_id=%TRACE_ID%&trace_start_ts=%TRACE_START_TS% + NOMOS_AWS_SES_CLIENT_ID= NOMOS_AWS_SES_SECRET= NOMOS_OAUTH_GITHUB_CLIENT=