diff --git a/Dockerfile b/Dockerfile index b01e051..5510abb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM cockroachdb/cockroach:v21.1.2 +FROM cockroachdb/cockroach:v21.2.4 + +RUN microdnf install bind-utils -ADD init_cluster.sh /cockroach/ ADD start_fly.sh /cockroach/ \ No newline at end of file diff --git a/README.md b/README.md index 10781bc..89398e2 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,73 @@ # CockroachDB on Fly.io -> **Note:** this app requires version 0.0.223 or greater of the Fly CLI. Run `flyctl version update` before you follow these instructions. +> **Note:** this app requires version 0.0.261 or greater of the Fly CLI. Run `flyctl version update` before you follow these instructions. This is an example CockroachDB cluster that runs on multiple Fly.io regions. Deploying this app is relatively simple: 1. Clone this repository -2. [Install the `fly` CLI](https://fly.io/docs/hands-on/installing/) -2. Run `fly launch`. +2. [Install the `cockroach` CLI](https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html) +3. [Install the `fly` CLI](https://fly.io/docs/hands-on/installing/) +4. Run `fly launch`. 1. Pick "yes" to copy existing configuration 2. When it asks if you want to deploy, say no -3. Create volumes: +5. Generate the Certificate Authority certificate and keypair (can reuse existing CA but don't share CA between production and non-production environments) ```bash - # one in Chicago - fly volumes create cdb_data --region ord --size 10 - # one in Paris - fly volumes create cdb_data --region cdg --size 10 - # one in Singapore - fly volumes create cdb_data --region sin --size 10 + > cockroach cert create-ca --certs-dir= --ca-key= ``` -4. Deploy an empty node +6. Generate the Node certificate and keypair + + _When generating the certificate, you can add external domains as well, e.g. `db.example.com`_ + + ```bash + > cockroach cert create-node --certs-dir= --ca-key= 127.0.0.1 localhost .internal "*..internal" "*.vm..internal" "*.nearest.of..internal" .fly.dev + ``` +7. Generate the root user certificate and keypair + ```bash + > cockroach cert create-client --certs-dir= --ca-key= root + ``` +8. Upload the certificates and keypair + ```bash + > base64 | fly secrets set DB_CA_CRT=- + > base64 | fly secrets set DB_NODE_CRT=- + > base64 | fly secrets set DB_NODE_KEY=- + ``` +9. Create volumes: + ```bash + # for single region, minimum 3 nodes required + > fly volumes create crdb_data --region --size 100 + > fly volumes create crdb_data --region --size 100 + > fly volumes create crdb_data --region --size 100 + + # for multi-region, minimum 3 regions required + > fly volumes create crdb_data --region --size 100 + > fly volumes create crdb_data --region --size 100 + > fly volumes create crdb_data --region --size 100 + ``` +10. Set VM size and scale to desired node count + ```bash + > fly scale vm --memory + Scaled VM Type to + + CPU Cores: + Memory: GB + > fly scale count + Count changed to + ``` +11. Deploy nodes ```bash > fly deploy ``` -5. Init the cluster: +12. Init the cluster: ```bash - > fly ssh console -C '/cockroach/init_cluster.sh' - Connecting to cockroachdb-example.internal... complete + > cockroach init --cluster-name= --host=.fly.dev --certs-dir= Cluster successfully initialized ``` -6. Scale to 3 nodes +13. View CockroachDB status ```bash - > fly scale count 3 - Count changed to 3 + > cockroach node status --host=.fly.dev --certs-dir= ``` -7. View CockroachDB status - ```bash - fly ssh console -C '/cockroach/cockroach node status --insecure' - ``` ## Hook up Grafana diff --git a/fly.toml b/fly.toml index 8b32398..7a81c38 100644 --- a/fly.toml +++ b/fly.toml @@ -1,15 +1,48 @@ app = "cockroachdb-example" kill_signal = "SIGINT" -kill_timeout = 60 +kill_timeout = 120 -[metrics] - port = 8080 - path = "/_status/vars" +[checks] -[mount] -source="cdb_data" -destination="/cockroach/cockroach-data" + [checks.crdb_live] + grace_period = "60s" + interval = "10s" + method = "get" + path = "/health" + port = 8080 + protocol = "https" + timeout = "5s" + tls_skip_verify = true + type = "http" [experimental] + auto_rollback = false cmd = ["shell", "start_fly.sh"] + private_network = true + +[metrics] + path = "/_status/vars" + port = 8080 + +[[mounts]] + destination = "/cockroach/cockroach-data" + source = "crdb_data" + +[[services]] + internal_port = 26257 + protocol = "tcp" + + [services.concurrency] + hard_limit = 1000 + soft_limit = 200 + type = "connections" + + [[services.ports]] + handlers = [] + port = 26257 + + [[services.tcp_checks]] + grace_period = "60s" + interval = 10000 + timeout = 5000 diff --git a/init_cluster.sh b/init_cluster.sh deleted file mode 100755 index 58858b9..0000000 --- a/init_cluster.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -set -eu - -exec /cockroach/cockroach init --insecure --cluster-name=$FLY_APP_NAME \ No newline at end of file diff --git a/start_fly.sh b/start_fly.sh index 953f54a..d90f2d9 100755 --- a/start_fly.sh +++ b/start_fly.sh @@ -2,14 +2,30 @@ set -eu +echo "Saving certificates to file system ..." +mkdir -p /cockroach/cockroach-certs +echo "${DB_CA_CRT}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/ca.crt +echo "${DB_NODE_CRT}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/node.crt +echo "${DB_NODE_KEY}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/node.key +chmod 0600 /cockroach/cockroach-certs/node.key + +echo "Building list of regional join nodes..." +JOIN_NODES=$(dig +short TXT regions.$FLY_APP_NAME.internal | sed -E 's/(")//g;s/([a-z]*)/\1.'"$FLY_APP_NAME"'.internal/g') + echo "Starting on Fly ..." echo exec /cockroach/cockroach start \ - --insecure \ - --locality=fly_region=$FLY_REGION \ + --logtostderr \ + --certs-dir=/cockroach/cockroach-certs \ --cluster-name=$FLY_APP_NAME \ - --join=$FLY_APP_NAME.internal + --locality=region=$FLY_REGION \ + --advertise-addr=$(hostname -s).vm.$FLY_APP_NAME.internal \ + --http-addr 0.0.0.0 \ + --join=$JOIN_NODES,top10.nearest.of.$FLY_APP_NAME.internal,$FLY_APP_NAME.fly.dev exec /cockroach/cockroach start \ - --insecure \ - --locality=fly_region=$FLY_REGION \ + --logtostderr \ + --certs-dir=/cockroach/cockroach-certs \ --cluster-name=$FLY_APP_NAME \ - --join=$FLY_APP_NAME.internal \ No newline at end of file + --locality=region=$FLY_REGION \ + --advertise-addr=$(hostname -s).vm.$FLY_APP_NAME.internal \ + --http-addr 0.0.0.0 \ + --join=$JOIN_NODES,top10.nearest.of.$FLY_APP_NAME.internal,$FLY_APP_NAME.fly.dev \ No newline at end of file