feat: secure cluster

Dockerfile

@@ -1,4 +1,5 @@
-FROM cockroachdb/cockroach:v21.1.2
+FROM cockroachdb/cockroach:v21.2.4
+
+RUN microdnf install bind-utils
 
-ADD init_cluster.sh /cockroach/
 ADD start_fly.sh /cockroach/

README.md

@@ -1,44 +1,73 @@
 # CockroachDB on Fly.io
 
-> **Note:** this app requires version 0.0.223 or greater of the Fly CLI. Run `flyctl version update` before you follow these instructions.
+> **Note:** this app requires version 0.0.261 or greater of the Fly CLI. Run `flyctl version update` before you follow these instructions.
 
 This is an example CockroachDB cluster that runs on multiple Fly.io regions.
 
 Deploying this app is relatively simple:
 
 1. Clone this repository
-2. [Install the `fly` CLI](https://fly.io/docs/hands-on/installing/)
-2. Run `fly launch`.
+2. [Install the `cockroach` CLI](https://www.cockroachlabs.com/docs/stable/install-cockroachdb.html)
+3. [Install the `fly` CLI](https://fly.io/docs/hands-on/installing/)
+4. Run `fly launch`.
     1. Pick "yes" to copy existing configuration
     2. When it asks if you want to deploy, say no
-3. Create volumes:
+5. Generate the Certificate Authority certificate and keypair (can reuse existing CA but don't share CA between production and non-production environments)
     ```bash
-    # one in Chicago
-    fly volumes create cdb_data --region ord --size 10
-    # one in Paris
-    fly volumes create cdb_data --region cdg --size 10
-    # one in Singapore
-    fly volumes create cdb_data --region sin --size 10
+    > cockroach cert create-ca --certs-dir=<absolute_path_to_certificates_directory> --ca-key=<absolute_path_to_ca_key_file>
     ```
-4. Deploy an empty node
+6. Generate the Node certificate and keypair
+
+    _When generating the certificate, you can add external domains as well, e.g. `db.example.com`_
+
+    ```bash
+    > cockroach cert create-node --certs-dir=<absolute_path_to_certificates_directory> --ca-key=<absolute_path_to_ca_key_file> 127.0.0.1 localhost <app_name>.internal "*.<app_name>.internal" "*.vm.<app_name>.internal" "*.nearest.of.<app_name>.internal" <app_name>.fly.dev
+    ```
+7. Generate the root user certificate and keypair
+    ```bash
+    > cockroach cert create-client --certs-dir=<absolute_path_to_certificates_directory> --ca-key=<absolute_path_to_ca_key_file> root
+    ```
+8. Upload the certificates and keypair
+    ```bash
+    > base64 <path_to_ca.crt> | fly secrets set DB_CA_CRT=-
+    > base64 <path_to_node.crt> | fly secrets set DB_NODE_CRT=-
+    > base64 <path_to_node.key> | fly secrets set DB_NODE_KEY=-
+    ```
+9. Create volumes:
+    ```bash
+    # for single region, minimum 3 nodes required
+    > fly volumes create crdb_data --region <region> --size 100
+    > fly volumes create crdb_data --region <region> --size 100
+    > fly volumes create crdb_data --region <region> --size 100
+
+    # for multi-region, minimum 3 regions required
+    > fly volumes create crdb_data --region <region1> --size 100
+    > fly volumes create crdb_data --region <region2> --size 100
+    > fly volumes create crdb_data --region <region3> --size 100
+    ```
+10. Set VM size and scale to desired node count
+    ```bash
+    > fly scale vm <size> --memory <memory_in_megabytes>
+    Scaled VM Type to
+     <size>
+      CPU Cores: <number_of_cores>
+         Memory: <memory> GB
+    > fly scale count <node_count>
+    Count changed to <node_count>
+    ```
+11. Deploy nodes
     ```bash
     > fly deploy
     ```
-5. Init the cluster:
+12. Init the cluster:
     ```bash
-    > fly ssh console -C '/cockroach/init_cluster.sh'
-    Connecting to cockroachdb-example.internal... complete
+    > cockroach init --cluster-name=<app_name> --host=<app_name>.fly.dev --certs-dir=<absolute_path_to_certificates_directory>
     Cluster successfully initialized
     ```
-6. Scale to 3 nodes
+13. View CockroachDB status
     ```bash
-    > fly scale count 3
-    Count changed to 3
+    > cockroach node status --host=<app_name>.fly.dev --certs-dir=<absolute_path_to_certificates>
     ```
-7. View CockroachDB status
-   ```bash
-   fly ssh console -C '/cockroach/cockroach node status --insecure'
-   ```
 
 ## Hook up Grafana
 

fly.toml

@@ -1,15 +1,48 @@
 app = "cockroachdb-example"
 
 kill_signal = "SIGINT"
-kill_timeout = 60
+kill_timeout = 120
 
-[metrics]
-    port = 8080
-    path = "/_status/vars"
+[checks]
 
-[mount]
-source="cdb_data"
-destination="/cockroach/cockroach-data"
+  [checks.crdb_live]
+    grace_period = "60s"
+    interval = "10s"
+    method = "get"
+    path = "/health"
+    port = 8080
+    protocol = "https"
+    timeout = "5s"
+    tls_skip_verify = true
+    type = "http"
 
 [experimental]
+  auto_rollback = false
   cmd = ["shell", "start_fly.sh"]
+  private_network = true
+
+[metrics]
+  path = "/_status/vars"
+  port = 8080
+
+[[mounts]]
+  destination = "/cockroach/cockroach-data"
+  source = "crdb_data"
+
+[[services]]
+  internal_port = 26257
+  protocol = "tcp"
+
+  [services.concurrency]
+    hard_limit = 1000
+    soft_limit = 200
+    type = "connections"
+
+  [[services.ports]]
+    handlers = []
+    port = 26257
+
+  [[services.tcp_checks]]
+    grace_period = "60s"
+    interval = 10000
+    timeout = 5000

start_fly.sh

@@ -2,14 +2,30 @@
 
 set -eu
 
+echo "Saving certificates to file system ..."
+mkdir -p /cockroach/cockroach-certs
+echo "${DB_CA_CRT}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/ca.crt
+echo "${DB_NODE_CRT}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/node.crt
+echo "${DB_NODE_KEY}" | base64 --decode --ignore-garbage > /cockroach/cockroach-certs/node.key
+chmod 0600 /cockroach/cockroach-certs/node.key
+
+echo "Building list of regional join nodes..."
+JOIN_NODES=$(dig +short TXT regions.$FLY_APP_NAME.internal | sed -E 's/(")//g;s/([a-z]*)/\1.'"$FLY_APP_NAME"'.internal/g')
+
 echo "Starting on Fly ..."
 echo exec /cockroach/cockroach start \
-  --insecure \
-  --locality=fly_region=$FLY_REGION \
+  --logtostderr \
+  --certs-dir=/cockroach/cockroach-certs \
   --cluster-name=$FLY_APP_NAME \
-  --join=$FLY_APP_NAME.internal
+  --locality=region=$FLY_REGION \
+  --advertise-addr=$(hostname -s).vm.$FLY_APP_NAME.internal \
+  --http-addr 0.0.0.0 \
+  --join=$JOIN_NODES,top10.nearest.of.$FLY_APP_NAME.internal,$FLY_APP_NAME.fly.dev
 exec /cockroach/cockroach start \
-  --insecure \
-  --locality=fly_region=$FLY_REGION \
+  --logtostderr \
+  --certs-dir=/cockroach/cockroach-certs \
   --cluster-name=$FLY_APP_NAME \
-  --join=$FLY_APP_NAME.internal
+  --locality=region=$FLY_REGION \
+  --advertise-addr=$(hostname -s).vm.$FLY_APP_NAME.internal \
+  --http-addr 0.0.0.0 \
+  --join=$JOIN_NODES,top10.nearest.of.$FLY_APP_NAME.internal,$FLY_APP_NAME.fly.dev