From d585e5d321e1efebcadb9cb5c9336d18c403addd Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 18 Oct 2024 16:47:39 -0400 Subject: [PATCH] feat: added ceramic-one-mainnet k8s example --- k8s/ceramic-one-mainnnet/README.md | 80 ++++++ k8s/ceramic-one-mainnnet/envs/ceramic-one.env | 10 + k8s/ceramic-one-mainnnet/envs/js-ceramic.env | 6 + .../files/daemon-config.json | 40 +++ .../files/js-ceramic-init.sh | 10 + k8s/ceramic-one-mainnnet/kustomization.yaml | 34 +++ .../manifests/ceramic-one-lb.yaml | 35 +++ .../manifests/ceramic-one-sts.yaml | 140 ++++++++++ .../manifests/js-ceramic-lb.yaml | 35 +++ .../manifests/js-ceramic-sts.yaml | 239 ++++++++++++++++++ .../manifests/postgres-sts.yaml | 90 +++++++ .../scripts/create-secrets.sh | 12 + 12 files changed, 731 insertions(+) create mode 100644 k8s/ceramic-one-mainnnet/README.md create mode 100644 k8s/ceramic-one-mainnnet/envs/ceramic-one.env create mode 100644 k8s/ceramic-one-mainnnet/envs/js-ceramic.env create mode 100644 k8s/ceramic-one-mainnnet/files/daemon-config.json create mode 100644 k8s/ceramic-one-mainnnet/files/js-ceramic-init.sh create mode 100644 k8s/ceramic-one-mainnnet/kustomization.yaml create mode 100644 k8s/ceramic-one-mainnnet/manifests/ceramic-one-lb.yaml create mode 100644 k8s/ceramic-one-mainnnet/manifests/ceramic-one-sts.yaml create mode 100644 k8s/ceramic-one-mainnnet/manifests/js-ceramic-lb.yaml create mode 100644 k8s/ceramic-one-mainnnet/manifests/js-ceramic-sts.yaml create mode 100644 k8s/ceramic-one-mainnnet/manifests/postgres-sts.yaml create mode 100755 k8s/ceramic-one-mainnnet/scripts/create-secrets.sh diff --git a/k8s/ceramic-one-mainnnet/README.md b/k8s/ceramic-one-mainnnet/README.md new file mode 100644 index 0000000..a99b03a --- /dev/null +++ b/k8s/ceramic-one-mainnnet/README.md @@ -0,0 +1,80 @@ +# ceramic-one-mainnet + +This overlay configures the `ceramic-one` deployment to connect to the Ceramic mainnet network. + +## Step 1: Create a namespace + +The example kustomization uses namespace `ceramic-one-mainnet`. + +Update the namespace in the kustomization file to your desired namespace. + +Create the namespace with the following command: + +``` +export CERAMIC_NAMESPACE=ceramic-one-mainnet +kubectl create namespace ${CERAMIC_NAMESPACE} +``` + +## Step 2: Create secrets + +The k8s manifests require 2 secrets to be set up before deploying. + +### ceramic-admin + +This secret contains the following 3 keys: + +#### eth-rpc-urls + +This should be a valid Ethereum mainnetRPC URL that the Ceramic node can use to connect to the Ethereum network. + +#### node-private-key + +A private key for the Ceramic node to authenticate with CAS. + +If you don't already have a private key, you can generate one with `openssl rand -hex 32`. + +From this key, a DID will be derived. Use the `composedb did:from-private-key` command to derive a DID from the private key. + +Use this DID to register with CAS as documented https://developers.ceramic.network/docs/composedb/guides/composedb-server/access-mainnet + +#### admin-private-key + +A private key for the Admin DID to use the deployed Ceramic API. + +If you don't already have a private key, you can generate one with `openssl rand -hex 32`. + +#### Create the ceramic-admin secret +Create the secret with the following command (as example): + +``` +kubectl create secret generic ceramic-admin \ + --namespace ${CERAMIC_NAMESPACE} \ + --from-literal=admin-private-key=$(openssl rand -hex 32) \ + --from-literal=node-private-key=$(openssl rand -hex 32) \ + --from-literal=eth-rpc-urls= +``` + +### ceramic-postgres-auth + +This example deployment also deploys a postgres instance with a database for the Ceramic node to use. + +#### Create the ceramic-postgres-auth secret + +Create the secret with the following command (as example): + +``` +kubectl create secret generic ceramic-postgres-auth \ + --namespace ${CERAMIC_NAMESPACE} \ + --from-literal=db=ceramic-mainnet \ + --from-literal=host=postgres \ + --from-literal=password=ceramic \ + --from-literal=username=ceramic +``` + +## Step 3: Deploy + +From this directory, apply the kustomization: + +``` +kubectl apply -k . +``` diff --git a/k8s/ceramic-one-mainnnet/envs/ceramic-one.env b/k8s/ceramic-one-mainnnet/envs/ceramic-one.env new file mode 100644 index 0000000..21a6af5 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/envs/ceramic-one.env @@ -0,0 +1,10 @@ +CERAMIC_ONE_BIND_ADDRESS=0.0.0.0:5101 +CERAMIC_ONE_KADEMLIA_PARALLELISM=1 +CERAMIC_ONE_KADEMLIA_REPLICATION=6 +CERAMIC_ONE_LOCAL_NETWORK_ID=0 +CERAMIC_ONE_METRICS_BIND_ADDRESS=0.0.0.0:9465 +CERAMIC_ONE_NETWORK=mainnet +CERAMIC_ONE_RECON=true +CERAMIC_ONE_STORE_DIR=/data/ceramic-one +CERAMIC_ONE_SWARM_ADDRESSES=/ip4/0.0.0.0/tcp/4001 +RUST_LOG=info,ceramic_one=debug,multipart=error diff --git a/k8s/ceramic-one-mainnnet/envs/js-ceramic.env b/k8s/ceramic-one-mainnnet/envs/js-ceramic.env new file mode 100644 index 0000000..a21f8d1 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/envs/js-ceramic.env @@ -0,0 +1,6 @@ +CERAMIC_RECON_MODE=true +CERAMIC_NETWORK=mainnet +CERAMIC_SQLITE_PATH=/js-ceramic-data/ceramic.db +CERAMIC_STATE_STORE_PATH=/js-ceramic-data/statestore +CERAMIC_CORS_ALLOWED_ORIGINS=.* +CERAMIC_LOG_LEVEL=2 \ No newline at end of file diff --git a/k8s/ceramic-one-mainnnet/files/daemon-config.json b/k8s/ceramic-one-mainnnet/files/daemon-config.json new file mode 100644 index 0000000..ea97693 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/files/daemon-config.json @@ -0,0 +1,40 @@ +{ + "anchor": { + "auth-method": "did" + }, + "http-api": { + "cors-allowed-origins": [ + "${CERAMIC_CORS_ALLOWED_ORIGINS}" + ], + "admin-dids": [ + "${CERAMIC_ADMIN_DID}" + ] + }, + "ipfs": { + "mode": "remote", + "host": "${CERAMIC_IPFS_HOST}" + }, + "logger": { + "log-level": ${CERAMIC_LOG_LEVEL}, + "log-to-files": false + }, + "metrics": { + "metrics-exporter-enabled": false, + "prometheus-exporter-enabled": true, + "prometheus-exporter-port": 9464 + }, + "network": { + "name": "${CERAMIC_NETWORK}" + }, + "node": { + "privateSeedUrl": "inplace:ed25519#${CERAMIC_NODE_PRIVATE_SEED}" + }, + "state-store": { + "mode": "fs", + "local-directory": "${CERAMIC_STATE_STORE_PATH}" + }, + "indexing": { + "db": "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}", + "allow-queries-before-historical-sync": true + } +} \ No newline at end of file diff --git a/k8s/ceramic-one-mainnnet/files/js-ceramic-init.sh b/k8s/ceramic-one-mainnnet/files/js-ceramic-init.sh new file mode 100644 index 0000000..5d5b5e9 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/files/js-ceramic-init.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eo pipefail + +export MY_POD_INDEX=$(echo $MY_POD_NAME | grep -o '[0-9]\+$') +# Assuming http://ceramic-one-1:5101 format +export CERAMIC_IPFS_HOST=http://ceramic-one-${MY_POD_INDEX}:5101 +export CERAMIC_ADMIN_DID=$(composedb did:from-private-key ${CERAMIC_ADMIN_PRIVATE_KEY}) + +CERAMIC_ADMIN_DID=$CERAMIC_ADMIN_DID envsubst < /js-ceramic-init/daemon-config.json > /config/daemon-config.json diff --git a/k8s/ceramic-one-mainnnet/kustomization.yaml b/k8s/ceramic-one-mainnnet/kustomization.yaml new file mode 100644 index 0000000..330d7b2 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/kustomization.yaml @@ -0,0 +1,34 @@ +namespace: ceramic-one-mainnet + +resources: + - ./manifests/js-ceramic-sts.yaml + - ./manifests/ceramic-one-sts.yaml + - ./manifests/ceramic-one-lb.yaml + - ./manifests/js-ceramic-lb.yaml + - ./manifests/postgres-sts.yaml + +images: + - name: ceramicnetwork/composedb-cli + newName: ceramicnetwork/composedb-cli + newTag: "latest" + - name: ceramicnetwork/js-ceramic + newName: ceramicnetwork/js-ceramic + newTag: "latest" + - name: ceramicnetwork/ceramic-one + newName: public.ecr.aws/r5b3e0r5/3box/ceramic-one + newTag: "0.40.0" + - name: postgres + newName: postgres + newTag: 15-alpine + +configMapGenerator: + - name: js-ceramic-env + envs: + - ./envs/js-ceramic.env + - name: ceramic-one-env + envs: + - ./envs/ceramic-one.env + - name: js-ceramic-init + files: + - ./files/daemon-config.json + - ./files/js-ceramic-init.sh diff --git a/k8s/ceramic-one-mainnnet/manifests/ceramic-one-lb.yaml b/k8s/ceramic-one-mainnnet/manifests/ceramic-one-lb.yaml new file mode 100644 index 0000000..4ac6e28 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/manifests/ceramic-one-lb.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: ceramic-one-lb-0 + annotations: + cloud.google.com/l4-rbs: "enabled" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: ceramic-one + statefulset.kubernetes.io/pod-name: ceramic-one-0 + ports: + - name: swarm-tcp + protocol: TCP + port: 4101 + targetPort: 4101 +--- +apiVersion: v1 +kind: Service +metadata: + name: ceramic-one-lb-1 + annotations: + cloud.google.com/l4-rbs: "enabled" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: ceramic-one + statefulset.kubernetes.io/pod-name: ceramic-one-1 + ports: + - name: swarm-tcp + protocol: TCP + port: 4101 + targetPort: 4101 diff --git a/k8s/ceramic-one-mainnnet/manifests/ceramic-one-sts.yaml b/k8s/ceramic-one-mainnnet/manifests/ceramic-one-sts.yaml new file mode 100644 index 0000000..3e9f4b1 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/manifests/ceramic-one-sts.yaml @@ -0,0 +1,140 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ceramic-one +spec: + replicas: 1 + selector: + matchLabels: + app: ceramic-one + serviceName: ceramic-one + template: + metadata: + labels: + app: ceramic-one + spec: + containers: + - name: ceramic-one + envFrom: + - configMapRef: + name: ceramic-one-env + env: + - name: CERAMIC_ONE_ETHEREUM_RPC_URLS + valueFrom: + secretKeyRef: + name: ceramic-admin + key: eth-rpc-urls + image: ceramicnetwork/ceramic-one + imagePullPolicy: Always + ports: + - containerPort: 4101 + name: swarm-tcp + protocol: TCP + - containerPort: 5101 + name: rpc + protocol: TCP + - containerPort: 9465 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /ceramic/liveness + port: rpc + scheme: HTTP + readinessProbe: + httpGet: + path: /ceramic/liveness + port: rpc + scheme: HTTP + resources: + limits: + cpu: "1" + ephemeral-storage: 1Gi + memory: 1Gi + requests: + cpu: "1" + ephemeral-storage: 1Gi + memory: 1Gi + volumeMounts: + - mountPath: /data/ceramic-one + name: ceramic-one-data + volumes: + - name: ceramic-one-data + persistentVolumeClaim: + claimName: ceramic-one-pvc + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ceramic-one-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: ceramic-one +spec: + ports: + - name: swarm-tcp + port: 4101 + protocol: TCP + targetPort: swarm-tcp + - name: rpc + port: 5101 + protocol: TCP + targetPort: rpc + - name: metrics + port: 9465 + protocol: TCP + targetPort: metrics + selector: + app: ceramic-one +--- +apiVersion: v1 +kind: Service +metadata: + name: ceramic-one-0 +spec: + ports: + - name: swarm-tcp + port: 4101 + protocol: TCP + targetPort: swarm-tcp + - name: rpc + port: 5101 + protocol: TCP + targetPort: rpc + - name: metrics + port: 9465 + protocol: TCP + targetPort: metrics + selector: + app: ceramic-one + statefulset.kubernetes.io/pod-name: ceramic-one-0 +--- +apiVersion: v1 +kind: Service +metadata: + name: ceramic-one-1 +spec: + ports: + - name: swarm-tcp + port: 4101 + protocol: TCP + targetPort: swarm-tcp + - name: rpc + port: 5101 + protocol: TCP + targetPort: rpc + - name: metrics + port: 9465 + protocol: TCP + targetPort: metrics + selector: + app: ceramic-one + statefulset.kubernetes.io/pod-name: ceramic-one-1 diff --git a/k8s/ceramic-one-mainnnet/manifests/js-ceramic-lb.yaml b/k8s/ceramic-one-mainnnet/manifests/js-ceramic-lb.yaml new file mode 100644 index 0000000..09925cd --- /dev/null +++ b/k8s/ceramic-one-mainnnet/manifests/js-ceramic-lb.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: Service +metadata: + name: js-ceramic-lb-0 + annotations: + cloud.google.com/l4-rbs: "enabled" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: js-ceramic + statefulset.kubernetes.io/pod-name: js-ceramic-0 + ports: + - name: api + protocol: TCP + port: 7007 + targetPort: 7007 +--- +apiVersion: v1 +kind: Service +metadata: + name: js-ceramic-lb-1 + annotations: + cloud.google.com/l4-rbs: "enabled" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: js-ceramic + statefulset.kubernetes.io/pod-name: js-ceramic-1 + ports: + - name: api + protocol: TCP + port: 7007 + targetPort: 7007 \ No newline at end of file diff --git a/k8s/ceramic-one-mainnnet/manifests/js-ceramic-sts.yaml b/k8s/ceramic-one-mainnnet/manifests/js-ceramic-sts.yaml new file mode 100644 index 0000000..363b876 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/manifests/js-ceramic-sts.yaml @@ -0,0 +1,239 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: js-ceramic +spec: + replicas: 1 + selector: + matchLabels: + app: js-ceramic + serviceName: js-ceramic + template: + metadata: + labels: + app: js-ceramic + spec: + containers: + - name: js-ceramic + command: + - /js-ceramic/packages/cli/bin/ceramic.js + - daemon + - --config + - /config/daemon-config.json + # - /bin/bash + # - -c + # - "sleep infinity" + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: db + name: ceramic-postgres-auth + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + key: host + name: ceramic-postgres-auth + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: ceramic-postgres-auth + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: username + name: ceramic-postgres-auth + - name: ETHEREUM_RPC_URL + valueFrom: + secretKeyRef: + name: ceramic-admin + key: eth-rpc-urls + - name: CERAMIC_NODE_PRIVATE_SEED + valueFrom: + secretKeyRef: + name: ceramic-admin + key: node-private-key + envFrom: + - configMapRef: + name: js-ceramic-env + image: ceramicnetwork/js-ceramic + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /api/v0/node/healthcheck + port: api + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 30 + ports: + - containerPort: 7007 + name: api + protocol: TCP + - containerPort: 9464 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /api/v0/node/healthcheck + port: api + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 30 + resources: + limits: + ephemeral-storage: 3Gi + requests: + ephemeral-storage: 3Gi + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /js-ceramic-data + name: js-ceramic-data + initContainers: + - name: init-ceramic-config + command: + - /bin/bash + - -c + - /js-ceramic-init/js-ceramic-init.sh + env: + - name: CERAMIC_ADMIN_PRIVATE_KEY + valueFrom: + secretKeyRef: + key: admin-private-key + name: ceramic-admin + - name: CERAMIC_NODE_PRIVATE_SEED + valueFrom: + secretKeyRef: + name: ceramic-admin + key: node-private-key + - name: ETHEREUM_RPC_URL + valueFrom: + secretKeyRef: + name: ceramic-admin + key: eth-rpc-urls + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: db + name: ceramic-postgres-auth + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + key: host + name: ceramic-postgres-auth + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: ceramic-postgres-auth + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: username + name: ceramic-postgres-auth + envFrom: + - configMapRef: + name: js-ceramic-env + image: ceramicnetwork/composedb-cli + imagePullPolicy: Always + resources: + limits: + ephemeral-storage: 3Gi + requests: + ephemeral-storage: 3Gi + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /js-ceramic-init + name: js-ceramic-init + securityContext: + fsGroup: 70 + volumes: + - emptyDir: {} + name: config-volume + - configMap: + defaultMode: 493 + name: js-ceramic-init + name: js-ceramic-init + - name: js-ceramic-data + persistentVolumeClaim: + claimName: js-ceramic-data + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: js-ceramic-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + volumeMode: Filesystem +--- +apiVersion: v1 +kind: Service +metadata: + name: js-ceramic +spec: + ports: + - name: api + port: 7007 + protocol: TCP + targetPort: api + - name: metrics + port: 9464 + protocol: TCP + targetPort: metrics + selector: + app: js-ceramic + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + name: js-ceramic-0 +spec: + ports: + - name: api + port: 7007 + protocol: TCP + targetPort: api + - name: metrics + port: 9464 + protocol: TCP + targetPort: metrics + selector: + app: js-ceramic + statefulset.kubernetes.io/pod-name: js-ceramic-0 + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + name: js-ceramic-1 +spec: + ports: + - name: api + port: 7007 + protocol: TCP + targetPort: api + - name: metrics + port: 9464 + protocol: TCP + targetPort: metrics + selector: + app: js-ceramic + statefulset.kubernetes.io/pod-name: js-ceramic-1 + type: ClusterIP \ No newline at end of file diff --git a/k8s/ceramic-one-mainnnet/manifests/postgres-sts.yaml b/k8s/ceramic-one-mainnnet/manifests/postgres-sts.yaml new file mode 100644 index 0000000..e04e71a --- /dev/null +++ b/k8s/ceramic-one-mainnnet/manifests/postgres-sts.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + key: db + name: ceramic-postgres-auth + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + key: host + name: ceramic-postgres-auth + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: ceramic-postgres-auth + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + key: username + name: ceramic-postgres-auth + image: postgres + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5432 + name: postgres + protocol: TCP + resources: + limits: + cpu: 250m + ephemeral-storage: 1Gi + memory: 1Gi + requests: + cpu: 250m + ephemeral-storage: 1Gi + memory: 1Gi + securityContext: + runAsGroup: 70 + runAsUser: 70 + volumeMounts: + - mountPath: /var/lib/postgresql + name: postgres-data + subPath: ceramic_data + securityContext: + fsGroup: 70 + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: postgres-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + ports: + - name: postgres + port: 5432 + protocol: TCP + targetPort: postgres + selector: + app: postgres diff --git a/k8s/ceramic-one-mainnnet/scripts/create-secrets.sh b/k8s/ceramic-one-mainnnet/scripts/create-secrets.sh new file mode 100755 index 0000000..b73eb64 --- /dev/null +++ b/k8s/ceramic-one-mainnnet/scripts/create-secrets.sh @@ -0,0 +1,12 @@ +kubectl create secret generic ceramic-admin \ + --namespace ${CERAMIC_NAMESPACE} \ + --from-literal=admin-private-key=$(openssl rand -hex 32) \ + --from-literal=node-private-key=$(openssl rand -hex 32) \ + --from-literal=eth-rpc-urls= + +kubectl create secret generic ceramic-postgres-auth \ + --namespace ${CERAMIC_NAMESPACE} \ + --from-literal=db=ceramic-mainnet \ + --from-literal=host=postgres \ + --from-literal=password=ceramic \ + --from-literal=username=ceramic