From 6840fd7c16fab88ef865de33cb5ec6b5fee7b1b3 Mon Sep 17 00:00:00 2001 From: hdahme Date: Fri, 18 Nov 2022 18:10:34 -0800 Subject: [PATCH 1/2] + terraform config and gcloud setup scripts --- .env.copy | 1 + .gitignore | 13 ++++- main.py | 2 +- ops/gcp/cloudrun/.dockerignore | 1 + ops/gcp/cloudrun/Dockerfile | 18 +++++++ ops/gcp/cloudrun/Readme.md | 42 ++++++++++++++++ ops/gcp/terraform/Readme.md | 72 ++++++++++++++++++++++++++++ ops/gcp/terraform/backend.tf | 3 ++ ops/gcp/terraform/gce.tf | 30 ++++++++++++ ops/gcp/terraform/iam.tf | 22 +++++++++ ops/gcp/terraform/outputs.tf | 3 ++ ops/gcp/terraform/provider.template | 4 ++ ops/gcp/terraform/variables.template | 29 +++++++++++ ops/gcp/terraform/vpc.tf | 21 ++++++++ requirements.txt | 4 -- 15 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 .env.copy create mode 100644 ops/gcp/cloudrun/.dockerignore create mode 100644 ops/gcp/cloudrun/Dockerfile create mode 100644 ops/gcp/cloudrun/Readme.md create mode 100644 ops/gcp/terraform/Readme.md create mode 100644 ops/gcp/terraform/backend.tf create mode 100644 ops/gcp/terraform/gce.tf create mode 100644 ops/gcp/terraform/iam.tf create mode 100644 ops/gcp/terraform/outputs.tf create mode 100644 ops/gcp/terraform/provider.template create mode 100644 ops/gcp/terraform/variables.template create mode 100644 ops/gcp/terraform/vpc.tf diff --git a/.env.copy b/.env.copy new file mode 100644 index 0000000..a9aaafc --- /dev/null +++ b/.env.copy @@ -0,0 +1 @@ +export MNEMONIC="" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8971528..e070c85 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,15 @@ .vscode .pytest_cache *.pyc -/junk/ \ No newline at end of file +/junk/ + + +# juno-arb +.env +ops/gcp/terraform/.terraform* + +ops/gcp/terraform/provider.tf +ops/gcp/terraform/variables.tf + +Dockerfile +.dockerignore diff --git a/main.py b/main.py index 177e68f..a5e01f8 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ """#############################################""" """@USER TODO: CHOOSE ENVIRONMENT VARIABLES PATH""" -ENV_FILE_PATH = "envs/juno.env" +ENV_FILE_PATH = ".env" #ENV_FILE_PATH = "envs/terra.env" """#############################################""" diff --git a/ops/gcp/cloudrun/.dockerignore b/ops/gcp/cloudrun/.dockerignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/ops/gcp/cloudrun/.dockerignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/ops/gcp/cloudrun/Dockerfile b/ops/gcp/cloudrun/Dockerfile new file mode 100644 index 0000000..613ad6b --- /dev/null +++ b/ops/gcp/cloudrun/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.10-slim + +RUN apt-get clean \ + && apt-get -y update + +RUN apt-get -y install \ + nginx \ + python3-dev \ + build-essential + +WORKDIR /app + +COPY requirements.txt /app/requirements.txt +RUN pip install -r requirements.txt --src /usr/local/src + +COPY . . + +CMD [ "python3", "main.py" ] \ No newline at end of file diff --git a/ops/gcp/cloudrun/Readme.md b/ops/gcp/cloudrun/Readme.md new file mode 100644 index 0000000..b7747b7 --- /dev/null +++ b/ops/gcp/cloudrun/Readme.md @@ -0,0 +1,42 @@ +Cloudrun is if you want the easiest, lowest overhead way of running a bot. If you want more flexibility and customizability, check out the `terraform` config. + +Since a job times out after a maximum of 1 hour, we get around that by scheduling this job to run every hour, at minute 0 + +Because these commands are fairly symmetric with Cloud Build, you should be able to plug them into your CI/CD process with little/no lift + +Firstly, copy the `.dockerignore` and `Dockerfile` to the root of the bot. Then submit the build to cloudrun + +```bash +export REGION="us-central1" +export PROJECT=$(gcloud config get-value project) +export PROJECT_NUMBER=`gcloud projects list | grep $PROJECT | awk '{print $(NF)}'` +export IMAGE="us-docker.pkg.dev/$PROJECT/gcr.io/skip-mev/juno-arb:latest" +export JOB_NAME="skip-mev-juno-arb" +source .env + +# Build the image +gcloud builds submit --tag $IMAGE + +# Create the job +gcloud beta run jobs create $JOB_NAME \ + --image $IMAGE \ + --region $REGION \ + --task-timeout 1h \ + --set-env-vars MNEMONIC="$MNEMONIC" + +# Either run the job now +gcloud beta run jobs execute $JOB_NAME --region $REGION + +# Or schedule the job +gcloud scheduler jobs create http $JOB_NAME \ + --location $REGION \ + --schedule "0 * * * *" \ + --uri="https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT/jobs/$JOB_NAME:run" \ + --http-method POST \ + --oauth-service-account-email $PROJECT_NUMBER-compute@developer.gserviceaccount.com + +# To delete the job and cancel any scheduled executions +gcloud beta run jobs delete $JOB_NAME +``` + +You can monitor job executions at https://console.cloud.google.com/run and view logs at https://console.cloud.google.com/run/jobs/details/$REGION/$JOB_NAME/executions \ No newline at end of file diff --git a/ops/gcp/terraform/Readme.md b/ops/gcp/terraform/Readme.md new file mode 100644 index 0000000..03131c2 --- /dev/null +++ b/ops/gcp/terraform/Readme.md @@ -0,0 +1,72 @@ +If you want to use a persistent GCE instance to manage your bot, use this config. A persistent GCE instance offers more flexibility in line with more overhead, say if you want to run sidecar services and so on. Use the cloudrun config if you're just looking for the quick and dirty. + +Firstly, set up your environment and create the VM + +```bash +export TF_VAR_zone="us-central1-c" +export TF_VAR_instance_name="skip-mev-juno-arb" +export PROJECT=$(gcloud config get-value project) + +# Copy the .template files +cp variables.template variables.tf +cp provider.template provider.tf + +# Dynamic substitution for the terraform config +sed -i "" s/PROJECT_ID/$PROJECT/g variables.tf +sed -i "" s/PROJECT_ID/$PROJECT/g provider.tf +sed -i "" s/ZONE/$TF_VAR_zone/g variables.tf +sed -i "" s/ZONE/$TF_VAR_zone/g provider.tf +sed -i "" s/INSTANCE_NAME/$TF_VAR_instance_name/g variables.tf + +# You can always reset to the template with the following +# sed -i "" s/$PROJECT/PROJECT_ID/g variables.template +# sed -i "" s/$PROJECT/PROJECT_ID/g provider.template +# sed -i "" s/$TF_VAR_zone/ZONE/g variables.template +# sed -i "" s/$TF_VAR_zone/ZONE/g provider.template +# sed -i "" s/$TF_VAR_instance_name/INSTANCE_NAME/g variables.template + +# Initialize terraform state in GCS +terraform init \ + -backend-config="bucket=$PROJECT-tfstate" \ + -backend-config="prefix=$TF_VAR_instance_name" +terraform apply +``` + +Now, we can copy the bot to the instance. Make sure the mnemonic in your `.env` is popualted correctly. + +```bash +gcloud compute scp --recurse ../../../*.py ../../../*.json ../../../*.txt ../../../.env ubuntu@$TF_VAR_instance_name:/home/ubuntu --zone $TF_VAR_zone + +# SSH onto the machine and follow setup instructions for the bot, namely +gcloud compute ssh --zone $TF_VAR_zone ubuntu@$TF_VAR_instance_name --tunnel-through-iap --project $PROJECT + +# Set up dependencies to run from the root, so that the startup script can function +sudo apt update +sudo apt-get install python3-pip -y +sudo mkdir -p /app && sudo cp ~/.env ~/* /app +cd /app +sudo pip install -r requirements.txt +exit +``` + +[Optionally, but recommended] install the cloudwatch ops agent +*You can also install via the console, by visiting https://console.cloud.google.com/monitoring/dashboards/resourceList/gce_instance* + +```bash +cd ~/ && curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +``` + +Finally, restart the instance. The bot is initialized via the startup script specified in the terraform config + +```bash +gcloud compute instances reset $TF_VAR_instance_name --zone $TF_VAR_zone +``` + +To verify the startup script worked, you can run `sudo journalctl -u google-startup-scripts.service` and can view logs by querying the following in the logs explorer + +```bash +resource.type="gce_instance" +sourceLocation.function="main.setupAndRunScript" +resource.labels.instance_id="THE_INSTANCE_ID_RETURNED_FROM_TERRAFORM" +``` \ No newline at end of file diff --git a/ops/gcp/terraform/backend.tf b/ops/gcp/terraform/backend.tf new file mode 100644 index 0000000..f174bec --- /dev/null +++ b/ops/gcp/terraform/backend.tf @@ -0,0 +1,3 @@ +terraform { + backend "gcs" {} +} \ No newline at end of file diff --git a/ops/gcp/terraform/gce.tf b/ops/gcp/terraform/gce.tf new file mode 100644 index 0000000..fdac2b3 --- /dev/null +++ b/ops/gcp/terraform/gce.tf @@ -0,0 +1,30 @@ +resource "google_compute_instance" "default" { + name = var.instance_name + machine_type = var.machine_type + zone = var.zone + tags = [var.instance_name, "ssh"] + project = var.project + + boot_disk { + initialize_params { + image = "ubuntu-os-cloud/ubuntu-2204-lts" + } + } + + metadata_startup_script = "cd /app && source .env && python3 main.py &" + + network_interface { + network = "default" + + access_config { + // Ephemeral public IP + } + } + + service_account { + email = google_service_account.default.email + scopes = ["cloud-platform"] + } + + allow_stopping_for_update = true +} diff --git a/ops/gcp/terraform/iam.tf b/ops/gcp/terraform/iam.tf new file mode 100644 index 0000000..a3a5ede --- /dev/null +++ b/ops/gcp/terraform/iam.tf @@ -0,0 +1,22 @@ +resource "google_service_account" "default" { + account_id = "${var.instance_name}-sa" + display_name = "Service Account" +} + +resource "google_project_iam_binding" "logs-writer-iam" { + role = "roles/logging.logWriter" + project = var.project + + members = [ + "serviceAccount:${google_service_account.default.email}", + ] +} + +resource "google_project_iam_binding" "metrics-writer-iam" { + role = "roles/monitoring.metricWriter" + project = var.project + + members = [ + "serviceAccount:${google_service_account.default.email}", + ] +} \ No newline at end of file diff --git a/ops/gcp/terraform/outputs.tf b/ops/gcp/terraform/outputs.tf new file mode 100644 index 0000000..e58820e --- /dev/null +++ b/ops/gcp/terraform/outputs.tf @@ -0,0 +1,3 @@ +output "instance_id" { + value = "${google_compute_instance.default.instance_id}" +} \ No newline at end of file diff --git a/ops/gcp/terraform/provider.template b/ops/gcp/terraform/provider.template new file mode 100644 index 0000000..61c202f --- /dev/null +++ b/ops/gcp/terraform/provider.template @@ -0,0 +1,4 @@ +provider "google" { + project = "PROJECT_ID" + zone = "ZONE" +} \ No newline at end of file diff --git a/ops/gcp/terraform/variables.template b/ops/gcp/terraform/variables.template new file mode 100644 index 0000000..ad992dd --- /dev/null +++ b/ops/gcp/terraform/variables.template @@ -0,0 +1,29 @@ +variable project { + type = string + default = "PROJECT_ID" + description = "Project ID" +} + +variable "zone" { + type = string + default = "ZONE" + description = "Zone" +} + +variable "region" { + type = string + default = "us-central1" + description = "Region" +} + +variable "instance_name" { + type = string + default = "INSTANCE_NAME" + description = "Instance Name" +} + +variable "machine_type" { + type = string + default = "e2-small" + description = "Machine Type" +} \ No newline at end of file diff --git a/ops/gcp/terraform/vpc.tf b/ops/gcp/terraform/vpc.tf new file mode 100644 index 0000000..251c01b --- /dev/null +++ b/ops/gcp/terraform/vpc.tf @@ -0,0 +1,21 @@ +resource "google_compute_network" "skip-mev" { + name = "skip-mev" + auto_create_subnetworks = false + mtu = 1460 + project = var.project + +} + +resource "google_compute_firewall" "ssh" { + name = "allow-ssh" + project = var.project + allow { + ports = ["22"] + protocol = "tcp" + } + direction = "INGRESS" + network = google_compute_network.skip-mev.id + priority = 1000 + source_ranges = ["0.0.0.0/0"] + target_tags = ["ssh"] +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 92b6df1..2f8575f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,8 +40,6 @@ google-api-python-client==2.65.0 google-auth==2.14.1 google-auth-httplib2==0.1.0 googleapis-common-protos==1.56.4 -grpcio==1.47.0 -grpclib==0.4.3 h11==0.12.0 h2==4.1.0 hpack==4.0.0 @@ -58,7 +56,6 @@ jedi==0.18.1 jsonschema==4.17.0 jupyter_client==7.4.4 jupyter_core==5.0.0 -lazy-object-proxy==1.9.0 MarkupSafe==2.0.1 matplotlib-inline==0.1.6 mccabe==0.7.0 @@ -76,7 +73,6 @@ pickleshare==0.7.5 platformdirs==2.5.3 pluggy==1.0.0 prompt-toolkit==3.0.32 -protobuf==3.20.3 psutil==5.9.4 ptyprocess==0.7.0 pure-eval==0.2.2 From 79f05d2428f8ddaafa165e1786558af5e54dc7e0 Mon Sep 17 00:00:00 2001 From: hdahme Date: Thu, 2 Feb 2023 09:23:13 -0800 Subject: [PATCH 2/2] Updated deployment scripts for the new repo format --- ops/gcp/terraform/Readme.md | 5 +++-- ops/gcp/terraform/gce.tf | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ops/gcp/terraform/Readme.md b/ops/gcp/terraform/Readme.md index 03131c2..aeaab9c 100644 --- a/ops/gcp/terraform/Readme.md +++ b/ops/gcp/terraform/Readme.md @@ -35,7 +35,8 @@ terraform apply Now, we can copy the bot to the instance. Make sure the mnemonic in your `.env` is popualted correctly. ```bash -gcloud compute scp --recurse ../../../*.py ../../../*.json ../../../*.txt ../../../.env ubuntu@$TF_VAR_instance_name:/home/ubuntu --zone $TF_VAR_zone +gcloud compute scp --recurse * ./* ubuntu@$TF_VAR_instance_name:/home/ubuntu --zone $TF_VAR_zone +gcloud compute scp --recurse .env ubuntu@$TF_VAR_instance_name:/home/ubuntu --zone $TF_VAR_zone # SSH onto the machine and follow setup instructions for the bot, namely gcloud compute ssh --zone $TF_VAR_zone ubuntu@$TF_VAR_instance_name --tunnel-through-iap --project $PROJECT @@ -43,7 +44,7 @@ gcloud compute ssh --zone $TF_VAR_zone ubuntu@$TF_VAR_instance_name --tunnel-thr # Set up dependencies to run from the root, so that the startup script can function sudo apt update sudo apt-get install python3-pip -y -sudo mkdir -p /app && sudo cp ~/.env ~/* /app +sudo mkdir -p /app && sudo cp -r ~/.env ~/* /app cd /app sudo pip install -r requirements.txt exit diff --git a/ops/gcp/terraform/gce.tf b/ops/gcp/terraform/gce.tf index fdac2b3..f7a39d9 100644 --- a/ops/gcp/terraform/gce.tf +++ b/ops/gcp/terraform/gce.tf @@ -11,7 +11,7 @@ resource "google_compute_instance" "default" { } } - metadata_startup_script = "cd /app && source .env && python3 main.py &" + metadata_startup_script = "cd /app && python3 main.py &" network_interface { network = "default"