From cff0f6a9a14c8a3424d9715b1b274035cc16a7fc Mon Sep 17 00:00:00 2001
From: supahcraig <craignelson7007@gmail.com>
Date: Tue, 20 May 2025 15:03:19 -0500
Subject: [PATCH 1/2] Added networking terraform to GCP, unified ssh & tfvars
 across AWS/GCP, and ammended the README to give better cloud-specific
 instructions.

---
 driver-redpanda/README.md                     |  67 +++++-
 driver-redpanda/deploy/ansible.cfg            |   1 -
 .../deploy/aws/provision-redpanda-aws.tf      |   8 +-
 .../deploy/aws/terraform.tfvars.example       |   3 +-
 driver-redpanda/deploy/deploy.yaml            |   2 +-
 .../deploy/gcp/provision-redpanda-gcp.tf      | 225 ++++++++++++------
 .../deploy/gcp/terraform.tfvars.example       |  22 ++
 driver-redpanda/deploy/hosts_ini.tpl          |   1 +
 8 files changed, 244 insertions(+), 85 deletions(-)
 create mode 100644 driver-redpanda/deploy/gcp/terraform.tfvars.example

diff --git a/driver-redpanda/README.md b/driver-redpanda/README.md
index 41e0c54d..42a21fbb 100644
--- a/driver-redpanda/README.md
+++ b/driver-redpanda/README.md
@@ -14,29 +14,61 @@
 - The [terraform inventory plugin](https://github.com/adammck/terraform-inventory)
 
 - aws-cli
+- gcloud
 
 ## Setup
 
-1. In the top level directory run `mvn clean package`. This will build the benchmark client needed during deployment.
+1.  In the top level directory run `mvn clean package`. This will build the benchmark client needed during deployment.
 
-2. Create an ssh key for the benchmark using the following: `ssh-keygen -f ~/.ssh/redpanda_aws`. Set the password to blank.
+2.  Create an environment variable for the cloud provider you're going to deploy on:
 
-3. In the `driver-redpanda/deploy` directory set an environment variable for your cloud provider and then run the terraform apply. 
+```bash
+        export REDPANDA_CLOUD_PROVIDER=<aws | gcp | azure>
+````
 
+3.  Create an ssh key for the benchmark by running the following:
+
+ ```bash
+        ssh-keygen -f ~/.ssh/redpanda_${REDPANDA_CLOUD_PROVIDER} 
 ```
-        export REDPANDA_CLOUD_PROVIDER=aws
-```
+Set the password to blank when prompted.
+
+4.  Copy & edit `terraform.tfvars` for your specific needs around instance types & quantities:
 
+```bash
+        cp ${REDPANDA_CLOUD_PROVIDER}/terraform.tfvars.example ${REDPANDA_CLOUD_PROVIDER}/terraform.tfvars
 ```
-	cp ${REDPANDA_CLOUD_PROVIDER}/terraform.tfvars.example ${REDPANDA_CLOUD_PROVIDER}/terraform.tfvars
+
+Hint:  if you're planning to benchmark against an existing Redpanda cluster, set the `num_instances` of `redpanda` to 0 in `terraform.tfvars`
+
+
+5.  In the `driver-redpanda/deploy` directory run terraform apply. 
+
+### AWS
+
+```bash
         terraform -chdir=${REDPANDA_CLOUD_PROVIDER} init
+        terraform -chdir=${REDPANDA_CLOUD_PROVIDER} plan
         aws sts get-caller-identity || aws sso login
         terraform -chdir=${REDPANDA_CLOUD_PROVIDER} apply --auto-approve
 ```
 
-4. To setup the deployed nodes. Run the ansible playbook.  If running locally include `--ask-become-pass` and supply your admin password when prompted.   If running on a cloud VM run the command as `sudo` instead.
+### GCP
 
+```bash
+        terraform -chdir=${REDPANDA_CLOUD_PROVIDER} init
+        terraform -chdir=${REDPANDA_CLOUD_PROVIDER} plan
+        gcloud auth print-access-token || gcloud auth login
+        terraform -chdir=${REDPANDA_CLOUD_PROVIDER} apply --auto-approve
 ```
+
+### Azure
+
+_coming soon_
+
+6.  To setup the deployed nodes, run the ansible playbook.  If running locally include `--ask-become-pass` and supply your admin password when prompted.   If running on a cloud VM run the command as `sudo` instead.
+
+```bash
         if [ "$(uname)" = "Darwin" ]; then export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES; fi
         if [ "$(uname)" = "Darwin" ]; then brew install gnu-tar; fi # https://github.com/prometheus-community/ansible/issues/186
         ansible-galaxy install -r requirements.yaml
@@ -44,9 +76,20 @@
         ansible-playbook --inventory ${REDPANDA_CLOUD_PROVIDER}/hosts.ini --ask-become-pass deploy.yaml
 ```
 
-To instead use an existing BYOC cluster, run the ansible playbook as follows.
+To instead use an existing Redpanda BYOC/Dedicated cluster, you'll need to add several things to the command:
 
+An extra variable to enable TLS (required by Redpanda cloud clusters), and then a SASL username & password for a user already created on your cluster.
+```bash
+        -e "tls_enabled=true sasl_enabled=true sasl_username=<YOUR SASL USER> sasl_password=<YOUR SASL PASSWORD>" \
 ```
+
+An extra variable to identify the bootstrap server address (e.g. `http://seed-abc123.redpanda.com:9092`)
+```bash
+        -e bootstrapServers="<YOUR BYOC KAFKA API ENDPOINT>" \
+```
+
+So the complete ansible-playbook command would look like this:
+```bash
         ansible-playbook --inventory  ${REDPANDA_CLOUD_PROVIDER}/hosts.ini \
         --ask-become-pass \
         -e "tls_enabled=true sasl_enabled=true sasl_username=<YOUR SASL USER> sasl_password=<YOUR SASL PASSWORD>" \
@@ -56,6 +99,7 @@ To instead use an existing BYOC cluster, run the ansible playbook as follows.
 
 ---
 
+
 ## Running the benchmark
 
 1. SSH to the client machine. 
@@ -68,9 +112,14 @@ To instead use an existing BYOC cluster, run the ansible playbook as follows.
 
 3. Run a benchmark using a specific driver and workload, for example: 
 
-        sudo bin/benchmark -d driver-redpanda/redpanda-ack-all-group-linger-10ms.yaml \
+         bin/benchmark -d driver-redpanda/redpanda-ack-all-group-linger-10ms.yaml \
             driver-redpanda/deploy/workloads/1-topic-100-partitions-1kb-4-producers-500k-rate.yaml
 
+While the benchmark is running, you can observe the cluster performance in Grafana, by navigating to:
+`http://<prometheus.ip.address:3000`
+
+The default username & password is admin/admin.
+
 ## Generating charts
 
 Once you have ran a benchmark, a json file will be generated in the data directory. You can use `bin/generate_charts.py` to generate a a visual representation of this data.
diff --git a/driver-redpanda/deploy/ansible.cfg b/driver-redpanda/deploy/ansible.cfg
index 0c05d8ca..24a8df22 100644
--- a/driver-redpanda/deploy/ansible.cfg
+++ b/driver-redpanda/deploy/ansible.cfg
@@ -1,6 +1,5 @@
 [defaults]
 host_key_checking=false
-private_key_file=~/.ssh/redpanda_aws
 interpreter_python=auto
 inventory = hosts.ini
 pipelining = true
diff --git a/driver-redpanda/deploy/aws/provision-redpanda-aws.tf b/driver-redpanda/deploy/aws/provision-redpanda-aws.tf
index a95717a8..a9b342ad 100644
--- a/driver-redpanda/deploy/aws/provision-redpanda-aws.tf
+++ b/driver-redpanda/deploy/aws/provision-redpanda-aws.tf
@@ -15,6 +15,11 @@ provider "aws" {
 provider "random" {
 }
 
+variable "private_key_path" {
+  description = "Path to the private SSH key used for Ansible"
+  type        = string
+}
+
 variable "public_key_path" {
   description = <<DESCRIPTION
 Path to the SSH public key to be used for authentication.
@@ -280,7 +285,8 @@ resource "local_file" "hosts_ini" {
       control_public_ips   = aws_instance.client.*.public_ip
       control_private_ips  = aws_instance.client.*.private_ip
       instance_type	   = var.instance_types["redpanda"]
-      ssh_user              = "ubuntu"
+      ssh_user             = "ubuntu"
+      private_key_path     = var.private_key_path
     }
   )
   filename = "${path.module}/hosts.ini"
diff --git a/driver-redpanda/deploy/aws/terraform.tfvars.example b/driver-redpanda/deploy/aws/terraform.tfvars.example
index a0e4700a..12661432 100644
--- a/driver-redpanda/deploy/aws/terraform.tfvars.example
+++ b/driver-redpanda/deploy/aws/terraform.tfvars.example
@@ -1,4 +1,5 @@
-public_key_path = "~/.ssh/redpanda_aws.pub"
+public_key_path  = "~/.ssh/redpanda_aws.pub"
+private_key_path = "~/.ssh/redpanda_aws"
 
 region          = "us-west-2"
 # arm64 ubuntu focal
diff --git a/driver-redpanda/deploy/deploy.yaml b/driver-redpanda/deploy/deploy.yaml
index b16e5bcf..cfec9755 100644
--- a/driver-redpanda/deploy/deploy.yaml
+++ b/driver-redpanda/deploy/deploy.yaml
@@ -469,7 +469,7 @@
     grafana_version: 9.5.14
     grafana_security:
       admin_user: admin
-      admin_password: "{{ grafana_admin_pass | default('enter_your_secure_password', true) }}"
+      admin_password: "{{ grafana_admin_pass | default('admin', true) }}"
     grafana_datasources:
     - name: prometheus
       type: prometheus
diff --git a/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf b/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
index e4372b30..fb93cc7f 100644
--- a/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
+++ b/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
@@ -1,8 +1,77 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.0"
+    }
+  }
+}
+
+
 provider "google" {
-  project     = var.project_name
+  #project     = var.project_name
   region      = var.region
 }
 
+provider "random" {
+}
+
+variable "private_key_path" {
+  description = "Path to the private SSH key used for Ansible"
+  type        = string
+}
+
+variable "public_key_path" {
+  description = <<DESCRIPTION
+Path to the SSH public key to be used for authentication.
+Ensure this keypair is added to your local SSH agent so provisioners can
+connect.
+
+Example: ~/.ssh/redpanda_gcp.pub
+DESCRIPTION
+  type = string
+}
+
+variable "subnet_cidr_range" {
+  type        = string
+  description = "CIDR range for Redpanda subnet"
+  default     = "10.10.0.0/16"
+}
+
+variable "instance_types" {
+  type = map(string)
+  default = {
+    redpanda = "n2-standard-8"
+    monitor  = "n2-standard-4"
+    client   = "n2-standard-16"
+  }
+}
+
+variable "num_instances" {
+  description = "Map of instance counts by role"
+  type        = map(number)
+  default     = {
+    redpanda = 3
+    client   = 4
+    monitor  = 1
+  }
+}
+
+variable "redpanda_disk_size_gb" {
+  type        = number
+  description = "Size (in GB) of the local NVMe disk for Redpanda.  Must be in 375GB increments."
+  default     = 375
+}
+
+data "http" "my_ip" {
+  url = "https://ipv4.icanhazip.com"
+}
+
+
+resource "random_id" "hash" {
+  byte_length = 8
+}
+
 resource "random_uuid" "cluster" {}
 
 locals {
@@ -10,33 +79,77 @@ locals {
   deployment_id = random_uuid.cluster.result
 }
 
+
+# Create a new VPC
+resource "google_compute_network" "redpanda_vpc" {
+  name                    = "redpanda-vpc-${random_id.hash.hex}"
+  auto_create_subnetworks = false
+}
+
+
+# Create subnets
+resource "google_compute_subnetwork" "redpanda_subnet" {
+  name          = "redpanda-subnet-${random_id.hash.hex}"
+  ip_cidr_range = var.subnet_cidr_range
+  region        = var.region
+  network       = google_compute_network.redpanda_vpc.id
+}
+
+
+# Allow traffic on Redpanda, Prometheus, and Grafana ports
+resource "google_compute_firewall" "allow_ssh" {
+  name    = "allow-external-access"
+  network = google_compute_network.redpanda_vpc.name
+
+  allow {
+    protocol = "tcp"
+    ports    = ["22", "3000", "9090"]
+  }
+
+  source_ranges = ["${chomp(data.http.my_ip.response_body)}/32"]
+}
+
+
+# Allow traffic on Redpanda, Prometheus, and Grafana ports
+resource "google_compute_firewall" "allow_redpanda" {
+  name    = "allow-redpanda"
+  network = google_compute_network.redpanda_vpc.name
+
+  allow {
+    protocol = "tcp"
+    ports    = ["9092", "9644", "8081", "8082", "33145", "3000", "9090"] # Kafka API + Admin API + Prometheus/Grafana
+  }
+
+  source_ranges = [var.subnet_cidr_range]
+}
+
+
+
 resource "google_compute_resource_policy" "redpanda-rp" {
   name   = "redpanda-rp"
   region = var.region
   group_placement_policy {
-    availability_domain_count = var.ha ? max(3, var.nodes) : 1
+    availability_domain_count = var.ha ? max(3, var.num_instances["redpanda"]) : 1
   }
   count = var.ha ? 1 : 0
 }
 
 resource "google_compute_instance" "redpanda" {
-  count             = var.nodes
+  count             = var.num_instances["redpanda"]
   name              = "rp-node-${count.index}-${local.deployment_id}"
   tags              = ["rp-cluster", "tf-deployment-${local.deployment_id}"]
   zone              = "${var.region}-${var.availability_zone[count.index % length(var.availability_zone)]}"
-  machine_type      = var.machine_type
+  machine_type      = var.instance_types["redpanda"]
   // GCP does not give you visibility nor control over which failure domain a resource has been placed into
   // (https://issuetracker.google.com/issues/256993209?pli=1). So the only way that we can guarantee that
   // specific nodes are in separate racks is to put them into entirely separate failure domains - basically one
   // broker per failure domain, and we are limited by the number of failure domains (at the moment 8).
-  resource_policies = (var.ha && var.nodes <= 8) ? [
+  resource_policies = (var.ha && var.num_instances["redpanda"] <= 8) ? [
     google_compute_resource_policy.redpanda-rp[0].id
   ] : null
 
   metadata = {
-    ssh-keys = <<KEYS
-${var.ssh_user}:${file(abspath(var.public_key_path))}
-KEYS
+    ssh-keys = "${var.ssh_user}:${file(var.public_key_path)}"
   }
 
   boot_disk {
@@ -46,7 +159,7 @@ KEYS
   }
 
   dynamic "scratch_disk" {
-    for_each = range(var.disks)
+    for_each = range(floor(var.redpanda_disk_size_gb / 375))
     content {
       // 375 GB local SSD drive.
       interface = "NVME"
@@ -54,7 +167,7 @@ KEYS
   }
 
   network_interface {
-    subnetwork = var.subnet
+    subnetwork = google_compute_subnetwork.redpanda_subnet.id
     access_config {
     }
   }
@@ -63,16 +176,14 @@ KEYS
 }
 
 resource "google_compute_instance" "monitor" {
-  count        = 1
+  count        = var.num_instances["monitor"]
   name         = "rp-monitor-${local.deployment_id}"
   tags         = ["rp-cluster", "tf-deployment-${local.deployment_id}"]
-  machine_type = var.monitor_machine_type
+  machine_type = var.instance_types["monitor"]
   zone         = "${var.region}-${var.availability_zone[0]}"
 
   metadata = {
-    ssh-keys = <<KEYS
-${var.ssh_user}:${file(abspath(var.public_key_path))}
-KEYS
+    ssh-keys = "${var.ssh_user}:${file(var.public_key_path)}"
   }
 
   boot_disk {
@@ -87,7 +198,7 @@ KEYS
   }
 
   network_interface {
-    subnetwork = var.subnet
+    subnetwork = google_compute_subnetwork.redpanda_subnet.id
     access_config {
     }
   }
@@ -96,16 +207,14 @@ KEYS
 }
 
 resource "google_compute_instance" "client" {
-  count        = var.client_nodes
+  count        = var.num_instances["client"]
   name         = "rp-client-${count.index}-${local.deployment_id}"
   tags         = ["rp-cluster", "tf-deployment-${local.deployment_id}"]
-  machine_type = var.client_machine_type
+  machine_type = var.instance_types["client"]
   zone         = "${var.region}-${var.availability_zone[count.index % length(var.availability_zone)]}"
 
   metadata = {
-    ssh-keys = <<KEYS
-${var.ssh_user}:${file(abspath(var.public_key_path))}
-KEYS
+    ssh-keys = "${var.ssh_user}:${file(var.public_key_path)}"
   }
 
   boot_disk {
@@ -123,13 +232,22 @@ KEYS
   }
 
   network_interface {
-    subnetwork = var.subnet
+    subnetwork = google_compute_subnetwork.redpanda_subnet.id
     access_config {
     }
   }
   labels = tomap(var.labels)
 }
 
+resource "google_compute_route" "internet_access" {
+  name              = "redpanda-default-route"
+  network           = google_compute_network.redpanda_vpc.name
+  dest_range        = "0.0.0.0/0"
+  next_hop_gateway  = "default-internet-gateway"
+  priority          = 1000
+}
+
+
 resource "google_compute_instance_group" "redpanda" {
   name      = "redpanda-group-${local.deployment_id}"
   count     = length(var.availability_zone)
@@ -145,17 +263,18 @@ resource "google_compute_instance_group" "redpanda" {
 resource "local_file" "hosts_ini" {
   content = templatefile("${path.module}/../hosts_ini.tpl",
     {
-      redpanda_public_ips        = google_compute_instance.redpanda[*].network_interface.0.access_config.0.nat_ip
-      redpanda_private_ips       = google_compute_instance.redpanda[*].network_interface.0.network_ip
-      clients_public_ips         = google_compute_instance.client[*].network_interface.0.access_config.0.nat_ip
-      clients_private_ips        = google_compute_instance.client[*].network_interface.0.network_ip
-      control_public_ips         = google_compute_instance.client[*].network_interface.0.access_config.0.nat_ip
-      control_private_ips        = google_compute_instance.client[*].network_interface.0.network_ip
-      prometheus_host_public_ips = google_compute_instance.monitor[*].network_interface.0.access_config.0.nat_ip
-      prometheus_host_private_ips= google_compute_instance.monitor[*].network_interface.0.network_ip
-      ssh_user                   = var.ssh_user
-      instance_type              = var.machine_type
-    }
+      redpanda_public_ips         = google_compute_instance.redpanda[*].network_interface.0.access_config.0.nat_ip
+      redpanda_private_ips        = google_compute_instance.redpanda[*].network_interface.0.network_ip
+      clients_public_ips          = google_compute_instance.client[*].network_interface.0.access_config.0.nat_ip
+      clients_private_ips         = google_compute_instance.client[*].network_interface.0.network_ip
+      control_public_ips          = google_compute_instance.client[*].network_interface.0.access_config.0.nat_ip
+      control_private_ips         = google_compute_instance.client[*].network_interface.0.network_ip
+      prometheus_host_public_ips  = google_compute_instance.monitor[*].network_interface.0.access_config.0.nat_ip
+      prometheus_host_private_ips = google_compute_instance.monitor[*].network_interface.0.network_ip
+      instance_type               = var.instance_types["redpanda"]
+      ssh_user                    = var.ssh_user
+      private_key_path            = var.private_key_path 
+   }
   )
   filename = "${path.module}/hosts.ini"
 }
@@ -195,32 +314,12 @@ variable "instance_group_name" {
   default     = "redpanda-group"
 }
 
-variable "subnet" {
-  description = "The name of the existing subnet where the machines will be deployed"
-}
-
-variable "project_name" {
-  description = "The project name on GCP."
-}
-
-variable "nodes" {
-  description = "The number of nodes to deploy."
-  type        = number
-  default     = "3"
-}
-
 variable "ha" {
   description = "Whether to use placement groups to create an HA topology"
   type        = bool
   default     = false
 }
 
-variable "client_nodes" {
-  description = "The number of clients to deploy."
-  type        = number
-  default     = "4"
-}
-
 variable "disks" {
   description = "The number of local disks on each machine."
   type        = number
@@ -239,24 +338,6 @@ variable "image" {
   default = "ubuntu-os-cloud/ubuntu-2204-lts"
 }
 
-variable machine_type {
-  # List of available machines per region/ zone:
-  # https://cloud.google.com/compute/docs/regions-zones#available
-  default = "n2-standard-8"
-}
-
-variable monitor_machine_type {
-  default = "n2-standard-4"
-}
-
-variable client_machine_type {
-  default = "n2-standard-16"
-}
-
-variable "public_key_path" {
-  description = "The ssh key."
-}
-
 variable "ssh_user" {
   description = "The ssh user. Must match the one in the public ssh key's comments."
 }
@@ -268,7 +349,7 @@ variable "enable_monitoring" {
 variable "labels" {
   description = "passthrough of GCP labels"
   default     = {
-    "purpose"      = "redpanda-cluster"
+    "purpose"      = "redpanda-cluster-via-omb"
     "created-with" = "terraform"
   }
 }
diff --git a/driver-redpanda/deploy/gcp/terraform.tfvars.example b/driver-redpanda/deploy/gcp/terraform.tfvars.example
new file mode 100644
index 00000000..9af022df
--- /dev/null
+++ b/driver-redpanda/deploy/gcp/terraform.tfvars.example
@@ -0,0 +1,22 @@
+public_key_path  = "~/.ssh/redpanda_gcp.pub"
+private_key_path = "~/.ssh/redpanda_gcp"
+
+ssh_user         = "ubuntu"
+region           = "us-central1"
+
+subnet_cidr_range      = "10.10.0.0/16"
+redpanda_disk_size_gb  = 750    # must be in 375GB increments
+
+instance_types = {
+  "redpanda"      = "n2d-standard-8"
+  "client"        = "n2-standard-16"
+  "monitor"       = "n2-standard-4"
+}
+
+# client instances may need to be larger than redpanda broker count
+# to provide enough message volume for testing
+num_instances = {
+  "client"     = 4
+  "redpanda"   = 3
+  "monitor"    = 1
+}
diff --git a/driver-redpanda/deploy/hosts_ini.tpl b/driver-redpanda/deploy/hosts_ini.tpl
index 30a41c27..4da0c0cb 100644
--- a/driver-redpanda/deploy/hosts_ini.tpl
+++ b/driver-redpanda/deploy/hosts_ini.tpl
@@ -18,3 +18,4 @@ ${ ip } ansible_user=${ ssh_user } ansible_become=True private_ip=${prometheus_h
 
 [all:vars]
 instance_type=${instance_type}
+ansible_ssh_private_key_file=${private_key_path}

From 9b26f6932d88a6bb3e75488ae483a5e191bbeec6 Mon Sep 17 00:00:00 2001
From: supahcraig <craignelson7007@gmail.com>
Date: Tue, 20 May 2025 20:15:45 -0500
Subject: [PATCH 2/2] Added firewall rule for ingress on port 8080

---
 driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf b/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
index fb93cc7f..3b4af2ba 100644
--- a/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
+++ b/driver-redpanda/deploy/gcp/provision-redpanda-gcp.tf
@@ -117,7 +117,7 @@ resource "google_compute_firewall" "allow_redpanda" {
 
   allow {
     protocol = "tcp"
-    ports    = ["9092", "9644", "8081", "8082", "33145", "3000", "9090"] # Kafka API + Admin API + Prometheus/Grafana
+    ports    = ["9092", "9644", "8080", "8081", "8082", "33145", "3000", "9090"] # Kafka API + Admin API + Prometheus/Grafana
   }
 
   source_ranges = [var.subnet_cidr_range]