From 60f871de9c6fc5e5e37e01dd354745115a2c8b3f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Dec 2025 17:24:36 +0100 Subject: [PATCH 1/3] Add volume mount in elastic-agent for credential_source file --- .../_static/docker-agent-base.yml.tmpl | 9 +++++ internal/agentdeployer/agent.go | 33 +++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml.tmpl b/internal/agentdeployer/_static/docker-agent-base.yml.tmpl index c31e12fa0..bd4d7fdb5 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml.tmpl +++ b/internal/agentdeployer/_static/docker-agent-base.yml.tmpl @@ -6,6 +6,7 @@ {{- $agent_version := fact "agent_version" }} {{- $agent_image := fact "agent_image" }} {{- $enrollment_token := fact "enrollment_token" }} +{{- $google_credential_source_file := fact "google_credential_source_file" -}} services: elastic-agent: hostname: ${AGENT_HOSTNAME} @@ -62,5 +63,13 @@ services: source: ${SERVICE_LOGS_DIR} target: /run/service_logs/ read_only: false + {{ if ne $google_credential_source_file "" }} + # Mount Google credential source file (token file) provided + # Required for external accounts authentication when creating resources via terraform in GCP + - type: bind + source: {{ $google_credential_source_file }} + target: {{ $google_credential_source_file }} + read_only: true + {{ end }} extra_hosts: - "host.docker.internal:host-gateway" diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 794db8c1e..5a0c09ac6 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -306,21 +306,28 @@ func (d *DockerComposeAgentDeployer) installDockerCompose(ctx context.Context, a return "", nil } + googleCredentialSourceFile := os.Getenv("GOOGLE_CREDENTIAL_SOURCE_FILE") + if _, err := os.Stat(googleCredentialSourceFile); err != nil { + logger.Warn("GOOGLE_CREDENTIAL_SOURCE_FILE environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") + googleCredentialSourceFile = "" + } + resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ - "agent_image": agentImage, - "user": agentInfo.Agent.User, - "capabilities": strings.Join(agentInfo.Agent.LinuxCapabilities, ","), - "runtime": agentInfo.Agent.Runtime, - "pid_mode": agentInfo.Agent.PidMode, - "ports": strings.Join(agentInfo.Agent.Ports, ","), - "dockerfile_hash": hex.EncodeToString(hashDockerfile), - "agent_version": agentVersion, - "fleet_url": fleetURL, - "kibana_host": stack.DockerInternalHost(kibanaHost), - "elasticsearch_username": config.ElasticsearchUsername, - "elasticsearch_password": config.ElasticsearchPassword, - "enrollment_token": enrollmentToken, + "agent_image": agentImage, + "user": agentInfo.Agent.User, + "capabilities": strings.Join(agentInfo.Agent.LinuxCapabilities, ","), + "runtime": agentInfo.Agent.Runtime, + "pid_mode": agentInfo.Agent.PidMode, + "ports": strings.Join(agentInfo.Agent.Ports, ","), + "dockerfile_hash": hex.EncodeToString(hashDockerfile), + "agent_version": agentVersion, + "fleet_url": fleetURL, + "kibana_host": stack.DockerInternalHost(kibanaHost), + "elasticsearch_username": config.ElasticsearchUsername, + "elasticsearch_password": config.ElasticsearchPassword, + "enrollment_token": enrollmentToken, + "google_credential_source_file": googleCredentialSourceFile, }) resourceManager.RegisterProvider("file", &resource.FileProvider{ From 1506bba9f48643923fcf468e9b236dea0bc52c48 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 Dec 2025 18:42:38 +0100 Subject: [PATCH 2/3] Add file set in GOOGLE_APPLICATION_CREDENTIALS too --- .../_static/docker-agent-base.yml.tmpl | 9 ++++ internal/agentdeployer/agent.go | 45 ++++++++++++------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/internal/agentdeployer/_static/docker-agent-base.yml.tmpl b/internal/agentdeployer/_static/docker-agent-base.yml.tmpl index bd4d7fdb5..baf97441c 100644 --- a/internal/agentdeployer/_static/docker-agent-base.yml.tmpl +++ b/internal/agentdeployer/_static/docker-agent-base.yml.tmpl @@ -6,6 +6,7 @@ {{- $agent_version := fact "agent_version" }} {{- $agent_image := fact "agent_image" }} {{- $enrollment_token := fact "enrollment_token" }} +{{- $google_application_credentials := fact "google_application_credentials" -}} {{- $google_credential_source_file := fact "google_credential_source_file" -}} services: elastic-agent: @@ -63,6 +64,14 @@ services: source: ${SERVICE_LOGS_DIR} target: /run/service_logs/ read_only: false + {{ if ne $google_application_credentials "" }} + # Mount Google Application Credentials file provided + # Required for external accounts authentication when creating resources via terraform in GCP + - type: bind + source: {{ $google_application_credentials }} + target: {{ $google_application_credentials }} + read_only: true + {{ end }} {{ if ne $google_credential_source_file "" }} # Mount Google credential source file (token file) provided # Required for external accounts authentication when creating resources via terraform in GCP diff --git a/internal/agentdeployer/agent.go b/internal/agentdeployer/agent.go index 5a0c09ac6..67652a4dc 100644 --- a/internal/agentdeployer/agent.go +++ b/internal/agentdeployer/agent.go @@ -306,28 +306,39 @@ func (d *DockerComposeAgentDeployer) installDockerCompose(ctx context.Context, a return "", nil } + googleApplicationCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + if googleApplicationCredentials != "" { + if _, err := os.Stat(googleApplicationCredentials); err != nil { + logger.Warn("GOOGLE_APPLICATION_CREDENTIALS environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") + googleApplicationCredentials = "" + } + } + googleCredentialSourceFile := os.Getenv("GOOGLE_CREDENTIAL_SOURCE_FILE") - if _, err := os.Stat(googleCredentialSourceFile); err != nil { - logger.Warn("GOOGLE_CREDENTIAL_SOURCE_FILE environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") - googleCredentialSourceFile = "" + if googleCredentialSourceFile != "" { + if _, err := os.Stat(googleCredentialSourceFile); err != nil { + logger.Warn("GOOGLE_CREDENTIAL_SOURCE_FILE environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") + googleCredentialSourceFile = "" + } } resourceManager := resource.NewManager() resourceManager.AddFacter(resource.StaticFacter{ - "agent_image": agentImage, - "user": agentInfo.Agent.User, - "capabilities": strings.Join(agentInfo.Agent.LinuxCapabilities, ","), - "runtime": agentInfo.Agent.Runtime, - "pid_mode": agentInfo.Agent.PidMode, - "ports": strings.Join(agentInfo.Agent.Ports, ","), - "dockerfile_hash": hex.EncodeToString(hashDockerfile), - "agent_version": agentVersion, - "fleet_url": fleetURL, - "kibana_host": stack.DockerInternalHost(kibanaHost), - "elasticsearch_username": config.ElasticsearchUsername, - "elasticsearch_password": config.ElasticsearchPassword, - "enrollment_token": enrollmentToken, - "google_credential_source_file": googleCredentialSourceFile, + "agent_image": agentImage, + "user": agentInfo.Agent.User, + "capabilities": strings.Join(agentInfo.Agent.LinuxCapabilities, ","), + "runtime": agentInfo.Agent.Runtime, + "pid_mode": agentInfo.Agent.PidMode, + "ports": strings.Join(agentInfo.Agent.Ports, ","), + "dockerfile_hash": hex.EncodeToString(hashDockerfile), + "agent_version": agentVersion, + "fleet_url": fleetURL, + "kibana_host": stack.DockerInternalHost(kibanaHost), + "elasticsearch_username": config.ElasticsearchUsername, + "elasticsearch_password": config.ElasticsearchPassword, + "enrollment_token": enrollmentToken, + "google_credential_source_file": googleCredentialSourceFile, + "google_application_credentials": googleApplicationCredentials, }) resourceManager.RegisterProvider("file", &resource.FileProvider{ From cb672f3d41e1c9a73506b73cf1984893294c4827 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 Dec 2025 13:00:40 +0100 Subject: [PATCH 3/3] Add volumes into internal terraform docker-compose Set the terraform docker-compose scenario as a template in order to be able to set the volumes just when the environment variables are defined. This will help when executing elastic-package in local scenarios. --- .../_static/terraform_deployer.yml | 16 -------- .../_static/terraform_deployer.yml.tmpl | 34 +++++++++++++++++ internal/servicedeployer/terraform.go | 37 ++++++++++++++++--- 3 files changed, 65 insertions(+), 22 deletions(-) delete mode 100644 internal/servicedeployer/_static/terraform_deployer.yml create mode 100644 internal/servicedeployer/_static/terraform_deployer.yml.tmpl diff --git a/internal/servicedeployer/_static/terraform_deployer.yml b/internal/servicedeployer/_static/terraform_deployer.yml deleted file mode 100644 index f775bb93e..000000000 --- a/internal/servicedeployer/_static/terraform_deployer.yml +++ /dev/null @@ -1,16 +0,0 @@ -services: - terraform: - build: . - tty: true - stop_grace_period: 5m - environment: - - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached} - - TF_VAR_CREATED_DATE=${CREATED_DATE:-unknown} - - TF_VAR_BRANCH=${BRANCH_NAME_LOWER_CASE:-unknown} - - TF_VAR_BUILD_ID=${BUILD_ID:-unknown} - - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} - - TF_VAR_REPO=${REPO:-unknown} - volumes: - - ${TF_DIR}:/stage - - ${TF_OUTPUT_DIR}:/output - - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ diff --git a/internal/servicedeployer/_static/terraform_deployer.yml.tmpl b/internal/servicedeployer/_static/terraform_deployer.yml.tmpl new file mode 100644 index 000000000..928b9040b --- /dev/null +++ b/internal/servicedeployer/_static/terraform_deployer.yml.tmpl @@ -0,0 +1,34 @@ +{{- $google_application_credentials := fact "google_application_credentials" -}} +{{- $google_credential_source_file := fact "google_credential_source_file" -}} +services: + terraform: + build: . + tty: true + stop_grace_period: 5m + environment: + - TF_VAR_TEST_RUN_ID=${TF_VAR_TEST_RUN_ID:-detached} + - TF_VAR_CREATED_DATE=${CREATED_DATE:-unknown} + - TF_VAR_BRANCH=${BRANCH_NAME_LOWER_CASE:-unknown} + - TF_VAR_BUILD_ID=${BUILD_ID:-unknown} + - TF_VAR_ENVIRONMENT=${ENVIRONMENT:-unknown} + - TF_VAR_REPO=${REPO:-unknown} + volumes: + - ${TF_DIR}:/stage + - ${TF_OUTPUT_DIR}:/output + - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ + {{ if ne $google_application_credentials "" }} + # Mount Google Application Credentials file provided + # Required for external accounts authentication when creating resources via terraform in GCP + - type: bind + source: {{ $google_application_credentials }} + target: {{ $google_application_credentials }} + read_only: true + {{ end }} + {{ if ne $google_credential_source_file "" }} + # Mount Google credential source file (token file) provided + # Required for external accounts authentication when creating resources via terraform in GCP + - type: bind + source: {{ $google_credential_source_file }} + target: {{ $google_credential_source_file }} + read_only: true + {{ end }} diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 54b62154d..dadd0ef97 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -6,7 +6,7 @@ package servicedeployer import ( "context" - _ "embed" + "embed" "encoding/json" "fmt" "os" @@ -29,14 +29,18 @@ const ( terraformDeployerRun = "run.sh" terraformOutputPrefix = "TF_OUTPUT_" terraformOutputJSONFile = "tfOutputValues.json" -) -//go:embed _static/terraform_deployer.yml -var terraformDeployerYmlContent string + terraformDeployerYmlTemplatePath = "_static/terraform_deployer.yml.tmpl" +) //go:embed _static/terraform_deployer_run.sh var terraformDeployerRunContent string +//go:embed _static +var tfStatic embed.FS + +var tfStaticSource = resource.NewSourceFS(tfStatic) + //go:embed _static/Dockerfile.terraform_deployer var terraformDeployerDockerfileContent string @@ -178,12 +182,34 @@ func (tsd TerraformServiceDeployer) installDockerfile(folder string) (string, er return "", fmt.Errorf("failed to find the configuration directory: %w", err) } + googleApplicationCredentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + if googleApplicationCredentials != "" { + if _, err := os.Stat(googleApplicationCredentials); err != nil { + logger.Warn("GOOGLE_APPLICATION_CREDENTIALS environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") + googleApplicationCredentials = "" + } + } + + googleCredentialSourceFile := os.Getenv("GOOGLE_CREDENTIAL_SOURCE_FILE") + if googleCredentialSourceFile != "" { + if _, err := os.Stat(googleCredentialSourceFile); err != nil { + logger.Warn("GOOGLE_CREDENTIAL_SOURCE_FILE environment variable is set, but the file does not exist. Skipping inclusion in agent configuration.") + googleCredentialSourceFile = "" + } + } + + resourceManager := resource.NewManager() + resourceManager.AddFacter(resource.StaticFacter{ + "google_credential_source_file": googleCredentialSourceFile, + "google_application_credentials": googleApplicationCredentials, + }) + tfDir := filepath.Join(locationManager.DeployerDir(), terraformDeployerDir, folder) resources := []resource.Resource{ &resource.File{ Path: terraformDeployerYml, - Content: resource.FileContentLiteral(terraformDeployerYmlContent), + Content: tfStaticSource.Template(terraformDeployerYmlTemplatePath), CreateParent: true, }, &resource.File{ @@ -198,7 +224,6 @@ func (tsd TerraformServiceDeployer) installDockerfile(folder string) (string, er }, } - resourceManager := resource.NewManager() resourceManager.RegisterProvider("file", &resource.FileProvider{ Prefix: tfDir, })