Skip to content

Adding api_request_count self-metric when running otlphttp exporter #2196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rafaelwestphal wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Adding api_request_count self-metric when running otlphttp exporter #2196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cddc711
Adding equivalent self-metric when running otlphttp exporter
rafaelwestphal Feb 4, 2026
525f259
Filter self-metric
rafaelwestphal Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
60 changes: 48 additions & 12 deletions confgenerator/agentmetrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package confgenerator

import (
"context"
"fmt"
"path/filepath"
"time"
Expand Down Expand Up @@ -56,15 +57,32 @@ var grpcToHTTPStatus = map[string]string{
"DEADLINE_EXCEEDED": "504",
}

func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]otel.ReceiverPipeline, pipelines map[string]otel.Pipeline) {
// Source: https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto
// HTTPStringToGRPC maps string representations of HTTP status codes to gRPC codes.
var HTTPStringToGRPC = map[string]string{
"200": "OK",
"400": "INVALID_ARGUMENT", // Chosen over FAILED_PRECONDITION, OUT_OF_RANGE
"401": "UNAUTHENTICATED",
"403": "PERMISSION_DENIED",
"404": "NOT_FOUND",
"409": "ABORTED", // Chosen over ALREADY_EXISTS
"429": "RESOURCE_EXHAUSTED",
"499": "CANCELLED",
"500": "INTERNAL", // Chosen over UNKNOWN, DATA_LOSS
"501": "UNIMPLEMENTED",
"503": "UNAVAILABLE",
"504": "DEADLINE_EXCEEDED",
}

func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]otel.ReceiverPipeline, pipelines map[string]otel.Pipeline, ctx context.Context) {
// Receiver pipelines names should have 1 underscore to avoid collision with user configurations.
receiverPipelines["agent_prometheus"] = r.PrometheusMetricsPipeline()

// Pipeline names should have no underscores to avoid collision with user configurations.
pipelines["otel"] = otel.Pipeline{
Type: "metrics",
ReceiverPipelineName: "agent_prometheus",
Processors: r.OtelPipelineProcessors(),
Processors: r.OtelPipelineProcessors(ctx),
}

pipelines["fluentbit"] = otel.Pipeline{
Expand Down Expand Up @@ -131,15 +149,37 @@ func (r AgentSelfMetrics) PrometheusMetricsPipeline() otel.ReceiverPipeline {
}
}

func (r AgentSelfMetrics) OtelPipelineProcessors() []otel.Component {
func (r AgentSelfMetrics) OtelPipelineProcessors(ctx context.Context) []otel.Component {
apiRequestCount :=
otel.RenameMetric("grpc.client.attempt.duration_count", "agent/api_request_count",
otel.RenameLabel("grpc.status", "state"),
// delete grpc_client_method dimension & service.version label, retaining only state
otel.AggregateLabels("sum", "state"),
)
metricFilter := otel.MetricsOTTLFilter([]string{}, []string{
// Filter out histogram datapoints where the grpc.target is not related to monitoring.
`metric.name == "grpc.client.attempt.duration_count" and (not IsMatch(datapoint.attributes["grpc.target"], "monitoring.googleapis"))`,
})
expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
if expOtlpExporter {
apiRequestCount =
otel.RenameMetric("http_client_request_duration_count", "agent/api_request_count",
otel.RenameLabel("http_response_status_code", "state"),
otel.RenameLabelValues("http_response_status_code", HTTPStringToGRPC),
// delete grpc_client_method dimension & service.version label, retaining only state
otel.AggregateLabels("sum", "state"),
)
metricFilter = otel.MetricsOTTLFilter([]string{}, []string{
// Filter out histogram datapoints where the server_address is not related to monitoring.
`metric.name == "http_client_request_duration_count" and (not IsMatch(datapoint.attributes["server_address"], "telemetry.googleapis.com"))`,
})
}

return []otel.Component{
otel.Transform("metric", "metric",
ottl.ExtractCountMetric(true, "grpc.client.attempt.duration"),
),
otel.MetricsOTTLFilter([]string{}, []string{
// Filter out histogram datapoints where the grpc.target is not related to monitoring.
`metric.name == "grpc.client.attempt.duration_count" and (not IsMatch(datapoint.attributes["grpc.target"], "monitoring.googleapis"))`,
}),
metricFilter,
otel.MetricsFilter(
"include",
"strict",
Expand All @@ -160,11 +200,7 @@ func (r AgentSelfMetrics) OtelPipelineProcessors() []otel.Component {
// remove service.version label
otel.AggregateLabels("sum"),
),
otel.RenameMetric("grpc.client.attempt.duration_count", "agent/api_request_count",
otel.RenameLabel("grpc.status", "state"),
// delete grpc_client_method dimension & service.version label, retaining only state
otel.AggregateLabels("sum", "state"),
),
apiRequestCount,
otel.RenameMetric("googlecloudmonitoring/point_count", "agent/monitoring/point_count",
// change data type from double -> int64
otel.ToggleScalarDataType,
Expand Down
2 changes: 1 addition & 1 deletion confgenerator/confgenerator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDi
OtelRuntimeDir: outDir,
OtelLogging: uc.Logging.Service.OTelLogging,
}
agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines)
agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx)

otelConfig, err := otel.ModularConfig{
LogLevel: uc.getOTelLogLevel(),
Expand Down
1 change: 1 addition & 0 deletions confgenerator/otel/modular.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ func (c ModularConfig) Generate(ctx context.Context) (string, error) {
"pipelines": pipelines,
"telemetry": map[string]interface{}{
"metrics": map[string]interface{}{
"level": "detailed",
"readers": []map[string]interface{}{{
"pull": map[string]interface{}{
"exporter": map[string]interface{}{
Expand Down
1 change: 1 addition & 0 deletions ...stdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...r/testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...ata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...testdata/goldens/all-backward_compatible_with_explicit_exporters/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,6 +749,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-custom_use_built_in_receivers/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -701,6 +701,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-quoted_map_keys/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-user_config_file_deleted/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/all-user_config_file_deleted/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/builtin/golden/linux-dataproc/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/builtin/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/builtin/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/builtin/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/builtin/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,7 @@ service:
- prometheus/agent_prometheus
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -830,6 +830,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,6 +790,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,6 +925,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,6 +925,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions .../testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...ator/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...stdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...or/testdata/goldens/combined-receiver_otlp_googlecloudmonitoring/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...ns/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...oldens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...dens/combined-receiver_otlp_googlecloudmonitoring_with_processor/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...estdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...or/testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...data/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows-2012/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -881,6 +881,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions .../testdata/goldens/combined-receiver_otlp_googlemanagedprometheus/golden/windows/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -881,6 +881,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions ...generator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux-gpu/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
1 change: 1 addition & 0 deletions confgenerator/testdata/goldens/combined-receiver_otlp_grpcendpoint/golden/linux/otel.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,7 @@ service:
- otlp/otlp
telemetry:
metrics:
level: detailed
readers:
- pull:
exporter:
Expand Down
Loading
Loading