From be6856efe095c064cf8f1169888b10a901381580 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Fri, 6 Feb 2026 12:29:24 +0000 Subject: [PATCH 1/2] Add support for 'otel.java.metrics.cardinality.limit' system property --- .../shim/metrics/data/OtelMetricStorage.java | 8 ++++---- .../datadog/trace/api/ConfigDefaults.java | 5 +++-- .../datadog/trace/api/config/OtlpConfig.java | 1 + .../main/java/datadog/trace/api/Config.java | 20 +++++++++++++++++++ .../provider/OtelEnvironmentConfigSource.java | 5 +++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java index 39b6edd9fd6..daa81e0bfc2 100644 --- a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java +++ b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java @@ -23,7 +23,7 @@ public final class OtelMetricStorage { private static final RatelimitedLogger RATELIMITED_LOGGER = new RatelimitedLogger(LOGGER, 5, TimeUnit.MINUTES); - private static final int DEFAULT_MAX_CARDINALITY = 2_000; + private static final int CARDINALITY_LIMIT = Config.get().getMetricsOtelCardinalityLimit(); private static final Attributes CARDINALITY_OVERFLOW = Attributes.builder().put("otel.metric.overflow", true).build(); @@ -102,11 +102,11 @@ private OtelAggregator aggregator( if (null != aggregator) { return aggregator; } - if (aggregators.size() >= DEFAULT_MAX_CARDINALITY) { + if (aggregators.size() >= CARDINALITY_LIMIT) { RATELIMITED_LOGGER.warn( "Instrument {} has exceeded the maximum allowed cardinality ({}).", descriptor.getName(), - DEFAULT_MAX_CARDINALITY); + CARDINALITY_LIMIT); attributes = CARDINALITY_OVERFLOW; // write data to overflow } return aggregators.computeIfAbsent(attributes, aggregatorSupplier); @@ -153,7 +153,7 @@ private void doCollectAndReset(OtelInstrumentVisitor visitor) { Map aggregators = recording.aggregators; // avoid churn: only remove empty aggregators if we're over cardinality - if (aggregators.size() >= DEFAULT_MAX_CARDINALITY) { + if (aggregators.size() >= CARDINALITY_LIMIT) { aggregators.values().removeIf(OtelAggregator::isEmpty); } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index ecdd8dca43e..9fa15e8f241 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -104,9 +104,10 @@ public final class ConfigDefaults { public static final boolean DEFAULT_METRICS_OTEL_ENABLED = false; // Default recommended by Datadog; it differs from Otel’s default of 60000 (60s) - static final int DEFAULT_METRICS_OTEL_INTERVAL = 10000; // ms + static final int DEFAULT_METRICS_OTEL_INTERVAL = 10_000; // ms // Default recommended by Datadog; it differs from Otel’s default of 30000 (30s) - static final int DEFAULT_METRICS_OTEL_TIMEOUT = 7500; // ms + static final int DEFAULT_METRICS_OTEL_TIMEOUT = 7_500; // ms + static final int DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT = 2_000; static final String DEFAULT_OTLP_HTTP_METRIC_ENDPOINT = "v1/metrics"; static final String DEFAULT_OTLP_HTTP_PORT = "4318"; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java index 29d7e144ca4..32c59f9301a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java @@ -5,6 +5,7 @@ public final class OtlpConfig { public static final String METRICS_OTEL_ENABLED = "metrics.otel.enabled"; public static final String METRICS_OTEL_INTERVAL = "metrics.otel.interval"; public static final String METRICS_OTEL_TIMEOUT = "metrics.otel.timeout"; + public static final String METRICS_OTEL_CARDINALITY_LIMIT = "metrics.otel.cardinality.limit"; public static final String OTLP_METRICS_ENDPOINT = "otlp.metrics.endpoint"; public static final String OTLP_METRICS_HEADERS = "otlp.metrics.headers"; diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index e54aa7bb3b6..88f28845246 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -111,6 +111,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_JMX_FETCH_MULTIPLE_RUNTIME_SERVICES_LIMIT; import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_AGENTLESS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_LOGS_INJECTION_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT; import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_INTERVAL; import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_TIMEOUT; @@ -444,6 +445,7 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS; import static datadog.trace.api.config.LlmObsConfig.LLMOBS_AGENTLESS_ENABLED; import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ML_APP; +import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_CARDINALITY_LIMIT; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_ENABLED; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT; @@ -913,6 +915,7 @@ public static String getHostName() { private final boolean metricsOtelEnabled; private final int metricsOtelInterval; private final int metricsOtelTimeout; + private final int metricsOtelCardinalityLimit; private final String otlpMetricsEndpoint; private final Map otlpMetricsHeaders; private final OtlpConfig.Protocol otlpMetricsProtocol; @@ -1887,6 +1890,17 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins metricsOtelEnabled = configProvider.getBoolean(METRICS_OTEL_ENABLED, DEFAULT_METRICS_OTEL_ENABLED); + int cardinalityLimit = + configProvider.getInteger( + METRICS_OTEL_CARDINALITY_LIMIT, DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT); + if (cardinalityLimit < 0) { + log.warn( + "Invalid OTel metrics cardinality limit: {}. The value must be positive", + cardinalityLimit); + cardinalityLimit = DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT; + } + metricsOtelCardinalityLimit = cardinalityLimit; + int otelInterval = configProvider.getInteger(METRICS_OTEL_INTERVAL, DEFAULT_METRICS_OTEL_INTERVAL); if (otelInterval < 0) { @@ -5189,6 +5203,10 @@ public boolean isMetricsOtelEnabled() { return metricsOtelEnabled; } + public int getMetricsOtelCardinalityLimit() { + return metricsOtelCardinalityLimit; + } + public int getMetricsOtelInterval() { return metricsOtelInterval; } @@ -6232,6 +6250,8 @@ public String toString() { + metricsOtelInterval + ", metricsOtelTimeout=" + metricsOtelTimeout + + ", metricsOtelCardinalityLimit=" + + metricsOtelCardinalityLimit + ", otlpMetricsEndpoint=" + otlpMetricsEndpoint + ", otlpMetricsHeaders=" diff --git a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java index d8b12c81280..d7d3e07fa67 100644 --- a/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java +++ b/utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java @@ -8,6 +8,7 @@ import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME; import static datadog.trace.api.config.GeneralConfig.TAGS; import static datadog.trace.api.config.GeneralConfig.VERSION; +import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_CARDINALITY_LIMIT; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_ENABLED; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL; import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT; @@ -144,6 +145,10 @@ private void setupMetricsOtelEnvironment() { capture( METRICS_OTEL_TIMEOUT, getOtelProperty("otel.metric.export.timeout", "dd." + METRICS_OTEL_TIMEOUT)); + capture( + METRICS_OTEL_CARDINALITY_LIMIT, + getOtelProperty( + "otel.java.metrics.cardinality.limit", "dd." + METRICS_OTEL_CARDINALITY_LIMIT)); String exporter = getOtelProperty("otel.metrics.exporter"); if (exporter == null || "otlp".equalsIgnoreCase(exporter)) { From 81424cbc4b88538c7089b3d06a8915517e60a0c1 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Fri, 6 Feb 2026 15:42:30 +0000 Subject: [PATCH 2/2] Add OTel/DD cardinality limit config keys --- metadata/supported-configurations.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 7880bc2f843..8dca3dee711 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -2265,6 +2265,14 @@ "aliases": [] } ], + "DD_METRICS_OTEL_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "int", + "default": "2000", + "aliases": [] + } + ], "DD_OBFUSCATION_QUERY_STRING_REGEXP": [ { "version": "A", @@ -11096,6 +11104,14 @@ "default": null, "aliases": [] } + ], + "OTEL_JAVA_METRICS_CARDINALITY_LIMIT": [ + { + "version": "A", + "type": "int", + "default": "2000", + "aliases": [] + } ] }, "deprecations": {}