From 17cfa85f8c3146ef3afaeae8e6fb497ce4f66940 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 21 Jul 2025 14:08:10 +0000 Subject: [PATCH 1/7] generated test files --- apps/elasticsearch.go | 1 + confgenerator/testdata/feature/golden.csv | 1 + .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../config.yaml | 1 + .../input.log | 15 ++ .../output_fluentbit.yaml | 201 ++++++++++++++++++ .../output_otel.yaml | 1 + 10 files changed, 224 insertions(+), 4 deletions(-) create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-json/config.yaml create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-json/input.log create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-json/output_fluentbit.yaml create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index 357ccf9c74..26dd23e8ff 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -332,6 +332,7 @@ func (r LoggingReceiverElasticsearchGC) Components(ctx context.Context, tag stri } func init() { + confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorElasticsearchJson{} }) confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchJson{} }) confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchGC{} }) } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 6095bd2c9a..21960af399 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -17,6 +17,7 @@ App,Field,Override, *apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingProcessorCouchdb,confgenerator.ConfigComponent.Type, +*apps.LoggingProcessorElasticsearchJson,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorHbaseSystem,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorIisAccess,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorJettyAccess,confgenerator.ConfigComponent.Type, diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 10751c0cf5..4f3c8f92e8 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 10751c0cf5..4f3c8f92e8 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 1649e4e323..13924fadc6 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 1649e4e323..13924fadc6 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/transformation_test/testdata/logging_processor-elasticsearch-json/config.yaml b/transformation_test/testdata/logging_processor-elasticsearch-json/config.yaml new file mode 100644 index 0000000000..947bbbb023 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-json/config.yaml @@ -0,0 +1 @@ +- type: elasticsearch_json diff --git a/transformation_test/testdata/logging_processor-elasticsearch-json/input.log b/transformation_test/testdata/logging_processor-elasticsearch-json/input.log new file mode 100644 index 0000000000..9d2090e657 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-json/input.log @@ -0,0 +1,15 @@ +{"type": "server", "timestamp": "2024-06-10T12:34:56,789Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "node-1", "message": "starting ..."} +{"type": "server", "timestamp": "2024-06-10T12:35:00,321Z", "level": "WARN", "component": "o.e.d.z.ZenDiscovery", "node.name": "node-1", "message": "failed to connect to master", "user.run_by.name": "admin", "event.action": "connect"} +{"type": "server", "timestamp": "2024-06-10T12:35:10,123Z", "level": "ERROR", "component": "o.e.b.Bootstrap", "node.name": "node-1", "message": "Exception", "error.type": "java.lang.IllegalStateException", "error.message": "failed to obtain node locks"} +{"type": "server", "timestamp": "2024-06-10T12:35:15,456Z", "level": "DEBUG", "component": "o.e.n.Node", "node.name": "node-1", "message": "stopping ..."} +{"type": "server", "timestamp": "2024-06-10T12:35:16,789Z", "level": "TRACE", "component": "o.e.n.Node", "node.name": "node-1", "message": "stopped"} +{"type": "server", "timestamp": "2024-06-10T12:35:17,123Z", "level": "DEPRECATION", "component": "o.e.n.Node", "node.name": "node-1", "message": "deprecated setting used"} +{"type": "server", "timestamp": "2024-06-10T12:35:18,456Z", "level": "CRITICAL", "component": "o.e.n.Node", "node.name": "node-1", "message": "critical error occurred"} +{"type": "server", "timestamp": "2024-06-10T12:35:19,456Z", "level": "FATAL", "component": "o.e.n.Node", "node.name": "node-1", "message": "fatal error occurred"} +{"type": "server", "timestamp": "2024-06-10T12:35:20,456Z", "level": "UNKNOWN", "component": "o.e.n.Node", "node.name": "node-1", "message": "unknown log level"} +{"type": "server", "timestamp": "2024-06-10T12:35:21,456Z", "level": "INFO", "message": "missing component and node name"} +{"type": "server", "timestamp": "2024-06-10T12:35:22,456Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "message": "missing node name"} +{"type": "server", "timestamp": "2024-06-10T12:35:23,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "missing cluster name"} +{"type": "server", "timestamp": "2024-06-10T12:35:24,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "nested fields", "user.run_by.name": "testuser", "authentication.token.id": "abc123"} +{"type": "server", "timestamp": "2024-06-10T12:35:25,456Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "node.name": "node-1", "message": "uncaught exception in thread [main]", "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key]", "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166)", "... 6 more"]} +{"type": "server", "timestamp": "2024-06-10T12:35:26,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "invalid timestamp", "timestamp": "not-a-timestamp"} diff --git a/transformation_test/testdata/logging_processor-elasticsearch-json/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-elasticsearch-json/output_fluentbit.yaml new file mode 100644 index 0000000000..87dfaf06ba --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-json/output_fluentbit.yaml @@ -0,0 +1,201 @@ +- entries: + - jsonPayload: + cluster: + name: elasticsearch + component: o.e.n.Node + level: INFO + message: starting ... + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:34:56.789000000Z + - jsonPayload: + component: o.e.d.z.ZenDiscovery + event: + action: connect + level: WARN + message: failed to connect to master + node: + name: node-1 + type: server + user: + run_by: + name: admin + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: 2024-06-10T12:35:00.321000000Z + - jsonPayload: + component: o.e.b.Bootstrap + error.message: failed to obtain node locks + error.type: java.lang.IllegalStateException + level: ERROR + message: Exception + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: 2024-06-10T12:35:10.123000000Z + - jsonPayload: + component: o.e.n.Node + level: DEBUG + message: stopping ... + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 100.0 + timestamp: 2024-06-10T12:35:15.456000000Z + - jsonPayload: + component: o.e.n.Node + level: TRACE + message: stopped + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 100.0 + timestamp: 2024-06-10T12:35:16.789000000Z + - jsonPayload: + component: o.e.n.Node + level: DEPRECATION + message: deprecated setting used + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: 2024-06-10T12:35:17.123000000Z + - jsonPayload: + component: o.e.n.Node + level: CRITICAL + message: critical error occurred + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: 2024-06-10T12:35:18.456000000Z + - jsonPayload: + component: o.e.n.Node + level: FATAL + message: fatal error occurred + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 600.0 + timestamp: 2024-06-10T12:35:19.456000000Z + - jsonPayload: + component: o.e.n.Node + level: UNKNOWN + message: unknown log level + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + timestamp: 2024-06-10T12:35:20.456000000Z + - jsonPayload: + level: INFO + message: missing component and node name + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:35:21.456000000Z + - jsonPayload: + cluster: + name: elasticsearch + component: o.e.n.Node + level: INFO + message: missing node name + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:35:22.456000000Z + - jsonPayload: + component: o.e.n.Node + level: INFO + message: missing cluster name + node: + name: node-1 + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:35:23.456000000Z + - jsonPayload: + authentication: + token: + id: abc123 + component: o.e.n.Node + level: INFO + message: nested fields + node: + name: node-1 + type: server + user: + run_by: + name: testuser + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:35:24.456000000Z + - jsonPayload: + component: o.e.b.ElasticsearchUncaughtExceptionHandler + level: ERROR + message: uncaught exception in thread [main] + node: + name: node-1 + stacktrace: + - "org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key]" + - at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) + - ... 6 more + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: 2024-06-10T12:35:25.456000000Z + - jsonPayload: + component: o.e.n.Node + level: INFO + message: invalid timestamp + node: + name: node-1 + timestamp: not-a-timestamp + type: server + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2024-06-10T12:35:26.456000000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml b/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml new file mode 100644 index 0000000000..b5cfe0a3c6 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml @@ -0,0 +1 @@ +- config_error: "not an OTel processor: &apps.LoggingProcessorElasticsearchJson{ConfigComponent:confgenerator.ConfigComponent{Type:\"elasticsearch_json\"}}" From 5315299e17a084e9b2eace2f68f6970c188c352b Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 21 Jul 2025 14:41:22 +0000 Subject: [PATCH 2/7] refactored elasticsearch json --- apps/elasticsearch.go | 106 ++++++++---------- confgenerator/logging_processors.go | 6 +- confgenerator/testdata/feature/golden.csv | 8 +- .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 8 ++ .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 8 ++ .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 8 ++ .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 8 ++ .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 12 ++ .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 12 ++ .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 12 ++ .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 12 ++ .../output_otel.yaml | 2 +- 20 files changed, 141 insertions(+), 77 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index 26dd23e8ff..5ecb362fef 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -149,38 +149,35 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverElasticsearch{} }) } -type LoggingProcessorElasticsearchJson struct { - confgenerator.ConfigComponent `yaml:",inline"` +type LoggingProcessorMacroElasticsearchJson struct { } -func (LoggingProcessorElasticsearchJson) Type() string { +func (LoggingProcessorMacroElasticsearchJson) Type() string { return "elasticsearch_json" } -func (p LoggingProcessorElasticsearchJson) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} - - // sample log line: - // {"type": "server", "timestamp": "2022-01-17T18:31:47,365Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "ubuntu-jammy", "message": "initialized" } - // Logs are formatted based on configuration (log4j); - // See https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/7.16.2/org/elasticsearch/common/logging/ESJsonLayout.html - // for general layout, and https://www.elastic.co/guide/en/elasticsearch/reference/current/logging.html for general configuration of logging - - jsonParser := &confgenerator.LoggingProcessorParseJson{ - ParserShared: confgenerator.ParserShared{ - TimeKey: "timestamp", - TimeFormat: "%Y-%m-%dT%H:%M:%S,%L%z", +func (p LoggingProcessorMacroElasticsearchJson) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + // // sample log line: + // // {"type": "server", "timestamp": "2022-01-17T18:31:47,365Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "ubuntu-jammy", "message": "initialized" } + // // Logs are formatted based on configuration (log4j); + // // See https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/7.16.2/org/elasticsearch/common/logging/ESJsonLayout.html + // // for general layout, and https://www.elastic.co/guide/en/elasticsearch/reference/current/logging.html for general configuration of logging + processors := []confgenerator.InternalLoggingProcessor{ + confgenerator.LoggingProcessorParseJson{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S,%L%z", + }, }, + p.severityParser(), } - c = append(c, jsonParser.Components(ctx, tag, uid)...) - c = append(c, p.severityParser(ctx, tag, uid)...) - c = append(c, p.nestingProcessors(ctx, tag, uid)...) + processors = append(processors, p.nestingProcessors()...) - return c + return processors } -func (p LoggingProcessorElasticsearchJson) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component { +func (p LoggingProcessorMacroElasticsearchJson) severityParser() confgenerator.InternalLoggingProcessor { return confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "severity": { @@ -199,10 +196,10 @@ func (p LoggingProcessorElasticsearchJson) severityParser(ctx context.Context, t }, InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, uid) + } } -func (p LoggingProcessorElasticsearchJson) nestingProcessors(ctx context.Context, tag, uid string) []fluentbit.Component { +func (p LoggingProcessorMacroElasticsearchJson) nestingProcessors() []confgenerator.InternalLoggingProcessor { // The majority of these prefixes come from here: // https://www.elastic.co/guide/en/elasticsearch/reference/7.16/audit-event-types.html#audit-event-attributes // Non-audit logs are formatted using the layout documented here, giving the "cluster" prefix: @@ -223,17 +220,17 @@ func (p LoggingProcessorElasticsearchJson) nestingProcessors(ctx context.Context "cluster", } - c := make([]fluentbit.Component, 0, len(prefixes)) + processors := make([]confgenerator.InternalLoggingProcessor, 0, len(prefixes)) for _, prefix := range prefixes { nestProcessor := confgenerator.LoggingProcessorNestWildcard{ Wildcard: fmt.Sprintf("%s.*", prefix), NestUnder: prefix, RemovePrefix: fmt.Sprintf("%s.", prefix), } - c = append(c, nestProcessor.Components(ctx, tag, uid)...) + processors = append(processors, nestProcessor) } - return c + return processors } type LoggingProcessorElasticsearchGC struct { @@ -271,47 +268,37 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui return c } -type LoggingReceiverElasticsearchJson struct { - LoggingProcessorElasticsearchJson `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` -} - -func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(r.ReceiverMixin.IncludePaths) == 0 { +func loggingReceiverFilesMixinElasticsearchJson() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ // Default JSON logs for Elasticsearch - r.ReceiverMixin.IncludePaths = []string{ + IncludePaths: []string{ "/var/log/elasticsearch/*_server.json", "/var/log/elasticsearch/*_deprecation.json", "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json", "/var/log/elasticsearch/*_audit.json", - } - } - - // When Elasticsearch emits stack traces, the json log may be spread across multiple lines, - // so we need this multiline parsing to properly parse the record. - // Example multiline log record: - // {"type": "server", "timestamp": "2022-01-20T15:46:00,131Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "brandon-testing-elasticsearch", "message": "uncaught exception in thread [main]", - // "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key] please check that any required plugins are installed, or check the breaking changes documentation for removed settings", - // -- snip -- - // "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]", - // "... 6 more"] } - - r.ReceiverMixin.MultilineRules = []confgenerator.MultilineRule{ - { - StateName: "start_state", - NextState: "cont", - Regex: `^{.*`, }, - { - StateName: "cont", - NextState: "cont", - Regex: `^[^{].*[,}]$`, + // When Elasticsearch emits stack traces, the json log may be spread across multiple lines, + // so we need this multiline parsing to properly parse the record. + // Example multiline log record: + // {"type": "server", "timestamp": "2022-01-20T15:46:00,131Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "brandon-testing-elasticsearch", "message": "uncaught exception in thread [main]", + // "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key] please check that any required plugins are installed, or check the breaking changes documentation for removed settings", + // -- snip -- + // "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]", + // "... 6 more"] } + MultilineRules: []confgenerator.MultilineRule{ + { + StateName: "start_state", + NextState: "cont", + Regex: `^{.*`, + }, + { + StateName: "cont", + NextState: "cont", + Regex: `^[^{].*[,}]$`, + }, }, } - - c := r.ReceiverMixin.Components(ctx, tag) - return append(c, r.LoggingProcessorElasticsearchJson.Components(ctx, tag, "elasticsearch_json")...) } type LoggingReceiverElasticsearchGC struct { @@ -332,7 +319,6 @@ func (r LoggingReceiverElasticsearchGC) Components(ctx context.Context, tag stri } func init() { - confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorElasticsearchJson{} }) - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchJson{} }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroElasticsearchJson](loggingReceiverFilesMixinElasticsearchJson) confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchGC{} }) } diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 74d5e0f1df..7f05eddb41 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -320,9 +320,9 @@ func (p LoggingProcessorParseRegexComplex) Components(ctx context.Context, tag, } type MultilineRule struct { - StateName string - Regex string - NextState string + StateName string `yaml:"state_name,omitempty"` + Regex string `yaml:"regex,omitempty"` + NextState string `yaml:"next_state,omitempty"` } func (r MultilineRule) AsString() string { diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 21960af399..7c8ee29993 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -17,7 +17,6 @@ App,Field,Override, *apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingProcessorCouchdb,confgenerator.ConfigComponent.Type, -*apps.LoggingProcessorElasticsearchJson,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorHbaseSystem,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorIisAccess,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorJettyAccess,confgenerator.ConfigComponent.Type, @@ -67,10 +66,6 @@ App,Field,Override, *apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, *apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, -*apps.LoggingReceiverElasticsearchJson,apps.LoggingProcessorElasticsearchJson.confgenerator.ConfigComponent.Type, -*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingReceiverHadoop,apps.LoggingProcessorHadoop.confgenerator.ConfigComponent.Type, *apps.LoggingReceiverHadoop,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, *apps.LoggingReceiverHadoop,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, @@ -250,10 +245,13 @@ App,Field,Override, *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_interval *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_timeout *confgenerator.PrometheusMetrics,config.[].scrape_configs.static_config_target_groups +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroRedis],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],ReceiverMacro, diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json index 7a8c10da4d..70d17cb2a1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml index b6100396aa..a5a83dfb2f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml @@ -22,3 +22,11 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json index 7a8c10da4d..70d17cb2a1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml index b6100396aa..a5a83dfb2f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml @@ -22,3 +22,11 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json index 7a8c10da4d..70d17cb2a1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml index b6100396aa..a5a83dfb2f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml @@ -22,3 +22,11 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json index 7a8c10da4d..70d17cb2a1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml index b6100396aa..a5a83dfb2f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml @@ -22,3 +22,11 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[1].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json index dfc3c268e1..bad5b9731b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml index d3d7484567..e93c89703d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml @@ -34,7 +34,19 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" +- module: logging + feature: receivers:elasticsearch_json + key: "[2].-.__length" + value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json index dfc3c268e1..bad5b9731b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml index d3d7484567..e93c89703d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml @@ -34,7 +34,19 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" +- module: logging + feature: receivers:elasticsearch_json + key: "[2].-.__length" + value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json index dfc3c268e1..bad5b9731b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml index d3d7484567..e93c89703d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml @@ -34,7 +34,19 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" +- module: logging + feature: receivers:elasticsearch_json + key: "[2].-.__length" + value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].-.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json index dfc3c268e1..bad5b9731b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml index d3d7484567..e93c89703d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml @@ -34,7 +34,19 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" +- module: logging + feature: receivers:elasticsearch_json + key: "[2].-.__length" + value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].include_paths.__length" + value: "5" +- module: logging + feature: receivers:elasticsearch_json + key: "[3].-.__length" + value: "2" diff --git a/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml b/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml index b5cfe0a3c6..bff2febbfa 100644 --- a/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-elasticsearch-json/output_otel.yaml @@ -1 +1 @@ -- config_error: "not an OTel processor: &apps.LoggingProcessorElasticsearchJson{ConfigComponent:confgenerator.ConfigComponent{Type:\"elasticsearch_json\"}}" +- config_error: "failed generating OTel processor: &confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]{ConfigComponent:confgenerator.ConfigComponent{Type:\"elasticsearch_json\"}, ProcessorMacro:apps.LoggingProcessorMacroElasticsearchJson{}}, err: unimplemented" From b603b9a6af028b62b2fdd8a8ae642f36dcf4ec15 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 16:03:11 +0000 Subject: [PATCH 3/7] removed double comments --- apps/elasticsearch.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index 5ecb362fef..b59e4ee20f 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -157,11 +157,11 @@ func (LoggingProcessorMacroElasticsearchJson) Type() string { } func (p LoggingProcessorMacroElasticsearchJson) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { - // // sample log line: - // // {"type": "server", "timestamp": "2022-01-17T18:31:47,365Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "ubuntu-jammy", "message": "initialized" } - // // Logs are formatted based on configuration (log4j); - // // See https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/7.16.2/org/elasticsearch/common/logging/ESJsonLayout.html - // // for general layout, and https://www.elastic.co/guide/en/elasticsearch/reference/current/logging.html for general configuration of logging + // sample log line: + // {"type": "server", "timestamp": "2022-01-17T18:31:47,365Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "ubuntu-jammy", "message": "initialized" } + // Logs are formatted based on configuration (log4j); + // See https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/7.16.2/org/elasticsearch/common/logging/ESJsonLayout.html + // for general layout, and https://www.elastic.co/guide/en/elasticsearch/reference/current/logging.html for general configuration of logging processors := []confgenerator.InternalLoggingProcessor{ confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ From b3b6f00a00de763168012ffea2ae5154225a596f Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 17:16:51 +0000 Subject: [PATCH 4/7] move elasticsearch processor init to transform tests only --- apps/elasticsearch.go | 58 +++++++++++-------- confgenerator/testdata/feature/golden.csv | 7 ++- .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 8 --- .../golden/linux-gpu/fluent_bit_main.conf | 21 ------- .../golden/linux-gpu/fluent_bit_parser.conf | 7 --- .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 8 --- .../golden/linux/fluent_bit_main.conf | 21 ------- .../golden/linux/fluent_bit_parser.conf | 7 --- .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 8 --- .../golden/windows-2012/fluent_bit_main.conf | 21 ------- .../windows-2012/fluent_bit_parser.conf | 7 --- .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 8 --- .../golden/windows/fluent_bit_main.conf | 21 ------- .../golden/windows/fluent_bit_parser.conf | 7 --- .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 12 ---- .../golden/linux-gpu/fluent_bit_main.conf | 43 -------------- .../golden/linux-gpu/fluent_bit_parser.conf | 14 ----- .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 12 ---- .../golden/linux/fluent_bit_main.conf | 43 -------------- .../golden/linux/fluent_bit_parser.conf | 14 ----- .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 12 ---- .../golden/windows-2012/fluent_bit_main.conf | 43 -------------- .../windows-2012/fluent_bit_parser.conf | 14 ----- .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 12 ---- .../golden/windows/fluent_bit_main.conf | 43 -------------- .../golden/windows/fluent_bit_parser.conf | 14 ----- transformation_test/transformation_test.go | 3 + 39 files changed, 54 insertions(+), 458 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index b59e4ee20f..095b4b6ba6 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -268,37 +268,47 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui return c } -func loggingReceiverFilesMixinElasticsearchJson() confgenerator.LoggingReceiverFilesMixin { - return confgenerator.LoggingReceiverFilesMixin{ +type LoggingReceiverMacroElasticsearchJson struct { + LoggingProcessorMacroElasticsearchJson `yaml:",inline"` + confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` + IncludePaths []string `yaml:"include_paths,omitempty"` + MultilineRules []confgenerator.MultilineRule `yaml:"multiline_rules,omitempty"` +} + +func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { + if len(r.IncludePaths) == 0 { // Default JSON logs for Elasticsearch - IncludePaths: []string{ + r.IncludePaths = []string{ "/var/log/elasticsearch/*_server.json", "/var/log/elasticsearch/*_deprecation.json", "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json", "/var/log/elasticsearch/*_audit.json", + } + } + + // When Elasticsearch emits stack traces, the json log may be spread across multiple lines, + // so we need this multiline parsing to properly parse the record. + // Example multiline log record: + // {"type": "server", "timestamp": "2022-01-20T15:46:00,131Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "brandon-testing-elasticsearch", "message": "uncaught exception in thread [main]", + // "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key] please check that any required plugins are installed, or check the breaking changes documentation for removed settings", + // -- snip -- + // "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]", + // "... 6 more"] } + r.MultilineRules = []confgenerator.MultilineRule{ + { + StateName: "start_state", + NextState: "cont", + Regex: `^{.*`, }, - // When Elasticsearch emits stack traces, the json log may be spread across multiple lines, - // so we need this multiline parsing to properly parse the record. - // Example multiline log record: - // {"type": "server", "timestamp": "2022-01-20T15:46:00,131Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "brandon-testing-elasticsearch", "message": "uncaught exception in thread [main]", - // "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key] please check that any required plugins are installed, or check the breaking changes documentation for removed settings", - // -- snip -- - // "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]", - // "... 6 more"] } - MultilineRules: []confgenerator.MultilineRule{ - { - StateName: "start_state", - NextState: "cont", - Regex: `^{.*`, - }, - { - StateName: "cont", - NextState: "cont", - Regex: `^[^{].*[,}]$`, - }, + { + StateName: "cont", + NextState: "cont", + Regex: `^[^{].*[,}]$`, }, } + + return &r.LoggingReceiverFilesMixin, r.LoggingProcessorMacroElasticsearchJson.Expand(ctx) } type LoggingReceiverElasticsearchGC struct { @@ -319,6 +329,8 @@ func (r LoggingReceiverElasticsearchGC) Components(ctx context.Context, tag stri } func init() { - confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroElasticsearchJson](loggingReceiverFilesMixinElasticsearchJson) + confgenerator.RegisterLoggingReceiverMacro(func() LoggingReceiverMacroElasticsearchJson { + return LoggingReceiverMacroElasticsearchJson{} + }) confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchGC{} }) } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 7c8ee29993..075f3eb5da 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -245,13 +245,10 @@ App,Field,Override, *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_interval *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_timeout *confgenerator.PrometheusMetrics,config.[].scrape_configs.static_config_target_groups -*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroRedis],confgenerator.ConfigComponent.Type, -*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]],ReceiverMacro, -*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],ReceiverMacro, @@ -260,3 +257,7 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroRedis]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroRedis]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],confgenerator.ConfigComponent.Type, diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 4f3c8f92e8..10751c0cf5 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 4f3c8f92e8..10751c0cf5 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 13924fadc6..1649e4e323 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 13924fadc6..1649e4e323 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json index 70d17cb2a1..7a8c10da4d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml index a5a83dfb2f..b6100396aa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/features.yaml @@ -22,11 +22,3 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf index dc41310771..07552bf16c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf @@ -47,22 +47,6 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_elasticsearch_json - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch.elasticsearch_json - multiline.parser multiline.elasticsearch.elasticsearch_json - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -137,11 +121,6 @@ call process script f931d345d17c583089b53f72102e12d3.lua -[FILTER] - Match elasticsearch.elasticsearch_json - Name modify - Rename log message - [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf index 1ca2652bac..93fd540e87 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf @@ -25,10 +25,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch.elasticsearch_json - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json index 70d17cb2a1..7a8c10da4d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml index a5a83dfb2f..b6100396aa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/features.yaml @@ -22,11 +22,3 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf index dc41310771..07552bf16c 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf @@ -47,22 +47,6 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_elasticsearch_json - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch.elasticsearch_json - multiline.parser multiline.elasticsearch.elasticsearch_json - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -137,11 +121,6 @@ call process script f931d345d17c583089b53f72102e12d3.lua -[FILTER] - Match elasticsearch.elasticsearch_json - Name modify - Rename log message - [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf index 1ca2652bac..93fd540e87 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf @@ -25,10 +25,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch.elasticsearch_json - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json index 70d17cb2a1..7a8c10da4d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml index a5a83dfb2f..b6100396aa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/features.yaml @@ -22,11 +22,3 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf index 42c6fd0a52..4af6a5f297 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf @@ -40,22 +40,6 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_elasticsearch_json - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch.elasticsearch_json - multiline.parser multiline.elasticsearch.elasticsearch_json - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -180,11 +164,6 @@ call process script f931d345d17c583089b53f72102e12d3.lua -[FILTER] - Match elasticsearch.elasticsearch_json - Name modify - Rename log message - [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf index 69e547b1e0..04820fe916 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf @@ -32,10 +32,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch.elasticsearch_json - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json index 70d17cb2a1..7a8c10da4d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml index a5a83dfb2f..b6100396aa 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/features.yaml @@ -22,11 +22,3 @@ feature: receivers:elasticsearch_json key: "[1].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[1].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf index 42c6fd0a52..4af6a5f297 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf @@ -40,22 +40,6 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_elasticsearch_json - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch.elasticsearch_json - multiline.parser multiline.elasticsearch.elasticsearch_json - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -180,11 +164,6 @@ call process script f931d345d17c583089b53f72102e12d3.lua -[FILTER] - Match elasticsearch.elasticsearch_json - Name modify - Rename log message - [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf index 69e547b1e0..04820fe916 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf @@ -32,10 +32,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch.elasticsearch_json - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json index bad5b9731b..dfc3c268e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml index e93c89703d..d3d7484567 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/features.yaml @@ -34,19 +34,7 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" -- module: logging - feature: receivers:elasticsearch_json - key: "[2].-.__length" - value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf index 783c8d9eaf..174395d526 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf @@ -48,23 +48,6 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /logs/es/example.json - Read_from_Head True - Refresh_Interval 55 - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_custom.elasticsearch_json_custom - multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom - storage.type filesystem - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -80,22 +63,6 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_default.elasticsearch_json_default - multiline.parser multiline.elasticsearch_default.elasticsearch_json_default - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -170,11 +137,6 @@ call process script f45355f629f3e258617784f330d248fe.lua -[FILTER] - Match elasticsearch_custom.elasticsearch_json_custom - Name modify - Rename log message - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -341,11 +303,6 @@ call process script eea5053c24348f1499dd165b7b305c37.lua -[FILTER] - Match elasticsearch_default.elasticsearch_json_default - Name modify - Rename log message - [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf index 11b906ac1f..3811862a53 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -39,17 +39,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_custom.elasticsearch_json_custom - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_default.elasticsearch_json_default - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json index bad5b9731b..dfc3c268e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml index e93c89703d..d3d7484567 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/features.yaml @@ -34,19 +34,7 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" -- module: logging - feature: receivers:elasticsearch_json - key: "[2].-.__length" - value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf index 783c8d9eaf..174395d526 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf @@ -48,23 +48,6 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /logs/es/example.json - Read_from_Head True - Refresh_Interval 55 - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_custom.elasticsearch_json_custom - multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom - storage.type filesystem - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -80,22 +63,6 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_default.elasticsearch_json_default - multiline.parser multiline.elasticsearch_default.elasticsearch_json_default - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -170,11 +137,6 @@ call process script f45355f629f3e258617784f330d248fe.lua -[FILTER] - Match elasticsearch_custom.elasticsearch_json_custom - Name modify - Rename log message - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -341,11 +303,6 @@ call process script eea5053c24348f1499dd165b7b305c37.lua -[FILTER] - Match elasticsearch_default.elasticsearch_json_default - Name modify - Rename log message - [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf index 11b906ac1f..3811862a53 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf @@ -39,17 +39,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_custom.elasticsearch_json_custom - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_default.elasticsearch_json_default - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json index bad5b9731b..dfc3c268e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml index e93c89703d..d3d7484567 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/features.yaml @@ -34,19 +34,7 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" -- module: logging - feature: receivers:elasticsearch_json - key: "[2].-.__length" - value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf index 420399af41..7e6460e95a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf @@ -41,23 +41,6 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /logs/es/example.json - Read_from_Head True - Refresh_Interval 55 - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_custom.elasticsearch_json_custom - multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom - storage.type filesystem - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,22 +56,6 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_default.elasticsearch_json_default - multiline.parser multiline.elasticsearch_default.elasticsearch_json_default - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -213,11 +180,6 @@ call process script f45355f629f3e258617784f330d248fe.lua -[FILTER] - Match elasticsearch_custom.elasticsearch_json_custom - Name modify - Rename log message - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -384,11 +346,6 @@ call process script eea5053c24348f1499dd165b7b305c37.lua -[FILTER] - Match elasticsearch_default.elasticsearch_json_default - Name modify - Rename log message - [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf index aba8422797..a195818ec9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf @@ -46,17 +46,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_custom.elasticsearch_json_custom - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_default.elasticsearch_json_default - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json index bad5b9731b..dfc3c268e1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].-.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_gc"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:elasticsearch_json"}},{"key":"key","value":{"stringValue":"[3].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml index e93c89703d..d3d7484567 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/features.yaml @@ -34,19 +34,7 @@ feature: receivers:elasticsearch_json key: "[2].include_paths.__length" value: "1" -- module: logging - feature: receivers:elasticsearch_json - key: "[2].-.__length" - value: "2" - module: logging feature: receivers:elasticsearch_json key: "[3].enabled" value: "true" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].include_paths.__length" - value: "5" -- module: logging - feature: receivers:elasticsearch_json - key: "[3].-.__length" - value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf index 420399af41..7e6460e95a 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf @@ -41,23 +41,6 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /logs/es/example.json - Read_from_Head True - Refresh_Interval 55 - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_custom.elasticsearch_json_custom - multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom - storage.type filesystem - [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -73,22 +56,6 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem -[INPUT] - Buffer_Chunk_Size 512k - Buffer_Max_Size 2M - DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default - DB.locking true - Key message - Mem_Buf_Limit 10M - Name tail - Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json - Read_from_Head True - Rotate_Wait 30 - Skip_Long_Lines On - Tag elasticsearch_default.elasticsearch_json_default - multiline.parser multiline.elasticsearch_default.elasticsearch_json_default - storage.type filesystem - [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -213,11 +180,6 @@ call process script f45355f629f3e258617784f330d248fe.lua -[FILTER] - Match elasticsearch_custom.elasticsearch_json_custom - Name modify - Rename log message - [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -384,11 +346,6 @@ call process script eea5053c24348f1499dd165b7b305c37.lua -[FILTER] - Match elasticsearch_default.elasticsearch_json_default - Name modify - Rename log message - [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf index aba8422797..a195818ec9 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf @@ -46,17 +46,3 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_custom.elasticsearch_json_custom - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" - -[MULTILINE_PARSER] - flush_timeout 5000 - name multiline.elasticsearch_default.elasticsearch_json_default - type regex - rule "start_state" "^{.*" "cont" - rule "cont" "^[^{].*[,}]$" "cont" diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index e9c8d4084d..5d3864a257 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -33,6 +33,7 @@ import ( "time" logpb "cloud.google.com/go/logging/apiv2/loggingpb" + "github.com/GoogleCloudPlatform/ops-agent/apps" _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" @@ -593,4 +594,6 @@ func sanitizeStacktrace(t *testing.T, input string) string { func init() { // The processors registered here are only meant to be used in transformation tests. confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &confgenerator.LoggingProcessorWindowsEventLogV1{} }) + confgenerator.RegisterLoggingProcessorMacro[apps.LoggingProcessorMacroElasticsearchJson]() + } From 992497f4b2930385c37713a3dedf4eb880fad274 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 20:34:56 +0000 Subject: [PATCH 5/7] updated to use ReceiverMixin --- apps/elasticsearch.go | 14 +++--- .../golden/linux-gpu/fluent_bit_main.conf | 21 +++++++++ .../golden/linux-gpu/fluent_bit_parser.conf | 7 +++ .../golden/linux/fluent_bit_main.conf | 21 +++++++++ .../golden/linux/fluent_bit_parser.conf | 7 +++ .../golden/windows-2012/fluent_bit_main.conf | 21 +++++++++ .../windows-2012/fluent_bit_parser.conf | 7 +++ .../golden/windows/fluent_bit_main.conf | 21 +++++++++ .../golden/windows/fluent_bit_parser.conf | 7 +++ .../golden/linux-gpu/fluent_bit_main.conf | 43 +++++++++++++++++++ .../golden/linux-gpu/fluent_bit_parser.conf | 14 ++++++ .../golden/linux/fluent_bit_main.conf | 43 +++++++++++++++++++ .../golden/linux/fluent_bit_parser.conf | 14 ++++++ .../golden/windows-2012/fluent_bit_main.conf | 43 +++++++++++++++++++ .../windows-2012/fluent_bit_parser.conf | 14 ++++++ .../golden/windows/fluent_bit_main.conf | 43 +++++++++++++++++++ .../golden/windows/fluent_bit_parser.conf | 14 ++++++ 17 files changed, 346 insertions(+), 8 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index 095b4b6ba6..144f09cbac 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -269,16 +269,14 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui } type LoggingReceiverMacroElasticsearchJson struct { - LoggingProcessorMacroElasticsearchJson `yaml:",inline"` - confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` - IncludePaths []string `yaml:"include_paths,omitempty"` - MultilineRules []confgenerator.MultilineRule `yaml:"multiline_rules,omitempty"` + LoggingProcessorMacroElasticsearchJson `yaml:",inline"` + ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` } func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { - if len(r.IncludePaths) == 0 { + if len(r.ReceiverMixin.IncludePaths) == 0 { // Default JSON logs for Elasticsearch - r.IncludePaths = []string{ + r.ReceiverMixin.IncludePaths = []string{ "/var/log/elasticsearch/*_server.json", "/var/log/elasticsearch/*_deprecation.json", "/var/log/elasticsearch/*_index_search_slowlog.json", @@ -295,7 +293,7 @@ func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (conf // -- snip -- // "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]", // "... 6 more"] } - r.MultilineRules = []confgenerator.MultilineRule{ + r.ReceiverMixin.MultilineRules = []confgenerator.MultilineRule{ { StateName: "start_state", NextState: "cont", @@ -308,7 +306,7 @@ func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (conf }, } - return &r.LoggingReceiverFilesMixin, r.LoggingProcessorMacroElasticsearchJson.Expand(ctx) + return &r.ReceiverMixin, r.LoggingProcessorMacroElasticsearchJson.Expand(ctx) } type LoggingReceiverElasticsearchGC struct { diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf index 07552bf16c..dc41310771 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_main.conf @@ -47,6 +47,22 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_elasticsearch_json + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch.elasticsearch_json + multiline.parser multiline.elasticsearch.elasticsearch_json + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -121,6 +137,11 @@ call process script f931d345d17c583089b53f72102e12d3.lua +[FILTER] + Match elasticsearch.elasticsearch_json + Name modify + Rename log message + [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf index 93fd540e87..1ca2652bac 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux-gpu/fluent_bit_parser.conf @@ -25,3 +25,10 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch.elasticsearch_json + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf index 07552bf16c..dc41310771 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_main.conf @@ -47,6 +47,22 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_elasticsearch_json + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch.elasticsearch_json + multiline.parser multiline.elasticsearch.elasticsearch_json + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -121,6 +137,11 @@ call process script f931d345d17c583089b53f72102e12d3.lua +[FILTER] + Match elasticsearch.elasticsearch_json + Name modify + Rename log message + [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf index 93fd540e87..1ca2652bac 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/linux/fluent_bit_parser.conf @@ -25,3 +25,10 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch.elasticsearch_json + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf index 4af6a5f297..42c6fd0a52 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_main.conf @@ -40,6 +40,22 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_elasticsearch_json + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch.elasticsearch_json + multiline.parser multiline.elasticsearch.elasticsearch_json + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -164,6 +180,11 @@ call process script f931d345d17c583089b53f72102e12d3.lua +[FILTER] + Match elasticsearch.elasticsearch_json + Name modify + Rename log message + [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf index 04820fe916..69e547b1e0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows-2012/fluent_bit_parser.conf @@ -32,3 +32,10 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch.elasticsearch_json + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf index 4af6a5f297..42c6fd0a52 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_main.conf @@ -40,6 +40,22 @@ Tag elasticsearch.elasticsearch_gc storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_elasticsearch_json + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch.elasticsearch_json + multiline.parser multiline.elasticsearch.elasticsearch_json + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -164,6 +180,11 @@ call process script f931d345d17c583089b53f72102e12d3.lua +[FILTER] + Match elasticsearch.elasticsearch_json + Name modify + Rename log message + [FILTER] Match elasticsearch.elasticsearch_json Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf index 04820fe916..69e547b1e0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch/golden/windows/fluent_bit_parser.conf @@ -32,3 +32,10 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch.elasticsearch_json + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf index 174395d526..783c8d9eaf 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_main.conf @@ -48,6 +48,23 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /logs/es/example.json + Read_from_Head True + Refresh_Interval 55 + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_custom.elasticsearch_json_custom + multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom + storage.type filesystem + [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -63,6 +80,22 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_default.elasticsearch_json_default + multiline.parser multiline.elasticsearch_default.elasticsearch_json_default + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -137,6 +170,11 @@ call process script f45355f629f3e258617784f330d248fe.lua +[FILTER] + Match elasticsearch_custom.elasticsearch_json_custom + Name modify + Rename log message + [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -303,6 +341,11 @@ call process script eea5053c24348f1499dd165b7b305c37.lua +[FILTER] + Match elasticsearch_default.elasticsearch_json_default + Name modify + Rename log message + [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf index 3811862a53..11b906ac1f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux-gpu/fluent_bit_parser.conf @@ -39,3 +39,17 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_custom.elasticsearch_json_custom + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_default.elasticsearch_json_default + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf index 174395d526..783c8d9eaf 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_main.conf @@ -48,6 +48,23 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /logs/es/example.json + Read_from_Head True + Refresh_Interval 55 + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_custom.elasticsearch_json_custom + multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom + storage.type filesystem + [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -63,6 +80,22 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_default.elasticsearch_json_default + multiline.parser multiline.elasticsearch_default.elasticsearch_json_default + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -137,6 +170,11 @@ call process script f45355f629f3e258617784f330d248fe.lua +[FILTER] + Match elasticsearch_custom.elasticsearch_json_custom + Name modify + Rename log message + [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -303,6 +341,11 @@ call process script eea5053c24348f1499dd165b7b305c37.lua +[FILTER] + Match elasticsearch_default.elasticsearch_json_default + Name modify + Rename log message + [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf index 3811862a53..11b906ac1f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/linux/fluent_bit_parser.conf @@ -39,3 +39,17 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_custom.elasticsearch_json_custom + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_default.elasticsearch_json_default + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf index 7e6460e95a..420399af41 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_main.conf @@ -41,6 +41,23 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /logs/es/example.json + Read_from_Head True + Refresh_Interval 55 + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_custom.elasticsearch_json_custom + multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom + storage.type filesystem + [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,6 +73,22 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_default.elasticsearch_json_default + multiline.parser multiline.elasticsearch_default.elasticsearch_json_default + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -180,6 +213,11 @@ call process script f45355f629f3e258617784f330d248fe.lua +[FILTER] + Match elasticsearch_custom.elasticsearch_json_custom + Name modify + Rename log message + [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -346,6 +384,11 @@ call process script eea5053c24348f1499dd165b7b305c37.lua +[FILTER] + Match elasticsearch_default.elasticsearch_json_default + Name modify + Rename log message + [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf index a195818ec9..aba8422797 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows-2012/fluent_bit_parser.conf @@ -46,3 +46,17 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_custom.elasticsearch_json_custom + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_default.elasticsearch_json_default + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf index 7e6460e95a..420399af41 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_main.conf @@ -41,6 +41,23 @@ Tag elasticsearch_custom.elasticsearch_gc_custom storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_custom_elasticsearch_json_custom + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /logs/es/example.json + Read_from_Head True + Refresh_Interval 55 + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_custom.elasticsearch_json_custom + multiline.parser multiline.elasticsearch_custom.elasticsearch_json_custom + storage.type filesystem + [INPUT] Buffer_Chunk_Size 512k Buffer_Max_Size 2M @@ -56,6 +73,22 @@ Tag elasticsearch_default.elasticsearch_gc_default storage.type filesystem +[INPUT] + Buffer_Chunk_Size 512k + Buffer_Max_Size 2M + DB ${buffers_dir}/elasticsearch_default_elasticsearch_json_default + DB.locking true + Key message + Mem_Buf_Limit 10M + Name tail + Path /var/log/elasticsearch/*_server.json,/var/log/elasticsearch/*_deprecation.json,/var/log/elasticsearch/*_index_search_slowlog.json,/var/log/elasticsearch/*_index_indexing_slowlog.json,/var/log/elasticsearch/*_audit.json + Read_from_Head True + Rotate_Wait 30 + Skip_Long_Lines On + Tag elasticsearch_default.elasticsearch_json_default + multiline.parser multiline.elasticsearch_default.elasticsearch_json_default + storage.type filesystem + [INPUT] Dummy {"code": "LogPingOpsAgent", "severity": "DEBUG"} Interval_NSec 0 @@ -180,6 +213,11 @@ call process script f45355f629f3e258617784f330d248fe.lua +[FILTER] + Match elasticsearch_custom.elasticsearch_json_custom + Name modify + Rename log message + [FILTER] Match elasticsearch_custom.elasticsearch_json_custom Name lua @@ -346,6 +384,11 @@ call process script eea5053c24348f1499dd165b7b305c37.lua +[FILTER] + Match elasticsearch_default.elasticsearch_json_default + Name modify + Rename log message + [FILTER] Match elasticsearch_default.elasticsearch_json_default Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf index a195818ec9..aba8422797 100644 --- a/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_elasticsearch_custom/golden/windows/fluent_bit_parser.conf @@ -46,3 +46,17 @@ Name ops-agent-health.health-checks-json Time_Format %Y-%m-%dT%H:%M:%S%z Time_Key time + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_custom.elasticsearch_json_custom + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" + +[MULTILINE_PARSER] + flush_timeout 5000 + name multiline.elasticsearch_default.elasticsearch_json_default + type regex + rule "start_state" "^{.*" "cont" + rule "cont" "^[^{].*[,}]$" "cont" From b16bece12f3e5b7f8b5e7371b0ec52d560ae1cec Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Wed, 23 Jul 2025 13:12:09 +0000 Subject: [PATCH 6/7] removed duplicate import --- transformation_test/transformation_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index 5d3864a257..234ac708d8 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -34,7 +34,6 @@ import ( logpb "cloud.google.com/go/logging/apiv2/loggingpb" "github.com/GoogleCloudPlatform/ops-agent/apps" - _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" From 025d2230a186c4497cb70e75246cea751f0deda5 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 28 Jul 2025 18:21:36 +0000 Subject: [PATCH 7/7] updated yaml tags --- apps/elasticsearch.go | 2 +- confgenerator/logging_processors.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index 144f09cbac..ea9937e5c9 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -270,7 +270,7 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui type LoggingReceiverMacroElasticsearchJson struct { LoggingProcessorMacroElasticsearchJson `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` + ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` } func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 7f05eddb41..74d5e0f1df 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -320,9 +320,9 @@ func (p LoggingProcessorParseRegexComplex) Components(ctx context.Context, tag, } type MultilineRule struct { - StateName string `yaml:"state_name,omitempty"` - Regex string `yaml:"regex,omitempty"` - NextState string `yaml:"next_state,omitempty"` + StateName string + Regex string + NextState string } func (r MultilineRule) AsString() string {