From 4571f5153a9b17baf22a92fd16831e8fbb907998 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 18 Aug 2025 15:08:04 +0000 Subject: [PATCH 1/5] generated golden tests --- .../config.yaml | 1 + .../input.log | 35 ++ .../output_fluentbit.yaml | 311 ++++++++++++++++++ .../output_otel.yaml | 1 + transformation_test/transformation_test.go | 3 +- 5 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-gc/config.yaml create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-gc/input.log create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-gc/output_fluentbit.yaml create mode 100644 transformation_test/testdata/logging_processor-elasticsearch-gc/output_otel.yaml diff --git a/transformation_test/testdata/logging_processor-elasticsearch-gc/config.yaml b/transformation_test/testdata/logging_processor-elasticsearch-gc/config.yaml new file mode 100644 index 0000000000..fe0d31d3d5 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-gc/config.yaml @@ -0,0 +1 @@ +- type: elasticsearch_gc diff --git a/transformation_test/testdata/logging_processor-elasticsearch-gc/input.log b/transformation_test/testdata/logging_processor-elasticsearch-gc/input.log new file mode 100644 index 0000000000..5c9c443ba5 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-gc/input.log @@ -0,0 +1,35 @@ +[2025-08-18T10:14:37.240+0000][31245][gc,start ] GC(0) Pause Young (Normal) (G1 Evacuation Pause) +[2025-08-18T10:14:37.242+0000][31245][gc,heap ] GC(0) Eden regions: 48->0(47) +[2025-08-18T10:14:37.242+0000][31245][gc,heap ] GC(0) Survivor regions: 0->1(7) +[2025-08-18T10:14:37.242+0000][31245][gc,heap ] GC(0) Old regions: 0->2 +[2025-08-18T10:14:37.242+0000][31245][gc,metaspace] GC(0) Metaspace: 18000K->18000K(1064960K) +[2025-08-18T10:14:37.243+0000][31245][gc ] GC(0) Pause Young (Normal) (G1 Evacuation Pause) 48M->10M(1024M) 3.456ms +[2025-08-18T10:20:12.555+0000][31245][gc,start ] GC(1) Pause Young (Concurrent Start) (G1 Evacuation Pause) +[2025-08-18T10:20:12.559+0000][31245][gc,heap ] GC(1) Eden regions: 47->0(45) +[2025-08-18T10:20:12.559+0000][31245][gc,heap ] GC(1) Survivor regions: 1->2(7) +[2025-08-18T10:20:12.559+0000][31245][gc,heap ] GC(1) Old regions: 2->3 +[2025-08-18T10:20:12.559+0000][31245][gc,metaspace] GC(1) Metaspace: 18250K->18250K(1064960K) +[2025-08-18T10:20:12.561+0000][31245][gc ] GC(1) Pause Young (Concurrent Start) (G1 Evacuation Pause) 58M->15M(1024M) 5.623ms +[2025-08-18T10:20:12.561+0000][31245][gc ] GC(1) Concurrent Cycle +[2025-08-18T10:20:12.561+0000][31245][gc,marking ] GC(1) Concurrent Clear Claimed Marks +[2025-08-18T10:20:12.561+0000][31245][gc,marking ] GC(1) Concurrent Scan Root Regions +[2025-08-18T10:20:12.563+0000][31245][gc,marking ] GC(1) Concurrent Mark +[2025-08-18T10:20:12.567+0000][31245][gc,marking ] GC(1) Concurrent Preclean +[2025-08-18T10:20:12.568+0000][31245][gc,start ] GC(1) Pause Remark +[2025-08-18T10:20:12.569+0000][31245][gc ] GC(1) Pause Remark 20M->20M(1024M) 1.234ms +[2025-08-18T10:20:12.569+0000][31245][gc,marking ] GC(1) Concurrent Mark Cleanup +[2025-08-18T10:20:12.570+0000][31245][gc,start ] GC(1) Pause Cleanup +[2025-08-18T10:20:12.570+0000][31245][gc ] GC(1) Pause Cleanup 20M->20M(1024M) 0.234ms +[2025-08-18T10:20:12.571+0000][31245][gc ] GC(1) Concurrent Cycle 10.123ms +[2025-08-18T10:45:45.125+0000][31245][gc,start ] GC(2) Pause Young (Normal) (G1 Evacuation Pause) +[2025-08-18T10:45:45.129+0000][31245][gc,heap ] GC(2) Eden regions: 46->0(48) +[2025-08-18T10:45:45.129+0000][31245][gc,heap ] GC(2) Survivor regions: 2->3(7) +[2025-08-18T10:45:45.129+0000][31245][gc,heap ] GC(2) Old regions: 3->3 +[2025-08-18T10:45:45.129+0000][31245][gc,metaspace] GC(2) Metaspace: 18500K->18500K(1064960K) +[2025-08-18T10:45:45.130+0000][31245][gc ] GC(2) Pause Young (Normal) (G1 Evacuation Pause) 75M->25M(1024M) 4.543ms +[2025-08-18T11:02:10.341+0000][31245][gc,start ] GC(3) Pause Full (System.gc()) +[2025-08-18T11:02:10.348+0000][31245][gc,heap ] GC(3) Eden regions: 45->0(45) +[2025-08-18T11:02:10.348+0000][31245][gc,heap ] GC(3) Survivor regions: 3->0(7) +[2025-08-18T11:02:10.348+0000][31245][gc,heap ] GC(3) Old regions: 3->1 +[2025-08-18T11:02:10.349+0000][31245][gc,metaspace] GC(3) Metaspace: 18700K->18650K(1064960K) +[2025-08-18T11:02:10.350+0000][31245][gc ] GC(3) Pause Full (System.gc()) 100M->15M(1024M) 8.765ms \ No newline at end of file diff --git a/transformation_test/testdata/logging_processor-elasticsearch-gc/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-elasticsearch-gc/output_fluentbit.yaml new file mode 100644 index 0000000000..dc27ca0c4f --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-gc/output_fluentbit.yaml @@ -0,0 +1,311 @@ +- entries: + - jsonPayload: + gc_run: 0.0 + message: Pause Young (Normal) (G1 Evacuation Pause) + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.240000000Z + - jsonPayload: + gc_run: 0.0 + message: "Eden regions: 48->0(47)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.242000000Z + - jsonPayload: + gc_run: 0.0 + message: "Survivor regions: 0->1(7)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.242000000Z + - jsonPayload: + gc_run: 0.0 + message: "Old regions: 0->2" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.242000000Z + - jsonPayload: + gc_run: 0.0 + message: "Metaspace: 18000K->18000K(1064960K)" + type: gc,metaspace + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.242000000Z + - jsonPayload: + gc_run: 0.0 + message: Pause Young (Normal) (G1 Evacuation Pause) 48M->10M(1024M) 3.456ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:14:37.243000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Young (Concurrent Start) (G1 Evacuation Pause) + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.555000000Z + - jsonPayload: + gc_run: 1.0 + message: "Eden regions: 47->0(45)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.559000000Z + - jsonPayload: + gc_run: 1.0 + message: "Survivor regions: 1->2(7)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.559000000Z + - jsonPayload: + gc_run: 1.0 + message: "Old regions: 2->3" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.559000000Z + - jsonPayload: + gc_run: 1.0 + message: "Metaspace: 18250K->18250K(1064960K)" + type: gc,metaspace + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.559000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Young (Concurrent Start) (G1 Evacuation Pause) 58M->15M(1024M) 5.623ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.561000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Cycle + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.561000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Clear Claimed Marks + type: gc,marking + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.561000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Scan Root Regions + type: gc,marking + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.561000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Mark + type: gc,marking + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.563000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Preclean + type: gc,marking + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.567000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Remark + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.568000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Remark 20M->20M(1024M) 1.234ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.569000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Mark Cleanup + type: gc,marking + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.569000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Cleanup + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.570000000Z + - jsonPayload: + gc_run: 1.0 + message: Pause Cleanup 20M->20M(1024M) 0.234ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.570000000Z + - jsonPayload: + gc_run: 1.0 + message: Concurrent Cycle 10.123ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:20:12.571000000Z + - jsonPayload: + gc_run: 2.0 + message: Pause Young (Normal) (G1 Evacuation Pause) + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.125000000Z + - jsonPayload: + gc_run: 2.0 + message: "Eden regions: 46->0(48)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.129000000Z + - jsonPayload: + gc_run: 2.0 + message: "Survivor regions: 2->3(7)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.129000000Z + - jsonPayload: + gc_run: 2.0 + message: "Old regions: 3->3" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.129000000Z + - jsonPayload: + gc_run: 2.0 + message: "Metaspace: 18500K->18500K(1064960K)" + type: gc,metaspace + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.129000000Z + - jsonPayload: + gc_run: 2.0 + message: Pause Young (Normal) (G1 Evacuation Pause) 75M->25M(1024M) 4.543ms + type: gc + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T10:45:45.130000000Z + - jsonPayload: + gc_run: 3.0 + message: Pause Full (System.gc()) + type: gc,start + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T11:02:10.341000000Z + - jsonPayload: + gc_run: 3.0 + message: "Eden regions: 45->0(45)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T11:02:10.348000000Z + - jsonPayload: + gc_run: 3.0 + message: "Survivor regions: 3->0(7)" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T11:02:10.348000000Z + - jsonPayload: + gc_run: 3.0 + message: "Old regions: 3->1" + type: gc,heap + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T11:02:10.348000000Z + - jsonPayload: + gc_run: 3.0 + message: "Metaspace: 18700K->18650K(1064960K)" + type: gc,metaspace + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_gc + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T11:02:10.349000000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-elasticsearch-gc/output_otel.yaml b/transformation_test/testdata/logging_processor-elasticsearch-gc/output_otel.yaml new file mode 100644 index 0000000000..4a8920aad8 --- /dev/null +++ b/transformation_test/testdata/logging_processor-elasticsearch-gc/output_otel.yaml @@ -0,0 +1 @@ +- config_error: processor "processor0" not supported in pipeline "transformation_test" diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index 236bebdbab..53b42547b7 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -32,7 +32,7 @@ import ( "time" logpb "cloud.google.com/go/logging/apiv2/loggingpb" - _ "github.com/GoogleCloudPlatform/ops-agent/apps" + apps "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" @@ -595,4 +595,5 @@ func sanitizeStacktrace(t *testing.T, input string) string { func init() { // The processors registered here are only meant to be used in transformation tests. confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &confgenerator.LoggingProcessorWindowsEventLogV1{} }) + confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &apps.LoggingProcessorElasticsearchGC{} }) } From 1e3df24ed7f4330ce85db6e333971fa9e5259da5 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 18 Aug 2025 15:14:04 +0000 Subject: [PATCH 2/5] refactored elasticsearch_gc --- apps/elasticsearch.go | 54 ++++++++++------------ transformation_test/transformation_test.go | 2 +- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go index f6d9cb6ce0..81298bb007 100644 --- a/apps/elasticsearch.go +++ b/apps/elasticsearch.go @@ -19,7 +19,6 @@ import ( "fmt" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -255,39 +254,33 @@ func (p LoggingProcessorMacroElasticsearchJson) nestingProcessors() []confgenera return processors } -type LoggingProcessorElasticsearchGC struct { - confgenerator.ConfigComponent `yaml:",inline"` +type LoggingProcessorMacroElasticsearchGC struct { } -func (LoggingProcessorElasticsearchGC) Type() string { +func (LoggingProcessorMacroElasticsearchGC) Type() string { return "elasticsearch_gc" } -func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} - - regexParser := confgenerator.LoggingProcessorParseRegex{ - // Sample log line: - // [2022-01-17T18:31:37.240+0000][652141][gc,start ] GC(0) Pause Young (Normal) (G1 Evacuation Pause) - Regex: `\[(?