Skip to content

Commit 1e04e00

Browse files
nickytdnickytd
committed
test: add fetcher to e2e kind test setup
1 parent c2b0ce8 commit 1e04e00

File tree

14 files changed

+942
-174
lines changed

14 files changed

+942
-174
lines changed

‎cmd/fluent-bit-output-plugin/config_dump.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ func dumpConfiguration(conf *config.Config) {
6262
logger.V(1).Info("[flb-go]", "DQueBatchProcessorExportBufferSize", fmt.Sprintf("%+v", conf.OTLPConfig.DQueBatchProcessorExportBufferSize))
6363
// OTLP general configuration
6464
logger.V(1).Info("[flb-go]", "Endpoint", fmt.Sprintf("%+v", conf.OTLPConfig.Endpoint))
65+
logger.V(1).Info("[flb-go]", "EndpointUrl", fmt.Sprintf("%+v", conf.OTLPConfig.EndpointURL))
66+
logger.V(1).Info("[flb-go]", "EndpointUrlPath", fmt.Sprintf("%+v", conf.OTLPConfig.EndpointURLPath))
67+
6568
logger.V(1).Info("[flb-go]", "Insecure", fmt.Sprintf("%+v", conf.OTLPConfig.Insecure))
6669
logger.V(1).Info("[flb-go]", "Compression", fmt.Sprintf("%+v", conf.OTLPConfig.Compression))
6770
logger.V(1).Info("[flb-go]", "Timeout", fmt.Sprintf("%+v", conf.OTLPConfig.Timeout))

‎cmd/fluent-bit-output-plugin/plugin_config.go‎

Lines changed: 60 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -31,103 +31,105 @@ func (c *pluginConfig) toStringMap() map[string]string {
3131
// Define all possible configuration keys based on the structs and documentation
3232
configKeys := []string{
3333
// Client types
34-
"SeedType", "seedType",
35-
"ShootType", "shootType",
34+
"SeedType", "seedType", "seed_type",
35+
"ShootType", "shootType", "shoot_type",
3636

3737
// Plugin config
38-
"DynamicHostPath", "dynamicHostPath",
39-
"DynamicHostPrefix", "dynamicHostPrefix",
40-
"DynamicHostSuffix", "dynamicHostSuffix",
41-
"DynamicHostRegex", "dynamicHostRegex",
38+
"DynamicHostPath", "dynamicHostPath", "dynamic_host_path",
39+
"DynamicHostPrefix", "dynamicHostPrefix", "dynamic_host_prefix",
40+
"DynamicHostSuffix", "dynamicHostSuffix", "dynamic_host_suffix",
41+
"DynamicHostRegex", "dynamicHostRegex", "dynamic_host_regex",
4242

43-
"HostnameValue", "hostnameValue",
43+
"HostnameValue", "hostnameValue", "hostname_value",
4444
"Origin", "origin",
4545

4646
// Kubernetes metadata - TODO: revisit how to handle kubernetes metadata. Simplify?
47-
"FallbackToTagWhenMetadataIsMissing", "fallbackToTagWhenMetadataIsMissing",
48-
"DropLogEntryWithoutK8sMetadata", "dropLogEntryWithoutK8sMetadata",
49-
"TagKey", "tagKey",
50-
"TagPrefix", "tagPrefix",
51-
"TagExpression", "tagExpression",
47+
"FallbackToTagWhenMetadataIsMissing", "fallbackToTagWhenMetadataIsMissing", "fallback_to_tag_when_metadata_is_missing",
48+
"DropLogEntryWithoutK8sMetadata", "dropLogEntryWithoutK8sMetadata", "drop_log_entry_without_k8s_metadata",
49+
"TagKey", "tagKey", "tag_key",
50+
"TagPrefix", "tagPrefix", "tag_prefix",
51+
"TagExpression", "tagExpression", "tag_expression",
5252

5353
// Dque config
54-
"DQueDir", "dqueDir",
55-
"DQueSegmentSize", "dqueSegmentSize",
56-
"DQueSync", "dqueSync",
57-
"DQueName", " dqueName",
54+
"DQueDir", "dqueDir", "dque_dir",
55+
"DQueSegmentSize", "dqueSegmentSize", "dque_segment_size",
56+
"DQueSync", "dqueSync", "dque_sync",
57+
"DQueName", " dqueName", "dque_name",
5858

5959
// Controller config
60-
"DeletedClientTimeExpiration", "deletedClientTimeExpiration",
61-
"ControllerSyncTimeout", "controllerSyncTimeout",
60+
"DeletedClientTimeExpiration", "deletedClientTimeExpiration", "deleted_client_time_expiration",
61+
"ControllerSyncTimeout", "controllerSyncTimeout", "controller_sync_timeout",
6262

6363
// Log flows depending on cluster state
6464
// Shoot client config
65-
"SendLogsToShootWhenIsInCreationState", "sendLogsToShootWhenIsInCreationState",
66-
"SendLogsToShootWhenIsInReadyState", "sendLogsToShootWhenIsInReadyState",
67-
"SendLogsToShootWhenIsInHibernatingState", "sendLogsToShootWhenIsInHibernatingState",
68-
"SendLogsToShootWhenIsInHibernatedState", "sendLogsToShootWhenIsInHibernatedState",
69-
"SendLogsToShootWhenIsInWakingState", "sendLogsToShootWhenIsInWakingState",
70-
"SendLogsToShootWhenIsInDeletionState", "sendLogsToShootWhenIsInDeletionState",
71-
"SendLogsToShootWhenIsInDeletedState", "sendLogsToShootWhenIsInDeletedState",
72-
"SendLogsToShootWhenIsInRestoreState", "sendLogsToShootWhenIsInRestoreState",
73-
"SendLogsToShootWhenIsInMigrationState", "sendLogsToShootWhenIsInMigrationState",
65+
"SendLogsToShootWhenIsInCreationState", "sendLogsToShootWhenIsInCreationState", "send_logs_to_shoot_when_is_in_creation_state",
66+
"SendLogsToShootWhenIsInReadyState", "sendLogsToShootWhenIsInReadyState", "send_logs_to_shoot_when_is_in_ready_state",
67+
"SendLogsToShootWhenIsInHibernatingState", "sendLogsToShootWhenIsInHibernatingState", "send_logs_to_shoot_when_is_in_hibernating_state",
68+
"SendLogsToShootWhenIsInHibernatedState", "sendLogsToShootWhenIsInHibernatedState", "send_logs_to_shoot_when_is_in_hibernated_state",
69+
"SendLogsToShootWhenIsInWakingState", "sendLogsToShootWhenIsInWakingState", "send_logs_to_shoot_when_is_in_waking_state",
70+
"SendLogsToShootWhenIsInDeletionState", "sendLogsToShootWhenIsInDeletionState", "send_logs_to_shoot_when_is_in_deletion_state",
71+
"SendLogsToShootWhenIsInDeletedState", "sendLogsToShootWhenIsInDeletedState", "send_logs_to_shoot_when_is_in_deleted_state",
72+
"SendLogsToShootWhenIsInRestoreState", "sendLogsToShootWhenIsInRestoreState", "send_logs_to_shoot_when_is_in_restore_state",
73+
"SendLogsToShootWhenIsInMigrationState", "sendLogsToShootWhenIsInMigrationState", "send_logs_to_shoot_when_is_in_migration_state",
7474

7575
// Seed client config for shoots with dynamic hostnames
76-
"SendLogsToSeedWhenShootIsInCreationState", "sendLogsToSeedWhenShootIsInCreationState",
77-
"SendLogsToSeedWhenShootIsInReadyState", "sendLogsToSeedWhenShootIsInReadyState",
78-
"SendLogsToSeedWhenShootIsInHibernatingState", "sendLogsToSeedWhenShootIsInHibernatingState",
79-
"SendLogsToSeedWhenShootIsInHibernatedState", "sendLogsToSeedWhenShootIsInHibernatedState",
80-
"SendLogsToSeedWhenShootIsInWakingState", "sendLogsToSeedWhenShootIsInWakingState",
81-
"SendLogsToSeedWhenShootIsInDeletionState", "sendLogsToSeedWhenShootIsInDeletionState",
82-
"SendLogsToSeedWhenShootIsInDeletedState", "sendLogsToSeedWhenShootIsInDeletedState",
83-
"SendLogsToSeedWhenShootIsInRestoreState", "sendLogsToSeedWhenShootIsInRestoreState",
84-
"SendLogsToSeedWhenShootIsInMigrationState", "sendLogsToSeedWhenShootIsInMigrationState",
76+
"SendLogsToSeedWhenShootIsInCreationState", "sendLogsToSeedWhenShootIsInCreationState", "send_logs_to_seed_when_shoot_is_in_creation_state",
77+
"SendLogsToSeedWhenShootIsInReadyState", "sendLogsToSeedWhenShootIsInReadyState", "send_logs_to_seed_when_shoot_is_in_ready_state",
78+
"SendLogsToSeedWhenShootIsInHibernatingState", "sendLogsToSeedWhenShootIsInHibernatingState", "send_logs_to_seed_when_shoot_is_in_hibernating_state",
79+
"SendLogsToSeedWhenShootIsInHibernatedState", "sendLogsToSeedWhenShootIsInHibernatedState", "send_logs_to_seed_when_shoot_is_in_hibernated_state",
80+
"SendLogsToSeedWhenShootIsInWakingState", "sendLogsToSeedWhenShootIsInWakingState", "send_logs_to_seed_when_shoot_is_in_waking_state",
81+
"SendLogsToSeedWhenShootIsInDeletionState", "sendLogsToSeedWhenShootIsInDeletionState", "send_logs_to_seed_when_shoot_is_in_deletion_state",
82+
"SendLogsToSeedWhenShootIsInDeletedState", "sendLogsToSeedWhenShootIsInDeletedState", "send_logs_to_seed_when_shoot_is_in_deleted_state",
83+
"SendLogsToSeedWhenShootIsInRestoreState", "sendLogsToSeedWhenShootIsInRestoreState", "send_logs_to_seed_when_shoot_is_in_restore_state",
84+
"SendLogsToSeedWhenShootIsInMigrationState", "sendLogsToSeedWhenShootIsInMigrationState", "send_logs_to_seed_when_shoot_is_in_migration_state",
8585

8686
// Common OTLP configs
8787
"Endpoint", "endpoint",
88+
"EndpointUrl", "endpointUrl", "endpoint_url",
89+
"EndpointUrlPath:", "endpointUrlPath", "endpoint_url_path",
8890
"Insecure", "insecure",
8991
"Compression", "compression",
9092
"Timeout", "timeout",
9193
"Headers", "headers",
9294

9395
// OTLP Retry configs
94-
"RetryEnabled", "retryEnabled",
95-
"RetryInitialInterval", "retryInitialInterval",
96-
"RetryMaxInterval", "retryMaxInterval",
97-
"RetryMaxElapsedTime", "retryMaxElapsedTime",
96+
"RetryEnabled", "retryEnabled", "retry_enabled",
97+
"RetryInitialInterval", "retryInitialInterval", "retry_initial_interval",
98+
"RetryMaxInterval", "retryMaxInterval", "retry_max_interval",
99+
"RetryMaxElapsedTime", "retryMaxElapsedTime", "retry_max_elapsed_time",
98100

99101
// OTLP HTTP specific configs
100-
"HTTPPath", "httpPath",
101-
"HTTPProxy", "httpProxy",
102+
"HTTPPath", "httpPath", "http_path",
103+
"HTTPProxy", "httpProxy", "http_proxy",
102104

103105
// OTLP TLS configs
104-
"TLSCertFile", "tlsCertFile",
105-
"TLSKeyFile", "tlsKeyFile",
106-
"TLSCAFile", "tlsCAFile",
107-
"TLSServerName", "tlsServerName",
108-
"TLSInsecureSkipVerify", "tlsInsecureSkipVerify",
109-
"TLSMinVersion", "tlsMinVersion",
110-
"TLSMaxVersion", "tlsMaxVersion",
106+
"TLSCertFile", "tlsCertFile", "tls_cert_file",
107+
"TLSKeyFile", "tlsKeyFile", "tls_key_file",
108+
"TLSCAFile", "tlsCAFile", "tls_ca_file",
109+
"TLSServerName", "tlsServerName", "tls_server_name",
110+
"TLSInsecureSkipVerify", "tlsInsecureSkipVerify", "tls_insecure_skip_verify",
111+
"TLSMinVersion", "tlsMinVersion", "tls_min_version",
112+
"TLSMaxVersion", "tlsMaxVersion", "tls_max_version",
111113

112-
"ThrottleEnabled", "throttleEnabled",
113-
"ThrottleRequestsPerSec", "throttleRequestsPerSec",
114+
"ThrottleEnabled", "throttleEnabled", "throttle_enabled",
115+
"ThrottleRequestsPerSec", "throttleRequestsPerSec", "throttle_requests_per_sec",
114116

115117
// OTLP Batch Processor configs
116-
"DQueBatchProcessorMaxQueueSize", "dqueBatchProcessorMaxQueueSize",
117-
"DQueBatchProcessorMaxBatchSize", "dqueBatchProcessorMaxBatchSize",
118-
"DQueBatchProcessorExportTimeout", "dqueBatchProcessorExportTimeout",
119-
"DQueBatchProcessorExportInterval", "dqueBatchProcessorExportInterval",
120-
"DQueBatchProcessorExportBufferSize", "dqueBatchProcessorExportBufferSize",
118+
"DQueBatchProcessorMaxQueueSize", "dqueBatchProcessorMaxQueueSize", "dque_batch_processor_max_queue_size",
119+
"DQueBatchProcessorMaxBatchSize", "dqueBatchProcessorMaxBatchSize", "dque_batch_processor_max_batch_size",
120+
"DQueBatchProcessorExportTimeout", "dqueBatchProcessorExportTimeout", "dque_batch_processor_export_timeout",
121+
"DQueBatchProcessorExportInterval", "dqueBatchProcessorExportInterval", "dque_batch_processor_export_interval",
122+
"DQueBatchProcessorExportBufferSize", "dqueBatchProcessorExportBufferSize", "dque_batch_processor_export_buffer_size",
121123

122124
// General config
123-
"LogLevel", "logLevel",
125+
"LogLevel", "logLevel", "log_level",
124126
"Pprof", "pprof",
125127
}
126128

127129
// Extract values for all known keys
128130
for _, key := range configKeys {
129131
if value := c.Get(key); value != "" {
130-
configMap[strings.ToLower(key)] = value
132+
configMap[strings.ToLower(strings.ReplaceAll(key, "_", ""))] = value
131133
}
132134
}
133135

‎pkg/client/otlp_config_builder.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,9 @@ func NewOTLPHTTPConfigBuilder(cfg config.Config) *OTLPHTTPConfigBuilder {
8989

9090
// Build constructs the exporter options
9191
func (b *OTLPHTTPConfigBuilder) Build() []otlploghttp.Option {
92-
opts := []otlploghttp.Option{
93-
otlploghttp.WithEndpoint(b.cfg.OTLPConfig.Endpoint),
94-
}
92+
opts := []otlploghttp.Option{}
9593

94+
b.configureEndpoint(&opts)
9695
b.configureTLS(&opts)
9796
b.configureHeaders(&opts)
9897
b.configureTimeout(&opts)
@@ -103,7 +102,7 @@ func (b *OTLPHTTPConfigBuilder) Build() []otlploghttp.Option {
103102
}
104103

105104
func (b *OTLPHTTPConfigBuilder) configureTLS(opts *[]otlploghttp.Option) {
106-
if b.cfg.OTLPConfig.Insecure {
105+
if b.cfg.OTLPConfig.Insecure && b.cfg.OTLPConfig.EndpointURL == "" {
107106
*opts = append(*opts, otlploghttp.WithInsecure())
108107
} else if b.cfg.OTLPConfig.TLSConfig != nil {
109108
*opts = append(*opts, otlploghttp.WithTLSClientConfig(b.cfg.OTLPConfig.TLSConfig))
@@ -139,6 +138,17 @@ func (b *OTLPHTTPConfigBuilder) configureRetry(opts *[]otlploghttp.Option) {
139138
}
140139
}
141140

141+
func (b *OTLPHTTPConfigBuilder) configureEndpoint(opts *[]otlploghttp.Option) {
142+
// TODO: check the correct order of precedence for EndpointURL vs Endpoint
143+
*opts = append(*opts, otlploghttp.WithURLPath(b.cfg.OTLPConfig.EndpointURLPath))
144+
145+
if b.cfg.OTLPConfig.EndpointURL != "" {
146+
*opts = append(*opts, otlploghttp.WithEndpointURL(b.cfg.OTLPConfig.EndpointURL))
147+
} else {
148+
*opts = append(*opts, otlploghttp.WithEndpoint(b.cfg.OTLPConfig.Endpoint))
149+
}
150+
}
151+
142152
// compressionToString maps compression integer to string
143153
func compressionToString(compression int) string {
144154
switch compression {

‎pkg/config/config.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,22 @@ func processOTLPConfig(config *Config, configMap map[string]any) error {
327327
config.OTLPConfig.Endpoint = endpoint
328328
}
329329

330+
if endpointURL, ok := configMap["endpointurl"].(string); ok && endpointURL != "" {
331+
// check that it starts with http:// or https://
332+
if !strings.HasPrefix(endpointURL, "http://") && !strings.HasPrefix(endpointURL, "https://") {
333+
return fmt.Errorf("invalid EndpointURL: %s", endpointURL)
334+
}
335+
config.OTLPConfig.EndpointURL = endpointURL
336+
}
337+
338+
if endpointURLPath, ok := configMap["endpointurlpath"].(string); ok && endpointURLPath != "" {
339+
// check that it starts with a slash and there is no whitespace or colon
340+
if !strings.HasPrefix(endpointURLPath, "/") || strings.ContainsAny(endpointURLPath, " :") {
341+
return fmt.Errorf("invalid EndpointURLPath: %s", endpointURLPath)
342+
}
343+
config.OTLPConfig.EndpointURLPath = endpointURLPath
344+
}
345+
330346
// Process Insecure
331347
if insecure, ok := configMap["insecure"].(string); ok && insecure != "" {
332348
boolVal, err := strconv.ParseBool(insecure)

‎pkg/config/otlp.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@ var DefaultDQueConfig = DQueConfig{
2525

2626
// OTLPConfig holds configuration for otlp endpoint
2727
type OTLPConfig struct {
28-
Endpoint string `mapstructure:"Endpoint"`
29-
Insecure bool `mapstructure:"Insecure"`
30-
Compression int `mapstructure:"Compression"`
31-
Timeout time.Duration `mapstructure:"Timeout"`
32-
Headers map[string]string `mapstructure:"-"` // Handled manually in processOTLPConfig
28+
Endpoint string `mapstructure:"Endpoint"`
29+
EndpointURL string `mapstructure:"EndpointURL"`
30+
EndpointURLPath string `mapstructure:"EndpointURLPath"`
31+
Insecure bool `mapstructure:"Insecure"`
32+
Compression int `mapstructure:"Compression"`
33+
Timeout time.Duration `mapstructure:"Timeout"`
34+
Headers map[string]string `mapstructure:"-"` // Handled manually in processOTLPConfig
3335

3436
DQueConfig DQueConfig `mapstructure:",squash"`
3537

@@ -69,6 +71,8 @@ type OTLPConfig struct {
6971
// DefaultOTLPConfig holds the default configuration for OTLP
7072
var DefaultOTLPConfig = OTLPConfig{
7173
Endpoint: "localhost:4317",
74+
EndpointURL: "",
75+
EndpointURLPath: "/v1/logs",
7276
Insecure: false,
7377
Compression: 0, // No compression by default
7478
Timeout: 30 * time.Second,

‎tests/e2e/config/fluent-bit.yaml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ pipeline:
88
- name: tail
99
tag: kubernetes.*
1010
path: "/var/log/containers/*.log"
11+
exclude_path:
12+
- "/var/log/containers/fluent-bit-*.log"
1113
readFromHead: true
1214
parser: "containerd-parser"
1315
processors:
@@ -45,7 +47,7 @@ pipeline:
4547
- _SYSTEMD_UNIT=kubelet.service
4648
db: /var/run/fluentbit/flb_systemd.db
4749
db.sync: normal
48-
read_from_tail: true
50+
read_from_tail: false
4951
processors:
5052
logs:
5153
- name: lua
@@ -79,11 +81,18 @@ pipeline:
7981
record:
8082
- unit containerd.service
8183
- name: record_modifier
82-
match: "systemd.kubelete.*"
84+
match: "systemd.kubelet.*"
8385
record:
8486
- unit kubelet.service
8587

8688
outputs:
87-
- name: stdout
89+
- name: gardener
8890
match: "systemd.*"
89-
format: json_lines
91+
seed_type: "otlphttp"
92+
log_level: "debug"
93+
endpoint: "victoria-logs-0.victoria-logs.fluent-bit.svc.cluster.local:9428"
94+
endpoint_url_path: "/insert/opentelemetry/v1/logs"
95+
insecure: true
96+
origin: "systemd_journal"
97+
hostname_value: "${NODE_NAME}"
98+
fallback_to_tag_when_metadata_is_missing: false

‎tests/e2e/fetcher/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
fetcher
2+

‎tests/e2e/fetcher/Dockerfile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2025 SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM golang:1.25-alpine AS builder
5+
6+
WORKDIR /workspace
7+
8+
COPY go.mod go.mod
9+
COPY main.go main.go
10+
11+
RUN go mod download
12+
RUN CGO_ENABLED=0 go build -a -o fetcher main.go
13+
14+
FROM gcr.io/distroless/static:nonroot
15+
16+
WORKDIR /
17+
18+
COPY --from=builder /workspace/fetcher .
19+
20+
USER 65532:65532
21+
22+
ENTRYPOINT ["/fetcher"]
23+

‎tests/e2e/fetcher/go.mod‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
module github.com/gardener/logging/v1/tests/e2e/fetcher
2+
3+
go 1.23
4+
5+
toolchain go1.23.4
6+

0 commit comments

Comments
 (0)