diff --git a/pkg/sloop/server/internal/config/config.go b/pkg/sloop/server/internal/config/config.go index 2933299f..3dc1eda8 100644 --- a/pkg/sloop/server/internal/config/config.go +++ b/pkg/sloop/server/internal/config/config.go @@ -20,6 +20,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" + "github.com/salesforce/sloop/pkg/sloop/server/server_metrics" "github.com/salesforce/sloop/pkg/sloop/webserver" ) @@ -29,9 +30,10 @@ type SloopConfig struct { // These fields can only come from command line ConfigFile string // These fields can only come from file because they use complex types - LeftBarLinks []webserver.LinkTemplate `json:"leftBarLinks"` - ResourceLinks []webserver.ResourceLinkTemplate `json:"resourceLinks"` - ExclusionRules map[string][]any `json:"exclusionRules"` + LeftBarLinks []webserver.LinkTemplate `json:"leftBarLinks"` + ResourceLinks []webserver.ResourceLinkTemplate `json:"resourceLinks"` + ExclusionRules map[string][]any `json:"exclusionRules"` + UserMetricsHeaders []server_metrics.UserMetricsConfig `json:"userMetricsHeaders"` // Normal fields that can come from file or cmd line DisableKubeWatcher bool `json:"disableKubeWatch"` KubeWatchResyncInterval time.Duration `json:"kubeWatchResyncInterval"` @@ -76,6 +78,7 @@ type SloopConfig struct { BadgerVLogTruncate bool `json:"badgerVLogTruncate"` EnableDeleteKeys bool `json:"enableDeleteKeys"` EnableGranularMetrics bool `json:"enableGranularMetrics"` + EnableUserMetrics bool `json:"enableUserMetrics"` PrivilegedAccess bool `json:"PrivilegedAccess"` BadgerDetailLogEnabled bool `json:"badgerDetailLogEnabled"` } @@ -124,6 +127,7 @@ func registerFlags(fs *flag.FlagSet, config *SloopConfig) { fs.BoolVar(&config.BadgerSyncWrites, "badger-sync-writes", config.BadgerSyncWrites, "Sync Writes ensures writes are synced to disk if set to true") fs.BoolVar(&config.EnableDeleteKeys, "enable-delete-keys", config.EnableDeleteKeys, "Use delete prefixes instead of dropPrefix for GC") fs.BoolVar(&config.EnableGranularMetrics, "enable-granular-metrics", config.EnableGranularMetrics, "default:True , allows metrics of event kind to be granular with reason, type and name") + fs.BoolVar(&config.EnableUserMetrics, "enable-user-metrics", config.EnableUserMetrics, "Enable collection of user metrics from request headers") fs.BoolVar(&config.PrivilegedAccess, "allow-privileged-access", config.PrivilegedAccess, "default:True , allows sloop to access kube root path") fs.BoolVar(&config.BadgerVLogFileIOMapping, "badger-vlog-fileIO-mapping", config.BadgerVLogFileIOMapping, "Indicates which file loading mode should be used for the value log data, in memory constrained environments the value is recommended to be true") fs.BoolVar(&config.BadgerVLogTruncate, "badger-vlog-truncate", config.BadgerVLogTruncate, "Truncate value log if badger db offset is different from badger db size") @@ -176,6 +180,7 @@ func getDefaultConfig() *SloopConfig { BadgerVLogTruncate: true, EnableDeleteKeys: false, EnableGranularMetrics: false, + EnableUserMetrics: false, PrivilegedAccess: true, BadgerDetailLogEnabled: false, ExclusionRules: map[string][]any{}, diff --git a/pkg/sloop/server/server.go b/pkg/sloop/server/server.go index 3b6eb9d9..a33a0949 100644 --- a/pkg/sloop/server/server.go +++ b/pkg/sloop/server/server.go @@ -18,6 +18,7 @@ import ( "github.com/salesforce/sloop/pkg/sloop/ingress" "github.com/salesforce/sloop/pkg/sloop/server/internal/config" + "github.com/salesforce/sloop/pkg/sloop/server/server_metrics" "github.com/salesforce/sloop/pkg/sloop/store/typed" "github.com/salesforce/sloop/pkg/sloop/store/untyped" @@ -146,18 +147,24 @@ func RealMain() error { displayContext = conf.DisplayContext } + // Initialize user metrics if enabled + if conf.EnableUserMetrics { + server_metrics.InitUserMetrics(conf.UserMetricsHeaders) + } + webConfig := webserver.WebConfig{ - BindAddress: conf.BindAddress, - Port: conf.Port, - WebFilesPath: conf.WebFilesPath, - ConfigYaml: conf.ToYaml(), - MaxLookback: conf.MaxLookback, - DefaultNamespace: conf.DefaultNamespace, - DefaultLookback: conf.DefaultLookback, - DefaultResources: conf.DefaultKind, - ResourceLinks: conf.ResourceLinks, - LeftBarLinks: conf.LeftBarLinks, - CurrentContext: displayContext, + BindAddress: conf.BindAddress, + Port: conf.Port, + WebFilesPath: conf.WebFilesPath, + ConfigYaml: conf.ToYaml(), + MaxLookback: conf.MaxLookback, + DefaultNamespace: conf.DefaultNamespace, + DefaultLookback: conf.DefaultLookback, + DefaultResources: conf.DefaultKind, + ResourceLinks: conf.ResourceLinks, + LeftBarLinks: conf.LeftBarLinks, + CurrentContext: displayContext, + EnableUserMetrics: conf.EnableUserMetrics, } err = webserver.Run(webConfig, tables) if err != nil { diff --git a/pkg/sloop/server/server_metrics/server_metrics.go b/pkg/sloop/server/server_metrics/server_metrics.go new file mode 100644 index 00000000..1926031b --- /dev/null +++ b/pkg/sloop/server/server_metrics/server_metrics.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2019, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +package server_metrics + +import ( + "net/http" + "sort" + "sync" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + // Metric names + MetricUserRequests = "user_requests" + + // Built-in request context label keys (always included) + LabelHandler = "handler" + LabelCluster = "cluster" + LabelQuery = "query" + LabelNamespace = "namespace" + LabelKind = "kind" + LabelName = "name" + LabelLookback = "lookback" +) + +// UserMetricsConfig defines a single header extraction rule for user metrics +type UserMetricsConfig struct { + // Label is the Prometheus metric label name (e.g., "username", "email", "team") + Label string `json:"label"` + // Headers is a list of HTTP header names to check, in order of preference + // The first non-empty header value found will be used + Headers []string `json:"headers"` +} + +var ( + userRequestCounter *prometheus.CounterVec + configuredLabels []string + configuredHeaderRules []UserMetricsConfig + initOnce sync.Once + + // Built-in labels that are always included for request context + builtInLabels = []string{LabelHandler, LabelCluster, LabelQuery, LabelNamespace, LabelKind, LabelName, LabelLookback} +) + +// GetDefaultUserMetricsConfig returns the default configuration for user metrics (username and user_agent) +func GetDefaultUserMetricsConfig() []UserMetricsConfig { + return []UserMetricsConfig{ + { + Label: "username", + Headers: []string{"X-Username", "x-username"}, + }, + { + Label: "user_agent", + Headers: []string{"User-Agent"}, + }, + } +} + +// InitUserMetrics initializes the user metrics with the given configuration +// This should be called once during application startup before serving requests +// If config is nil or empty, default configuration will be used +func InitUserMetrics(config []UserMetricsConfig) { + initOnce.Do(func() { + // Use default config if none provided + if len(config) == 0 { + config = GetDefaultUserMetricsConfig() + glog.Infof("No user metrics configuration provided, using defaults") + } + + // Store the configuration + configuredHeaderRules = config + + // Start with built-in labels for request context + labelSet := make(map[string]bool) + for _, label := range builtInLabels { + labelSet[label] = true + } + + // Add header-based labels from configuration + for _, rule := range config { + if rule.Label != "" { + labelSet[rule.Label] = true + } + } + + // Sort labels for consistent ordering + configuredLabels = make([]string, 0, len(labelSet)) + for label := range labelSet { + configuredLabels = append(configuredLabels, label) + } + sort.Strings(configuredLabels) + + // Create the counter with configured labels + userRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: MetricUserRequests, + Help: "Counter of requests broken out by user and request information.", + }, + configuredLabels, + ) + + // Register with Prometheus + prometheus.MustRegister(userRequestCounter) + + glog.Infof("User metrics initialized with labels: %v", configuredLabels) + glog.Infof(" Built-in request labels: %v", builtInLabels) + for _, rule := range config { + glog.Infof(" Header label '%s': headers=%v", rule.Label, rule.Headers) + } + }) +} + +// ============================================================================ +// Header Information Extraction Functions +// ============================================================================ + +// getHeaderWithFallback tries multiple header names and returns the first non-empty value +func getHeaderWithFallback(headers *http.Header, headerNames []string) string { + for _, headerName := range headerNames { + if value := headers.Get(headerName); value != "" { + return value + } + } + return "" +} + +// extractRequesterInfo extracts user information from request headers based on configuration +func extractRequesterInfo(headers *http.Header) map[string]string { + requesterInfo := make(map[string]string) + + for _, rule := range configuredHeaderRules { + if rule.Label != "" && len(rule.Headers) > 0 { + requesterInfo[rule.Label] = getHeaderWithFallback(headers, rule.Headers) + } + } + + return requesterInfo +} + +// mergeMaps merges two maps, with values from additionalTags taking precedence +func mergeMaps(base, additional map[string]string) map[string]string { + result := make(map[string]string) + + for k, v := range base { + result[k] = v + } + + for k, v := range additional { + result[k] = v + } + + return result +} + +// PublishHeaderMetrics publishes header metrics to the metrics server +func PublishHeaderMetrics(headers *http.Header, additionalTags map[string]string, enableUserMetrics bool) { + if !enableUserMetrics { + return + } + + userInfo := extractRequesterInfo(headers) + allLabels := mergeMaps(userInfo, additionalTags) + + // Build label values in the correct order + labelValues := make([]string, len(configuredLabels)) + for i, label := range configuredLabels { + labelValues[i] = allLabels[label] + } + + userRequestCounter.WithLabelValues(labelValues...).Inc() + + // Log user request at V(2) verbosity level for debugging + glog.V(2).Infof("User request made with labels: %v", allLabels) +} diff --git a/pkg/sloop/server/server_metrics/server_metrics_test.go b/pkg/sloop/server/server_metrics/server_metrics_test.go new file mode 100644 index 00000000..994c347e --- /dev/null +++ b/pkg/sloop/server/server_metrics/server_metrics_test.go @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2019, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +package server_metrics + +import ( + "net/http" + "sync" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +// testOnce ensures metrics are only initialized once across all tests +var testOnce sync.Once + +// initTestMetrics initializes metrics once for all tests using the default config +func initTestMetrics() { + testOnce.Do(func() { + // Reset in case previous test run left state + if userRequestCounter != nil { + prometheus.Unregister(userRequestCounter) + } + initOnce = sync.Once{} + configuredLabels = nil + configuredHeaderRules = nil + userRequestCounter = nil + + InitUserMetrics(GetDefaultUserMetricsConfig()) + }) +} + +func TestGetHeaderWithFallback(t *testing.T) { + tests := []struct { + name string + headers map[string]string + headerNames []string + want string + }{ + { + name: "no headers", + headers: map[string]string{}, + headerNames: []string{"x-email", "X-Email"}, + want: "", + }, + { + name: "first header present", + headers: map[string]string{"x-email": "test@example.com"}, + headerNames: []string{"x-email", "X-Email"}, + want: "test@example.com", + }, + { + name: "second header present", + headers: map[string]string{"X-Email": "uppercase@example.com"}, + headerNames: []string{"x-email", "X-Email"}, + want: "uppercase@example.com", + }, + { + name: "empty header value", + headers: map[string]string{"x-email": ""}, + headerNames: []string{"x-email", "X-Email"}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + headers := &http.Header{} + + for k, v := range tt.headers { + headers.Set(k, v) + } + + got := getHeaderWithFallback(headers, tt.headerNames) + if got != tt.want { + t.Errorf("getHeaderWithFallback() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetDefaultUserMetricsConfig(t *testing.T) { + config := GetDefaultUserMetricsConfig() + + if len(config) != 2 { + t.Errorf("GetDefaultUserMetricsConfig() returned %d configs, want 2", len(config)) + } + + // Check username config + usernameFound := false + for _, c := range config { + if c.Label == "username" { + usernameFound = true + if len(c.Headers) < 1 { + t.Errorf("username config should have at least one header") + } + } + } + if !usernameFound { + t.Errorf("GetDefaultUserMetricsConfig() should include username label") + } + + // Check user_agent config + userAgentFound := false + for _, c := range config { + if c.Label == "user_agent" { + userAgentFound = true + } + } + if !userAgentFound { + t.Errorf("GetDefaultUserMetricsConfig() should include user_agent label") + } +} + +func TestInitUserMetricsIncludesBuiltInLabels(t *testing.T) { + initTestMetrics() + + // Check that built-in labels are included + labelMap := make(map[string]bool) + for _, label := range configuredLabels { + labelMap[label] = true + } + + if !labelMap[LabelHandler] { + t.Errorf("configuredLabels should include '%s'", LabelHandler) + } + if !labelMap[LabelCluster] { + t.Errorf("configuredLabels should include '%s'", LabelCluster) + } + if !labelMap[LabelQuery] { + t.Errorf("configuredLabels should include '%s'", LabelQuery) + } + if !labelMap[LabelNamespace] { + t.Errorf("configuredLabels should include '%s'", LabelNamespace) + } + if !labelMap[LabelKind] { + t.Errorf("configuredLabels should include '%s'", LabelKind) + } + if !labelMap[LabelName] { + t.Errorf("configuredLabels should include '%s'", LabelName) + } + if !labelMap[LabelLookback] { + t.Errorf("configuredLabels should include '%s'", LabelLookback) + } + + // Check that header-based labels are also included + if !labelMap["username"] { + t.Errorf("configuredLabels should include 'username'") + } + if !labelMap["user_agent"] { + t.Errorf("configuredLabels should include 'user_agent'") + } +} + +func TestMergeMaps(t *testing.T) { + tests := []struct { + name string + base map[string]string + additional map[string]string + want map[string]string + }{ + { + name: "merge with override", + base: map[string]string{"key1": "value1", "key2": "value2"}, + additional: map[string]string{"key2": "newvalue2", "key3": "value3"}, + want: map[string]string{"key1": "value1", "key2": "newvalue2", "key3": "value3"}, + }, + { + name: "empty additional", + base: map[string]string{"key1": "value1"}, + additional: map[string]string{}, + want: map[string]string{"key1": "value1"}, + }, + { + name: "empty base", + base: map[string]string{}, + additional: map[string]string{"key1": "value1"}, + want: map[string]string{"key1": "value1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mergeMaps(tt.base, tt.additional) + if len(got) != len(tt.want) { + t.Errorf("mergeMaps() map length = %d, want %d", len(got), len(tt.want)) + } + for k, v := range tt.want { + if got[k] != v { + t.Errorf("mergeMaps() [%s] = %v, want %v", k, got[k], v) + } + } + }) + } +} + +func TestUserMetricsConfigJSON(t *testing.T) { + // Test that the config structure can be represented correctly + config := UserMetricsConfig{ + Label: "email", + Headers: []string{"X-Email", "X-User-Email", "x-email"}, + } + + if config.Label != "email" { + t.Errorf("Expected label 'email', got '%s'", config.Label) + } + if len(config.Headers) != 3 { + t.Errorf("Expected 3 headers, got %d", len(config.Headers)) + } +} + +func TestExtractRequesterInfo(t *testing.T) { + initTestMetrics() + + // Test with no headers + headers := &http.Header{} + info := extractRequesterInfo(headers) + if info["username"] != "" { + t.Errorf("Expected username to be empty, got %s", info["username"]) + } + if info["user_agent"] != "" { + t.Errorf("Expected user_agent to be empty, got %s", info["user_agent"]) + } + + // Test with headers + headers.Set("X-Username", "testuser") + headers.Set("User-Agent", "Mozilla/5.0") + + info = extractRequesterInfo(headers) + if info["username"] != "testuser" { + t.Errorf("Expected username to be 'testuser', got %s", info["username"]) + } + if info["user_agent"] != "Mozilla/5.0" { + t.Errorf("Expected user_agent to be 'Mozilla/5.0', got %s", info["user_agent"]) + } +} + +func TestExtractRequesterInfoEmptyHeaders(t *testing.T) { + initTestMetrics() + + headers := &http.Header{} + headers.Set("X-Username", "") + headers.Set("User-Agent", "") + + info := extractRequesterInfo(headers) + if info["username"] != "" { + t.Errorf("Expected username to be empty for empty header, got %s", info["username"]) + } + if info["user_agent"] != "" { + t.Errorf("Expected user_agent to be empty for empty header, got %s", info["user_agent"]) + } +} + +func TestPublishHeaderMetrics(t *testing.T) { + initTestMetrics() + + // Test with metrics disabled - should not panic + headers := &http.Header{} + headers.Set("X-Username", "testuser") + additionalTags := map[string]string{ + LabelHandler: "test", + LabelCluster: "test-cluster", + LabelQuery: "EventHeatMap", + LabelNamespace: "default", + LabelKind: "Pod", + LabelName: "", + LabelLookback: "1h", + } + + // Should not panic when disabled + PublishHeaderMetrics(headers, additionalTags, false) + + // Should not panic when enabled + PublishHeaderMetrics(headers, additionalTags, true) +} + +func TestLabelCount(t *testing.T) { + initTestMetrics() + + // Default config has 2 header labels (username, user_agent) + 7 built-in labels = 9 total + expectedCount := 2 + len(builtInLabels) + if len(configuredLabels) != expectedCount { + t.Errorf("Expected %d configured labels with default config, got %d: %v", + expectedCount, len(configuredLabels), configuredLabels) + } +} diff --git a/pkg/sloop/webserver/middleware.go b/pkg/sloop/webserver/middleware.go index 910fa348..1fb7e546 100644 --- a/pkg/sloop/webserver/middleware.go +++ b/pkg/sloop/webserver/middleware.go @@ -11,12 +11,15 @@ import ( "context" "fmt" "net/http" + "strings" "time" "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/salesforce/sloop/pkg/sloop/server/server_metrics" ) const requestIDKey string = "reqId" @@ -71,6 +74,36 @@ func glogMiddleware(next http.Handler) http.Handler { }) } +// userMetricsMiddleware collects user metrics from request headers +func userMetricsMiddleware(handlerName string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Call the next handler first + next.ServeHTTP(w, r) + + // Extract cluster/context from path (first segment after leading slash) + cluster := "" + pathParts := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") + if len(pathParts) > 0 && pathParts[0] != "" { + cluster = pathParts[0] + } + + // Extract query parameters - always include all built-in labels (empty string if not present) + // Prometheus requires all labels to be provided for each metric observation + queryParams := r.URL.Query() + additionalTags := map[string]string{ + server_metrics.LabelHandler: handlerName, + server_metrics.LabelCluster: cluster, + server_metrics.LabelQuery: queryParams.Get("query"), + server_metrics.LabelNamespace: queryParams.Get("namespace"), + server_metrics.LabelKind: queryParams.Get("kind"), + server_metrics.LabelName: queryParams.Get("namematch"), + server_metrics.LabelLookback: queryParams.Get("lookback"), + } + + server_metrics.PublishHeaderMetrics(&r.Header, additionalTags, enableUserMetrics) + }) +} + func middlewareChain(handlerName string, next http.Handler) http.HandlerFunc { return promhttp.InstrumentHandlerCounter( metricWebServerRequestCount.MustCurryWith(prometheus.Labels{"handler": handlerName}), @@ -78,7 +111,7 @@ func middlewareChain(handlerName string, next http.Handler) http.HandlerFunc { metricWebServerRequestDuration.MustCurryWith(prometheus.Labels{"handler": handlerName}), traceMiddleware( glogMiddleware( - next, + userMetricsMiddleware(handlerName, next), ), ), ), @@ -87,5 +120,6 @@ func middlewareChain(handlerName string, next http.Handler) http.HandlerFunc { func metricCountMiddleware(handlerName string, next http.Handler) http.HandlerFunc { return promhttp.InstrumentHandlerCounter( - metricWebServerRequestCount.MustCurryWith(prometheus.Labels{"handler": handlerName}), next) + metricWebServerRequestCount.MustCurryWith(prometheus.Labels{"handler": handlerName}), + userMetricsMiddleware(handlerName, next)) } diff --git a/pkg/sloop/webserver/webserver.go b/pkg/sloop/webserver/webserver.go index c39d8e6f..cbabd742 100644 --- a/pkg/sloop/webserver/webserver.go +++ b/pkg/sloop/webserver/webserver.go @@ -51,22 +51,24 @@ const ( ) type WebConfig struct { - BindAddress string - Port int - WebFilesPath string - DefaultNamespace string - DefaultLookback string - DefaultResources string - MaxLookback time.Duration - ConfigYaml string - ResourceLinks []ResourceLinkTemplate - LeftBarLinks []LinkTemplate - CurrentContext string + BindAddress string + Port int + WebFilesPath string + DefaultNamespace string + DefaultLookback string + DefaultResources string + MaxLookback time.Duration + ConfigYaml string + ResourceLinks []ResourceLinkTemplate + LeftBarLinks []LinkTemplate + CurrentContext string + EnableUserMetrics bool } // This is not going to change and we don't want to pass it to every function // so use a static for now var webFilesPath string +var enableUserMetrics bool func logWebError(err error, note string, r *http.Request, w http.ResponseWriter) { message := fmt.Sprintf("Error rendering url: %q. Note: %v. Error: %v", r.URL, note, err) @@ -221,6 +223,7 @@ func registerRoutes(mux *http.ServeMux, config WebConfig, tables typed.Tables) { func Run(config WebConfig, tables typed.Tables) error { webFilesPath = config.WebFilesPath + enableUserMetrics = config.EnableUserMetrics mux := http.NewServeMux() registerRoutes(mux, config, tables)