diff --git a/integration_test/metadata/integration_metadata.go b/integration_test/metadata/integration_metadata.go index 085c9328ca..19806b759c 100644 --- a/integration_test/metadata/integration_metadata.go +++ b/integration_test/metadata/integration_metadata.go @@ -17,6 +17,8 @@ package metadata import ( "bytes" "fmt" + "net" + "os" "reflect" "regexp" "slices" @@ -303,12 +305,30 @@ func assertMetricLabels(metric *ExpectedMetric, series *monitoringpb.TimeSeries) } // All expected labels must be present and match the given pattern for _, expectedLabel := range metric.Labels { + actualValue, ok := series.Metric.Labels[expectedLabel.Name] if !ok { err = multierr.Append(err, fmt.Errorf("expected label not found: %s", expectedLabel)) continue } - match, matchErr := regexp.MatchString(fmt.Sprintf("^(?:%s)$", expectedLabel.ValueRegex), actualValue) + var matchErr error + // public_ip and private_ip labels will be different according to where the test is running (kokoro or cloudtop). + // https://github.com/GoogleCloudPlatform/opentelemetry-operations-collector/blob/32b5f554a82ff840e690eaf325d83cbff83f3f62/integration_test/gce-testing-internal/gce/gce_testing.go#L2158 + useInternalIP := os.Getenv("USE_INTERNAL_IP") != "" + if (expectedLabel.Name == "public_ip" && useInternalIP) || (expectedLabel.Name == "private_ip" && !useInternalIP) { + // just check if it is a valid IP address + validIP := net.ParseIP(actualValue) + if validIP == nil { + err = multierr.Append(err, fmt.Errorf("error parsing pattern. label=%s, pattern=%s, err=%v", + expectedLabel.Name, + expectedLabel.ValueRegex, + matchErr, + )) + } + continue + } + var match bool + match, matchErr = regexp.MatchString(fmt.Sprintf("^(?:%s)$", expectedLabel.ValueRegex), actualValue) if matchErr != nil { err = multierr.Append(err, fmt.Errorf("error parsing pattern. label=%s, pattern=%s, err=%v", expectedLabel.Name, diff --git a/integration_test/ops_agent_test/main_test.go b/integration_test/ops_agent_test/main_test.go index b79553fa59..cbdd23b590 100644 --- a/integration_test/ops_agent_test/main_test.go +++ b/integration_test/ops_agent_test/main_test.go @@ -46,7 +46,6 @@ import ( "embed" "encoding/json" "encoding/xml" - "errors" "fmt" "io" "log" @@ -2948,54 +2947,31 @@ func TestPrometheusMetrics(t *testing.T) { // under normal circumstances. Based on some experiments, 2 minutes // is normal; wait a bit longer to be on the safe side. time.Sleep(3 * time.Minute) - - existingMetric := "prometheus.googleapis.com/fluentbit_uptime/counter" - window := time.Minute - metric, err := gce.WaitForMetric(ctx, logger, vm, existingMetric, window, nil, true) - if err != nil { - t.Fatal(fmt.Errorf("failed to find metric %q in VM %q: %w", existingMetric, vm.Name, err)) + expectedLabels := append([]*metadata.MetricLabel{ + {Name: "instance_id", ValueRegex: fmt.Sprintf("%d", vm.ID)}, + {Name: "instance_project_id", ValueRegex: vm.Project}, + {Name: "zone", ValueRegex: vm.Zone}, + {Name: "public_ip", ValueRegex: vm.IPAddress}, + {Name: "hostname", ValueRegex: vm.Name}, + {Name: "private_ip", ValueRegex: vm.IPAddress}, + {Name: "network", ValueRegex: fmt.Sprintf("^projects/[0-9]+/networks/%s$", vm.Network)}, + }, getCommonLabels(vm)...) + + expectedMetric := metadata.ExpectedMetric{ + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/fluentbit_uptime/counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 5.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: expectedLabels, + }, + Optional: false, } - var multiErr error - metricValueType := metric.ValueType.String() - metricKind := metric.MetricKind.String() - metricResource := metric.Resource.Type - metricLabels := metric.Metric.Labels - - if metricValueType != "DOUBLE" { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected value type %q", existingMetric, metricValueType)) - } - if metricKind != "CUMULATIVE" { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected kind %q", existingMetric, metricKind)) - } - if metricResource != "prometheus_target" { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected resource type %q", existingMetric, metricResource)) - } - if metricLabels["instance_name"] != vm.Name { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected instance_name label %q. But expected %q", existingMetric, metricLabels["instance_name"], vm.Name)) - } - if metricLabels["instance_id"] != fmt.Sprintf("%d", vm.ID) { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected instance_id label %q. But expected %q", existingMetric, metricLabels["instance_id"], fmt.Sprintf("%d", vm.ID))) - } - expectedMachineType := regexp.MustCompile(fmt.Sprintf("^projects/[0-9]+/machineTypes/%s$", vm.MachineType)) - if !expectedMachineType.MatchString(metricLabels["machine_type"]) { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected machine_type label %q. But expected %q", existingMetric, metricLabels["machine_type"], vm.MachineType)) - } - if metricLabels["instance_project_id"] != vm.Project { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected instance_project_id label %q. But expected %q", existingMetric, metricLabels["instance_project_id"], vm.Project)) - } - if metricLabels["zone"] != vm.Zone { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected zone label %q. But expected %q", existingMetric, metricLabels["zone"], vm.Zone)) - } - expectedNetworkURL := regexp.MustCompile(fmt.Sprintf("^projects/[0-9]+/networks/%s$", vm.Network)) - if !expectedNetworkURL.MatchString(metricLabels["network"]) { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q has unexpected network label %q. But expected %q", existingMetric, metricLabels["network"], vm.Network)) - } - if metricLabels["public_ip"] != vm.IPAddress && metricLabels["private_ip"] != vm.IPAddress { - multiErr = multierr.Append(multiErr, fmt.Errorf("metric %q doesn't hace VM IP %q. Public IP %q Private IP %q", existingMetric, vm.IPAddress, metricLabels["public_ip"], metricLabels["private_ip"])) - } - if multiErr != nil { - t.Error(multiErr) + err := waitForAndAssertMetric(ctx, logger, vm, time.Hour, &expectedMetric, nil, true) + if err != nil { + t.Error(err) } expectedFeatures := []*feature_tracking_metadata.FeatureTracking{ @@ -3109,17 +3085,25 @@ func TestPrometheusMetricsWithMetadata(t *testing.T) { // under normal circumstances. Based on some experiments, 2 minutes // is normal; wait a bit longer to be on the safe side. time.Sleep(3 * time.Minute) + expectedLabels := append([]*metadata.MetricLabel{ + {Name: "hostname", ValueRegex: vm.Name}, + {Name: "test", ValueRegex: escapedMetadataValue}, + }, getCommonLabels(vm)...) - existingMetric := "prometheus.googleapis.com/fluentbit_uptime/counter" - window := time.Minute - metric, err := gce.WaitForMetric(ctx, logger.ToMainLog(), vm, existingMetric, window, nil, true) - if err != nil { - t.Fatal(fmt.Errorf("failed to find metric %q in VM %q: %w", existingMetric, vm.Name, err)) + expectedMetric := metadata.ExpectedMetric{ + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/fluentbit_uptime/counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 5.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: expectedLabels, + }, + Optional: false, } - - metricLabels := metric.Metric.Labels - if metricLabels[metadataKey] != escapedMetadataValue { - t.Errorf("metric %q has VM metadata %q set to %q instead of %q", existingMetric, metadataKey, metricLabels[metadataKey], escapedMetadataValue) + err := waitForAndAssertMetric(ctx, logger.ToMainLog(), vm, time.Minute, &expectedMetric, nil, true) + if err != nil { + t.Error(err) } }) } @@ -3249,7 +3233,6 @@ func TestPrometheusMetricsWithJSONExporter(t *testing.T) { // under normal circumstances. Based on some experiments, 2 minutes // is normal; wait a bit longer to be on the safe side. time.Sleep(3 * time.Minute) - window := time.Minute tests := []metadata.ExpectedMetric{ { MetricSpec: metadata.MetricSpec{ @@ -3308,7 +3291,7 @@ func TestPrometheusMetricsWithJSONExporter(t *testing.T) { var multiErr error for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) + multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, time.Minute, &test, nil, true)) } if multiErr != nil { t.Error(multiErr) @@ -3348,19 +3331,23 @@ func TestPrometheusRelabelConfigs(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - if pts, err := gce.WaitForMetric(ctx, logger, vm, "prometheus.googleapis.com/test_metric/gauge", window, nil, true); err != nil { - return err - } else { - labelValue, ok := pts.Metric.Labels["test_label"] - if !ok { - return errors.New("test_label label not found in metric") - } - if labelValue != "group/capture" { - return fmt.Errorf("Expected test_label to be 'group/capture' but got %s", labelValue) - } - } return nil }, + expectedMetrics: []metadata.ExpectedMetric{{ + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_metric/gauge", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 1000.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + // metric_relabel_configs does not remove source label + {Name: "test_label", ValueRegex: "capture@group"}, + {Name: "destination", ValueRegex: "group/capture"}, + }, + }, + Optional: false, + }}, }) testPrometheusMetrics(t, config, testChecks) } @@ -3392,83 +3379,75 @@ func TestPrometheusUntypedMetrics(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - tests := []metadata.ExpectedMetric{ - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/explicit_untyped_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 1.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "c"}, - {Name: "label_b", ValueRegex: "d"}, - }, getCommonLabels(vm)...), + // Ensure that the gauge-casted metric isn't reported when the unknown typed one is. + if err := gce.AssertMetricMissing(ctx, logger, vm, "prometheus.googleapis.com/explicit_untyped_metric/gauge", true, window); err != nil { + return err + } + if err := gce.AssertMetricMissing(ctx, logger, vm, "prometheus.googleapis.com/missing_type_hint_metric/gauge", true, window); err != nil { + return err + + } + return nil + }, + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/explicit_untyped_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 1.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "c"}, + {Name: "label_b", ValueRegex: "d"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/missing_type_hint_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 10.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "c"}, - {Name: "label_b", ValueRegex: "d"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/missing_type_hint_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 10.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "c"}, + {Name: "label_b", ValueRegex: "d"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - // Since we are sending the same number at every time point, - // the cumulative counter metric will return 0 as no change in values - Type: "prometheus.googleapis.com/explicit_untyped_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 0.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "c"}, - {Name: "label_b", ValueRegex: "d"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + // Since we are sending the same number at every time point, + // the cumulative counter metric will return 0 as no change in values + Type: "prometheus.googleapis.com/explicit_untyped_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 0.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "c"}, + {Name: "label_b", ValueRegex: "d"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/missing_type_hint_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 0.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "c"}, - {Name: "label_b", ValueRegex: "d"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/missing_type_hint_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 0.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "c"}, + {Name: "label_b", ValueRegex: "d"}, }, - Optional: false, }, - } - - var multiErr error - for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) - } - if multiErr != nil { - t.Error(multiErr) - } - - // Ensure that the gauge-casted metric isn't reported when the unknown typed one is. - if err := gce.AssertMetricMissing(ctx, logger, vm, "prometheus.googleapis.com/explicit_untyped_metric/gauge", true, window); err != nil { - t.Error(err) - } - if err := gce.AssertMetricMissing(ctx, logger, vm, "prometheus.googleapis.com/missing_type_hint_metric/gauge", true, window); err != nil { - t.Error(err) - } - return multiErr + Optional: false, + }, }, }) testPrometheusMetrics(t, config, testChecks) @@ -3501,43 +3480,36 @@ func TestPrometheusUntypedMetricsReset(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - tests := []metadata.ExpectedMetric{ - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 10.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + return nil + }, + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 10.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 0.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 0.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - } - var multiErr error - for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) - } - if multiErr != nil { - t.Error(multiErr) - } - return multiErr + Optional: false, + }, }, }) @@ -3548,43 +3520,36 @@ func TestPrometheusUntypedMetricsReset(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - tests := []metadata.ExpectedMetric{ - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 100.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + return nil + }, + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 100.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 90.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 90.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - } - var multiErr error - for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) - } - if multiErr != nil { - t.Error(multiErr) - } - return multiErr + Optional: false, + }, }, }) @@ -3596,42 +3561,35 @@ func TestPrometheusUntypedMetricsReset(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - tests := []metadata.ExpectedMetric{ - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 10.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + return nil + }, + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 10.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 0.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 0.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - } - var multiErr error - for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) - } - if multiErr != nil { - t.Error(multiErr) - } - return multiErr + Optional: false, + }, }, }) testChecks = append(testChecks, mockPrometheusCheck{ @@ -3640,43 +3598,35 @@ func TestPrometheusUntypedMetricsReset(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - tests := []metadata.ExpectedMetric{ - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown", - Kind: metric.MetricDescriptor_GAUGE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 1000.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + return nil + }, + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown", + Kind: metric.MetricDescriptor_GAUGE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 1000.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - { - MetricSpec: metadata.MetricSpec{ - Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Value: 990.0, - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "label_a", ValueRegex: "b"}, - }, getCommonLabels(vm)...), + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/untyped_metric/unknown:counter", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Value: 990.0, + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "label_a", ValueRegex: "b"}, }, - Optional: false, }, - } - - var multiErr error - for _, test := range tests { - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true)) - } - if multiErr != nil { - t.Error(multiErr) - } - return multiErr + Optional: false, + }, }, }) testPrometheusMetrics(t, config, testChecks) @@ -3769,19 +3719,38 @@ func TestPrometheusHistogramMetrics(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - return assertPrometheusHistogramMetric(ctx, logger, vm, "test_histogram", window, stepOneExpected) + return nil }, - }) + expectedMetrics: []metadata.ExpectedMetric{{ + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_histogram/histogram", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DISTRIBUTION.String(), + Value: stepOneExpected, + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + }}) testChecks = append(testChecks, mockPrometheusCheck{ fileToUpload: fileToUpload{ local: path.Join(prometheusTestdata, "sample_histogram_step_2"), remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - return assertPrometheusHistogramMetric(ctx, logger, vm, "test_histogram", window, stepTwoExpected) + return nil }, - }) - + expectedMetrics: []metadata.ExpectedMetric{{ + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_histogram/histogram", + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + ValueType: metric.MetricDescriptor_DISTRIBUTION.String(), + Value: stepTwoExpected, + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + }}) testPrometheusMetrics(t, config, testChecks) } @@ -3857,18 +3826,149 @@ func TestPrometheusSummaryMetrics(t *testing.T) { remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - return assertPrometheusSummaryMetric(ctx, logger, vm, "test_summary", window, stepOneExpected) + return nil }, - }) + expectedMetrics: []metadata.ExpectedMetric{ + // There is no direct mapping of Prometheus summary type. Instead, GCM + // would store the quantiles into prometheus.googleapis.com/NAME/summary + // with the actual quantile as a metric label, of type gauge + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 40, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.5"}, + }, + }, + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 80, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.9"}, + }, + }, + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 90, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.99"}, + }, + }, + Optional: false, + }, + // The count value in Prometheus summary goes to + // prometheus.googleapis.com/NAME_count/summary of type cumulative + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary_count/summary", + Value: stepOneExpected.Count, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + // The sum value in Prometheus summary goes to + // prometheus.googleapis.com/NAME_sum/summary:counter of type cumulative + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary_sum/summary:counter", + Value: stepOneExpected.Sum, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + }}) testChecks = append(testChecks, mockPrometheusCheck{ fileToUpload: fileToUpload{ local: path.Join(prometheusTestdata, "sample_summary_step_2"), remote: path.Join(remoteWorkDir, "data"), }, check: func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error { - return assertPrometheusSummaryMetric(ctx, logger, vm, "test_summary", window, stepTwoExpected) + return nil }, - }) + expectedMetrics: []metadata.ExpectedMetric{ + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 40, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.5"}, + }, + }, + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 80, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.9"}, + }, + }, + Optional: false, + }, + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary/summary", + Value: 90, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_GAUGE.String(), + MonitoredResources: []string{"prometheus_target"}, + Labels: []*metadata.MetricLabel{ + {Name: "quantile", ValueRegex: "0.99"}, + }, + }, + Optional: false, + }, + // The count value in Prometheus summary goes to + // prometheus.googleapis.com/NAME_count/summary of type cumulative + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary_count/summary", + Value: stepTwoExpected.Count, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + // The sum value in Prometheus summary goes to + // prometheus.googleapis.com/NAME_sum/summary:counter of type cumulative + { + MetricSpec: metadata.MetricSpec{ + Type: "prometheus.googleapis.com/test_summary_sum/summary:counter", + Value: stepTwoExpected.Sum, + ValueType: metric.MetricDescriptor_DOUBLE.String(), + Kind: metric.MetricDescriptor_CUMULATIVE.String(), + MonitoredResources: []string{"prometheus_target"}, + }, + Optional: false, + }, + }}) testPrometheusMetrics(t, config, testChecks) } @@ -3950,6 +4050,14 @@ func testPrometheusMetrics(t *testing.T, opsAgentConfig string, testChecks []moc time.Sleep(3 * time.Minute) window := time.Minute var multiErr error + commonLabels := getCommonLabels(vm) + for _, expectedMetric := range firstTest.expectedMetrics { + expectedMetric.Labels = append(expectedMetric.Labels, commonLabels...) + multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &expectedMetric, nil, true)) + if multiErr != nil { + t.Error(multiErr) + } + } multiErr = multierr.Append(multiErr, firstTest.check(ctx, logger, vm, window)) for _, test := range restTests { @@ -3961,6 +4069,13 @@ func testPrometheusMetrics(t *testing.T, opsAgentConfig string, testChecks []moc // Wait until the new points have arrived and complete the check for this test input. time.Sleep(3 * time.Minute) + for _, expectedMetric := range test.expectedMetrics { + expectedMetric.Labels = append(expectedMetric.Labels, commonLabels...) + multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &expectedMetric, nil, true)) + if multiErr != nil { + t.Error(multiErr) + } + } multiErr = multierr.Append(multiErr, test.check(ctx, logger, vm, window)) if multiErr != nil { @@ -3970,24 +4085,6 @@ func testPrometheusMetrics(t *testing.T, opsAgentConfig string, testChecks []moc }) } -// assertPrometheusHistogramMetric Check if the last point of the time series is -// the expected Prometheus histogram metric point -func assertPrometheusHistogramMetric(ctx context.Context, logger *log.Logger, vm *gce.VM, name string, window time.Duration, expected *distribution.Distribution) error { - // GCM map Prometheus histogram to cumulative distribution - test := metadata.ExpectedMetric{ - MetricSpec: metadata.MetricSpec{ - Type: fmt.Sprintf("prometheus.googleapis.com/%s/histogram", name), - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - ValueType: metric.MetricDescriptor_DISTRIBUTION.String(), - Value: expected, - MonitoredResources: []string{"prometheus_target"}, - Labels: getCommonLabels(vm), - }, - Optional: false, - } - return waitForAndAssertMetric(ctx, logger, vm, window, &test, nil, true) -} - // A sample of the Prometheus summary metric with name 'test_summary': // # HELP test_summary Test Summary. // # TYPE test_summary summary @@ -4001,62 +4098,6 @@ type prometheusSummaryMetric struct { Count, Sum float64 } -// assertPrometheusSummaryMetric checks if the last point of the time series is -// the expected prometheus summary metric point -func assertPrometheusSummaryMetric(ctx context.Context, logger *log.Logger, vm *gce.VM, name string, window time.Duration, expected prometheusSummaryMetric) error { - var multiErr error - // There is no direct mapping of Prometheus summary type. Instead, GCM - // would store the quantiles into prometheus.googleapis.com/NAME/summary - // with the actual quantile as a metric label, of type gauge - - for quantile, value := range expected.Quantiles { - test := metadata.ExpectedMetric{ - MetricSpec: metadata.MetricSpec{ - Type: fmt.Sprintf("prometheus.googleapis.com/%s/summary", name), - Value: value, - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Kind: metric.MetricDescriptor_GAUGE.String(), - MonitoredResources: []string{"prometheus_target"}, - Labels: append([]*metadata.MetricLabel{ - {Name: "quantile", ValueRegex: quantile}, - }, getCommonLabels(vm)...), - }, - Optional: false, - } - extraFilters := []string{fmt.Sprintf(`metric.labels.quantile = "%s"`, quantile)} - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &test, extraFilters, true)) - } - // The count value in Prometheus summary goes to - // prometheus.googleapis.com/NAME_count/summary of type cumulative - testCount := metadata.ExpectedMetric{ - MetricSpec: metadata.MetricSpec{ - Type: fmt.Sprintf("prometheus.googleapis.com/%s_count/summary", name), - Value: expected.Count, - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - MonitoredResources: []string{"prometheus_target"}, - Labels: getCommonLabels(vm), - }, - Optional: false, - } - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &testCount, nil, true)) - // The sum value in Prometheus summary goes to - // prometheus.googleapis.com/NAME_sum/summary:counter of type cumulative - testSummary := metadata.ExpectedMetric{ - MetricSpec: metadata.MetricSpec{ - Type: fmt.Sprintf("prometheus.googleapis.com/%s_sum/summary:counter", name), - Value: expected.Sum, - ValueType: metric.MetricDescriptor_DOUBLE.String(), - Kind: metric.MetricDescriptor_CUMULATIVE.String(), - MonitoredResources: []string{"prometheus_target"}, - Labels: getCommonLabels(vm), - }, - Optional: false, - } - multiErr = multierr.Append(multiErr, waitForAndAssertMetric(ctx, logger, vm, window, &testSummary, nil, true)) - return multiErr -} - func waitForAndAssertMetric(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration, metric *metadata.ExpectedMetric, extraFilters []string, isPrometheus bool) error { series, err := gce.WaitForMetric(ctx, logger, vm, metric.Type, window, extraFilters, isPrometheus) if err != nil { @@ -4077,8 +4118,9 @@ type fileToUpload struct { // by an http server at endpoint localhost:8000/metrics. It also contains a check that // be used to validate that the ingested metrics are collected correctly. type mockPrometheusCheck struct { - fileToUpload fileToUpload - check func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error + fileToUpload fileToUpload + check func(ctx context.Context, logger *log.Logger, vm *gce.VM, window time.Duration) error + expectedMetrics []metadata.ExpectedMetric } // uploadFiles upload files from fs embedded file system to vm @@ -5004,8 +5046,7 @@ traces: if err := installGolang(ctx, logger, vm); err != nil { t.Fatal(err) } - serviceName := "otlp-metric-googlemanagedprometheus-test" - if err = runGoCode(ctx, logger, vm, metricFile, "-service_name", serviceName); err != nil { + if err = runGoCode(ctx, logger, vm, metricFile); err != nil { t.Fatal(err) } expectedLabels := []*metadata.MetricLabel{