diff --git a/integration-tests/common/metric.go b/integration-tests/common/metric.go index 95990a042475..fc3b6fee74ee 100644 --- a/integration-tests/common/metric.go +++ b/integration-tests/common/metric.go @@ -5,6 +5,11 @@ import ( "fmt" ) +type MetricsResponse struct { + Status string `json:"status"` + Data []Metric `json:"data"` +} + type MetricResponse struct { Status string `json:"status"` Data MetricData `json:"data"` @@ -53,6 +58,10 @@ func (m *MetricResponse) Unmarshal(data []byte) error { return json.Unmarshal(data, m) } +func (m *MetricsResponse) Unmarshal(data []byte) error { + return json.Unmarshal(data, m) +} + func (h *HistogramRawData) UnmarshalJSON(b []byte) error { var arr []json.RawMessage if err := json.Unmarshal(b, &arr); err != nil { diff --git a/integration-tests/common/metrics_assert.go b/integration-tests/common/metrics_assert.go index 5797a74271a0..51f2d97daa51 100644 --- a/integration-tests/common/metrics_assert.go +++ b/integration-tests/common/metrics_assert.go @@ -6,9 +6,10 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" +const promURL = "http://localhost:9009/prometheus/api/v1/" // Default metrics list according to what the prom-gen app is generating. var PromDefaultMetrics = []string{ @@ -39,13 +40,19 @@ var OtelDefaultHistogramMetrics = []string{ "example_exponential_float_histogram", } -// MetricQuery returns a formatted Prometheus metric query with a given metricName and a given label. +// MetricQuery returns a formatted Prometheus metric query with a given metricName and the given test_name label. func MetricQuery(metricName string, testName string) string { - return fmt.Sprintf("%s%s{test_name='%s'}", promURL, metricName, testName) + return fmt.Sprintf("%squery?query=%s{test_name='%s'}", promURL, metricName, testName) +} + +// MetricsQuery returns the list of available metrics matching the given test_name label. +func MetricsQuery(testName string) string { + return fmt.Sprintf("%sseries?match[]={test_name='%s'}", promURL, testName) } // MimirMetricsTest checks that all given metrics are stored in Mimir. func MimirMetricsTest(t *testing.T, metrics []string, histogramMetrics []string, testName string) { + AssertMetricsAvailable(t, metrics, histogramMetrics, testName) for _, metric := range metrics { metric := metric t.Run(metric, func(t *testing.T) { @@ -62,6 +69,40 @@ func MimirMetricsTest(t *testing.T, metrics []string, histogramMetrics []string, } } +// AssertMetricsAvailable performs a Prometheus query and expect the result to eventually contain the list of expected metrics. +func AssertMetricsAvailable(t *testing.T, metrics []string, histogramMetrics []string, testName string) { + var missingMetrics []string + expectedMetrics := append(metrics, histogramMetrics...) + query := MetricsQuery(testName) + require.EventuallyWithT(t, func(c *assert.CollectT) { + var metricsResponse MetricsResponse + err := FetchDataFromURL(query, &metricsResponse) + assert.NoError(c, err) + missingMetrics = checkMissingMetrics(expectedMetrics, metricsResponse.Data) + msg := fmt.Sprintf("Some metrics are missing: %v", missingMetrics) + if len(missingMetrics) == len(expectedMetrics) { + msg = "All metrics are missing" + } + assert.Empty(c, missingMetrics, msg) + }, DefaultTimeout, DefaultRetryInterval) +} + +// checkMissingMetrics returns the expectedMetrics which are not contained in actualMetrics. +func checkMissingMetrics(expectedMetrics []string, actualMetrics []Metric) []string { + metricSet := make(map[string]struct{}, len(actualMetrics)) + for _, metric := range actualMetrics { + metricSet[metric.Name] = struct{}{} + } + + var missingMetrics []string + for _, expectedMetric := range expectedMetrics { + if _, exists := metricSet[expectedMetric]; !exists { + missingMetrics = append(missingMetrics, expectedMetric) + } + } + return missingMetrics +} + // AssertHistogramData performs a Prometheus query and expect the result to eventually contain the expected histogram. // The count and sum metrics should be greater than 10 before the timeout triggers. func AssertHistogramData(t *testing.T, query string, expectedMetric string, testName string) { diff --git a/integration-tests/utils.go b/integration-tests/utils.go index 248e56eaaf8c..9e9280f0247c 100644 --- a/integration-tests/utils.go +++ b/integration-tests/utils.go @@ -106,7 +106,6 @@ func runAllTests() { }(testDir, i) } wg.Wait() - close(logChan) } func cleanUpEnvironment() { @@ -119,6 +118,9 @@ func cleanUpEnvironment() { func reportResults() { testsFailed := 0 + // It's ok to close the channel here because all tests are finished. + // If the channel would not be closed, the for loop would wait forever. + close(logChan) for log := range logChan { fmt.Printf("Failure detected in %s:\n", log.TestDir) fmt.Println("Test output:", log.TestOutput)