Integration tests improvements (#6298)

* close channel in the reportResults to prevent deadlock when running only one test

* Add an assert that checks if all expected metrics are available in Mimir

This change makes it easir to debug when a lot of metrics are missing.

integration-tests/common/metric.go

@@ -5,6 +5,11 @@ import (
 	"fmt"
 )
 
+type MetricsResponse struct {
+	Status string   `json:"status"`
+	Data   []Metric `json:"data"`
+}
+
 type MetricResponse struct {
 	Status string     `json:"status"`
 	Data   MetricData `json:"data"`
@@ -53,6 +58,10 @@ func (m *MetricResponse) Unmarshal(data []byte) error {
 	return json.Unmarshal(data, m)
 }
 
+func (m *MetricsResponse) Unmarshal(data []byte) error {
+	return json.Unmarshal(data, m)
+}
+
 func (h *HistogramRawData) UnmarshalJSON(b []byte) error {
 	var arr []json.RawMessage
 	if err := json.Unmarshal(b, &arr); err != nil {

integration-tests/common/metrics_assert.go

@@ -6,9 +6,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-const promURL = "http://localhost:9009/prometheus/api/v1/query?query="
+const promURL = "http://localhost:9009/prometheus/api/v1/"
 
 // Default metrics list according to what the prom-gen app is generating.
 var PromDefaultMetrics = []string{
@@ -39,13 +40,19 @@ var OtelDefaultHistogramMetrics = []string{
 	"example_exponential_float_histogram",
 }
 
-// MetricQuery returns a formatted Prometheus metric query with a given metricName and a given label.
+// MetricQuery returns a formatted Prometheus metric query with a given metricName and the given test_name label.
 func MetricQuery(metricName string, testName string) string {
-	return fmt.Sprintf("%s%s{test_name='%s'}", promURL, metricName, testName)
+	return fmt.Sprintf("%squery?query=%s{test_name='%s'}", promURL, metricName, testName)
+}
+
+// MetricsQuery returns the list of available metrics matching the given test_name label.
+func MetricsQuery(testName string) string {
+	return fmt.Sprintf("%sseries?match[]={test_name='%s'}", promURL, testName)
 }
 
 // MimirMetricsTest checks that all given metrics are stored in Mimir.
 func MimirMetricsTest(t *testing.T, metrics []string, histogramMetrics []string, testName string) {
+	AssertMetricsAvailable(t, metrics, histogramMetrics, testName)
 	for _, metric := range metrics {
 		metric := metric
 		t.Run(metric, func(t *testing.T) {
@@ -62,6 +69,40 @@ func MimirMetricsTest(t *testing.T, metrics []string, histogramMetrics []string,
 	}
 }
 
+// AssertMetricsAvailable performs a Prometheus query and expect the result to eventually contain the list of expected metrics.
+func AssertMetricsAvailable(t *testing.T, metrics []string, histogramMetrics []string, testName string) {
+	var missingMetrics []string
+	expectedMetrics := append(metrics, histogramMetrics...)
+	query := MetricsQuery(testName)
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		var metricsResponse MetricsResponse
+		err := FetchDataFromURL(query, &metricsResponse)
+		assert.NoError(c, err)
+		missingMetrics = checkMissingMetrics(expectedMetrics, metricsResponse.Data)
+		msg := fmt.Sprintf("Some metrics are missing: %v", missingMetrics)
+		if len(missingMetrics) == len(expectedMetrics) {
+			msg = "All metrics are missing"
+		}
+		assert.Empty(c, missingMetrics, msg)
+	}, DefaultTimeout, DefaultRetryInterval)
+}
+
+// checkMissingMetrics returns the expectedMetrics which are not contained in actualMetrics.
+func checkMissingMetrics(expectedMetrics []string, actualMetrics []Metric) []string {
+	metricSet := make(map[string]struct{}, len(actualMetrics))
+	for _, metric := range actualMetrics {
+		metricSet[metric.Name] = struct{}{}
+	}
+
+	var missingMetrics []string
+	for _, expectedMetric := range expectedMetrics {
+		if _, exists := metricSet[expectedMetric]; !exists {
+			missingMetrics = append(missingMetrics, expectedMetric)
+		}
+	}
+	return missingMetrics
+}
+
 // AssertHistogramData performs a Prometheus query and expect the result to eventually contain the expected histogram.
 // The count and sum metrics should be greater than 10 before the timeout triggers.
 func AssertHistogramData(t *testing.T, query string, expectedMetric string, testName string) {

integration-tests/utils.go

@@ -106,7 +106,6 @@ func runAllTests() {
 		}(testDir, i)
 	}
 	wg.Wait()
-	close(logChan)
 }
 
 func cleanUpEnvironment() {
@@ -119,6 +118,9 @@ func cleanUpEnvironment() {
 
 func reportResults() {
 	testsFailed := 0
+	// It's ok to close the channel here because all tests are finished.
+	// If the channel would not be closed, the for loop would wait forever.
+	close(logChan)
 	for log := range logChan {
 		fmt.Printf("Failure detected in %s:\n", log.TestDir)
 		fmt.Println("Test output:", log.TestOutput)