feat: use check label in metrics

pkg/metrics/metrics.go

@@ -1,12 +1,16 @@
 package metrics
 
 import (
+	"fmt"
+	"sort"
+	"strings"
 	"time"
 
 	"github.com/asecurityteam/rolling"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 	"github.com/flanksource/canary-checker/pkg/runner"
+	"github.com/flanksource/commons/collections"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/types"
 	cmap "github.com/orcaman/concurrent-map"
@@ -31,14 +35,6 @@ var (
 		[]string{"type", "endpoint", "canary_name", "canary_namespace", "owner", "severity", "key", "name"},
 	)
 
-	CanaryCheckInfo = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "canary_check_info",
-			Help: "Information about the canary check",
-		},
-		[]string{"type", "endpoint", "canary_name", "canary_namespace", "owner", "severity", "key", "name"},
-	)
-
 	OpsSuccessCount = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "canary_check_success_count",
@@ -98,12 +94,14 @@ var (
 	)
 )
 
-var failed = cmap.New()
-var passed = cmap.New()
-var latencies = cmap.New()
+var (
+	failed    = cmap.New()
+	passed    = cmap.New()
+	latencies = cmap.New()
+)
 
 func init() {
-	prometheus.MustRegister(Gauge, CanaryCheckInfo, OpsCount, OpsSuccessCount, OpsFailedCount, RequestLatency, GenericGauge, GenericCounter, GenericHistogram)
+	prometheus.MustRegister(Gauge, OpsCount, OpsSuccessCount, OpsFailedCount, RequestLatency, GenericGauge, GenericCounter, GenericHistogram)
 	CustomCounters = make(map[string]*prometheus.CounterVec)
 	CustomGauges = make(map[string]*prometheus.GaugeVec)
 	CustomHistograms = make(map[string]*prometheus.HistogramVec)
@@ -202,10 +200,22 @@ func Record(ctx context.Context, canary v1.Canary, result *pkg.CheckResult) (_up
 		latency.Append(float64(result.Duration))
 	}
 
+	metricSuffix, keyValuePairs := metricLabels(result.Check.GetLabels())
+	labels := append([]string{
+		"type", checkType,
+		"endpoint", endpoint,
+		"canary_name", canaryName,
+		"canary_namespace", canaryNamespace,
+		"owner", owner,
+		"severity", severity,
+		"key", key,
+		"name", name,
+	}, keyValuePairs...)
+
 	if result.Pass {
 		pass.Append(1)
 		Gauge.WithLabelValues(key, checkType, canaryName, canaryNamespace, name).Set(0)
-		CanaryCheckInfo.WithLabelValues(checkType, endpoint, canaryName, canaryNamespace, owner, severity, key, name).Set(0)
+		ctx.Gauge(metricName("canary_check_info", metricSuffix), labels...).Set(0)
 		OpsSuccessCount.WithLabelValues(checkType, endpoint, canaryName, canaryNamespace, owner, severity, key, name).Inc()
 		// always add a failed count to ensure the metric is present in prometheus
 		// for an uptime calculation
@@ -231,7 +241,7 @@ func Record(ctx context.Context, canary v1.Canary, result *pkg.CheckResult) (_up
 	} else {
 		fail.Append(1)
 		Gauge.WithLabelValues(key, checkType, canaryName, canaryNamespace, name).Set(1)
-		CanaryCheckInfo.WithLabelValues(checkType, endpoint, canaryName, canaryNamespace, owner, severity, key, name).Set(1)
+		ctx.Gauge(metricName("canary_check_info", metricSuffix), labels...).Set(1)
 		OpsFailedCount.WithLabelValues(checkType, endpoint, canaryName, canaryNamespace, owner, severity, key, name).Inc()
 	}
 
@@ -244,6 +254,31 @@ func Record(ctx context.Context, canary v1.Canary, result *pkg.CheckResult) (_up
 	return _uptime, _latency
 }
 
+func metricName(metric, sortedLabelKeys string) string {
+	if sortedLabelKeys == "" {
+		return metric
+	}
+
+	return fmt.Sprintf("%s_%s", metric, sortedLabelKeys)
+}
+
+func metricLabels(m map[string]string) (string, []string) {
+	if len(m) == 0 {
+		return "", nil
+	}
+
+	keys := collections.MapKeys(m)
+	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+	var sortedKeyValue []string
+	for _, k := range keys {
+		sortedKeyValue = append(sortedKeyValue, k)
+		sortedKeyValue = append(sortedKeyValue, m[k])
+	}
+
+	return strings.Join(keys, "_"), sortedKeyValue
+}
+
 func getOrCreateGauge(m pkg.Metric) error {
 	var gauge *prometheus.GaugeVec
 	var ok bool