Add the kubernetes cluster UUID as a label on kubernetes metrics (#108)

Add the kubernetes cluster UUID as a label on kubernetes metrics

Gopkg.toml

@@ -36,3 +36,7 @@
 [[constraint]]
   version = "0.16.0"
   name = "github.com/prometheus/node_exporter"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/digitalocean/go-metadata"

cmd/do-agent/config.go

@@ -4,10 +4,13 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/digitalocean/go-metadata"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/digitalocean/do-agent/internal/log"
@@ -41,6 +44,11 @@ var (
 	// disabledCollectors is a hash used by disableCollectors to prevent
 	// duplicate entries
 	disabledCollectors = map[string]interface{}{}
+
+	kubernetesClusterUUIDUserDataPrefix = "k8saas_cluster_uuid: "
+	kubernetesClusterUUIDLabel = "kubernetes_cluster_uuid"
+
+	errClusterUUIDNotFound = errors.New("kubernetes cluster UUID not found")
 )
 
 const (
@@ -141,6 +149,27 @@ func newTimeseriesClient() (*WrappedTSClient, error) {
 	return wrappedTSClient, nil
 }
 
+// getKubernetesClusterUUID retrieves the k8s cluster UUID from the droplet metadata
+func getKubernetesClusterUUID() (string, error) {
+	client := metadata.NewClient(metadata.WithBaseURL(config.metadataURL))
+	userData, err := client.UserData()
+	if err != nil {
+		return "", fmt.Errorf("failed to get user data: %+v", err)
+	}
+	return parseKubernetesClusterUUID(userData)
+}
+
+// parseKubernetesClusterUUID parses the user data and returns the value of the kubernetes cluster UUID
+func parseKubernetesClusterUUID(userData string) (string, error) {
+	userDataSplit := strings.Split(userData, "\n")
+	for _, line := range userDataSplit {
+		if strings.HasPrefix(line, kubernetesClusterUUIDUserDataPrefix) {
+			return strings.Trim(strings.TrimPrefix(line, kubernetesClusterUUIDUserDataPrefix), "\""), nil
+		}
+	}
+	return "", errClusterUUIDNotFound
+}
+
 // initCollectors initializes the prometheus collectors. By default this
 // includes node_exporter and buildInfo for each remote target
 func initCollectors() []prometheus.Collector {
@@ -154,16 +183,11 @@ func initCollectors() []prometheus.Collector {
 	}
 
 	if config.kubernetes != "" {
-		k, err := collector.NewScraper("dokubernetes", config.kubernetes, k8sWhitelist, defaultTimeout)
-		if err != nil {
-			log.Error("Failed to initialize DO Kubernetes metrics: %+v", err)
-		} else {
-			cols = append(cols, k)
-		}
+		cols = appendKubernetesCollectors(cols)
 	}
 
 	if config.dbaas != "" {
-		k, err := collector.NewScraper("dodbaas", config.dbaas, dbaasWhitelist, defaultTimeout)
+		k, err := collector.NewScraper("dodbaas", config.dbaas, nil, dbaasWhitelist, defaultTimeout)
 		if err != nil {
 			log.Error("Failed to initialize DO DBaaS metrics collector: %+v", err)
 		} else {
@@ -189,6 +213,24 @@ func initCollectors() []prometheus.Collector {
 	return cols
 }
 
+// appendKubernetesCollectors appends a kubernetes metrics collector if it can be initialized successfully
+func appendKubernetesCollectors(cols []prometheus.Collector) []prometheus.Collector {
+	kubernetesClusterUUID, err := getKubernetesClusterUUID()
+	if err != nil {
+		log.Error("Failed to get cluster UUID when initializing DO Kubernetes metrics: %+v", err)
+		return cols
+	}
+	var kubernetesLabels []*dto.LabelPair
+	kubernetesLabels = append(kubernetesLabels, &dto.LabelPair{Name: &kubernetesClusterUUIDLabel, Value: &kubernetesClusterUUID})
+	k, err := collector.NewScraper("dokubernetes", config.kubernetes, kubernetesLabels, k8sWhitelist, defaultTimeout)
+	if err != nil {
+		log.Error("Failed to initialize DO Kubernetes metrics: %+v", err)
+		return cols
+	}
+	cols = append(cols, k)
+	return cols
+}
+
 // disableCollectors disables collectors by names by adding a list of
 // --no-collector.<name> flags to additionalParams
 func disableCollectors(names ...string) {

cmd/do-agent/config_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDisableCollectorsAddsCorrectFlags(t *testing.T) {
@@ -35,3 +36,39 @@ func TestDisableCollectorsIsIdempotent(t *testing.T) {
 	disableCollectors(items...)
 	assert.EqualValues(t, flags, additionalParams)
 }
+
+func TestParseKubuernetesClusterUUID(t *testing.T) {
+	userData := `k8saas_role: kubelet
+k8saas_master_domain_name: "ok.bou.ke"
+k8saas_bootstrap_token: "123"
+k8saas_proxy_token: "456"
+k8saas_ca_cert: "CERT CERT CERT"
+k8saas_etcd_ca: "CERT2 CERT2 CERT2"
+k8saas_etcd_key: "ok\nwhatever"
+k8saas_etcd_cert: "NEAT"
+k8saas_overlay_subnet: "GOOD"
+k8saas_cluster_uuid: "11111111-2222-3333-4444-555555555555"
+k8saas_dns_service_ip: "YES"`
+
+	parsed, err := parseKubernetesClusterUUID(userData)
+	require.NoError(t, err)
+	assert.EqualValues(t, "11111111-2222-3333-4444-555555555555", parsed)
+}
+
+func TestParseKubuernetesClusterUUIDMissing(t *testing.T) {
+	userData := `k8saas_role: kubelet
+k8saas_master_domain_name: "ok.bou.ke"
+k8saas_bootstrap_token: "123"
+k8saas_proxy_token: "456"
+k8saas_ca_cert: "CERT CERT CERT"
+k8saas_etcd_ca: "CERT2 CERT2 CERT2"
+k8saas_etcd_key: "ok\nwhatever"
+k8saas_etcd_cert: "NEAT"
+k8saas_overlay_subnet: "GOOD"
+k8saas_dns_service_ip: "YES"`
+
+	parsed, err := parseKubernetesClusterUUID(userData)
+	require.Error(t, err)
+	require.Equal(t, err, errClusterUUIDNotFound)
+	assert.EqualValues(t, "", parsed)
+}

pkg/clients/tsclient/metric_definition.go

@@ -112,7 +112,7 @@ func NewDefinitionFromMap(m map[string]string) *Definition {
 	return NewDefinition(name, WithCommonLabels(m))
 }
 
-// GetLFM returns an lfm corresponding to a defitnition
+// GetLFM returns an lfm corresponding to a definition
 func GetLFM(def *Definition, labels []string) (string, error) {
 	lfm := []string{def.name}
 	for _, x := range def.sortedLabels {

pkg/collector/scraper.go

@@ -10,16 +10,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/digitalocean/do-agent/internal/log"
-	"github.com/digitalocean/do-agent/pkg/clients"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
+
+	"github.com/digitalocean/do-agent/internal/log"
+	"github.com/digitalocean/do-agent/pkg/clients"
 )
 
 // NewScraper creates a new scraper to scrape metrics from the provided host
-func NewScraper(name, metricsEndpoint string, whitelist map[string]bool, timeout time.Duration) (*Scraper, error) {
+func NewScraper(name, metricsEndpoint string, extraMetricLabels []*dto.LabelPair, whitelist map[string]bool, timeout time.Duration) (*Scraper, error) {
 	metricsEndpoint = strings.TrimRight(metricsEndpoint, "/")
 	req, err := http.NewRequest("GET", metricsEndpoint, nil)
 	if err != nil {
@@ -31,11 +32,12 @@ func NewScraper(name, metricsEndpoint string, whitelist map[string]bool, timeout
 	req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", fmt.Sprintf("%f", timeout.Seconds()))
 
 	return &Scraper{
-		req:       req,
-		name:      name,
-		whitelist: whitelist,
-		timeout:   timeout,
-		client:    clients.NewHTTP(timeout),
+		req:               req,
+		name:              name,
+		extraMetricLabels: extraMetricLabels,
+		whitelist:         whitelist,
+		timeout:           timeout,
+		client:            clients.NewHTTP(timeout),
 		scrapeDurationDesc: prometheus.NewDesc(
 			prometheus.BuildFQName(name, "scrape", "collector_duration_seconds"),
 			fmt.Sprintf("%s: Duration of a collector scrape.", name),
@@ -58,6 +60,7 @@ type Scraper struct {
 	client             *http.Client
 	name               string
 	whitelist          map[string]bool
+	extraMetricLabels  []*dto.LabelPair
 	scrapeDurationDesc *prometheus.Desc
 	scrapeSuccessDesc  *prometheus.Desc
 }
@@ -138,7 +141,7 @@ func (s *Scraper) scrape(ctx context.Context, ch chan<- prometheus.Metric) (oute
 		if s.FilterMetric(mf) {
 			continue
 		}
-		convertMetricFamily(mf, ch)
+		convertMetricFamily(mf, ch, s.extraMetricLabels)
 	}
 
 	return nil
@@ -161,45 +164,17 @@ func (s *Scraper) FilterMetric(metricFamily *dto.MetricFamily) bool {
 // convertMetricFamily converts the dto metrics parsed from the expfmt package
 // into the prometheus.Metrics required to pass over the channel
 //
-// this was copied from github.com/prometheus/node_exporter
+// this was copied and extended/refactored from github.com/prometheus/node_exporter
 // see https://github.com/prometheus/node_exporter/blob/f56e8fcdf48ead56f1f149dbf1301ac028ef589b/collector/textfile.go#L63
 // for more details
-func convertMetricFamily(metricFamily *dto.MetricFamily, ch chan<- prometheus.Metric) {
+func convertMetricFamily(metricFamily *dto.MetricFamily, ch chan<- prometheus.Metric, extraLabels []*dto.LabelPair) {
 	var valType prometheus.ValueType
 	var val float64
 
-	allLabelNames := map[string]struct{}{}
-	for _, metric := range metricFamily.Metric {
-		labels := metric.GetLabel()
-		for _, label := range labels {
-			if _, ok := allLabelNames[label.GetName()]; !ok {
-				allLabelNames[label.GetName()] = struct{}{}
-			}
-		}
-	}
+	allLabelNames := getAllLabelNames(metricFamily, extraLabels)
 
 	for _, metric := range metricFamily.Metric {
-		labels := metric.GetLabel()
-		var names []string
-		var values []string
-		for _, label := range labels {
-			names = append(names, label.GetName())
-			values = append(values, label.GetValue())
-		}
-
-		for k := range allLabelNames {
-			present := false
-			for _, name := range names {
-				if k == name {
-					present = true
-					break
-				}
-			}
-			if !present {
-				names = append(names, k)
-				values = append(values, "")
-			}
-		}
+		names, values := getLabelNamesAndValues(metric, extraLabels, allLabelNames)
 
 		metricType := metricFamily.GetType()
 		switch metricType {
@@ -261,3 +236,48 @@ func convertMetricFamily(metricFamily *dto.MetricFamily, ch chan<- prometheus.Me
 		}
 	}
 }
+
+// getLabelNamesAndValues returns a slice of label names and a slice of label values from the metric and extra labels.
+func getLabelNamesAndValues(metric *dto.Metric, extraLabels []*dto.LabelPair, allLabelNames map[string]struct{}) ([]string, []string) {
+	labels := metric.GetLabel()
+	if extraLabels != nil {
+		labels = append(labels, extraLabels...)
+	}
+	var names []string
+	var values []string
+	for _, label := range labels {
+		names = append(names, label.GetName())
+		values = append(values, label.GetValue())
+	}
+	for k := range allLabelNames {
+		present := false
+		for _, name := range names {
+			if k == name {
+				present = true
+				break
+			}
+		}
+		if !present {
+			names = append(names, k)
+			values = append(values, "")
+		}
+	}
+	return names, values
+}
+
+// getAllLabelNames returns the map of all label names from the metric family including any extra labels provided.
+func getAllLabelNames(metricFamily *dto.MetricFamily, extraLabels []*dto.LabelPair) map[string]struct{} {
+	allLabelNames := map[string]struct{}{}
+	for _, metric := range metricFamily.Metric {
+		labels := metric.GetLabel()
+		if extraLabels != nil {
+			labels = append(labels, extraLabels...)
+		}
+		for _, label := range labels {
+			if _, ok := allLabelNames[label.GetName()]; !ok {
+				allLabelNames[label.GetName()] = struct{}{}
+			}
+		}
+	}
+	return allLabelNames
+}