Server metrics

Signed-off-by: Alan Clucas <[email protected]>

config/config.go

@@ -48,10 +48,14 @@ type Config struct {
 	// controller watches workflows and pods that *are not* labeled with an instance id.
 	InstanceID string `json:"instanceID,omitempty"`
 
-	// MetricsConfig specifies configuration for metrics emission. Metrics are enabled and emitted on localhost:9090/metrics
+	// MetricsConfig specifies configuration for the workflow controller metrics emission. Metrics are enabled and emitted on localhost:9090/metrics
 	// by default.
 	MetricsConfig MetricsConfig `json:"metricsConfig,omitempty"`
 
+	// ServerMetricsConfig specifies configuration for argo server metrics emission. Metrics are enabled and emitted on localhost:9090/metrics
+	// by default.
+	ServerMetricsConfig MetricsConfig `json:"serverMetricsConfig,omitempty"`
+
 	// TelemetryConfig specifies configuration for telemetry emission. Telemetry is enabled and emitted in the same endpoint
 	// as metrics by default, but can be overridden using this config.
 	TelemetryConfig MetricsConfig `json:"telemetryConfig,omitempty"`

manifests/quick-start/base/overlays/workflow-controller-configmap.yaml

@@ -26,6 +26,10 @@ data:
     enabled: true
     path: /metrics
     port: 9090
+  serverMetricsConfig: |
+    enabled: true
+    path: /metrics
+    port: 9091
   namespaceParallelism: "10"
   links: |
     - name: Workflow Link

server/apiserver/argoserver.go

@@ -51,6 +51,7 @@ import (
 	"github.com/argoproj/argo-workflows/v3/server/event"
 	"github.com/argoproj/argo-workflows/v3/server/eventsource"
 	"github.com/argoproj/argo-workflows/v3/server/info"
+	"github.com/argoproj/argo-workflows/v3/server/metrics"
 	"github.com/argoproj/argo-workflows/v3/server/sensor"
 	"github.com/argoproj/argo-workflows/v3/server/static"
 	"github.com/argoproj/argo-workflows/v3/server/types"
@@ -262,6 +263,9 @@ func (as *argoServer) Run(ctx context.Context, port int, browserOpenFunc func(st
 		conn = tls.NewListener(conn, as.tlsConfig)
 	}
 
+	metricsServer, err := metrics.New(ctx, `argo-server`, `argo_server`, metrics.GetServerConfig(config))
+	go metricsServer.RunPrometheusServer(ctx, false)
+
 	// Cmux is used to support servicing gRPC and HTTP1.1+JSON on the same port
 	tcpm := cmux.New(conn)
 	httpL := tcpm.Match(cmux.HTTP1Fast())

server/metrics/metrics.go

@@ -0,0 +1,60 @@
+package metrics
+
+import (
+	"context"
+
+	"github.com/argoproj/argo-workflows/v3/config"
+	"github.com/argoproj/argo-workflows/v3/util/telemetry"
+
+	metricsdk "go.opentelemetry.io/otel/sdk/metric"
+)
+
+type Metrics struct {
+	telemetry.Metrics
+}
+
+func New(ctx context.Context, serviceName, prometheusName string, config *telemetry.Config, extraOpts ...metricsdk.Option) (*Metrics, error) {
+	m, err := telemetry.NewMetrics(ctx, serviceName, prometheusName, config, extraOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	err = m.Populate(ctx,
+		telemetry.AddVersion,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	metrics := &Metrics{
+		Metrics: *m,
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	return metrics, nil
+}
+
+func GetServerConfig(config *config.Config) *telemetry.Config {
+	// Metrics config
+	modifiers := make(map[string]telemetry.Modifier)
+	for name, modifier := range config.ServerMetricsConfig.Modifiers {
+		modifiers[name] = telemetry.Modifier{
+			Disabled:           modifier.Disabled,
+			DisabledAttributes: modifier.DisabledAttributes,
+			HistogramBuckets:   modifier.HistogramBuckets,
+		}
+	}
+
+	metricsConfig := telemetry.Config{
+		Enabled:      config.ServerMetricsConfig.Enabled == nil || *config.ServerMetricsConfig.Enabled,
+		Path:         config.ServerMetricsConfig.Path,
+		Port:         config.ServerMetricsConfig.Port,
+		IgnoreErrors: config.ServerMetricsConfig.IgnoreErrors,
+		Secure:       config.ServerMetricsConfig.GetSecure(true),
+		Modifiers:    modifiers,
+		Temporality:  config.ServerMetricsConfig.Temporality,
+	}
+	return &metricsConfig
+}