feat: new cron workflow trigger counter metric

From #12589.

A new metric which counts how many times each cron workflow has
triggered. A simple enough counter which can be checked against
expectations for the cron.

Note to reviewers: this is part of a stack of reviews for metrics
changes. Please don't merge until the rest of the stack is also ready.

Signed-off-by: Alan Clucas <[email protected]>

docs/metrics.md

@@ -201,6 +201,15 @@ Metrics for the [Four Golden Signals](https://sre.google/sre-book/monitoring-dis
     Some metric attributes may have high cardinality and are marked with ⚠️ to warn you. You may need to disable this metric or disable the attribute.
 <!-- titles should be the exact metric name for deep-linking, alphabetical ordered -->
 <!-- titles are without argo_workflows prefix -->
+#### `cronworkflows_triggered_total`
+
+A counter of the number of times a CronWorkflow has been
+
+| attribute   | explanation                               |
+|-------------|-------------------------------------------|
+| `name`     | ⚠️ The name of the CronWorkflow. |
+| `namespace` | The namespace in which the pod is running |
+
 #### `gauge`
 
 A gauge of the number of workflows currently in the cluster in each phase. The `Running` count does not mean that a workflows pods are running, just that the controller has scheduled them. A workflow can be stuck in `Running` with pending pods for a long time.

docs/upgrading.md

@@ -25,6 +25,7 @@ These notes explain the differences in using the Prometheus `/metrics` endpoint
 
 The following are new metrics:
 
+* `cronworkflows_triggered_total`
 * `is_leader`
 * `k8s_request_duration`
 * `pods_total_count`

test/e2e/metrics_test.go

@@ -141,6 +141,22 @@ func (s *MetricsSuite) TestPodPendingMetric() {
 		WaitForWorkflowDeletion()
 }
 
+func (s *MetricsSuite) TestCronTriggeredCounter() {
+	s.Given().
+		CronWorkflow(`@testdata/cronworkflow-metrics.yaml`).
+		When().
+		CreateCronWorkflow().
+		Wait(1 * time.Minute). // This pattern is used in cron_test.go too
+		Then().
+		ExpectCron(func(t *testing.T, cronWf *wfv1.CronWorkflow) {
+			s.e(s.T()).GET("").
+				Expect().
+				Status(200).
+				Body().
+				Contains(`cronworkflows_triggered_total{name="test-cron-metric",namespace="argo"} 1`)
+		})
+}
+
 func TestMetricsSuite(t *testing.T) {
 	suite.Run(t, new(MetricsSuite))
 }

test/e2e/testdata/cronworkflow-metrics.yaml

@@ -0,0 +1,19 @@
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: test-cron-metric
+spec:
+  schedule: "* * * * *"
+  concurrencyPolicy: "Allow"
+  startingDeadlineSeconds: 0
+  workflowSpec:
+    metadata:
+      labels:
+        workflows.argoproj.io/test: "true"
+    podGC:
+      strategy: OnPodCompletion
+    entrypoint: whalesay
+    templates:
+      - name: whalesay
+        container:
+          image: argoproj/argosay:v2

workflow/cron/operator.go

@@ -80,6 +80,7 @@ func (woc *cronWfOperationCtx) run(ctx context.Context, scheduledRuntime time.Ti
 	defer woc.persistUpdate(ctx)
 
 	woc.log.Infof("Running %s", woc.name)
+	woc.metrics.CronWfTrigger(ctx, woc.name, woc.cronWf.ObjectMeta.Namespace)
 
 	// If the cron workflow has a schedule that was just updated, update its annotation
 	if woc.cronWf.IsUsingNewSchedule() {

workflow/metrics/counter_cronworkflow_trigger.go

@@ -0,0 +1,25 @@
+package metrics
+
+import (
+	"context"
+)
+
+const (
+	nameCronTriggered = `cronworkflows_triggered_total`
+)
+
+func addCronWfTriggerCounter(_ context.Context, m *Metrics) error {
+	return m.createInstrument(int64Counter,
+		nameCronTriggered,
+		"Total number of cron workflows triggered",
+		"{cronworkflow}",
+		withAsBuiltIn(),
+	)
+}
+
+func (m *Metrics) CronWfTrigger(ctx context.Context, name, namespace string) {
+	m.addInt(ctx, nameCronTriggered, 1, instAttribs{
+		{name: labelCronWFName, value: name},
+		{name: labelWorkflowNamespace, value: namespace},
+	})
+}

workflow/metrics/labels.go

@@ -10,6 +10,8 @@ const (
 	labelBuildGitTreeState string = `git_treestate`
 	labelBuildGitTag       string = `git_tag`
 
+	labelCronWFName string = `name`
+
 	labelErrorCause string = "cause"
 
 	labelLogLevel string = `level`

workflow/metrics/metrics.go

@@ -99,6 +99,7 @@ func New(ctx context.Context, serviceName string, config *Config, callbacks Call
 		addPodMissingCounter,
 		addPodPendingCounter,
 		addWorkflowPhaseGauge,
+		addCronWfTriggerCounter,
 		addOperationDurationHistogram,
 		addErrorCounter,
 		addLogCounter,