Add standard metrics for warning and error

This insects a span for the warning and error fields and produces a count metric
tagged with type:o11y.

We use field names with an _error suffix to indicate a problem span but only in
a path that can still make progress, e.g. a fail open path.

Since the field names are varied it is hard to add monitors, so this detects those
fields, adds a standard 'failure' field to the span with the prefix part as the value
and also produces a failure metric and uses the prefix part as a tag.

o11y/honeycomb/honeycomb.go

@@ -143,6 +143,8 @@ func stripMetrics(fields map[string]interface{}) {
 
 func extractAndSendMetrics(mp o11y.MetricsProvider) func(map[string]interface{}) {
 	return func(fields map[string]interface{}) {
+		standardErrorMetrics(mp, fields)
+
 		metrics, ok := fields[metricKey].([]o11y.Metric)
 		if !ok {
 			return
@@ -192,6 +194,40 @@ func extractAndSendMetrics(mp o11y.MetricsProvider) func(map[string]interface{})
 	}
 }
 
+func standardErrorMetrics(mp o11y.MetricsProvider, fields map[string]interface{}) {
+	// detect and map the fail same errors and add a metric for it if found
+	failClass := addFailure(fields)
+	if failClass != "" {
+		_ = mp.Count("failure", 1, []string{fmtTag("class", failClass)}, 1)
+	}
+	// add standard metric for error and warning
+	tag := []string{fmtTag("type", "o11y")}
+	if _, ok := fields["error"]; ok {
+		_ = mp.Count("error", 1, tag, 1)
+	}
+	if _, ok := fields["warning"]; ok {
+		_ = mp.Count("warning", 1, tag, 1)
+	}
+}
+
+// addFailure finds the first field suffixed with _error and adds the prefix as the value
+// to a failure field, if there is not already a failure field, and returns the prefix.
+// The original _error field is kept to retain details of its value.
+// If found the prefix part is returned.
+func addFailure(fields map[string]interface{}) string {
+	if _, ok := fields["failure"]; ok {
+		return ""
+	}
+	for k := range fields {
+		errClass := strings.TrimSuffix(k, "_error")
+		if errClass != k {
+			fields["failure"] = errClass
+			return errClass
+		}
+	}
+	return ""
+}
+
 func extractTagsFromFields(tags []string, fields map[string]interface{}) []string {
 	result := make([]string, 0, len(tags))
 	for _, name := range tags {

o11y/wrappers/o11ygin/o11ygin_test.go

@@ -106,6 +106,21 @@ func TestMiddleware(t *testing.T) {
 					},
 					Rate: 1,
 				},
+				{
+					Metric:   "count",
+					Name:     "error",
+					ValueInt: 1,
+					Tags:     []string{"type:o11y"},
+					Rate:     1,
+				},
+
+				{
+					Metric:   "count",
+					Name:     "warning",
+					ValueInt: 1,
+					Tags:     []string{"type:o11y"},
+					Rate:     1,
+				},
 			},
 			m.Calls(), fakemetrics.CMPMetrics, cmpopts.IgnoreFields(fakemetrics.MetricCall{}, "Value", "ValueInt")),
 		)

o11y/wrappers/o11ynethttp/o11ynethttp_test.go

@@ -33,6 +33,13 @@ func TestMiddleware(t *testing.T) {
 
 			assert.Check(t, cmp.DeepEqual(
 				[]fakemetrics.MetricCall{
+					{
+						Metric:   "count",
+						Name:     "warning",
+						ValueInt: 1,
+						Tags:     []string{"type:o11y"},
+						Rate:     1,
+					},
 					{
 						Metric: "timer",
 						Name:   "handler",

system/system_test.go

@@ -112,6 +112,13 @@ func TestSystem_Run(t *testing.T) {
 			Tags:   []string{"result:success"},
 			Rate:   1,
 		},
+		{
+			Metric:   "count",
+			Name:     "warning",
+			ValueInt: 1,
+			Tags:     []string{"type:o11y"},
+			Rate:     1,
+		},
 	}, cmpMetrics))
 }