Merge pull request #148 from appuio/feat/email-sending-metrics

Add metrics that indicate the status of email sending

controller.go

@@ -175,6 +175,10 @@ func setupManager(
 		&controllers.OrgBillingRefLinkMetric{
 			Client: mgr.GetClient(),
 		})
+	metrics.Registry.MustRegister(
+		&controllers.EmailPendingMetric{
+			Client: mgr.GetClient(),
+		})
 
 	ur := &controllers.UserReconciler{
 		Client:   mgr.GetClient(),
@@ -240,17 +244,20 @@ func setupManager(
 	if err = invclean.SetupWithManager(mgr); err != nil {
 		return nil, err
 	}
-	invmail := &controllers.InvitationEmailReconciler{
-		Client:         mgr.GetClient(),
-		Scheme:         mgr.GetScheme(),
-		Recorder:       mgr.GetEventRecorderFor("invitation-email-controller"),
-		BaseRetryDelay: invEmailBaseRetryDelay,
-		MailSender:     mailSender,
-	}
+
+	invmail := controllers.NewInvitationEmailReconciler(
+		mgr.GetClient(),
+		mgr.GetEventRecorderFor("invitation-email-controller"),
+		mgr.GetScheme(),
+		mailSender,
+		invEmailBaseRetryDelay,
+	)
 	if err = invmail.SetupWithManager(mgr); err != nil {
 		return nil, err
 	}
 
+	metrics.Registry.MustRegister(invmail.GetMetrics())
+
 	mgr.GetWebhookServer().Register("/validate-appuio-io-v1-user", &webhook.Admission{
 		Handler: &webhooks.UserValidator{},
 	})

controllers/invitation_email_controller.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/appuio/control-api/mailsenders"
+	"github.com/prometheus/client_golang/prometheus"
 
 	userv1 "github.com/appuio/control-api/apis/user/v1"
 )
@@ -33,6 +34,45 @@ type InvitationEmailReconciler struct {
 
 	MailSender     mailsenders.MailSender
 	BaseRetryDelay time.Duration
+
+	failureCounter prometheus.Counter
+	successCounter prometheus.Counter
+}
+
+func NewInvitationEmailReconciler(client client.Client, eventRecorder record.EventRecorder, scheme *runtime.Scheme, mailSender mailsenders.MailSender, baseRetryDelay time.Duration) InvitationEmailReconciler {
+	return InvitationEmailReconciler{
+		Client:         client,
+		Recorder:       eventRecorder,
+		Scheme:         scheme,
+		MailSender:     mailSender,
+		BaseRetryDelay: baseRetryDelay,
+		failureCounter: newFailureCounter(),
+		successCounter: newSuccessCounter(),
+	}
+
+}
+
+func newSuccessCounter() prometheus.Counter {
+	return prometheus.NewCounter(prometheus.CounterOpts{
+		Subsystem: "control_api_invitation_emails",
+		Name:      "sent_success_total",
+		Help:      "Total number of successfully sent invitation e-mails",
+	})
+}
+
+func newFailureCounter() prometheus.Counter {
+	return prometheus.NewCounter(prometheus.CounterOpts{
+		Subsystem: "control_api_invitation_emails",
+		Name:      "sent_failed_total",
+		Help:      "Total number of invitation e-mails which failed to send",
+	})
+}
+
+func (r *InvitationEmailReconciler) GetMetrics() prometheus.Collector {
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(r.failureCounter)
+	reg.MustRegister(r.successCounter)
+	return reg
 }
 
 //+kubebuilder:rbac:groups="rbac.appuio.io",resources=invitations,verbs=get;list;watch
@@ -66,7 +106,7 @@ func (r *InvitationEmailReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	id, err := r.MailSender.Send(ctx, email, inv)
 	if err != nil {
 		log.V(0).Error(err, "Error in e-mail backend")
-
+		r.failureCounter.Add(1)
 		apimeta.SetStatusCondition(&inv.Status.Conditions, metav1.Condition{
 			Type:    userv1.ConditionEmailSent,
 			Status:  metav1.ConditionFalse,
@@ -75,6 +115,7 @@ func (r *InvitationEmailReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		})
 		return ctrl.Result{}, multierr.Append(err, r.Client.Status().Update(ctx, &inv))
 	}
+	r.successCounter.Add(1)
 
 	var message string
 	if id != "" {

controllers/invitation_email_controller_test.go

@@ -3,9 +3,12 @@ package controllers_test
 import (
 	"context"
 	"errors"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -70,6 +73,62 @@ func Test_InvitationEmailReconciler_Reconcile_WithSendingFailure_Success(t *test
 	require.Equal(t, ReasonSendFailed, condition.Reason)
 }
 
+func Test_InvitationEmailReconciler_Reconcile_MetricsCorrect(t *testing.T) {
+	ctx := context.Background()
+
+	subject := baseInvitation()
+	apimeta.SetStatusCondition(&subject.Status.Conditions, metav1.Condition{
+		Type:   userv1.ConditionEmailSent,
+		Status: metav1.ConditionFalse,
+	})
+
+	c := prepareTest(t, subject)
+
+	r := invitationEmailReconciler(c)
+	_, err := r.Reconcile(ctx, requestFor(subject))
+	require.NoError(t, err)
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(r.GetMetrics())
+	require.NoError(t, testutil.CollectAndCompare(reg, strings.NewReader(`
+# HELP control_api_invitation_emails_sent_failed_total Total number of invitation e-mails which failed to send
+# TYPE control_api_invitation_emails_sent_failed_total counter
+control_api_invitation_emails_sent_failed_total 0
+# HELP control_api_invitation_emails_sent_success_total Total number of successfully sent invitation e-mails
+# TYPE control_api_invitation_emails_sent_success_total counter
+control_api_invitation_emails_sent_success_total 1
+`),
+	))
+}
+
+func Test_InvitationEmailReconciler_Reconcile_WithSendingFailure_MetricsCorrect(t *testing.T) {
+	ctx := context.Background()
+
+	subject := baseInvitation()
+	apimeta.SetStatusCondition(&subject.Status.Conditions, metav1.Condition{
+		Type:   userv1.ConditionEmailSent,
+		Status: metav1.ConditionFalse,
+	})
+
+	c := prepareTest(t, subject)
+
+	r := invitationEmailReconcilerWithFailingSender(c)
+	_, err := r.Reconcile(ctx, requestFor(subject))
+	require.Error(t, err)
+
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(r.GetMetrics())
+	require.NoError(t, testutil.CollectAndCompare(reg, strings.NewReader(`
+# HELP control_api_invitation_emails_sent_failed_total Total number of invitation e-mails which failed to send
+# TYPE control_api_invitation_emails_sent_failed_total counter
+control_api_invitation_emails_sent_failed_total 1
+# HELP control_api_invitation_emails_sent_success_total Total number of successfully sent invitation e-mails
+# TYPE control_api_invitation_emails_sent_success_total counter
+control_api_invitation_emails_sent_success_total 0
+`),
+	))
+}
+
 func Test_InvitationEmailReconciler_Reconcile_NoEmail_Success(t *testing.T) {
 	ctx := context.Background()
 
@@ -87,23 +146,25 @@ func Test_InvitationEmailReconciler_Reconcile_NoEmail_Success(t *testing.T) {
 }
 
 func invitationEmailReconciler(c client.WithWatch) *InvitationEmailReconciler {
-	return &InvitationEmailReconciler{
-		Client:         c,
-		Scheme:         c.Scheme(),
-		Recorder:       record.NewFakeRecorder(3),
-		MailSender:     &SenderWithConstantId{},
-		BaseRetryDelay: time.Minute,
-	}
+	r := NewInvitationEmailReconciler(
+		c,
+		record.NewFakeRecorder(3),
+		c.Scheme(),
+		&SenderWithConstantId{},
+		time.Minute,
+	)
+	return &r
 }
 
 func invitationEmailReconcilerWithFailingSender(c client.WithWatch) *InvitationEmailReconciler {
-	return &InvitationEmailReconciler{
-		Client:         c,
-		Scheme:         c.Scheme(),
-		Recorder:       record.NewFakeRecorder(3),
-		MailSender:     &FailingSender{},
-		BaseRetryDelay: time.Minute,
-	}
+	r := NewInvitationEmailReconciler(
+		c,
+		record.NewFakeRecorder(3),
+		c.Scheme(),
+		&FailingSender{},
+		time.Minute,
+	)
+	return &r
 }
 
 func baseInvitation() *userv1.Invitation {

controllers/invitation_email_pending_metric.go

@@ -0,0 +1,59 @@
+package controllers
+
+import (
+	"context"
+
+	"github.com/prometheus/client_golang/prometheus"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	userv1 "github.com/appuio/control-api/apis/user/v1"
+)
+
+//+kubebuilder:rbac:groups="rbac.appuio.io",resources=invitations,verbs=get;list;watch
+//+kubebuilder:rbac:groups="user.appuio.io",resources=invitations,verbs=get;list;watch
+//+kubebuilder:rbac:groups="rbac.appuio.io",resources=invitations/status,verbs=get
+//+kubebuilder:rbac:groups="user.appuio.io",resources=invitations/status,verbs=get
+
+var emailPendingDesc = prometheus.NewDesc(
+	"control_api_invitation_emails_pending_current",
+	"Amount of e-mails that have not been sent yet",
+	nil,
+	nil,
+)
+
+// EmailPendingMetric is a Prometheus collector that exposes the number of currently pending invitation e-mails
+type EmailPendingMetric struct {
+	client.Client
+}
+
+var _ prometheus.Collector = &EmailPendingMetric{}
+
+// Describe implements prometheus.Collector.
+// Sends the static description of the metric to the provided channel.
+func (e *EmailPendingMetric) Describe(ch chan<- *prometheus.Desc) {
+	ch <- emailPendingDesc
+}
+
+// Collect implements prometheus.Collector.
+// Sends a metric to the provided channel.
+func (e *EmailPendingMetric) Collect(ch chan<- prometheus.Metric) {
+	invs := &userv1.InvitationList{}
+
+	if err := e.List(context.Background(), invs); err != nil {
+		ch <- prometheus.NewInvalidMetric(emailPendingDesc, err)
+		return
+	}
+
+	var count float64 = 0
+	for _, inv := range invs.Items {
+		if !apimeta.IsStatusConditionTrue(inv.Status.Conditions, userv1.ConditionEmailSent) {
+			count++
+		}
+	}
+	ch <- prometheus.MustNewConstMetric(
+		emailPendingDesc,
+		prometheus.GaugeValue,
+		count,
+	)
+}

controllers/invitation_email_pending_metric_test.go

@@ -0,0 +1,37 @@
+package controllers_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	userv1 "github.com/appuio/control-api/apis/user/v1"
+	"github.com/appuio/control-api/controllers"
+)
+
+func TestEmailPendingMetric(t *testing.T) {
+	c := prepareTest(t, &userv1.Invitation{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-inv",
+		},
+		Spec:   userv1.InvitationSpec{},
+		Status: userv1.InvitationStatus{Conditions: []metav1.Condition{{Type: userv1.ConditionEmailSent, Status: metav1.ConditionTrue}}},
+	}, &userv1.Invitation{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-inv-2",
+		},
+		Spec: userv1.InvitationSpec{},
+	})
+
+	require.NoError(t,
+		testutil.CollectAndCompare(&controllers.EmailPendingMetric{c}, strings.NewReader(`
+# HELP control_api_invitation_emails_pending_current Amount of e-mails that have not been sent yet
+# TYPE control_api_invitation_emails_pending_current gauge
+control_api_invitation_emails_pending_current 1
+`),
+		),
+	)
+}