Prometheus Metrics: Duration for the watermarking/queueing

docker-compose.yml

@@ -58,6 +58,7 @@ services:
       - env_files/exam_marker.env
     networks:
       - backend-internal
+      - backend
     depends_on:
       - db
       - storage

exam_marker/Dockerfile

@@ -38,6 +38,7 @@ COPY server/graph ./server/graph
 COPY server/lti_utils ./server/lti_utils
 
 COPY exam_marker/exam_marker.go ./exam_marker/exam_marker.go
+COPY exam_marker/prometheus ./exam_marker/prometheus
 COPY utils ./utils
 RUN go build -a -o exam_marker.runnable ./exam_marker/exam_marker.go
 

exam_marker/exam_marker.go

@@ -11,11 +11,13 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/exam_marker/prometheus"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/utils"
 	"github.com/adjust/rmq/v3"
 	render "github.com/brunsgaard/go-pdfium-render"
@@ -24,6 +26,7 @@ import (
 	"github.com/minio/minio-go/v7"
 	pdfcpu_api "github.com/pdfcpu/pdfcpu/pkg/api"
 	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
@@ -43,15 +46,13 @@ var (
 
 type RMQConsumer struct {
 	name        string
-	count       int
 	before      time.Time
 	MinIOClient *minio.Client
 }
 
 func NewRMQConsumer(minioClient *minio.Client, tag int) *RMQConsumer {
 	return &RMQConsumer{
 		name:        fmt.Sprintf("consumer%d", tag),
-		count:       0,
 		before:      time.Now(),
 		MinIOClient: minioClient,
 	}
@@ -67,6 +68,10 @@ func (consumer *RMQConsumer) Consume(delivery rmq.Delivery) {
 	log.Printf("%s working on task %q", consumer.name, task.ExamUUID)
 	executeMarkerTask(consumer.MinIOClient, task)
 
+	taskDuration := time.Since(task.SubmitTime)
+	prometheus.WatermarkingTimeHistogram.Observe(float64(taskDuration.Seconds()))
+	log.Printf("%s took %v to work on task %q", consumer.name, taskDuration.Seconds(), task.ExamUUID)
+
 	if err := delivery.Ack(); err != nil {
 		log.Println(err)
 	}
@@ -259,6 +264,18 @@ func main() {
 		}
 	}
 
+	// Expose the registered metrics via HTTP.
+	go func() {
+		port := "8081"
+		http.Handle("/metrics", promhttp.Handler())
+		fmt.Print(
+			"==========================================\n",
+			"Started the exam_marker metrics listening on Port "+port+"\n",
+			"==========================================\n",
+		)
+		log.Fatal(http.ListenAndServe(":"+port, nil))
+	}()
+
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT)
 	defer signal.Stop(signals)

exam_marker/prometheus/prometheus.go

@@ -0,0 +1,14 @@
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// WatermarkingTimeHistogram is a prometheus metric for the total amount of exams in the database
+	WatermarkingTimeHistogram = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name: "altklausur_ausleihe_watermarking_histogram",
+		Help: "histogram for the time it takes an exam to get marked",
+	})
+)

frontend/nginx.conf

@@ -2,6 +2,10 @@ upstream backend-upstream {
     server backend:8081;
 }
 
+upstream exam_marker-upstream {
+    server exam_marker:8081;
+}
+
 server {
     listen       80;
     listen  [::]:80;
@@ -35,9 +39,13 @@ server {
         proxy_pass http://backend-upstream/adminlogin;
     }
 
-    location /metrics {
+    location /backend/metrics {
         proxy_pass http://backend-upstream/metrics;
     }
+
+    location /exam_marker/metrics {
+        proxy_pass http://exam_marker-upstream/metrics;
+    }
 
     error_page   500 502 503 504  /50x.html;
     location = /50x.html {

server/Dockerfile

@@ -8,6 +8,7 @@ RUN go mod download
 
 COPY server/graph ./server/graph
 COPY server/lti_utils ./server/lti_utils
+COPY server/prometheus ./server/prometheus
 COPY server/server.go server/dummylogin.html ./server/
 COPY utils ./utils
 

server/graph/schema.resolvers.go

@@ -22,6 +22,7 @@ import (
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/graph/generated"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/graph/model"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/lti_utils"
+	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/prometheus"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/utils"
 	"github.com/dustin/go-humanize"
 	"github.com/gabriel-vasile/mimetype"
@@ -139,7 +140,7 @@ func (r *mutationResolver) CreateExam(ctx context.Context, input model.NewExam)
 	}
 
 	// update the TotalExams metric
-	utils.UpdateTotalExamsMetric(r.DB)
+	prometheus.UpdateTotalExamsMetric(r.DB)
 
 	return &exam, nil
 }
@@ -193,6 +194,7 @@ func (r *mutationResolver) RequestMarkedExam(ctx context.Context, stringUUID str
 			UserID:       userInfos.ID,
 			TextLeft:     userInfos.PersonFamilyName + " - " + userInfos.PersonFamilyName,
 			TextDiagonal: userInfos.PersonPrimaryEmail,
+			SubmitTime:   time.Now(),
 		},
 	)
 
@@ -205,7 +207,7 @@ func (r *mutationResolver) RequestMarkedExam(ctx context.Context, stringUUID str
 		return nil, err
 	}
 
-	utils.ExamsMarkedMetric.Inc()
+	prometheus.ExamsMarkedMetric.Inc()
 
 	return &stringUUID, nil
 }

utils/prometheus.go → server/prometheus/prometheus.go

@@ -1,4 +1,4 @@
-package utils
+package prometheus
 
 import (
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/graph/model"

server/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/graph"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/graph/generated"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/lti_utils"
+	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/server/prometheus"
 	"github.com/FachschaftMathPhysInfo/altklausur-ausleihe/utils"
 	chiprometheus "github.com/edjumacator/chi-prometheus"
 	"github.com/go-chi/chi/v5"
@@ -107,7 +108,7 @@ func main() {
 	})
 
 	// set the TotalExams metric initially
-	utils.UpdateTotalExamsMetric(db)
+	prometheus.UpdateTotalExamsMetric(db)
 
 	router.Handle("/metrics", promhttp.Handler())
 

utils/utils.go

@@ -220,9 +220,11 @@ func GetExamCachePath(userID string, examUUID uuid.UUID) string {
 	return userID + "_" + examUUID.String()
 }
 
+// RMQMarkerTask models one of the tasks on the exam queue
 type RMQMarkerTask struct {
 	ExamUUID     uuid.UUID `json:"examuuid"`
 	UserID       string    `json:"userid"`
 	TextLeft     string    `json:"textleft"`
 	TextDiagonal string    `json:"textdiagonal"`
+	SubmitTime   time.Time `json:"submittime"`
 }