diff --git a/frontend/frontend.go b/frontend/frontend.go index 79c528f92..dc26abf08 100644 --- a/frontend/frontend.go +++ b/frontend/frontend.go @@ -17,6 +17,8 @@ import ( "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( @@ -35,6 +37,7 @@ type Frontend struct { cache Cache ready atomic.Value done chan struct{} + metrics metrics.Emitter } // MuxPattern forms a URL pattern suitable for passing to http.ServeMux. @@ -44,10 +47,11 @@ func MuxPattern(method string, segments ...string) string { return fmt.Sprintf("%s /%s", method, strings.ToLower(path.Join(segments...))) } -func NewFrontend(logger *slog.Logger, listener net.Listener) *Frontend { +func NewFrontend(logger *slog.Logger, listener net.Listener, emitter metrics.Emitter) *Frontend { f := &Frontend{ logger: logger, listener: listener, + metrics: emitter, server: http.Server{ ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), BaseContext: func(net.Listener) context.Context { @@ -60,12 +64,17 @@ func NewFrontend(logger *slog.Logger, listener net.Listener) *Frontend { subscriptionStateMuxValidator := NewSubscriptionStateMuxValidator(&f.cache) + // Setup metrics middleware + metricsMiddleware := MetricsMiddleware{Emitter: emitter} + mux := NewMiddlewareMux( MiddlewarePanic, MiddlewareLogging, MiddlewareBody, MiddlewareLowercase, - MiddlewareSystemData) + MiddlewareSystemData, + metricsMiddleware.Metrics(), + ) // Unauthenticated routes mux.HandleFunc("/", f.NotFound) @@ -73,6 +82,9 @@ func NewFrontend(logger *slog.Logger, listener net.Listener) *Frontend { // TODO: determine where in the auth chain we should allow for this endpoint to be called by ARM mux.HandleFunc(MuxPattern(http.MethodPut, PatternSubscriptions), f.ArmSubscriptionAction) + // Expose Prometheus metrics endpoint + mux.Handle(MuxPattern(http.MethodGet, "metrics"), promhttp.Handler()) + // Authenticated routes postMuxMiddleware := NewMiddleware( MiddlewareLoggingPostMux, @@ -102,6 +114,7 @@ func NewFrontend(logger *slog.Logger, listener net.Listener) *Frontend { mux.Handle( MuxPattern(http.MethodPost, PatternSubscriptions, PatternResourceGroups, PatternProviders, PatternResourceName, PatternActionName), postMuxMiddleware.HandlerFunc(f.ArmResourceAction)) + f.server.Handler = mux return f @@ -145,11 +158,18 @@ func (f *Frontend) NotFound(writer http.ResponseWriter, request *http.Request) { } func (f *Frontend) HealthzReady(writer http.ResponseWriter, request *http.Request) { + var healthStatus float64 if f.CheckReady() { writer.WriteHeader(http.StatusOK) + healthStatus = 1.0 } else { writer.WriteHeader(http.StatusInternalServerError) + healthStatus = 0.0 } + + f.metrics.EmitGauge("frontend_health", healthStatus, map[string]string{ + "endpoint": "/healthz/ready", + }) } func (f *Frontend) ArmResourceListBySubscription(writer http.ResponseWriter, request *http.Request) { diff --git a/frontend/go.mod b/frontend/go.mod index fd9ab3d22..0e16e6585 100644 --- a/frontend/go.mod +++ b/frontend/go.mod @@ -2,4 +2,13 @@ module github.com/Azure/ARO-HCP/frontend go 1.22 -require github.com/google/go-cmp v0.6.0 // indirect +require ( + github.com/google/go-cmp v0.6.0 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect + golang.org/x/sys v0.16.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) diff --git a/frontend/go.sum b/frontend/go.sum index 493949e04..25ca2fd0c 100644 --- a/frontend/go.sum +++ b/frontend/go.sum @@ -2,3 +2,17 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/frontend/main.go b/frontend/main.go index c5feea6c9..738746f09 100644 --- a/frontend/main.go +++ b/frontend/main.go @@ -36,7 +36,9 @@ func main() { os.Exit(1) } - frontend := NewFrontend(logger, listener) + // Init prometheus emitter + prometheusEmitter := NewPrometheusEmitter() + frontend := NewFrontend(logger, listener, prometheusEmitter) go frontend.Run(ctx, stop) sig := <-signalChannel diff --git a/frontend/metrics.go b/frontend/metrics.go new file mode 100644 index 000000000..352ce762f --- /dev/null +++ b/frontend/metrics.go @@ -0,0 +1,94 @@ +package main + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "net/http" + "strconv" + "time" + + "golang.org/x/exp/maps" + + "github.com/Azure/ARO-HCP/internal/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type PrometheusEmitter struct { + gauges map[string]*prometheus.GaugeVec + counters map[string]*prometheus.CounterVec +} + +func NewPrometheusEmitter() *PrometheusEmitter { + return &PrometheusEmitter{ + gauges: make(map[string]*prometheus.GaugeVec), + counters: make(map[string]*prometheus.CounterVec), + } +} + +func (pe *PrometheusEmitter) EmitGauge(name string, value float64, labels map[string]string) { + vec, exists := pe.gauges[name] + if !exists { + labelKeys := maps.Keys(labels) + vec = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name}, labelKeys) + prometheus.MustRegister(vec) + pe.gauges[name] = vec + } + vec.With(labels).Set(value) +} + +func (pe *PrometheusEmitter) EmitCounter(name string, value float64, labels map[string]string) { + vec, exists := pe.counters[name] + if !exists { + labelKeys := maps.Keys(labels) + vec = prometheus.NewCounterVec(prometheus.CounterOpts{Name: name}, labelKeys) + prometheus.MustRegister(vec) + pe.counters[name] = vec + } + vec.With(labels).Add(value) +} + +type MetricsMiddleware struct { + metrics.Emitter +} + +type logResponseWriter struct { + http.ResponseWriter + statusCode int +} + +// WriteHeader captures the status code sent to the client. +func (lrw *logResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + lrw.ResponseWriter.WriteHeader(code) +} + +// Metrics middleware to capture response time and status code +func (mm MetricsMiddleware) Metrics() MiddlewareFunc { + return func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + startTime := time.Now() + + lrw := &logResponseWriter{ResponseWriter: w} + + next(lrw, r) // Process the request + + // Get the route pattern that matched + routePattern := r.URL.Path + duration := time.Since(startTime).Milliseconds() + + // Emit metrics + mm.Emitter.EmitCounter("frontend_count", 1.0, map[string]string{ + "verb": r.Method, + "api_version": r.URL.Query().Get(APIVersionKey), + "code": strconv.Itoa(lrw.statusCode), + "route": routePattern, + }) + + mm.Emitter.EmitGauge("frontend_duration", float64(duration), map[string]string{ + "verb": r.Method, + "api_version": r.URL.Query().Get(APIVersionKey), + "code": strconv.Itoa(lrw.statusCode), + "route": routePattern, + }) + } +} diff --git a/go.work.sum b/go.work.sum index e7c2b2200..ef3d6051a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,7 +1,15 @@ +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= @@ -10,18 +18,29 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=