Merge pull request #2856 from sgayangi/go-enforcer-multienv

Update Prometheus implementation in go-enforcer

gateway/enforcer/cmd/main.go

@@ -66,6 +66,7 @@ func main() {
 	}
 	// Start the metrics server
 	if cfg.Metrics.Enabled && strings.EqualFold(cfg.Metrics.Type, "prometheus") {
+		metrics.RegisterDataSources(jwtTransformer, subAppDatastore)
 		go metrics.StartPrometheusMetricsServer(cfg.Metrics.Port)
 	}
 	// Wait forever

gateway/enforcer/go.mod

@@ -100,18 +100,17 @@ require (
 	golang.org/x/text v0.22.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.31.2 // indirect
-	k8s.io/apiextensions-apiserver v0.31.2 // indirect
-	k8s.io/apimachinery v0.31.2 // indirect
+	k8s.io/api v0.32.1 // indirect
+	k8s.io/apiextensions-apiserver v0.32.1 // indirect
+	k8s.io/apimachinery v0.32.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect
-	k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 // indirect
-	sigs.k8s.io/controller-runtime v0.19.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	sigs.k8s.io/controller-runtime v0.20.2 // indirect
 	sigs.k8s.io/gateway-api v1.2.1 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 

gateway/enforcer/go.sum

@@ -107,8 +107,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
-github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
@@ -163,10 +163,10 @@ github.com/moesif/moesifapi-go v1.0.5/go.mod h1:qlUk62wTdzP5a0NNoSsSUSwdYRSOcf+g
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
-github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
-github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
-github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
+github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
@@ -305,35 +305,34 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
-k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
-k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0=
-k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM=
-k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
-k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc=
+k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k=
+k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw=
+k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto=
+k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs=
+k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA=
-k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc=
-k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI=
-k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
+k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
 nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk=
-sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/controller-runtime v0.20.2 h1:/439OZVxoEc02psi1h4QO3bHzTgu49bb347Xp4gW1pc=
+sigs.k8s.io/controller-runtime v0.20.2/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY=
 sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM=
 sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
-sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

gateway/enforcer/internal/datastore/jwt_issuer_store.go

@@ -74,3 +74,14 @@ func (s *JWTIssuerStore) GetJWTISsuersByOrganization(organization string) map[st
 	}
 	return nil
 }
+
+// GetJWTIssuerCount obtains the total token issuer count for metrics purposes.
+func (s *JWTIssuerStore) GetJWTIssuerCount() int {
+	count := 0
+	if s.jwtIssuers != nil {
+		for _, orgWiseIssuers := range s.jwtIssuers {
+			count += len(orgWiseIssuers)
+		}
+	}
+	return count
+}

gateway/enforcer/internal/datastore/subs_app_datastore.go

@@ -401,3 +401,14 @@ func GetTLSConfig() (*tls.Config, error) {
 	tlsConfig := util.CreateTLSConfig(clientCert, certPool)
 	return tlsConfig, nil
 }
+
+// GetTotalSubscriptionCount obtains the total subscription count for metrics purposes.
+func (ds *SubscriptionApplicationDataStore) GetTotalSubscriptionCount() int {
+	ds.mu.Lock()
+	defer ds.mu.Unlock()
+	count := 0
+	for _, subscriptions := range ds.subscriptions {
+		count += len(subscriptions)
+	}
+	return count
+}

gateway/enforcer/internal/metrics/metrics.go

@@ -16,7 +16,7 @@
  * under the License.
  */
 
-// Package metrics holds the implementation for exposing adapter metrics to prometheus
+// Package metrics holds the implementation for exposing enforcer metrics to prometheus
 package metrics
 
 import (
@@ -26,13 +26,76 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	metrics "github.com/wso2/apk/common-go-libs/pkg/metrics"
+	commonmetrics "github.com/wso2/apk/common-go-libs/pkg/metrics"
+	"github.com/wso2/apk/gateway/enforcer/internal/datastore"
+	"github.com/wso2/apk/gateway/enforcer/internal/transformer"
 )
 
+var (
+	prometheusMetricRegistry = prometheus.NewRegistry()
+)
+
+var jwtTransformer *transformer.JWTTransformer
+var subAppDataStore *datastore.SubscriptionApplicationDataStore
+
+// enforcerCollector contains the descriptions of the custom metrics exposed by the adapter.
+// It also uses the metrics defined in common-go-libs
+type enforcerCollector struct {
+	commonmetrics.Collector
+	tokenIssuers  *prometheus.Desc
+	subscriptions *prometheus.Desc
+}
+
+func enforcerMetricsCollector() *enforcerCollector {
+	return &enforcerCollector{
+		Collector: *commonmetrics.CustomMetricsCollector(),
+		tokenIssuers: prometheus.NewDesc(
+			"token_issuer_count",
+			"Number of token issuers created.",
+			nil, nil,
+		),
+		subscriptions: prometheus.NewDesc(
+			"subscription_count",
+			"Number of subscriptions created.",
+			nil, nil,
+		),
+	}
+}
+
+// Describe sends all the descriptors of the metrics collected by this Collector
+// to the provided channel.
+func (collector *enforcerCollector) Describe(ch chan<- *prometheus.Desc) {
+	collector.Collector.Describe(ch)
+	ch <- collector.tokenIssuers
+	ch <- collector.subscriptions
+}
+
+// Collect collects all the relevant Prometheus metrics when Prometheus requests it
+func (collector *enforcerCollector) Collect(ch chan<- prometheus.Metric) {
+	collector.Collector.Collect(ch)
+	var tokenIssuerCount float64
+	var subscriptionCount float64
+	if jwtTransformer != nil {
+		tokenIssuerCount = float64(jwtTransformer.GetTokenIssuerCount())
+	}
+	if subAppDataStore != nil {
+		subscriptionCount = float64(subAppDataStore.GetTotalSubscriptionCount())
+	}
+
+	ch <- prometheus.MustNewConstMetric(collector.tokenIssuers, prometheus.GaugeValue, tokenIssuerCount)
+	ch <- prometheus.MustNewConstMetric(collector.subscriptions, prometheus.GaugeValue, subscriptionCount)
+}
+
+// RegisterDataSources registers the data sources that the metrics would be scraped from.
+func RegisterDataSources(transformer *transformer.JWTTransformer, dataStore *datastore.SubscriptionApplicationDataStore) {
+	jwtTransformer = transformer
+	subAppDataStore = dataStore
+}
+
 // StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus.
 func StartPrometheusMetricsServer(port int32) {
 
-	collector := metrics.CustomMetricsCollector()
+	collector := enforcerMetricsCollector()
 	prometheus.MustRegister(collector)
 	http.Handle("/metrics", promhttp.Handler())
 	err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil)

gateway/enforcer/internal/transformer/jwtTransformer.go

@@ -88,3 +88,8 @@ func (transformer *JWTTransformer) TransformJWTClaims(organization string, jwtAu
 	}
 	return &jwtValidationInfo
 }
+
+// GetTokenIssuerCount obtains the total token issuer count for metrics purposes.
+func (transformer *JWTTransformer) GetTokenIssuerCount() int {
+	return transformer.tokenissuerStore.GetJWTIssuerCount()
+}

helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml

@@ -101,6 +101,10 @@ spec:
               value: "3"
             - name: enforcer_admin_pwd
               value: admin
+            {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
+            - name: METRICS_ENABLED
+              value: "true"
+            {{- end }}
             - name: JAVA_OPTS
               value: {{ include "apk.javaOptions" . }}
             {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers }}
@@ -187,11 +191,6 @@ spec:
             {{- else }}
               subPath: tls.key
             {{- end }}
-            {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
-            - name: prometheus-jmx-config-volume
-              mountPath: /tmp/metrics/prometheus-jmx-config-enforcer.yml
-              subPath: prometheus-jmx-config-enforcer.yml
-            {{- end }}
             - name: enforcer-keystore-secret-volume
               mountPath: /home/wso2/security/keystore/enforcer.crt
             {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }}
@@ -440,11 +439,6 @@ spec:
         #     secretName: {{ template "apk-helm.resource.prefix" . }}-common-controller-server-cert
         #   {{- end }}
         #     defaultMode: 420
-        {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
-        - name: prometheus-jmx-config-volume
-          configMap: 
-            name: prometheus-jmx-config-enforcer
-        {{- end }}
         - name: enforcer-keystore-secret-volume
           secret: 
           {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }}