add query range cache

Signed-off-by: Thibault Mange <[email protected]>
rhobs · Dec 4, 2023 · b0bee3a · b0bee3a
1 parent dfddd12
commit b0bee3a
Show file tree

Hide file tree

Showing 6 changed files with 518 additions and 79 deletions.
diff --git a/go.mod b/go.mod
@@ -83,4 +83,4 @@ require (
 )
 
 // Delete when https://github.com/observatorium/observatorium/pull/543 is merged to main branch
-replace github.com/observatorium/observatorium => github.com/thibaultmg/observatorium v0.0.0-20231124164749-d47b398822ba
+replace github.com/observatorium/observatorium => github.com/thibaultmg/observatorium v0.0.0-20231204171717-54b9405e5fab
diff --git a/go.sum b/go.sum
@@ -1253,8 +1253,8 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
-github.com/thibaultmg/observatorium v0.0.0-20231124164749-d47b398822ba h1:C0lH8h/Dh6SCVG6FCNBsn1Rg075gQd3TPBVzjcdCIsk=
-github.com/thibaultmg/observatorium v0.0.0-20231124164749-d47b398822ba/go.mod h1:P+7t9O8AitkuZjUhXC4LHw4iwAzTpIrs0tHz8X3xTvM=
+github.com/thibaultmg/observatorium v0.0.0-20231204171717-54b9405e5fab h1:OLXEyHPt3e3Sz/w+MOU0uxwIPD7U1XRfHia1xs1TjuM=
+github.com/thibaultmg/observatorium v0.0.0-20231204171717-54b9405e5fab/go.mod h1:P+7t9O8AitkuZjUhXC4LHw4iwAzTpIrs0tHz8X3xTvM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=

diff --git a/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-query-frontend-template.yaml b/resources/services/app-sre-stage-01/rhobs/observatorium-metrics-query-frontend-template.yaml
@@ -4,6 +4,170 @@ metadata:
   creationTimestamp: null
   name: observatorium-thanos-query-frontend
 objects:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+      app.kubernetes.io/version: "1.5"
+    name: observatorium-thanos-query-frontend
+    namespace: rhobs
+  spec:
+    replicas: ${{REPLICAS}}
+    selector:
+      matchLabels:
+        app.kubernetes.io/component: memcached
+        app.kubernetes.io/instance: observatorium
+        app.kubernetes.io/name: memcached
+        app.kubernetes.io/part-of: observatorium
+    strategy: {}
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          app.kubernetes.io/component: memcached
+          app.kubernetes.io/instance: observatorium
+          app.kubernetes.io/name: memcached
+          app.kubernetes.io/part-of: observatorium
+          app.kubernetes.io/version: "1.5"
+        namespace: rhobs
+      spec:
+        affinity:
+          podAntiAffinity:
+            preferredDuringSchedulingIgnoredDuringExecution:
+            - podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                  - key: app.kubernetes.io/instance
+                    operator: In
+                    values:
+                    - observatorium
+                  - key: app.kubernetes.io/name
+                    operator: In
+                    values:
+                    - memcached
+                topologyKey: kubernetes.io/hostname
+              weight: 100
+        containers:
+        - args:
+          - --conn-limit=3072
+          - --max-item-size=5m
+          - --memory-limit=2048
+          - --verbose=true
+          image: quay.io/app-sre/memcached:1.5
+          imagePullPolicy: IfNotPresent
+          name: observatorium-thanos-query-frontend
+          ports:
+          - containerPort: 11211
+            name: client
+            protocol: TCP
+          resources:
+            limits:
+              memory: ${MEMORY_LIMIT}
+            requests:
+              cpu: ${CPU_REQUEST}
+              memory: ${MEMORY_REQUEST}
+          terminationMessagePolicy: FallbackToLogsOnError
+        - args:
+          - --memcached.address=localhost:0
+          - --web.listen-address=:9150
+          image: quay.io/prometheus/memcached-exporter:v0.13.0
+          imagePullPolicy: IfNotPresent
+          name: memcached-exporter
+          ports:
+          - containerPort: 9150
+            name: metrics
+            protocol: TCP
+          resources:
+            limits:
+              cpu: 200m
+              memory: 200Mi
+            requests:
+              cpu: 50m
+              memory: 50Mi
+          terminationMessagePolicy: FallbackToLogsOnError
+        nodeSelector:
+          kubernetes.io/os: linux
+        serviceAccountName: observatorium-thanos-query-frontend
+        terminationGracePeriodSeconds: 120
+- apiVersion: v1
+  kind: Service
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+      app.kubernetes.io/version: "1.5"
+    name: observatorium-thanos-query-frontend
+    namespace: rhobs
+  spec:
+    clusterIP: None
+    ports:
+    - name: client
+      port: 11211
+      protocol: TCP
+      targetPort: 11211
+    - name: metrics
+      port: 9150
+      protocol: TCP
+      targetPort: 9150
+    selector:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+- apiVersion: v1
+  imagePullSecrets:
+  - name: quay.io
+  kind: ServiceAccount
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+      app.kubernetes.io/version: "1.5"
+    name: observatorium-thanos-query-frontend
+    namespace: rhobs
+- apiVersion: monitoring.coreos.com/v1
+  kind: ServiceMonitor
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+      app.kubernetes.io/version: "1.5"
+      prometheus: app-sre
+    name: observatorium-thanos-query-frontend
+    namespace: openshift-customer-monitoring
+  spec:
+    endpoints:
+    - port: metrics
+      relabelings:
+      - action: replace
+        separator: /
+        sourceLabels:
+        - namespace
+        - pod
+        targetLabel: instance
+    namespaceSelector:
+      matchNames:
+      - rhobs
+    selector:
+      matchLabels:
+        app.kubernetes.io/component: memcached
+        app.kubernetes.io/instance: observatorium
+        app.kubernetes.io/name: memcached
+        app.kubernetes.io/part-of: observatorium
 - apiVersion: route.openshift.io/v1
   kind: Route
   metadata:
@@ -168,6 +332,18 @@ objects:
           - --query-frontend.downstream-url=http://observatorium-thanos-query.rhobs.svc.cluster.local:10902
           - --query-frontend.log-queries-longer-than=5s
           - --query-range.max-retries-per-request=0
+          - |
+            --query-range.response-cache-config=type: MEMCACHED
+            config:
+              addresses:
+              - dnssrv+_client._tcp.observatorium-thanos-query-range-cache-memcached.rhobs.svc
+              timeout: 2s
+              max_idle_connections: 1300
+              max_async_concurrency: 200
+              max_async_buffer_size: 2000000
+              max_get_multi_concurrency: 1000
+              max_item_size: 64MiB
+              max_get_multi_batch_size: 100
           - --query-range.split-interval=1d
           - |
             --tracing.config=type: JAEGER
@@ -293,6 +469,25 @@ objects:
         - name: tls
           secret:
             secretName: query-frontend-tls
+- apiVersion: policy/v1
+  kind: PodDisruptionBudget
+  metadata:
+    creationTimestamp: null
+    labels:
+      app.kubernetes.io/component: memcached
+      app.kubernetes.io/instance: observatorium
+      app.kubernetes.io/name: memcached
+      app.kubernetes.io/part-of: observatorium
+    name: observatorium-thanos-query-frontend
+    namespace: rhobs
+  spec:
+    maxUnavailable: 1
+    selector:
+      matchLabels:
+        app.kubernetes.io/component: memcached
+        app.kubernetes.io/instance: observatorium
+        app.kubernetes.io/name: memcached
+        app.kubernetes.io/part-of: observatorium
 parameters:
 - name: LOG_LEVEL
   value: warn