[CCXDEV-11589] Add clowdapp for dvo-writer

deploy/dvo-writer.yaml

@@ -0,0 +1,297 @@
+# Copyright 2023 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+kind: Template
+metadata:
+  name: dvo-writer
+objects:
+
+- kind: HorizontalPodAutoscaler
+  apiVersion: autoscaling/v1
+  metadata:
+    labels:
+      app: ccx-data-pipeline
+    name: dvo-writer
+  spec:
+    minReplicas: ${{MIN_REPLICAS}}
+    maxReplicas: ${{MAX_REPLICAS}}
+    scaleTargetRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: dvo-writer
+    targetCPUUtilizationPercentage: 80
+
+- apiVersion: cloud.redhat.com/v1alpha1
+  kind: ClowdApp
+  metadata:
+    name: dvo-writer
+  spec:
+    envName: ${ENV_NAME}
+    testing:
+      iqePlugin: ccx
+    deployments:
+      - name: instance
+        minReplicas: ${{DVO_WRITER_REPLICAS}}
+        webServices:
+          public:
+            enabled: false
+          private:
+            enabled: false
+          metrics:
+            enabled: true
+        podSpec:
+          initContainers:
+            - name: execute-migrations
+              image: ${IMAGE}:${IMAGE_TAG}
+              env:
+                - name: INSIGHTS_RESULTS_AGGREGATOR__STORAGE_BACKEND__USE
+                  value: "dvo_recommendations"
+                - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__ENABLED
+                  value: "false"
+                - name: INSIGHTS_RESULTS_AGGREGATOR__METRICS__ENABLED
+                  value: "false"
+                - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__DB_DRIVER
+                  value: postgres
+                - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__PG_PARAMS
+                  value: ${PG_PARAMS}
+                - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__LOG_SQL_QUERIES
+                  value: "true"
+                - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__TYPE
+                  value: "sql"
+                - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__LOGGING_TO_CLOUD_WATCH_ENABLED
+                  value: ${CLOUDWATCH_ENABLED}
+                - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__DEBUG
+                  value: ${CLOUDWATCH_DEBUG}
+                - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__STREAM_NAME
+                  value: ${LOG_STREAM}
+                - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__CREATE_STREAM_IF_NOT_EXISTS
+                  value: ${CREATE_STREAM_IF_NOT_EXISTS}
+                - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__AWS_REGION
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudwatch
+                      key: aws_region
+                      optional: true
+                - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__LOG_GROUP
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudwatch
+                      key: log_group_name
+                      optional: true
+                - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__AWS_ACCESS_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudwatch
+                      key: aws_access_key_id
+                      optional: true
+                - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__AWS_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudwatch
+                      key: aws_secret_access_key
+                      optional: true
+              resources:
+                limits:
+                  cpu: ${DVO_WRITER_CPU_LIMIT}
+                  memory: ${DVO_WRITER_MEMORY_LIMIT}
+                requests:
+                  cpu: ${DVO_WRITER_CPU_REQUEST}
+                  memory: ${DVO_WRITER_MEMORY_REQUEST}
+              command:
+                - ./insights-results-aggregator
+                - migration
+                - latest
+          env:
+            - name: INSIGHTS_RESULTS_AGGREGATOR__STORAGE_BACKEND__USE
+              value: "dvo_recommendations"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__ADDRESS
+              value: "${KAFKA_BOOTSTRAP_HOST}:${KAFKA_BOOTSTRAP_PORT}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__TIMEOUT
+              value: "${KAFKA_TIMEOUT}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__TOPIC
+              value: "${INCOMING_TOPIC}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__DEAD_LETTER_QUEUE_TOPIC
+              value: "${DEAD_LETTER_QUEUE_TOPIC}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__PAYLOAD_TRACKER_TOPIC
+              value: "${PAYLOAD_TRACKER_TOPIC}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__SERVICE_NAME
+              value: "${DVO_WRITER_SERVICE_NAME}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__GROUP
+              value: "${GROUP_ID}"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__ENABLED
+              value: "true"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__ENABLE_ORG_WHITELIST
+              value: "false"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__BROKER__DISPLAY_MESSAGE_WITH_WRONG_STRUCTURE
+              value: "true"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__PROCESSING__ORG_WHITELIST_FILE
+              value: "/data/org_whitelist.csv"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__METRICS__ENABLED
+              value: "true"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__DB_DRIVER
+              value: postgres
+            - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__PG_PARAMS
+              value: ${PG_PARAMS}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__DVO_RECOMMENDATIONS_STORAGE__TYPE
+              value: "sql"
+            - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__LOG_LEVEL
+              value: ${LOG_LEVEL}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__LOGGING__LOGGING_TO_CLOUD_WATCH_ENABLED
+              value: ${CLOUDWATCH_ENABLED}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__DEBUG
+              value: ${CLOUDWATCH_DEBUG}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__STREAM_NAME
+              value: ${LOG_STREAM}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__CREATE_STREAM_IF_NOT_EXISTS
+              value: ${CREATE_STREAM_IF_NOT_EXISTS}
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__AWS_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: cloudwatch
+                  key: aws_region
+                  optional: true
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__LOG_GROUP
+              valueFrom:
+                secretKeyRef:
+                  name: cloudwatch
+                  key: log_group_name
+                  optional: true
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__AWS_ACCESS_ID
+              valueFrom:
+                secretKeyRef:
+                  name: cloudwatch
+                  key: aws_access_key_id
+                  optional: true
+            - name: INSIGHTS_RESULTS_AGGREGATOR__CLOUDWATCH__AWS_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: cloudwatch
+                  key: aws_secret_access_key
+                  optional: true
+          image: ${IMAGE}:${IMAGE_TAG}
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: "${DVO_WRITER_API_PREFIX}metrics"
+              port: 9000
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 60
+            successThreshold: 1
+            timeoutSeconds: 60
+          readinessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: "${DVO_WRITER_API_PREFIX}metrics"
+              port: 9000
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 60
+            successThreshold: 1
+            timeoutSeconds: 60
+          resources:
+            limits:
+              cpu: ${DVO_WRITER_CPU_LIMIT}
+              memory: ${DVO_WRITER_MEMORY_LIMIT}
+            requests:
+              cpu: ${DVO_WRITER_CPU_REQUEST}
+              memory: ${DVO_WRITER_MEMORY_REQUEST}
+    database:
+      # the DB name should match to app-interface DB name without specifying environment
+      # https://gitlab.cee.redhat.com/service/app-interface/-/blob/ddd85c2ad79b40047391405b2d909eb38667bc43/data/services/insights/ccx-data-pipeline/namespaces/stage-ccx-data-pipeline-stage.yml#L60
+      name: ccx-data-pipeline
+      version: 15
+    kafkaTopics:
+      - replicas: 3
+        partitions: 1
+        topicName: ${INCOMING_TOPIC}
+      - replicas: 3
+        partitions: 1
+        topicName: ${PAYLOAD_TRACKER_TOPIC}
+      - replicas: 3
+        partitions: 1
+        topicName: ${DEAD_LETTER_QUEUE_TOPIC}
+- kind: Service
+  apiVersion: v1
+  metadata:
+    annotations:
+      prometheus.io/path: /metrics
+      prometheus.io/port: "9000"
+      prometheus.io/scheme: http
+      prometheus.io/scrape: "true"
+    name: dvo-writer-prometheus-exporter
+    labels:
+      app: dvo-writer
+  spec:
+    ports:
+      - name: web
+        port: 9001
+        protocol: TCP
+        targetPort: 9001
+    selector:
+      app: dvo-writer
+      pod: dvo-writer-instance
+
+parameters:
+- name: PG_PARAMS
+  value: sslmode=require
+- name: KAFKA_BOOTSTRAP_HOST
+  required: true
+  value: mq-kafka
+- name: KAFKA_BOOTSTRAP_PORT
+  required: true
+  value: "29092"
+- name: KAFKA_TIMEOUT
+  value: 300s
+- name: INCOMING_TOPIC
+  value: ccx.dvo.results
+  required: true
+- name: DEAD_LETTER_QUEUE_TOPIC
+  value: ccx.dvowriter.dead.letter.queue
+  required: true
+- name: PAYLOAD_TRACKER_TOPIC
+  value: platform.payload-status
+  required: true
+- name: DVO_WRITER_SERVICE_NAME
+  value: dvo-writer
+  required: true
+- name: GROUP_ID
+  value: dvo_writer_app
+  required: true
+- name: DVO_WRITER_API_PREFIX
+  required: true
+  value: /
+- name: LOG_STREAM
+  value: dvo-writer
+- name: DVO_WRITER_REPLICAS
+  description: The number of replicas to use for the dvo-writer deployment
+  value: "1"
+- name: DVO_WRITER_CPU_LIMIT
+  value: 200m
+- name: DVO_WRITER_MEMORY_LIMIT
+  value: 400Mi
+- name: DVO_WRITER_CPU_REQUEST
+  value: 100m
+- name: DVO_WRITER_MEMORY_REQUEST
+  value: 200Mi
+- name: CLOUDWATCH_DEBUG
+  value: "false"
+  required: true
+- name: CLOUDWATCH_ENABLED
+  value: "true"
+  required: true
+- name: CREATE_STREAM_IF_NOT_EXISTS
+  value: "true"