[MOSIP-35490] moved required helm and deploy scripts to modular repo (#…

…109) * [MOSIP-35490] moved required helm and deploy scripts to modular repo Signed-off-by: Chandra Keshav Mishra <[email protected]> * Update pom.xml Signed-off-by: Praful Rakhade <[email protected]> * Update pom.xml Signed-off-by: Praful Rakhade <[email protected]> * Update pom.xml Signed-off-by: Praful Rakhade <[email protected]> * Update pom.xml Signed-off-by: Praful Rakhade <[email protected]> * Update Dockerfile Signed-off-by: Praful Rakhade <[email protected]> * Update Dockerfile Signed-off-by: Praful Rakhade <[email protected]> * Update Dockerfile Signed-off-by: Praful Rakhade <[email protected]> * Update Dockerfile Signed-off-by: Praful Rakhade <[email protected]> --------- Signed-off-by: Chandra Keshav Mishra <[email protected]> Signed-off-by: Praful Rakhade <[email protected]> Co-authored-by: Praful Rakhade <[email protected]>
mosip · Dec 4, 2024 · 4988a0a · 4988a0a
1 parent 815b810
commit 4988a0a
Show file tree

Hide file tree

Showing 18 changed files with 612 additions and 1 deletion.
diff --git a/build/debezium-fixer/Dockerfile b/build/debezium-fixer/Dockerfile
@@ -1,5 +1,14 @@
 FROM alpine:3.15.4
 
+ARG SOURCE
+ARG COMMIT_HASH
+ARG COMMIT_ID
+ARG BUILD_TIME
+LABEL source=${SOURCE}
+LABEL commit_hash=${COMMIT_HASH}
+LABEL commit_id=${COMMIT_ID}
+LABEL build_time=${BUILD_TIME}
+
 ARG container_user=mosip
 ARG container_user_group=mosip
 ARG container_user_uid=1001

diff --git a/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile b/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile
@@ -1,5 +1,14 @@
 FROM confluentinc/cp-kafka-connect:7.0.0-1-ubi8
 
+ARG SOURCE
+ARG COMMIT_HASH
+ARG COMMIT_ID
+ARG BUILD_TIME
+LABEL source=${SOURCE}
+LABEL commit_hash=${COMMIT_HASH}
+LABEL commit_id=${COMMIT_ID}
+LABEL build_time=${BUILD_TIME}
+
 RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:11.1.3
 
 ADD ./target/kafka_connect_transforms-*.jar /usr/share/java/my_transforms

diff --git a/build/es-kafka-connecter/kafka_connect_transforms/pom.xml b/build/es-kafka-connecter/kafka_connect_transforms/pom.xml
@@ -30,6 +30,7 @@
     <groupId>io.mosip</groupId>
     <artifactId>kafka_connect_transforms</artifactId>
     <name>kafka_connect_transforms</name>
+    <description>Mosip Kafka Connect Transform Project</description>
     <version>1.2.1-SNAPSHOT</version>
     <url>https://github.com/mosip/reporting</url>
 
@@ -198,4 +199,4 @@
 
         </plugins>
     </build>
-</project>
+</project>
diff --git a/build/reporting-init/debezium-init/Dockerfile b/build/reporting-init/debezium-init/Dockerfile
@@ -1,5 +1,14 @@
 FROM alpine
 
+ARG SOURCE
+ARG COMMIT_HASH
+ARG COMMIT_ID
+ARG BUILD_TIME
+LABEL source=${SOURCE}
+LABEL commit_hash=${COMMIT_HASH}
+LABEL commit_id=${COMMIT_ID}
+LABEL build_time=${BUILD_TIME}
+
 RUN apk add curl bash vim
 
 ADD docker-entrypoint.sh /

diff --git a/build/reporting-init/es-kafka-connect-init/Dockerfile b/build/reporting-init/es-kafka-connect-init/Dockerfile
@@ -1,5 +1,14 @@
 FROM alpine
 
+ARG SOURCE
+ARG COMMIT_HASH
+ARG COMMIT_ID
+ARG BUILD_TIME
+LABEL source=${SOURCE}
+LABEL commit_hash=${COMMIT_HASH}
+LABEL commit_id=${COMMIT_ID}
+LABEL build_time=${BUILD_TIME}
+
 RUN apk add curl bash vim
 
 RUN mkdir /connectors /kibana_saved_objects

diff --git a/helm/reporting-init/.gitignore b/helm/reporting-init/.gitignore
@@ -0,0 +1 @@
+charts/
diff --git a/helm/reporting-init/.helmignore b/helm/reporting-init/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/helm/reporting-init/Chart.yaml b/helm/reporting-init/Chart.yaml
@@ -0,0 +1,9 @@
+apiVersion: v2
+name: reporting-init
+description: A Helm chart for MOSIP Reporting Initialization.
+type: application
+version: 0.0.1-develop
+appVersion: ""
+maintainers:
+  - email: [email protected]
+    name: MOSIP
diff --git a/helm/reporting-init/templates/debezium-job.yaml b/helm/reporting-init/templates/debezium-job.yaml
@@ -0,0 +1,64 @@
+{{ if .Values.debezium_connectors.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-debezium-job
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}-debezium-job
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ .Release.Name }}-debezium-job
+        sidecar.istio.io/inject: "false"
+    spec:
+      restartPolicy: Never  # This is one time job
+      containers:
+      - name: debezium-init
+        image: "{{ .Values.debezium_connectors.image }}:{{ .Values.debezium_connectors.imageTag }}"
+        imagePullPolicy: {{ .Values.debezium_connectors.imagePullPolicy }}
+        env:
+        - name: DB_PREFIX_INDEX
+          value: {{ .Values.base.db_prefix }}
+        - name: DB_HOSTNAME
+          value: {{ .Values.base.db_hostname }}
+        - name: DB_PORT
+          value: {{ .Values.base.db_port | quote }}
+        - name: DB_USER
+          value: {{ .Values.base.db_user }}
+        - name: DB_PASS
+          {{ if .Values.base.db_pass_direct }}
+          value: {{ .Values.base.db_pass_direct }}
+          {{ else }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.base.db_pass_secretref.name }}
+              key: {{ .Values.base.db_pass_secretref.key }}
+          {{ end }}
+        - name: ES_URL
+          value: {{ .Values.base.es_url }}
+        - name: KIBANA_URL
+          value: {{ .Values.base.kibana_url }}
+
+        - name: NUMBER_DEBEZ_CONN
+          value: {{ len .Values.debezium_connectors.dbs | quote }}
+        - name: DELAY_BETWEEN_CONNECTORS
+          value: {{ .Values.debezium_connectors.delay_between_connectors | quote }}
+        {{- range $i, $item := .Values.debezium_connectors.dbs }}
+        - name: DEBEZ_DB_NAME_{{ add $i 1 }}
+          value: {{ $item.name | quote }}
+        - name: DEBEZ_TABLE_LIST_{{ add $i 1 }}
+          value: {{ $item.tables | quote }}
+        {{- end }}
+        volumeMounts:
+        - name: debez-sample-file
+          mountPath: "/debez-connector"
+      volumes:
+      - name: debez-sample-file
+        configMap:
+          {{ if .Values.debezium_connectors.existingConfigMap }}
+          name: {{ .Values.debezium_connectors.existingConfigMap }}
+          {{ else }}
+          name: {{ .Release.Name }}-debezium-sample-conf
+          {{ end }}
+{{ end }}
diff --git a/helm/reporting-init/templates/es-kafka-connect-job.yaml b/helm/reporting-init/templates/es-kafka-connect-job.yaml
@@ -0,0 +1,56 @@
+{{ if .Values.es_kafka_connectors.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-es-kafka-conn-job
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}-es-kafka-conn-job
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ .Release.Name }}-es-kafka-conn-job
+        sidecar.istio.io/inject: "false"
+    spec:
+      restartPolicy: Never  # This is one time job
+      containers:
+      - name: {{ .Chart.Name }}
+        image: "{{ .Values.es_kafka_connectors.image }}:{{ .Values.es_kafka_connectors.imageTag }}"
+        imagePullPolicy: {{ .Values.es_kafka_connectors.imagePullPolicy }}
+        env:
+        - name: DB_PREFIX_INDEX
+          value: {{ .Values.base.db_prefix }}
+        - name: DB_HOSTNAME
+          value: {{ .Values.base.db_hostname }}
+        - name: DB_PORT
+          value: {{ .Values.base.db_port | quote }}
+        - name: DB_USER
+          value: {{ .Values.base.db_user }}
+        - name: DB_PASS
+          {{ if .Values.base.db_pass_direct }}
+          value: {{ .Values.base.db_pass_direct }}
+          {{ else }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.base.db_pass_secretref.name }}
+              key: {{ .Values.base.db_pass_secretref.key }}
+          {{ end }}
+        - name: ES_URL
+          value: {{ .Values.base.es_url }}
+        - name: KIBANA_URL
+          value: {{ .Values.base.kibana_url }}
+
+        - name: DELAY_BETWEEN_CONNECTORS
+          value: {{ .Values.es_kafka_connectors.delay_between_connectors | quote }}
+        volumeMounts:
+        - name: {{ .Release.Name }}-es-kafka-connectors
+          mountPath: "/connectors/{{ .Release.Name }}-es-kafka-connectors"
+      volumes:
+      - name: {{ .Release.Name }}-es-kafka-connectors
+        configMap:
+          {{ if .Values.es_kafka_connectors.existingConfigMap }}
+          name: {{ .Values.es_kafka_connectors.existingConfigMap }}
+          {{ else }}
+          name: {{ .Release.Name }}-es-kafka-connectors-conf
+          {{ end }}
+{{ end }}
diff --git a/helm/reporting-init/values.yaml b/helm/reporting-init/values.yaml
@@ -0,0 +1,43 @@
+base:
+  db_prefix: "mosip"
+  db_user: postgres
+  db_hostname: "postgres-postgresql.postgres"
+  db_port: "5432"
+  # if this is not false it will directly take pass from this variable, else it will take from db_pass_secretref
+  db_pass_direct: false
+  db_pass_secretref:
+    name: postgres-postgresql
+    key: postgresql-password
+  es_url: http://elasticsearch-master.cattle-logging-system:9200
+  kibana_url: http://kibana.cattle-logging-system:5601
+
+debezium_connectors:
+  image: mosipqa/debezium-init
+  imageTag: develop
+  imagePullPolicy: Always
+
+  enabled: true
+  existingConfigMap: null
+  delay_between_connectors: "0s"
+  dbs:
+  - name: "mosip_master"
+    tables: "master.doc_type,master.registration_center"
+  - name: "mosip_audit"
+    tables: "audit.app_audit_log"
+  - name: "mosip_prereg"
+    tables: "prereg.applicant_demographic,prereg.applicant_demographic_consumed,prereg.applicant_document,prereg.applicant_document_consumed,prereg.reg_appointment,prereg.reg_appointment_consumed,prereg.reg_available_slot,prereg.otp_transaction"
+  - name: "mosip_ida"
+    tables: "ida.auth_transaction"
+  - name: "mosip_regprc"
+    tables: "regprc.registration,regprc.registration_list"
+
+es_kafka_connectors:
+  image: mosipqa/es-kafka-connect-init
+  imageTag: develop
+  imagePullPolicy: Always
+
+  enabled: true
+  existingConfigMap: null
+  delay_between_connectors: "0s"
+
+kibana_enabled: {}
diff --git a/helm/reporting/.gitignore b/helm/reporting/.gitignore
@@ -0,0 +1 @@
+charts/
diff --git a/helm/reporting/.helmignore b/helm/reporting/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/helm/reporting/Chart.yaml b/helm/reporting/Chart.yaml
@@ -0,0 +1,30 @@
+apiVersion: v2
+name: reporting
+description: A Helm chart for MOSIP Reporting module.
+type: application
+version: 0.0.1-develop
+appVersion: ""
+dependencies:
+- name: common
+  repository: https://charts.bitnami.com/bitnami
+  tags:
+  - bitnami-common
+  version: 1.x.x
+- name: cp-helm-charts
+  alias: confluentinc
+  repository: "https://confluentinc.github.io/cp-helm-charts/"
+  version: ">= 0.6.1"
+  condition: confluentinc.enabled
+- name: kafka
+  alias: bitnami-kafka
+  repository: "https://charts.bitnami.com/bitnami"
+  version: "23.0.7"
+  condition: bitnami-kafka.enabled
+- name: elasticsearch
+  alias: bitnami-elasticsearch
+  repository: "https://charts.bitnami.com/bitnami"
+  version: ">= 17.0.0"
+  condition: bitnami-elasticsearch.enabled
+maintainers:
+  - email: [email protected]
+    name: MOSIP
diff --git a/helm/reporting/README.md b/helm/reporting/README.md
@@ -0,0 +1,19 @@
+# Reporting Framework
+
+Helm chart for installing reporting module
+
+## TL;DR
+
+```console
+$ helm repo add mosip https://mosip.github.io/mosip-helm
+$ helm install my-release mosip/reporting
+```
+
+## Contents
+
+This helm chart contains the following subcharts and they can be individually configured/installed/omitted, through the `values.yaml`.
+- Bitnami's Kafka
+- Debezium Kafka Connector
+- Elasticsearch Kafka Connector
+- A KafkaClient Pod for monitoring kafka topics, and making connect api calls
+