From 180e538c704dd9abcb542fc4ba5d123477e83b90 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Fri, 8 Dec 2023 16:00:27 -0600 Subject: [PATCH 1/4] Reconfigure playbook-dispatcher-connect in stage to connect to msk --- deploy/connect.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/connect.yaml b/deploy/connect.yaml index d3c19aae..19a0e4b8 100644 --- a/deploy/connect.yaml +++ b/deploy/connect.yaml @@ -196,7 +196,7 @@ objects: "-Xmx": ${XMX} "-Xms": ${XMS} - bootstrapServers: ${KAFKA_BOOTSTRAP_HOST}:${KAFKA_BOOTSTRAP_PORT} + bootstrapServers: ${KAFKA_BOOTSTRAP_HOST} config: group.id: playbook-dispatcher-connect config.providers: file @@ -207,6 +207,15 @@ objects: offset.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} status.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} config.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} + tls: + trustedCertificates: [] + authentication: + # this config is specific to PLAIN sasl mechanism + type: ${KAFKA_SASL_MECHANISM} + username: ${KAFKA_USERNAME} + passwordSecret: + secretName: app-auth + password: client_secret externalConfiguration: volumes: - name: rds-client-ca From 8eb832f93433594a628ef8d4caf2b6ee75604ecb Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Tue, 12 Dec 2023 11:27:55 -0600 Subject: [PATCH 2/4] Reconfigure again --- deploy/connect.yaml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/deploy/connect.yaml b/deploy/connect.yaml index 19a0e4b8..d3c19aae 100644 --- a/deploy/connect.yaml +++ b/deploy/connect.yaml @@ -196,7 +196,7 @@ objects: "-Xmx": ${XMX} "-Xms": ${XMS} - bootstrapServers: ${KAFKA_BOOTSTRAP_HOST} + bootstrapServers: ${KAFKA_BOOTSTRAP_HOST}:${KAFKA_BOOTSTRAP_PORT} config: group.id: playbook-dispatcher-connect config.providers: file @@ -207,15 +207,6 @@ objects: offset.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} status.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} config.storage.replication.factor: ${KAFKA_REPLICATION_FACTOR} - tls: - trustedCertificates: [] - authentication: - # this config is specific to PLAIN sasl mechanism - type: ${KAFKA_SASL_MECHANISM} - username: ${KAFKA_USERNAME} - passwordSecret: - secretName: app-auth - password: client_secret externalConfiguration: volumes: - name: rds-client-ca From f50583238a1041eefa74b119d45d3329e677ca08 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Tue, 12 Dec 2023 11:39:06 -0600 Subject: [PATCH 3/4] Reconfigure --- deploy/connect-rhosak.yaml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/deploy/connect-rhosak.yaml b/deploy/connect-rhosak.yaml index b3e5d5aa..dfb22955 100644 --- a/deploy/connect-rhosak.yaml +++ b/deploy/connect-rhosak.yaml @@ -196,7 +196,7 @@ objects: "-Xmx": ${XMX} "-Xms": ${XMS} - bootstrapServers: ${KAFKA_BOOTSTRAP_HOST}:${KAFKA_BOOTSTRAP_PORT} + bootstrapServers: ${KAFKA_BOOTSTRAP_HOST} config: group.id: playbook-dispatcher-connect config.providers: file @@ -214,8 +214,8 @@ objects: type: ${KAFKA_SASL_MECHANISM} username: ${KAFKA_USERNAME} passwordSecret: - secretName: clowder-oauth - password: password + secretName: app-auth + password: client_secret externalConfiguration: volumes: - name: rds-client-ca @@ -230,6 +230,10 @@ objects: imagePullSecrets: - name: quay-cloudservices-pull - name: rh-registry-pull + deployment: + metadata: + annotations: + ignore-check.kube-linter.io/minimum-three-replicas: "This deployment uses 1 pod, using more than 1 pod will produce more than 1 kafka messages for every db update" metricsConfig: type: jmxPrometheusExporter @@ -280,16 +284,19 @@ objects: plugin.name: pgoutput slot.max.retries: 999999999 database.server.name: playbook-dispatcher - table.include.list: public.runs + table.include.list: public.runs,public.run_hosts tombstones.on.delete: false key.converter: org.apache.kafka.connect.storage.StringConverter value.converter: org.apache.kafka.connect.storage.StringConverter - transforms: transformRunEvent + transforms: transformRunEvent,transformRunHostEvent transforms.transformRunEvent.type: com.redhat.cloud.platform.playbook_dispatcher.RunEventTransform transforms.transformRunEvent.table: runs transforms.transformRunEvent.topic: ${KAFKA_TOPIC_PREFIX}platform.playbook-dispatcher.runs + transforms.transformRunHostEvent.type: com.redhat.cloud.platform.playbook_dispatcher.RunHostEventTransform + transforms.transformRunHostEvent.table: run_hosts + transforms.transformRunHostEvent.topic: ${KAFKA_TOPIC_PREFIX}platform.playbook-dispatcher.run-hosts errors.tolerance: all errors.retry.delay.max.ms: 30000 @@ -307,6 +314,8 @@ objects: labels: app: playbook-dispatcher name: playbook-dispatcher-event-consumer + annotations: + ignore-check.kube-linter.io/minimum-three-replicas: "This deployment uses 1 pod, using more than 1 pod will produce more than 1 kafka messages for every db update" spec: replicas: ${{EVENT_CONSUMER_REPLICAS}} selector: @@ -324,7 +333,7 @@ objects: - command: - /opt/kafka/bin/kafka-console-consumer.sh - --bootstrap-server - - ${KAFKA_BOOTSTRAP_HOST}:${KAFKA_BOOTSTRAP_PORT} + - ${KAFKA_BOOTSTRAP_HOST} - --from-beginning - --group - ${EVENT_CONSUMER_GROUP} From c2d99c61c6c88fd316a7d9701377709620984307 Mon Sep 17 00:00:00 2001 From: Derek Horton Date: Tue, 12 Dec 2023 11:39:33 -0600 Subject: [PATCH 4/4] Rename file --- deploy/{connect-rhosak.yaml => connect-msk.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deploy/{connect-rhosak.yaml => connect-msk.yaml} (100%) diff --git a/deploy/connect-rhosak.yaml b/deploy/connect-msk.yaml similarity index 100% rename from deploy/connect-rhosak.yaml rename to deploy/connect-msk.yaml