From 89886679f4940a8656b2f74155b59c08ff3f9010 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 8 Jul 2024 20:52:37 +0200 Subject: [PATCH] Update snapshots of example pipelines --- .../test_generate/atm-fraud/pipeline.yaml | 336 +++++++++--------- .../test_generate/word-count/pipeline.yaml | 132 +++---- 2 files changed, 234 insertions(+), 234 deletions(-) diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index 528da26bb..76c9b1bf8 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -1,5 +1,16 @@ - _cleaner: - app: + debug: true + name: account-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 labels: @@ -15,34 +26,7 @@ outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: account-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: - image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud - prometheus: - jmx: - enabled: false - replicaCount: 1 - schedule: 0 12 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-producer-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - suspend: true debug: true name: account-producer namespace: ${NAMESPACE} @@ -59,9 +43,36 @@ configs: {} partitions_count: 3 type: producer-app + values: + image: ${DOCKER_REGISTRY}/atm-demo-accountproducer + imageTag: 1.0.0 + labels: + pipeline: atm-fraud + prometheus: + jmx: + enabled: false + replicaCount: 1 + schedule: 0 12 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-producer-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ + suspend: true version: 2.9.0 - _cleaner: - app: + debug: true + name: transaction-avro-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: commandLine: ITERATION: 20 REAL_TX: 19 @@ -80,19 +91,24 @@ outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: transaction-avro-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: + debug: true + name: transaction-avro-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-avro-producer-topic: + configs: {} + partitions_count: 3 + type: producer-app + values: commandLine: ITERATION: 20 REAL_TX: 19 @@ -111,25 +127,20 @@ outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: transaction-avro-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-avro-producer-topic: - configs: {} - partitions_count: 3 - type: producer-app version: 2.9.0 - _cleaner: - app: + debug: true + name: transaction-joiner + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic commandLine: @@ -153,19 +164,28 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: transaction-joiner - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: + debug: true + name: transaction-joiner + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-joiner-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-transaction-joiner-topic: + configs: {} + partitions_count: 3 + type: streams-app + values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic commandLine: @@ -189,29 +209,20 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: transaction-joiner - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-joiner-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-transaction-joiner-topic: - configs: {} - partitions_count: 3 - type: streams-app version: 2.9.0 - _cleaner: - app: + debug: true + name: fraud-detector + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic commandLine: @@ -235,19 +246,28 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: fraud-detector - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: + debug: true + name: fraud-detector + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-fraud-detector-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-fraud-detector-topic: + configs: {} + partitions_count: 3 + type: streams-app + values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic commandLine: @@ -271,29 +291,20 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: fraud-detector - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-fraud-detector-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-fraud-detector-topic: - configs: {} - partitions_count: 3 - type: streams-app version: 2.9.0 - _cleaner: - app: + debug: true + name: account-linker + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic commandLine: @@ -320,45 +331,7 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: account-linker - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - annotations: - consumerGroup: atm-accountlinker-atm-fraud-output-topic - commandLine: - PRODUCTIVE: false - image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: 1.0.0 - labels: - pipeline: atm-fraud - persistence: - enabled: false - prometheus: - jmx: - enabled: false - replicaCount: 1 - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-account-linker-dead-letter-topic - extraInputTopics: - accounts: - - atm-fraud-account-producer-topic - inputTopics: - - atm-fraud-fraud-detector-topic - optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-linker-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true from: components: @@ -386,13 +359,35 @@ configs: {} partitions_count: 3 type: streams-app + values: + annotations: + consumerGroup: atm-accountlinker-atm-fraud-output-topic + commandLine: + PRODUCTIVE: false + image: ${DOCKER_REGISTRY}/atm-demo-accountlinker + imageTag: 1.0.0 + labels: + pipeline: atm-fraud + persistence: + enabled: false + prometheus: + jmx: + enabled: false + replicaCount: 1 + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-account-linker-dead-letter-topic + extraInputTopics: + accounts: + - atm-fraud-account-producer-topic + inputTopics: + - atm-fraud-fraud-detector-topic + optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-linker-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: atm-fraud-postgresql-connector - connectorType: sink name: postgresql-connector namespace: ${NAMESPACE} prefix: atm-fraud- @@ -403,8 +398,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: atm-fraud-postgresql-connector + connectorType: sink version: 1.0.4 - app: + config: auto.create: true connection.ds.pool.size: 5 connection.password: AppPassword diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index 7e7d16fe8..e7afd3398 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -1,5 +1,16 @@ - _cleaner: - app: + debug: true + name: data-producer + namespace: ${NAMESPACE} + prefix: word-count- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: image: bakdata/kpops-demo-sentence-producer imageTag: 1.0.0 labels: @@ -13,32 +24,7 @@ optimizeLeaveGroupBehavior: false outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: data-producer - namespace: ${NAMESPACE} - prefix: word-count- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: - image: bakdata/kpops-demo-sentence-producer - imageTag: 1.0.0 - labels: - pipeline: word-count - prometheus: - jmx: - enabled: false - replicaCount: 1 - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - optimizeLeaveGroupBehavior: false - outputTopic: word-count-data-producer-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true name: data-producer namespace: ${NAMESPACE} @@ -55,9 +41,34 @@ configs: {} partitions_count: 3 type: producer-app + values: + image: bakdata/kpops-demo-sentence-producer + imageTag: 1.0.0 + labels: + pipeline: word-count + prometheus: + jmx: + enabled: false + replicaCount: 1 + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + optimizeLeaveGroupBehavior: false + outputTopic: word-count-data-producer-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: - app: + debug: true + name: word-counter + namespace: ${NAMESPACE} + prefix: word-count- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: commandLine: PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app @@ -79,40 +90,7 @@ optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: word-counter - namespace: ${NAMESPACE} - prefix: word-count- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - commandLine: - PRODUCTIVE: false - image: bakdata/kpops-demo-word-count-app - imageTag: 1.0.0 - labels: - pipeline: word-count - persistence: - enabled: false - prometheus: - jmx: - enabled: false - replicaCount: 1 - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: word-count-word-counter-dead-letter-topic - inputTopics: - - word-count-data-producer-topic - optimizeLeaveGroupBehavior: false - outputTopic: word-count-word-counter-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true name: word-counter namespace: ${NAMESPACE} @@ -135,13 +113,30 @@ partitions_count: 3 type: output type: streams-app + values: + commandLine: + PRODUCTIVE: false + image: bakdata/kpops-demo-word-count-app + imageTag: 1.0.0 + labels: + pipeline: word-count + persistence: + enabled: false + prometheus: + jmx: + enabled: false + replicaCount: 1 + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: word-count-word-counter-dead-letter-topic + inputTopics: + - word-count-data-producer-topic + optimizeLeaveGroupBehavior: false + outputTopic: word-count-word-counter-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: word-count-redis-sink-connector - connectorType: sink name: redis-sink-connector namespace: ${NAMESPACE} prefix: word-count- @@ -152,8 +147,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: word-count-redis-sink-connector + connectorType: sink version: 1.0.4 - app: + config: connector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector key.converter: org.apache.kafka.connect.storage.StringConverter name: word-count-redis-sink-connector