diff --git a/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml b/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml index a47ad6b50..20d30f041 100644 --- a/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml @@ -1,16 +1,16 @@ helm-app.yaml: -- app-helm-app.yaml +- values-helm-app.yaml - repo_config-helm-app.yaml kafka-app.yaml: - prefix.yaml - from_.yaml - to.yaml -- app-kafka-app.yaml +- values-kafka-app.yaml kafka-connector.yaml: - prefix.yaml - from_.yaml - to.yaml -- app-kafka-connector.yaml +- config-kafka-connector.yaml - resetter_values.yaml kafka-sink-connector.yaml: [] kafka-source-connector.yaml: @@ -21,9 +21,9 @@ kubernetes-app.yaml: - from_.yaml - to.yaml - namespace.yaml -- app-kubernetes-app.yaml +- values-kubernetes-app.yaml producer-app.yaml: - from_-producer-app.yaml -- app-producer-app.yaml +- values-producer-app.yaml streams-app.yaml: -- app-streams-app.yaml +- values-streams-app.yaml diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 380bfffe0..8705b752d 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -5,7 +5,7 @@ kpops_components_fields: - from_ - to - namespace - - app + - values - repo_config - version kafka-app: @@ -13,13 +13,13 @@ kpops_components_fields: - prefix - from_ - to - - app + - values kafka-connector: - name - prefix - from_ - to - - app + - config - resetter_namespace - resetter_values kafka-sink-connector: @@ -27,7 +27,7 @@ kpops_components_fields: - prefix - from_ - to - - app + - config - resetter_namespace - resetter_values kafka-source-connector: @@ -35,7 +35,7 @@ kpops_components_fields: - prefix - from_ - to - - app + - config - resetter_namespace - resetter_values - offset_topic @@ -45,7 +45,7 @@ kpops_components_fields: - from_ - to - namespace - - app + - values pipeline-component: - name - prefix @@ -57,7 +57,7 @@ kpops_components_fields: - from_ - to - namespace - - app + - values - repo_config - version streams-app: @@ -66,7 +66,7 @@ kpops_components_fields: - from_ - to - namespace - - app + - values - repo_config - version streams-bootstrap: @@ -75,7 +75,7 @@ kpops_components_fields: - from_ - to - namespace - - app + - values - repo_config - version kpops_components_inheritance_ref: diff --git a/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml b/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml index c7d08112c..0ff8788ad 100644 --- a/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml @@ -3,31 +3,31 @@ helm-app.yaml: - from_.yaml - to.yaml - namespace.yaml -- app-helm-app.yaml +- values-helm-app.yaml - repo_config-helm-app.yaml - version.yaml kafka-app.yaml: - prefix.yaml - from_.yaml - to.yaml -- app-kafka-app.yaml +- values-kafka-app.yaml kafka-connector.yaml: - prefix.yaml - from_.yaml - to.yaml -- app-kafka-connector.yaml +- config-kafka-connector.yaml - resetter_values.yaml kafka-sink-connector.yaml: - prefix.yaml - from_.yaml - to.yaml -- app-kafka-connector.yaml +- config-kafka-connector.yaml - resetter_values.yaml kafka-source-connector.yaml: - prefix.yaml - from_-kafka-source-connector.yaml - to.yaml -- app-kafka-connector.yaml +- config-kafka-connector.yaml - resetter_values.yaml - offset_topic-kafka-source-connector.yaml kubernetes-app.yaml: @@ -35,13 +35,13 @@ kubernetes-app.yaml: - from_.yaml - to.yaml - namespace.yaml -- app-kubernetes-app.yaml +- values-kubernetes-app.yaml producer-app.yaml: - prefix.yaml - from_-producer-app.yaml - to.yaml - namespace.yaml -- app-producer-app.yaml +- values-producer-app.yaml - repo_config-helm-app.yaml - version-kafka-app.yaml streams-app.yaml: @@ -49,6 +49,6 @@ streams-app.yaml: - from_.yaml - to.yaml - namespace.yaml -- app-streams-app.yaml +- values-streams-app.yaml - repo_config-helm-app.yaml - version-kafka-app.yaml diff --git a/docs/docs/resources/pipeline-components/helm-app.yaml b/docs/docs/resources/pipeline-components/helm-app.yaml index 6fee1c9f4..b08346107 100644 --- a/docs/docs/resources/pipeline-components/helm-app.yaml +++ b/docs/docs/resources/pipeline-components/helm-app.yaml @@ -44,9 +44,7 @@ models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/kafka-app.yaml b/docs/docs/resources/pipeline-components/kafka-app.yaml index 687e890f7..806662ef3 100644 --- a/docs/docs/resources/pipeline-components/kafka-app.yaml +++ b/docs/docs/resources/pipeline-components/kafka-app.yaml @@ -44,9 +44,7 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` can contain application-specific settings, hence the user is free to - # add the key-value pairs they need. - app: # required + values: # required streams: # required brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-components/kafka-connector.yaml b/docs/docs/resources/pipeline-components/kafka-connector.yaml index 90af66f7f..4a64cdf3a 100644 --- a/docs/docs/resources/pipeline-components/kafka-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-connector.yaml @@ -42,10 +42,8 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. diff --git a/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml b/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml index 33235894e..9253db0e0 100644 --- a/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml @@ -43,10 +43,8 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. diff --git a/docs/docs/resources/pipeline-components/kafka-source-connector.yaml b/docs/docs/resources/pipeline-components/kafka-source-connector.yaml index fc1f4e8c4..8fa6c51d5 100644 --- a/docs/docs/resources/pipeline-components/kafka-source-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-source-connector.yaml @@ -24,10 +24,8 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. diff --git a/docs/docs/resources/pipeline-components/kubernetes-app.yaml b/docs/docs/resources/pipeline-components/kubernetes-app.yaml index 2a8a431c8..10758d848 100644 --- a/docs/docs/resources/pipeline-components/kubernetes-app.yaml +++ b/docs/docs/resources/pipeline-components/kubernetes-app.yaml @@ -44,9 +44,7 @@ models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/pipeline.yaml b/docs/docs/resources/pipeline-components/pipeline.yaml index b2a98be32..835c989f1 100644 --- a/docs/docs/resources/pipeline-components/pipeline.yaml +++ b/docs/docs/resources/pipeline-components/pipeline.yaml @@ -44,9 +44,7 @@ models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example @@ -107,9 +105,7 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` can contain application-specific settings, hence the user is free to - # add the key-value pairs they need. - app: # required + values: # required streams: # required brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} @@ -160,10 +156,8 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. @@ -195,10 +189,8 @@ cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. @@ -253,9 +245,7 @@ models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example @@ -291,7 +281,7 @@ namespace: namespace # required # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app - app: # required + values: # required streams: # required, producer-app-specific brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} @@ -361,7 +351,7 @@ # No arbitrary keys are allowed under `app`here # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app - app: # required + values: # required # Streams Bootstrap streams section streams: # required, streams-app-specific brokers: ${config.kafka_brokers} # required diff --git a/docs/docs/resources/pipeline-components/producer-app.yaml b/docs/docs/resources/pipeline-components/producer-app.yaml index 784873617..9ddff3b47 100644 --- a/docs/docs/resources/pipeline-components/producer-app.yaml +++ b/docs/docs/resources/pipeline-components/producer-app.yaml @@ -30,7 +30,7 @@ namespace: namespace # required # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app - app: # required + values: # required streams: # required, producer-app-specific brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-components/sections/app-helm-app.yaml b/docs/docs/resources/pipeline-components/sections/app-helm-app.yaml deleted file mode 100644 index e2b6cbae0..000000000 --- a/docs/docs/resources/pipeline-components/sections/app-helm-app.yaml +++ /dev/null @@ -1,6 +0,0 @@ - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required - image: exampleImage # Example - debug: false # Example - commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/sections/app-kafka-connector.yaml b/docs/docs/resources/pipeline-components/sections/app-kafka-connector.yaml deleted file mode 100644 index 10b3f4cbe..000000000 --- a/docs/docs/resources/pipeline-components/sections/app-kafka-connector.yaml +++ /dev/null @@ -1,5 +0,0 @@ - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required - tasks.max: 1 diff --git a/docs/docs/resources/pipeline-components/sections/app-kubernetes-app.yaml b/docs/docs/resources/pipeline-components/sections/app-kubernetes-app.yaml deleted file mode 100644 index e2b6cbae0..000000000 --- a/docs/docs/resources/pipeline-components/sections/app-kubernetes-app.yaml +++ /dev/null @@ -1,6 +0,0 @@ - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required - image: exampleImage # Example - debug: false # Example - commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/sections/config-kafka-connector.yaml b/docs/docs/resources/pipeline-components/sections/config-kafka-connector.yaml new file mode 100644 index 000000000..c8402366d --- /dev/null +++ b/docs/docs/resources/pipeline-components/sections/config-kafka-connector.yaml @@ -0,0 +1,3 @@ + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required + tasks.max: 1 diff --git a/docs/docs/resources/pipeline-components/sections/values-helm-app.yaml b/docs/docs/resources/pipeline-components/sections/values-helm-app.yaml new file mode 100644 index 000000000..e2572fd73 --- /dev/null +++ b/docs/docs/resources/pipeline-components/sections/values-helm-app.yaml @@ -0,0 +1,4 @@ + values: # required + image: exampleImage # Example + debug: false # Example + commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml b/docs/docs/resources/pipeline-components/sections/values-kafka-app.yaml similarity index 67% rename from docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml rename to docs/docs/resources/pipeline-components/sections/values-kafka-app.yaml index 5ae8be6d6..e670fadf8 100644 --- a/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/values-kafka-app.yaml @@ -1,6 +1,4 @@ - # `app` can contain application-specific settings, hence the user is free to - # add the key-value pairs they need. - app: # required + values: # required streams: # required brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-components/sections/values-kubernetes-app.yaml b/docs/docs/resources/pipeline-components/sections/values-kubernetes-app.yaml new file mode 100644 index 000000000..e2572fd73 --- /dev/null +++ b/docs/docs/resources/pipeline-components/sections/values-kubernetes-app.yaml @@ -0,0 +1,4 @@ + values: # required + image: exampleImage # Example + debug: false # Example + commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml b/docs/docs/resources/pipeline-components/sections/values-producer-app.yaml similarity index 95% rename from docs/docs/resources/pipeline-components/sections/app-producer-app.yaml rename to docs/docs/resources/pipeline-components/sections/values-producer-app.yaml index 0fe6680cd..adc5f863f 100644 --- a/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/values-producer-app.yaml @@ -1,6 +1,6 @@ # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app - app: # required + values: # required streams: # required, producer-app-specific brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml b/docs/docs/resources/pipeline-components/sections/values-streams-app.yaml similarity index 99% rename from docs/docs/resources/pipeline-components/sections/app-streams-app.yaml rename to docs/docs/resources/pipeline-components/sections/values-streams-app.yaml index e3577aa5f..2d505f95c 100644 --- a/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/values-streams-app.yaml @@ -1,7 +1,7 @@ # No arbitrary keys are allowed under `app`here # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app - app: # required + values: # required # Streams Bootstrap streams section streams: # required, streams-app-specific brokers: ${config.kafka_brokers} # required diff --git a/docs/docs/resources/pipeline-components/streams-app.yaml b/docs/docs/resources/pipeline-components/streams-app.yaml index 33c190046..adb116a0c 100644 --- a/docs/docs/resources/pipeline-components/streams-app.yaml +++ b/docs/docs/resources/pipeline-components/streams-app.yaml @@ -48,7 +48,7 @@ # No arbitrary keys are allowed under `app`here # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app - app: # required + values: # required # Streams Bootstrap streams section streams: # required, streams-app-specific brokers: ${config.kafka_brokers} # required diff --git a/docs/docs/resources/pipeline-defaults/defaults-helm-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-helm-app.yaml index d08200203..021a1e8fc 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-helm-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-helm-app.yaml @@ -3,9 +3,7 @@ # Parent of: KafkaApp # Child of: KubernetesApp helm-app: - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml index e1938cb42..2e4c1dc82 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml @@ -45,9 +45,7 @@ kafka-app: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` can contain application-specific settings, hence the user is free to - # add the key-value pairs they need. - app: # required + values: # required streams: # required brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml index 0fd30c235..9ace5173b 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml @@ -45,10 +45,8 @@ kafka-connector: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. diff --git a/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml index 31f55aa45..8e9d1aef8 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml @@ -46,9 +46,7 @@ kubernetes-app: models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example diff --git a/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml index a5b4a0f6f..5a5b7f77c 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml @@ -8,7 +8,7 @@ producer-app: # `from` section, hence it does not support it. # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app - app: # required + values: # required streams: # required, producer-app-specific brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} diff --git a/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml index 4db627950..bd7ac9d0f 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml @@ -6,7 +6,7 @@ streams-app: # No arbitrary keys are allowed under `app`here # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app - app: # required + values: # required # Streams Bootstrap streams section streams: # required, streams-app-specific brokers: ${config.kafka_brokers} # required diff --git a/docs/docs/resources/pipeline-defaults/defaults.yaml b/docs/docs/resources/pipeline-defaults/defaults.yaml index 8547f6207..2eead5c37 100644 --- a/docs/docs/resources/pipeline-defaults/defaults.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults.yaml @@ -3,9 +3,7 @@ # Parent of: KafkaApp # Child of: KubernetesApp helm-app: - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example @@ -66,9 +64,7 @@ kafka-app: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` can contain application-specific settings, hence the user is free to - # add the key-value pairs they need. - app: # required + values: # required streams: # required brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} @@ -121,10 +117,8 @@ kafka-connector: cleanup.policy: compact models: # SchemaProvider is initiated with the values given here model: model - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. Extensive documentation on - # connectors: https://kafka.apache.org/documentation/#connectconfigs - app: # required + # Full documentation on connectors: https://kafka.apache.org/documentation/#connectconfigs + config: # required tasks.max: 1 # Overriding Kafka Connect Resetter Helm values. E.g. to override the # Image Tag etc. @@ -192,9 +186,7 @@ kubernetes-app: models: # SchemaProvider is initiated with the values given here model: model namespace: namespace # required - # `app` contains application-specific settings, hence it does not have a rigid - # structure. The fields below are just an example. - app: # required + values: # required image: exampleImage # Example debug: false # Example commandLine: {} # Example @@ -208,7 +200,7 @@ producer-app: # `from` section, hence it does not support it. # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app - app: # required + values: # required streams: # required, producer-app-specific brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${config.schema_registry.url} @@ -225,7 +217,7 @@ streams-app: # No arbitrary keys are allowed under `app`here # Allowed configs: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/streams-app - app: # required + values: # required # Streams Bootstrap streams section streams: # required, streams-app-specific brokers: ${config.kafka_brokers} # required diff --git a/docs/docs/resources/variables/variable_substitution.yaml b/docs/docs/resources/variables/variable_substitution.yaml index 8a4cf60ea..76e2eacb2 100644 --- a/docs/docs/resources/variables/variable_substitution.yaml +++ b/docs/docs/resources/variables/variable_substitution.yaml @@ -1,14 +1,14 @@ - type: scheduled-producer - app: + values: labels: app_type: "${component.type}" app_name: "${component.name}" - app_schedule: "${component.app.schedule}" + app_schedule: "${component.values.schedule}" commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -18,13 +18,13 @@ memory: 2G - type: filter name: "filter-app" - app: + values: labels: app_type: "${component.type}" app_name: "${component.name}" - app_resources_requests_memory: "${component.app.resources.requests.memory}" - ${component.type}: "${component.app.labels.app_name}-${component.app.labels.app_type}" - test_placeholder_in_placeholder: "${component.app.labels.${component.type}}" + app_resources_requests_memory: "${component.values.resources.requests.memory}" + ${component.type}: "${component.values.labels.app_name}-${component.values.labels.app_type}" + test_placeholder_in_placeholder: "${component.values.labels.${component.type}}" commandLine: TYPE: "nothing" resources: diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 77ceed301..e728d5fba 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -63,14 +63,6 @@ "additionalProperties": true, "description": "Kubernetes app managed through Helm with an associated Helm chart.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/HelmAppValues" - } - ], - "description": "Helm app values" - }, "from": { "anyOf": [ { @@ -128,6 +120,14 @@ "const": "helm-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/HelmAppValues" + } + ], + "description": "Helm app values" + }, "version": { "anyOf": [ { @@ -145,7 +145,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "HelmApp", @@ -222,14 +222,6 @@ "additionalProperties": true, "description": "Base component for Kafka-based components.\nProducer or streaming apps should inherit from this class.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/KafkaAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "anyOf": [ { @@ -265,11 +257,19 @@ ], "default": null, "description": "Topic(s) into which the component will write output" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/KafkaAppValues" + } + ], + "description": "Application-specific settings" } }, "required": [ "name", - "app" + "values" ], "title": "KafkaApp", "type": "object" @@ -311,13 +311,13 @@ "additionalProperties": true, "description": "Base class for all Kafka connectors.\nShould only be used to set defaults", "properties": { - "app": { + "config": { "allOf": [ { "$ref": "#/$defs/KafkaConnectorConfig" } ], - "description": "Application-specific settings" + "description": "Connector config" }, "from": { "anyOf": [ @@ -379,7 +379,7 @@ }, "required": [ "name", - "app" + "config" ], "title": "KafkaConnector", "type": "object" @@ -437,13 +437,13 @@ "additionalProperties": true, "description": "Kafka sink connector model.", "properties": { - "app": { + "config": { "allOf": [ { "$ref": "#/$defs/KafkaConnectorConfig" } ], - "description": "Application-specific settings" + "description": "Connector config" }, "from": { "anyOf": [ @@ -509,7 +509,7 @@ }, "required": [ "name", - "app", + "config", "type" ], "title": "KafkaSinkConnector", @@ -519,13 +519,13 @@ "additionalProperties": true, "description": "Kafka source connector model.", "properties": { - "app": { + "config": { "allOf": [ { "$ref": "#/$defs/KafkaConnectorConfig" } ], - "description": "Application-specific settings" + "description": "Connector config" }, "from": { "anyOf": [ @@ -604,7 +604,7 @@ }, "required": [ "name", - "app", + "config", "type" ], "title": "KafkaSourceConnector", @@ -664,14 +664,6 @@ "additionalProperties": true, "description": "Base class for all Kubernetes apps.\nAll built-in components are Kubernetes apps, except for the Kafka connectors.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/KubernetesAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "anyOf": [ { @@ -712,12 +704,20 @@ ], "default": null, "description": "Topic(s) into which the component will write output" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/KubernetesAppValues" + } + ], + "description": "Kubernetes app values" } }, "required": [ "name", "namespace", - "app" + "values" ], "title": "KubernetesApp", "type": "object" @@ -828,14 +828,6 @@ "additionalProperties": true, "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -893,6 +885,14 @@ "const": "producer-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, "version": { "anyOf": [ { @@ -910,7 +910,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "ProducerApp", @@ -1077,14 +1077,6 @@ "additionalProperties": true, "description": "StreamsApp component that configures a streams-bootstrap app.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "anyOf": [ { @@ -1149,6 +1141,14 @@ "const": "streams-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, "version": { "anyOf": [ { @@ -1166,7 +1166,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "StreamsApp", @@ -1339,14 +1339,6 @@ "additionalProperties": true, "description": "Base for components with a streams-bootstrap Helm chart.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/StreamsBootstrapValues" - } - ], - "description": "streams-bootstrap app values" - }, "from": { "anyOf": [ { @@ -1407,6 +1399,14 @@ "default": null, "description": "Topic(s) into which the component will write output" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsBootstrapValues" + } + ], + "description": "streams-bootstrap Helm values" + }, "version": { "anyOf": [ { diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index d8c953c82..e2acabfdd 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -63,14 +63,6 @@ "additionalProperties": true, "description": "Kubernetes app managed through Helm with an associated Helm chart.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/HelmAppValues" - } - ], - "description": "Helm app values" - }, "from": { "anyOf": [ { @@ -128,6 +120,14 @@ "const": "helm-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/HelmAppValues" + } + ], + "description": "Helm app values" + }, "version": { "anyOf": [ { @@ -145,7 +145,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "HelmApp", @@ -271,13 +271,13 @@ "additionalProperties": true, "description": "Kafka sink connector model.", "properties": { - "app": { + "config": { "allOf": [ { "$ref": "#/$defs/KafkaConnectorConfig" } ], - "description": "Application-specific settings" + "description": "Connector config" }, "from": { "anyOf": [ @@ -343,7 +343,7 @@ }, "required": [ "name", - "app", + "config", "type" ], "title": "KafkaSinkConnector", @@ -353,13 +353,13 @@ "additionalProperties": true, "description": "Kafka source connector model.", "properties": { - "app": { + "config": { "allOf": [ { "$ref": "#/$defs/KafkaConnectorConfig" } ], - "description": "Application-specific settings" + "description": "Connector config" }, "from": { "anyOf": [ @@ -438,7 +438,7 @@ }, "required": [ "name", - "app", + "config", "type" ], "title": "KafkaSourceConnector", @@ -496,14 +496,6 @@ "additionalProperties": true, "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "default": null, "description": "Producer doesn't support FromSection", @@ -561,6 +553,14 @@ "const": "producer-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, "version": { "anyOf": [ { @@ -578,7 +578,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "ProducerApp", @@ -745,14 +745,6 @@ "additionalProperties": true, "description": "StreamsApp component that configures a streams-bootstrap app.", "properties": { - "app": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "Application-specific settings" - }, "from": { "anyOf": [ { @@ -817,6 +809,14 @@ "const": "streams-app", "title": "Type" }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, "version": { "anyOf": [ { @@ -834,7 +834,7 @@ "required": [ "name", "namespace", - "app", + "values", "type" ], "title": "StreamsApp", diff --git a/docs/docs/user/migration-guide/v6-v7.md b/docs/docs/user/migration-guide/v6-v7.md index c08be2af8..1eb3261c5 100644 --- a/docs/docs/user/migration-guide/v6-v7.md +++ b/docs/docs/user/migration-guide/v6-v7.md @@ -9,41 +9,41 @@ As a result of the restructure, some imports need to be adjusted: **KPOps Python API** ```diff --import kpops -+import kpops.api as kpops +- import kpops ++ import kpops.api as kpops ``` **builtin KPOps components** ```diff --from kpops.components import ( -- HelmApp, -- KafkaApp, -- KafkaConnector, -- KafkaSinkConnector, -- KafkaSourceConnector, -- KubernetesApp, -- StreamsBootstrap, -- ProducerApp, -- StreamsApp, -- PipelineComponent, -- StreamsApp, -- ProducerApp, --) -+from kpops.components.base_components import ( -+ HelmApp, -+ KafkaApp, -+ KafkaConnector, -+ KafkaSinkConnector, -+ KafkaSourceConnector, -+ KubernetesApp, -+ PipelineComponent, -+) -+from kpops.components.streams_bootstrap import ( -+ StreamsBootstrap, -+ StreamsApp, -+ ProducerApp, -+) +- from kpops.components import ( +- HelmApp, +- KafkaApp, +- KafkaConnector, +- KafkaSinkConnector, +- KafkaSourceConnector, +- KubernetesApp, +- StreamsBootstrap, +- ProducerApp, +- StreamsApp, +- PipelineComponent, +- StreamsApp, +- ProducerApp, +- ) ++ from kpops.components.base_components import ( ++ HelmApp, ++ KafkaApp, ++ KafkaConnector, ++ KafkaSinkConnector, ++ KafkaSourceConnector, ++ KubernetesApp, ++ PipelineComponent, ++ ) ++ from kpops.components.streams_bootstrap import ( ++ StreamsBootstrap, ++ StreamsApp, ++ ProducerApp, ++ ) ``` ### your custom KPOps components @@ -51,14 +51,54 @@ As a result of the restructure, some imports need to be adjusted: #### config.yaml ```diff --components_module: components +- components_module: components ``` #### Python module ```diff --components/__init__.py -+kpops/components/custom/__init__.py +- components/__init__.py ++ kpops/components/custom/__init__.py +``` + +## [Rename app field](https://github.com/bakdata/kpops/pull/506) + +The `app` attribute of the builtin KPOps components has been renamed to better differentiate them. Both your `pipeline.yaml` and `defaults.yaml` files have to be updated, e.g.: + +#### defaults.yaml + +```diff + kubernetes-app: +- app: {} ++ values: {} + + helm-app: +- app: {} ++ values: {} + + kafka-app: +- app: {} ++ values: {} + + streams-app: +- app: {} ++ values: {} + + producer-app: +- app: {} ++ values: {} + + kafka-connector: +- app: {} ++ config: {} + + kafka-source-connector: +- app: {} ++ config: {} + + kafka-sink-connector: +- app: {} ++ config: {} ``` ## [Call destroy from inside of reset or clean](https://github.com/bakdata/kpops/pull/501) diff --git a/examples b/examples index f7613426d..f98a77f96 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f7613426dffe5a1d6332c9e3cc7f0bfb23396e68 +Subproject commit f98a77f962c8bf1188eba359245791feed22b3ff diff --git a/hooks/gen_docs/gen_docs_env_vars.py b/hooks/gen_docs/gen_docs_env_vars.py index 510d464b2..4b1297d0c 100644 --- a/hooks/gen_docs/gen_docs_env_vars.py +++ b/hooks/gen_docs/gen_docs_env_vars.py @@ -260,7 +260,7 @@ def fill_csv_pipeline_config(target: Path) -> None: strict=True, ): with suppress(KeyError): # In case the prefix is ever removed from KpopsConfig - env_var_name = KpopsConfig.model_config["env_prefix"] + env_var_name + env_var_name = KpopsConfig.model_config["env_prefix"] + env_var_name # pyright: ignore[reportTypedDictNotRequiredAccess] field_description: str = ( field_value.description or "No description available, please refer to the pipeline config documentation." diff --git a/hooks/gen_schema.py b/hooks/gen_schema.py index 62e154b29..73343c995 100644 --- a/hooks/gen_schema.py +++ b/hooks/gen_schema.py @@ -5,7 +5,7 @@ from pathlib import Path from hooks import ROOT -from kpops.api.file_type import KpopsFileType +from kpops.const.file_type import KpopsFileType from kpops.utils.gen_schema import ( gen_config_schema, gen_defaults_schema, diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 16d379456..28cca3a44 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -6,13 +6,18 @@ import typer import kpops.api as kpops -from kpops.api.file_type import CONFIG_YAML, DEFAULTS_YAML, PIPELINE_YAML, KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import ( collect_pipeline_paths, ) from kpops.config import ENV_PREFIX from kpops.const import KPOPS, __version__ +from kpops.const.file_type import ( + CONFIG_YAML, + DEFAULTS_YAML, + PIPELINE_YAML, + KpopsFileType, +) from kpops.utils.gen_schema import ( gen_config_schema, gen_defaults_schema, diff --git a/kpops/cli/utils.py b/kpops/cli/utils.py index 16a8af563..61f5f83e0 100644 --- a/kpops/cli/utils.py +++ b/kpops/cli/utils.py @@ -3,7 +3,7 @@ from collections.abc import Iterable, Iterator from pathlib import Path -from kpops.api.file_type import PIPELINE_YAML +from kpops.const.file_type import PIPELINE_YAML def collect_pipeline_paths(pipeline_paths: Iterable[Path]) -> Iterator[Path]: diff --git a/kpops/component_handlers/__init__.py b/kpops/component_handlers/__init__.py index fa296a574..243f4c797 100644 --- a/kpops/component_handlers/__init__.py +++ b/kpops/component_handlers/__init__.py @@ -11,6 +11,20 @@ class ComponentHandlers: + _instance: ComponentHandlers | None = None + + def __new__( + cls, + schema_handler, + connector_handler, + topic_handler, + *args, + **kwargs, + ) -> ComponentHandlers: + if not cls._instance: + cls._instance = super().__new__(cls, *args, **kwargs) + return cls._instance + def __init__( self, schema_handler: SchemaHandler | None, @@ -20,3 +34,10 @@ def __init__( self.schema_handler = schema_handler self.connector_handler = connector_handler self.topic_handler = topic_handler + + +def get_handlers() -> ComponentHandlers: + if not ComponentHandlers._instance: + msg = f"{ComponentHandlers.__name__} has not been initialized" + raise RuntimeError(msg) + return ComponentHandlers._instance diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 61e7cff56..0381d552e 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -81,7 +81,7 @@ async def upgrade_install( chart: str, dry_run: bool, namespace: str, - values: dict, + values: dict[str, Any], flags: HelmUpgradeInstallFlags | None = None, ) -> str: """Prepare and execute the `helm upgrade --install` command.""" @@ -158,7 +158,7 @@ def template( release_name: str, chart: str, namespace: str, - values: dict, + values: dict[str, Any], flags: HelmTemplateFlags | None = None, ) -> Resource: """From Helm: Render chart templates locally and display the output. diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 840df06c3..b86b1d359 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Literal +from typing import Any import pydantic from pydantic import ( @@ -12,10 +12,8 @@ from pydantic.json_schema import SkipJsonSchema from typing_extensions import override -from kpops.components.base_components.helm_app import HelmAppValues -from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.utils.pydantic import ( - CamelCaseConfigModel, DescConfigModel, by_alias, exclude_by_value, @@ -111,15 +109,3 @@ class KafkaConnectConfigErrorResponse(BaseModel): name: str error_count: int configs: list[KafkaConnectConfigDescription] - - -class KafkaConnectorResetterConfig(CamelCaseConfigModel): - brokers: str - connector: str - delete_consumer_group: bool | None = None - offset_topic: str | None = None - - -class KafkaConnectorResetterValues(HelmAppValues): - connector_type: Literal["source", "sink"] - config: KafkaConnectorResetterConfig diff --git a/kpops/component_handlers/topic/handler.py b/kpops/component_handlers/topic/handler.py index f79c552fc..fbf5824eb 100644 --- a/kpops/component_handlers/topic/handler.py +++ b/kpops/component_handlers/topic/handler.py @@ -17,7 +17,7 @@ parse_and_compare_topic_configs, parse_rest_proxy_topic_config, ) -from kpops.components.base_components.models.topic import KafkaTopic +from kpops.components.common.topic import KafkaTopic from kpops.utils.colorify import greenify, magentaify from kpops.utils.dict_differ import Diff, DiffType, render_diff diff --git a/kpops/components/base_components/__init__.py b/kpops/components/base_components/__init__.py index ff94dde1f..39dfabd0c 100644 --- a/kpops/components/base_components/__init__.py +++ b/kpops/components/base_components/__init__.py @@ -1,12 +1,12 @@ -from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kafka_app import KafkaApp -from kpops.components.base_components.kafka_connector import ( +from .helm_app import HelmApp +from .kafka_app import KafkaApp +from .kafka_connector import ( KafkaConnector, KafkaSinkConnector, KafkaSourceConnector, ) -from kpops.components.base_components.kubernetes_app import KubernetesApp -from kpops.components.base_components.pipeline_component import PipelineComponent +from .kubernetes_app import KubernetesApp +from .pipeline_component import PipelineComponent __all__ = ( "HelmApp", diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 3ad8ac894..661e54d92 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -19,9 +19,8 @@ ) from pydantic.json_schema import SkipJsonSchema -from kpops.api.file_type import KpopsFileType -from kpops.component_handlers import ComponentHandlers -from kpops.config import KpopsConfig +from kpops.config import KpopsConfig, get_config +from kpops.const.file_type import KpopsFileType from kpops.utils import cached_classproperty from kpops.utils.dataclasses import is_dataclass_instance from kpops.utils.dict_ops import ( @@ -51,8 +50,6 @@ class BaseDefaultsComponent(DescConfigModel, ABC): correctly to the component. :param enrich: Whether to enrich component with defaults, defaults to False - :param config: KPOps configuration to be accessed by this component - :param handlers: Component handlers to be accessed by this component :param validate: Whether to run custom validation on the component, defaults to True """ @@ -65,16 +62,6 @@ class BaseDefaultsComponent(DescConfigModel, ABC): description=describe_attr("enrich", __doc__), exclude=True, ) - config: SkipJsonSchema[KpopsConfig] = Field( - default=..., - description=describe_attr("config", __doc__), - exclude=True, - ) - handlers: SkipJsonSchema[ComponentHandlers] = Field( - default=..., - description=describe_attr("handlers", __doc__), - exclude=True, - ) validate_: SkipJsonSchema[bool] = Field( validation_alias=AliasChoices("validate", "validate_"), default=False, @@ -88,12 +75,10 @@ def __init__(self, **values: Any) -> None: values = cls.extend_with_defaults(**values) tmp_self = cls(**values, enrich=False) values = tmp_self.model_dump(mode="json", by_alias=True) - values = cls.substitute_in_component(tmp_self.config, **values) + values = cls.substitute_in_component(**values) self.__init__( enrich=False, validate=True, - config=tmp_self.config, - handlers=tmp_self.handlers, **values, ) else: @@ -131,14 +116,13 @@ def gen_parents(): return tuple(gen_parents()) @classmethod - def substitute_in_component( - cls, config: KpopsConfig, **component_data: Any - ) -> dict[str, Any]: + def substitute_in_component(cls, **component_data: Any) -> dict[str, Any]: """Substitute all $-placeholders in a component in dict representation. :param component_as_dict: Component represented as dict :return: Updated component """ + config = get_config() # Leftover variables that were previously introduced in the component by the substitution # functions, still hardcoded, because of their names. # TODO(Ivan Yordanov): Get rid of them @@ -167,15 +151,15 @@ def substitute_in_component( ) @classmethod - def extend_with_defaults(cls, config: KpopsConfig, **kwargs: Any) -> dict[str, Any]: + def extend_with_defaults(cls, **kwargs: Any) -> dict[str, Any]: """Merge parent components' defaults with own. :param config: KPOps configuration :param kwargs: The init kwargs for pydantic :returns: Enriched kwargs with inherited defaults """ + config = get_config() pipeline_path_str = ENV.get(PIPELINE_PATH) - kwargs["config"] = config if not pipeline_path_str: return kwargs pipeline_path = Path(pipeline_path_str) @@ -265,14 +249,14 @@ def get_defaults_file_paths( default_paths = [] if not pipeline_path.is_file(): - message = f"{pipeline_path} is not a valid pipeline file." - raise FileNotFoundError(message) + msg = f"{pipeline_path} is not a valid pipeline file." + raise FileNotFoundError(msg) path = pipeline_path.resolve() pipeline_base_dir = config.pipeline_base_dir.resolve() if pipeline_base_dir not in path.parents: - message = f"The given pipeline base path {pipeline_base_dir} is not part of the pipeline path {path}" - raise RuntimeError(message) + msg = f"The given pipeline base path {pipeline_base_dir} is not part of the pipeline path {path}" + raise RuntimeError(msg) while pipeline_base_dir != path: environment_default_file_path = ( path.parent / KpopsFileType.DEFAULTS.as_yaml_file(suffix=f"_{environment}") diff --git a/kpops/components/base_components/cleaner.py b/kpops/components/base_components/cleaner.py index 0376a6b7c..76a75247f 100644 --- a/kpops/components/base_components/cleaner.py +++ b/kpops/components/base_components/cleaner.py @@ -8,6 +8,7 @@ create_helm_release_name, ) from kpops.components.base_components.helm_app import HelmApp +from kpops.config import get_config class Cleaner(HelmApp, ABC): @@ -34,7 +35,7 @@ def helm_name_override(self) -> str: @override def helm_flags(self) -> HelmFlags: return HelmFlags( - create_namespace=self.config.create_namespace, + create_namespace=get_config().create_namespace, version=self.version, wait=True, wait_for_jobs=True, diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 94fa3e8cd..6959c578c 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -27,6 +27,7 @@ KubernetesAppValues, ) from kpops.components.base_components.models.resource import Resource +from kpops.config import get_config from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -65,7 +66,7 @@ class HelmApp(KubernetesApp): deploying the component, defaults to None this means that the command "helm repo add" is not called and Helm expects a path to local Helm chart. :param version: Helm chart version, defaults to None - :param app: Helm app values + :param values: Helm app values """ repo_config: HelmRepoConfig | None = Field( @@ -76,15 +77,15 @@ class HelmApp(KubernetesApp): default=None, description=describe_attr("version", __doc__), ) - app: HelmAppValues = Field( + values: HelmAppValues = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) @cached_property def helm(self) -> Helm: """Helm object that contains component-specific config such as repo.""" - helm = Helm(self.config.helm_config) + helm = Helm(get_config().helm_config) if self.repo_config is not None: helm.add_repo( self.repo_config.repository_name, @@ -96,11 +97,11 @@ def helm(self) -> Helm: @cached_property def helm_diff(self) -> HelmDiff: """Helm diff object of last and current release of this component.""" - return HelmDiff(self.config.helm_diff_config) + return HelmDiff(get_config().helm_diff_config) @cached_property def dry_run_handler(self) -> DryRunHandler: - helm_diff = HelmDiff(self.config.helm_diff_config) + helm_diff = HelmDiff(get_config().helm_diff_config) return DryRunHandler(self.helm, helm_diff, self.namespace) @property @@ -130,7 +131,7 @@ def helm_flags(self) -> HelmFlags: return HelmFlags( **auth_flags, version=self.version, - create_namespace=self.config.create_namespace, + create_namespace=get_config().create_namespace, ) @property @@ -138,7 +139,7 @@ def template_flags(self) -> HelmTemplateFlags: """Return flags for Helm template command.""" return HelmTemplateFlags( **self.helm_flags.model_dump(), - api_version=self.config.helm_config.api_version, + api_version=get_config().helm_config.api_version, ) @override @@ -181,13 +182,13 @@ async def destroy(self, dry_run: bool) -> None: log.info(magentaify(stdout)) def to_helm_values(self) -> dict[str, Any]: - """Generate a dictionary of values readable by Helm from `self.app`. + """Generate a dictionary of values readable by Helm from `self.values`. :returns: The values to be used by Helm """ - if self.app.name_override is None: - self.app.name_override = self.helm_name_override - return self.app.model_dump() + if self.values.name_override is None: + self.values.name_override = self.helm_name_override + return self.values.model_dump() def print_helm_diff(self, stdout: str) -> None: """Print the diff of the last and current release of this component. diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 0ab4806b9..fd9d30d1d 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -9,11 +9,13 @@ from pydantic import AliasChoices, ConfigDict, Field from typing_extensions import override +from kpops.component_handlers import get_handlers from kpops.components.base_components.cleaner import Cleaner from kpops.components.base_components.helm_app import HelmAppValues -from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.config import get_config from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, @@ -113,7 +115,7 @@ async def clean(self, dry_run: bool) -> None: log.info(f"Init cleanup job for {self.helm_release_name}") await self.deploy(dry_run) - if not self.config.retain_clean_jobs: + if not get_config().retain_clean_jobs: log.info(f"Uninstall cleanup job for {self.helm_release_name}") await self.destroy(dry_run) @@ -123,23 +125,21 @@ class KafkaApp(PipelineComponent, ABC): Producer or streaming apps should inherit from this class. - :param app: Application-specific settings + :param values: Application-specific settings """ - app: KafkaAppValues = Field( + values: KafkaAppValues = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) @override async def deploy(self, dry_run: bool) -> None: if self.to: for topic in self.to.kafka_topics: - await self.handlers.topic_handler.create_topic(topic, dry_run=dry_run) + await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) - if self.handlers.schema_handler: - await self.handlers.schema_handler.submit_schemas( - to_section=self.to, dry_run=dry_run - ) + if schema_handler := get_handlers().schema_handler: + await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) await super().deploy(dry_run) diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 3932fa3ef..c6f1f5511 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -3,28 +3,29 @@ import logging from abc import ABC from functools import cached_property -from typing import Any, NoReturn +from typing import Any, Literal, NoReturn import pydantic from pydantic import Field, PrivateAttr, ValidationInfo, computed_field, field_validator from typing_extensions import override +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.model import ( HelmRepoConfig, ) from kpops.component_handlers.kafka_connect.model import ( KafkaConnectorConfig, - KafkaConnectorResetterConfig, - KafkaConnectorResetterValues, KafkaConnectorType, ) from kpops.components.base_components.cleaner import Cleaner from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.models.from_section import FromTopic -from kpops.components.base_components.models.topic import KafkaTopic from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.topic import KafkaTopic +from kpops.config import get_config from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr +from kpops.utils.pydantic import CamelCaseConfigModel try: from typing import Self # pyright: ignore[reportAttributeAccessIssue] @@ -34,6 +35,18 @@ log = logging.getLogger("KafkaConnector") +class KafkaConnectorResetterConfig(CamelCaseConfigModel): + brokers: str + connector: str + delete_consumer_group: bool | None = None + offset_topic: str | None = None + + +class KafkaConnectorResetterValues(HelmAppValues): + connector_type: Literal["source", "sink"] + config: KafkaConnectorResetterConfig + + class KafkaConnectorResetter(Cleaner, ABC): """Helm app for resetting and cleaning a Kafka Connector. @@ -44,7 +57,7 @@ class KafkaConnectorResetter(Cleaner, ABC): from_: None = None to: None = None - app: KafkaConnectorResetterValues + values: KafkaConnectorResetterValues repo_config: HelmRepoConfig = Field( default=HelmRepoConfig( repository_name="bakdata-kafka-connect-resetter", @@ -72,19 +85,19 @@ async def reset(self, dry_run: bool) -> None: """ log.info( magentaify( - f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {self.app.config.connector}" + f"Connector Cleanup: uninstalling cleanup job Helm release from previous runs for {self.values.config.connector}" ) ) await self.destroy(dry_run) log.info( magentaify( - f"Connector Cleanup: deploy Connect {self.app.connector_type} resetter for {self.app.config.connector}" + f"Connector Cleanup: deploy Connect {self.values.connector_type} resetter for {self.values.config.connector}" ) ) await self.deploy(dry_run) - if not self.config.retain_clean_jobs: + if not get_config().retain_clean_jobs: log.info(magentaify("Connector Cleanup: uninstall Kafka Resetter.")) await self.destroy(dry_run) @@ -98,15 +111,15 @@ class KafkaConnector(PipelineComponent, ABC): Should only be used to set defaults - :param app: Application-specific settings + :param config: Connector config :param resetter_namespace: Kubernetes namespace in which the Kafka Connect resetter shall be deployed :param resetter_values: Overriding Kafka Connect resetter Helm values, e.g. to override the image tag etc., defaults to empty HelmAppValues """ - app: KafkaConnectorConfig = Field( + config: KafkaConnectorConfig = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("config", __doc__), ) resetter_namespace: str | None = Field( default=None, description=describe_attr("resetter_namespace", __doc__) @@ -117,22 +130,22 @@ class KafkaConnector(PipelineComponent, ABC): ) _connector_type: KafkaConnectorType = PrivateAttr() - @field_validator("app", mode="before") + @field_validator("config", mode="before") @classmethod def connector_config_should_have_component_name( cls, - app: KafkaConnectorConfig | dict[str, Any], + config: KafkaConnectorConfig | dict[str, Any], info: ValidationInfo, ) -> KafkaConnectorConfig: - if isinstance(app, KafkaConnectorConfig): - app = app.model_dump() + if isinstance(config, KafkaConnectorConfig): + config = config.model_dump() component_name: str = info.data["prefix"] + info.data["name"] - connector_name: str | None = app.get("name") + connector_name: str | None = config.get("name") if connector_name is not None and connector_name != component_name: msg = f"Connector name '{connector_name}' should be the same as component name '{component_name}'" raise ValueError(msg) - app["name"] = component_name - return KafkaConnectorConfig(**app) + config["name"] = component_name + return KafkaConnectorConfig(**config) @cached_property def _resetter(self) -> KafkaConnectorResetter: @@ -140,8 +153,6 @@ def _resetter(self) -> KafkaConnectorResetter: if self.resetter_namespace: kwargs["namespace"] = self.resetter_namespace return KafkaConnectorResetter( - config=self.config, - handlers=self.handlers, **kwargs, **self.model_dump( by_alias=True, @@ -149,16 +160,17 @@ def _resetter(self) -> KafkaConnectorResetter: "_resetter", "resetter_values", "resetter_namespace", - "app", + "values", + "config", "from_", "to", }, ), - app=KafkaConnectorResetterValues( + values=KafkaConnectorResetterValues( connector_type=self._connector_type.value, config=KafkaConnectorResetterConfig( connector=self.full_name, - brokers=self.config.kafka_brokers, + brokers=get_config().kafka_brokers, ), **self.resetter_values.model_dump(), ), @@ -169,21 +181,19 @@ async def deploy(self, dry_run: bool) -> None: """Deploy Kafka Connector (Source/Sink). Create output topics and register schemas if configured.""" if self.to: for topic in self.to.kafka_topics: - await self.handlers.topic_handler.create_topic(topic, dry_run=dry_run) + await get_handlers().topic_handler.create_topic(topic, dry_run=dry_run) - if self.handlers.schema_handler: - await self.handlers.schema_handler.submit_schemas( - to_section=self.to, dry_run=dry_run - ) + if schema_handler := get_handlers().schema_handler: + await schema_handler.submit_schemas(to_section=self.to, dry_run=dry_run) - await self.handlers.connector_handler.create_connector( - self.app, dry_run=dry_run + await get_handlers().connector_handler.create_connector( + self.config, dry_run=dry_run ) @override async def destroy(self, dry_run: bool) -> None: """Delete Kafka Connector (Source/Sink) from the Kafka connect cluster.""" - await self.handlers.connector_handler.destroy_connector( + await get_handlers().connector_handler.destroy_connector( self.full_name, dry_run=dry_run ) @@ -192,12 +202,10 @@ async def clean(self, dry_run: bool) -> None: """Delete Kafka Connector. If schema handler is enabled, then remove schemas. Delete all the output topics.""" await super().clean(dry_run) if self.to: - if self.handlers.schema_handler: - await self.handlers.schema_handler.delete_schemas( - to_section=self.to, dry_run=dry_run - ) + if schema_handler := get_handlers().schema_handler: + await schema_handler.delete_schemas(to_section=self.to, dry_run=dry_run) for topic in self.to.kafka_topics: - await self.handlers.topic_handler.delete_topic(topic, dry_run=dry_run) + await get_handlers().topic_handler.delete_topic(topic, dry_run=dry_run) class KafkaSourceConnector(KafkaConnector): @@ -218,7 +226,7 @@ class KafkaSourceConnector(KafkaConnector): @pydantic.model_validator(mode="after") def populate_offset_topic(self) -> Self: if self.offset_topic: - self._resetter.app.config.offset_topic = self.offset_topic + self._resetter.values.config.offset_topic = self.offset_topic return self @computed_field @@ -256,29 +264,29 @@ def _resetter(self) -> KafkaConnectorResetter: @property @override def input_topics(self) -> list[KafkaTopic]: - return self.app.topics + return self.config.topics @override def add_input_topics(self, topics: list[KafkaTopic]) -> None: - self.app.topics = KafkaTopic.deduplicate(self.app.topics + topics) + self.config.topics = KafkaTopic.deduplicate(self.config.topics + topics) @override def set_input_pattern(self, name: str) -> None: - self.app.topics_regex = name + self.config.topics_regex = name @override def set_error_topic(self, topic: KafkaTopic) -> None: - self.app.errors_deadletterqueue_topic_name = topic + self.config.errors_deadletterqueue_topic_name = topic @override async def reset(self, dry_run: bool) -> None: """Reset state. Keep consumer group and connector.""" - self._resetter.app.config.delete_consumer_group = False + self._resetter.values.config.delete_consumer_group = False await self._resetter.reset(dry_run) @override async def clean(self, dry_run: bool) -> None: """Delete connector and consumer group.""" await super().clean(dry_run) - self._resetter.app.config.delete_consumer_group = True + self._resetter.values.config.delete_consumer_group = True await self._resetter.clean(dry_run) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index e2ad7d39c..464b3418d 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -32,16 +32,16 @@ class KubernetesApp(PipelineComponent, ABC): All built-in components are Kubernetes apps, except for the Kafka connectors. :param namespace: Kubernetes namespace in which the component shall be deployed - :param app: Application-specific settings + :param values: Kubernetes app values """ namespace: str = Field( default=..., description=describe_attr("namespace", __doc__), ) - app: KubernetesAppValues = Field( + values: KubernetesAppValues = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) @override diff --git a/kpops/components/base_components/models/to_section.py b/kpops/components/base_components/models/to_section.py index 4e7e5b463..9ffdb199a 100644 --- a/kpops/components/base_components/models/to_section.py +++ b/kpops/components/base_components/models/to_section.py @@ -3,7 +3,7 @@ from pydantic import ConfigDict, Field from kpops.components.base_components.models import ModelName, ModelVersion, TopicName -from kpops.components.base_components.models.topic import KafkaTopic, TopicConfig +from kpops.components.common.topic import KafkaTopic, TopicConfig from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index ec49bce01..908a85086 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -17,7 +17,7 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, diff --git a/kpops/components/common/streams_bootstrap.py b/kpops/components/common/streams_bootstrap.py index ba45dbd30..a6061381e 100644 --- a/kpops/components/common/streams_bootstrap.py +++ b/kpops/components/common/streams_bootstrap.py @@ -45,15 +45,15 @@ class StreamsBootstrapValues(HelmAppValues): class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. - :param app: streams-bootstrap app values + :param values: streams-bootstrap Helm values :param repo_config: Configuration of the Helm chart repo to be used for deploying the component, defaults to streams-bootstrap Helm repo :param version: Helm chart version, defaults to "2.9.0" """ - app: StreamsBootstrapValues = Field( + values: StreamsBootstrapValues = Field( default_factory=StreamsBootstrapValues, - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) repo_config: HelmRepoConfig = Field( @@ -67,7 +67,7 @@ class StreamsBootstrap(HelmApp, ABC): @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: - if self.validate_ and self.app.image_tag == "latest": + if self.validate_ and self.values.image_tag == "latest": log.warning( f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." ) diff --git a/kpops/components/base_components/models/topic.py b/kpops/components/common/topic.py similarity index 100% rename from kpops/components/base_components/models/topic.py rename to kpops/components/common/topic.py diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index b4eb34b2f..19341d74b 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,6 +1,7 @@ from kpops.components.common.streams_bootstrap import StreamsBootstrap -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp + +from .producer.producer_app import ProducerApp +from .streams.streams_app import StreamsApp __all__ = ( "StreamsBootstrap", diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 2f4cb80c7..564487616 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -8,12 +8,12 @@ KafkaApp, KafkaAppCleaner, ) -from kpops.components.base_components.models.topic import ( +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr @@ -22,7 +22,7 @@ class ProducerAppCleaner(KafkaAppCleaner): - app: ProducerAppValues + values: ProducerAppValues @property @override @@ -40,13 +40,13 @@ class ProducerApp(KafkaApp, StreamsBootstrap): Note that the producer does not support error topics. - :param app: Application-specific settings + :param values: streams-bootstrap Helm values :param from_: Producer doesn't support FromSection, defaults to None """ - app: ProducerAppValues = Field( + values: ProducerAppValues = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) from_: None = Field( default=None, @@ -59,9 +59,7 @@ class ProducerApp(KafkaApp, StreamsBootstrap): @cached_property def _cleaner(self) -> ProducerAppCleaner: return ProducerAppCleaner( - config=self.config, - handlers=self.handlers, - **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}), + **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) ) @override @@ -76,20 +74,20 @@ def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: @property @override def output_topic(self) -> KafkaTopic | None: - return self.app.streams.output_topic + return self.values.streams.output_topic @property @override def extra_output_topics(self) -> dict[str, KafkaTopic]: - return self.app.streams.extra_output_topics + return self.values.streams.extra_output_topics @override def set_output_topic(self, topic: KafkaTopic) -> None: - self.app.streams.output_topic = topic + self.values.streams.output_topic = topic @override def add_extra_output_topic(self, topic: KafkaTopic, role: str) -> None: - self.app.streams.extra_output_topics[role] = topic + self.values.streams.extra_output_topics[role] = topic @property @override @@ -108,8 +106,8 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.app = self.app.model_validate(cluster_values) - self._cleaner.app.name_override = name_override + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override await super().destroy(dry_run) diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 675978396..87390f533 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -10,8 +10,8 @@ KafkaAppValues, KafkaStreamsConfig, ) -from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.components.common.streams_bootstrap import StreamsBootstrapValues +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 618824c6d..6e55931d9 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -7,8 +7,8 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaApp, KafkaAppCleaner -from kpops.components.base_components.models.topic import KafkaTopic from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, @@ -21,7 +21,7 @@ class StreamsAppCleaner(KafkaAppCleaner): from_: None = None to: None = None - app: StreamsAppValues + values: StreamsAppValues @property @override @@ -30,15 +30,15 @@ def helm_chart(self) -> str: @override async def reset(self, dry_run: bool) -> None: - self.app.streams.delete_output = False + self.values.streams.delete_output = False await super().clean(dry_run) @override async def clean(self, dry_run: bool) -> None: - self.app.streams.delete_output = True + self.values.streams.delete_output = True await super().clean(dry_run) - if self.app.stateful_set and self.app.persistence.enabled: + if self.values.stateful_set and self.values.persistence.enabled: await self.clean_pvcs(dry_run) async def clean_pvcs(self, dry_run: bool) -> None: @@ -55,70 +55,68 @@ async def clean_pvcs(self, dry_run: bool) -> None: class StreamsApp(KafkaApp, StreamsBootstrap): """StreamsApp component that configures a streams-bootstrap app. - :param app: Application-specific settings + :param values: streams-bootstrap Helm values """ - app: StreamsAppValues = Field( + values: StreamsAppValues = Field( default=..., - description=describe_attr("app", __doc__), + description=describe_attr("values", __doc__), ) @computed_field @cached_property def _cleaner(self) -> StreamsAppCleaner: return StreamsAppCleaner( - config=self.config, - handlers=self.handlers, - **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}), + **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) ) @property @override def input_topics(self) -> list[KafkaTopic]: - return self.app.streams.input_topics + return self.values.streams.input_topics @property @override def extra_input_topics(self) -> dict[str, list[KafkaTopic]]: - return self.app.streams.extra_input_topics + return self.values.streams.extra_input_topics @property @override def output_topic(self) -> KafkaTopic | None: - return self.app.streams.output_topic + return self.values.streams.output_topic @property @override def extra_output_topics(self) -> dict[str, KafkaTopic]: - return self.app.streams.extra_output_topics + return self.values.streams.extra_output_topics @override def add_input_topics(self, topics: list[KafkaTopic]) -> None: - self.app.streams.add_input_topics(topics) + self.values.streams.add_input_topics(topics) @override def add_extra_input_topics(self, role: str, topics: list[KafkaTopic]) -> None: - self.app.streams.add_extra_input_topics(role, topics) + self.values.streams.add_extra_input_topics(role, topics) @override def set_input_pattern(self, name: str) -> None: - self.app.streams.input_pattern = name + self.values.streams.input_pattern = name @override def add_extra_input_pattern(self, role: str, topic: str) -> None: - self.app.streams.extra_input_patterns[role] = topic + self.values.streams.extra_input_patterns[role] = topic @override def set_output_topic(self, topic: KafkaTopic) -> None: - self.app.streams.output_topic = topic + self.values.streams.output_topic = topic @override def set_error_topic(self, topic: KafkaTopic) -> None: - self.app.streams.error_topic = topic + self.values.streams.error_topic = topic @override def add_extra_output_topic(self, topic: KafkaTopic, role: str) -> None: - self.app.streams.extra_output_topics[role] = topic + self.values.streams.extra_output_topics[role] = topic @property @override @@ -133,8 +131,8 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.app = self.app.model_validate(cluster_values) - self._cleaner.app.name_override = name_override + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override await super().destroy(dry_run) diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 5a9f25a3e..488e0519d 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -2,8 +2,9 @@ import logging from pathlib import Path +from typing import ClassVar -from pydantic import AnyHttpUrl, Field, TypeAdapter +from pydantic import AnyHttpUrl, Field, PrivateAttr, TypeAdapter from pydantic_settings import ( BaseSettings, PydanticBaseSettingsSource, @@ -74,6 +75,8 @@ class KafkaConnectConfig(BaseSettings): class KpopsConfig(BaseSettings): """Global configuration for KPOps project.""" + _instance: ClassVar[KpopsConfig | None] = PrivateAttr() + pipeline_base_dir: Path = Field( default=Path(), description="Base directory to the pipelines (default is current working directory)", @@ -131,9 +134,10 @@ def create( cls.setup_logging_level(verbose) YamlConfigSettingsSource.config_dir = config YamlConfigSettingsSource.environment = environment - return KpopsConfig( + cls._instance = KpopsConfig( _env_file=dotenv # pyright: ignore[reportCallIssue] ) + return cls._instance @staticmethod def setup_logging_level(verbose: bool): @@ -162,3 +166,14 @@ def settings_customise_sources( dotenv_settings, file_secret_settings, ) + + +def get_config() -> KpopsConfig: + if not KpopsConfig._instance: + msg = f"{KpopsConfig.__name__} has not been initialized, call {KpopsConfig.__name__}.{KpopsConfig.create.__name__}" + raise RuntimeError(msg) + return KpopsConfig._instance + + +def set_config(config: KpopsConfig) -> None: + KpopsConfig._instance = config diff --git a/kpops/api/file_type.py b/kpops/const/file_type.py similarity index 100% rename from kpops/api/file_type.py rename to kpops/const/file_type.py diff --git a/kpops/pipeline/__init__.py b/kpops/pipeline/__init__.py index f0b1aa6b8..d0d4a6e77 100644 --- a/kpops/pipeline/__init__.py +++ b/kpops/pipeline/__init__.py @@ -305,11 +305,7 @@ def apply_component( :param component_class: Type of pipeline component :param component_data: Arguments for instantiation of pipeline component """ - component = component_class( - config=self.config, - handlers=self.handlers, - **component_data, - ) + component = component_class(**component_data) component = self.enrich_component_with_env(component) # inflate & enrich components for inflated_component in component.inflate(): # TODO: recursively @@ -349,11 +345,7 @@ def enrich_component_with_env( component.model_dump(mode="json", by_alias=True), ) - return component.__class__( - config=self.config, - handlers=self.handlers, - **env_component_as_dict, - ) + return component.__class__(**env_component_as_dict) @staticmethod def pipeline_filename_environment(pipeline_path: Path, environment: str) -> Path: diff --git a/kpops/utils/cli_commands.py b/kpops/utils/cli_commands.py index 614257b22..ba1b295ad 100644 --- a/kpops/utils/cli_commands.py +++ b/kpops/utils/cli_commands.py @@ -6,8 +6,8 @@ from pydantic.fields import FieldInfo from pydantic_core import PydanticUndefined -from kpops.api.file_type import KpopsFileType from kpops.config import KpopsConfig +from kpops.const.file_type import KpopsFileType from kpops.utils.docstring import describe_object from kpops.utils.json import is_jsonable from kpops.utils.pydantic import collect_fields, issubclass_patched diff --git a/tests/compiler/test_pipeline_name.py b/tests/compiler/test_pipeline_name.py index 24777dd1f..7e99733e4 100644 --- a/tests/compiler/test_pipeline_name.py +++ b/tests/compiler/test_pipeline_name.py @@ -2,7 +2,7 @@ import pytest -from kpops.api.file_type import KpopsFileType +from kpops.const.file_type import KpopsFileType from kpops.pipeline import PipelineGenerator from kpops.utils.environment import ENV diff --git a/tests/component_handlers/schema_handler/test_schema_handler.py b/tests/component_handlers/schema_handler/test_schema_handler.py index 716f2f482..fb23e225c 100644 --- a/tests/component_handlers/schema_handler/test_schema_handler.py +++ b/tests/component_handlers/schema_handler/test_schema_handler.py @@ -15,7 +15,7 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import OutputTopicTypes, TopicConfig +from kpops.components.common.topic import OutputTopicTypes, TopicConfig from kpops.config import KpopsConfig, SchemaRegistryConfig from kpops.utils.colorify import greenify, magentaify, yellowify from tests.pipeline.test_components import TestSchemaProvider diff --git a/tests/component_handlers/topic/test_topic_handler.py b/tests/component_handlers/topic/test_topic_handler.py index 5109652af..cbe26f61d 100644 --- a/tests/component_handlers/topic/test_topic_handler.py +++ b/tests/component_handlers/topic/test_topic_handler.py @@ -19,7 +19,7 @@ TopicResponse, TopicSpec, ) -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index f8fe12eec..759a18933 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -1,18 +1,16 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import MagicMock import pydantic import pytest -from kpops.api.file_type import DEFAULTS_YAML, PIPELINE_YAML, KpopsFileType -from kpops.component_handlers import ComponentHandlers from kpops.components.base_components.base_defaults_component import ( BaseDefaultsComponent, get_defaults_file_paths, ) -from kpops.config import KpopsConfig +from kpops.config import KpopsConfig, set_config +from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML, KpopsFileType from kpops.pipeline import PIPELINE_PATH from kpops.utils.environment import ENV from tests.components import PIPELINE_BASE_DIR, RESOURCES_PATH @@ -46,19 +44,11 @@ class EnvVarTest(BaseDefaultsComponent): name: str | None = None -@pytest.fixture() -def config() -> KpopsConfig: +@pytest.fixture(autouse=True) +def config() -> None: ENV[PIPELINE_PATH] = str(RESOURCES_PATH / "pipeline.yaml") - return KpopsConfig(pipeline_base_dir=PIPELINE_BASE_DIR) - - -@pytest.fixture() -def handlers() -> ComponentHandlers: - return ComponentHandlers( - schema_handler=MagicMock(), - connector_handler=MagicMock(), - topic_handler=MagicMock(), - ) + config = KpopsConfig(pipeline_base_dir=PIPELINE_BASE_DIR) + set_config(config) class TestBaseDefaultsComponent: @@ -123,9 +113,9 @@ def test_load_defaults_with_environment( == defaults ) - def test_inherit_defaults(self, config: KpopsConfig, handlers: ComponentHandlers): + def test_inherit_defaults(self): ENV["environment"] = "development" - component = Child(config=config, handlers=handlers) + component = Child() assert ( component.name == "fake-child-name" @@ -143,10 +133,8 @@ def test_inherit_defaults(self, config: KpopsConfig, handlers: ComponentHandlers component.hard_coded == "hard_coded_value" ), "Defaults in code should be kept for parents" - def test_inherit(self, config: KpopsConfig, handlers: ComponentHandlers): + def test_inherit(self): component = Child( - config=config, - handlers=handlers, name="name-defined-in-pipeline_parser", ) @@ -166,10 +154,8 @@ def test_inherit(self, config: KpopsConfig, handlers: ComponentHandlers): component.hard_coded == "hard_coded_value" ), "Defaults in code should be kept for parents" - def test_multiple_generations( - self, config: KpopsConfig, handlers: ComponentHandlers - ): - component = GrandChild(config=config, handlers=handlers) + def test_multiple_generations(self): + component = GrandChild() assert ( component.name == "fake-child-name" @@ -188,11 +174,9 @@ def test_multiple_generations( ), "Defaults in code should be kept for parents" assert component.grand_child == "grand-child-value" - def test_env_var_substitution( - self, config: KpopsConfig, handlers: ComponentHandlers - ): + def test_env_var_substitution(self): ENV["pipeline_name"] = RESOURCES_PATH.as_posix() - component = EnvVarTest(config=config, handlers=handlers) + component = EnvVarTest() assert component.name @@ -200,10 +184,8 @@ def test_env_var_substitution( Path(component.name) == RESOURCES_PATH ), "Environment variables should be substituted" - def test_merge_defaults(self, config: KpopsConfig, handlers: ComponentHandlers): - component = GrandChild( - config=config, handlers=handlers, nested=Nested(**{"bar": False}) - ) + def test_merge_defaults(self): + component = GrandChild(nested=Nested(**{"bar": False})) assert isinstance(component.nested, Nested) assert component.nested == Nested(**{"foo": "foo", "bar": False}) diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index 13c55dc78..0d0649747 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -4,38 +4,18 @@ from pytest_mock import MockerFixture from typing_extensions import override -from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.helm_wrapper.model import ( - HelmDiffConfig, HelmRepoConfig, HelmUpgradeInstallFlags, RepoAuthFlags, ) from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.helm_app import HelmApp, HelmAppValues -from kpops.config import KpopsConfig from kpops.utils.colorify import magentaify -from tests.components.test_base_defaults_component import ( - PIPELINE_BASE_DIR, -) @pytest.mark.usefixtures("mock_env") class TestHelmApp: - @pytest.fixture() - def config(self) -> KpopsConfig: - return KpopsConfig( - helm_diff_config=HelmDiffConfig(), pipeline_base_dir=PIPELINE_BASE_DIR - ) - - @pytest.fixture() - def handlers(self) -> ComponentHandlers: - return ComponentHandlers( - schema_handler=AsyncMock(), - connector_handler=AsyncMock(), - topic_handler=AsyncMock(), - ) - @pytest.fixture() def helm_mock(self, mocker: MockerFixture) -> MagicMock: async_mock = AsyncMock() @@ -58,16 +38,12 @@ def repo_config(self) -> HelmRepoConfig: @pytest.fixture() def helm_app( self, - config: KpopsConfig, - handlers: ComponentHandlers, app_values: HelmAppValues, repo_config: HelmRepoConfig, ) -> HelmApp: return HelmApp( name="test-helm-app", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", repo_config=repo_config, ) @@ -105,8 +81,6 @@ async def test_should_lazy_load_helm_wrapper_and_not_repo_add( @pytest.mark.asyncio() async def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( self, - config: KpopsConfig, - handlers: ComponentHandlers, helm_mock: MagicMock, mocker: MockerFixture, app_values: HelmAppValues, @@ -116,9 +90,7 @@ async def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( ) helm_app = HelmApp( name="test-helm-app", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", repo_config=repo_config, version="3.4.5", @@ -155,8 +127,6 @@ async def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( @pytest.mark.asyncio() async def test_should_deploy_app_with_local_helm_chart( self, - config: KpopsConfig, - handlers: ComponentHandlers, helm_mock: MagicMock, app_values: HelmAppValues, ): @@ -170,9 +140,7 @@ def helm_chart(self) -> str: app_with_local_chart = AppWithLocalChart( name="test-app-with-local-chart", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", ) @@ -262,16 +230,12 @@ async def test_should_call_helm_uninstall_when_cleaning_helm_app( def test_helm_name_override( self, - config: KpopsConfig, - handlers: ComponentHandlers, repo_config: HelmRepoConfig, ): helm_app = HelmApp( prefix="test-pipeline-prefix-with-a-long-name-", name="helm-app-name-is-very-long-as-well", - config=config, - handlers=handlers, - app=HelmAppValues(), + values=HelmAppValues(), namespace="test-namespace", repo_config=repo_config, ) diff --git a/tests/components/test_kafka_connector.py b/tests/components/test_kafka_connector.py index 0a9f9a0ad..559e95bd8 100644 --- a/tests/components/test_kafka_connector.py +++ b/tests/components/test_kafka_connector.py @@ -10,7 +10,7 @@ from kpops.components.base_components.kafka_connector import ( KafkaConnector, ) -from kpops.config import KpopsConfig, TopicNameConfig +from kpops.config import KpopsConfig, TopicNameConfig, set_config from tests.components import PIPELINE_BASE_DIR CONNECTOR_NAME = "test-connector-with-long-name-0123456789abcdefghijklmnop" @@ -28,9 +28,9 @@ @pytest.mark.usefixtures("mock_env") class TestKafkaConnector: - @pytest.fixture() - def config(self) -> KpopsConfig: - return KpopsConfig( + @pytest.fixture(autouse=True) + def config(self) -> None: + config = KpopsConfig( topic_name_config=TopicNameConfig( default_error_topic_name="${component.type}-error-topic", default_output_topic_name="${component.type}-output-topic", @@ -39,8 +39,9 @@ def config(self) -> KpopsConfig: helm_diff_config=HelmDiffConfig(), pipeline_base_dir=PIPELINE_BASE_DIR, ) + set_config(config) - @pytest.fixture() + @pytest.fixture(autouse=True) def handlers(self) -> ComponentHandlers: return ComponentHandlers( schema_handler=AsyncMock(), @@ -72,34 +73,26 @@ def connector_config(self) -> KafkaConnectorConfig: @pytest.fixture() def connector( self, - config: KpopsConfig, - handlers: ComponentHandlers, connector_config: KafkaConnectorConfig, ) -> KafkaConnector: return KafkaConnector( # HACK: not supposed to be instantiated, because ABC name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, ) def test_connector_config_name_override( self, connector: KafkaConnector, - config: KpopsConfig, - handlers: ComponentHandlers, ): - assert connector.app.name == CONNECTOR_FULL_NAME + assert connector.config.name == CONNECTOR_FULL_NAME connector = KafkaConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app={"connector.class": CONNECTOR_CLASS}, # type: ignore[reportGeneralTypeIssues], gets enriched + config={"connector.class": CONNECTOR_CLASS}, # type: ignore[reportGeneralTypeIssues], gets enriched resetter_namespace=RESETTER_NAMESPACE, ) - assert connector.app.name == CONNECTOR_FULL_NAME + assert connector.config.name == CONNECTOR_FULL_NAME with pytest.raises( ValueError, @@ -109,9 +102,7 @@ def test_connector_config_name_override( ): KafkaConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app={"connector.class": CONNECTOR_CLASS, "name": "different-name"}, # type: ignore[reportGeneralTypeIssues], gets enriched + config={"connector.class": CONNECTOR_CLASS, "name": "different-name"}, # type: ignore[reportGeneralTypeIssues], gets enriched ) with pytest.raises( @@ -122,7 +113,5 @@ def test_connector_config_name_override( ): KafkaConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app={"connector.class": CONNECTOR_CLASS, "name": ""}, # type: ignore[reportGeneralTypeIssues], gets enriched + config={"connector.class": CONNECTOR_CLASS, "name": ""}, # type: ignore[reportGeneralTypeIssues], gets enriched ) diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 5594f44a2..7aa3889fc 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture -from kpops.component_handlers import ComponentHandlers +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.model import ( HelmUpgradeInstallFlags, RepoAuthFlags, @@ -25,12 +25,11 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.config import KpopsConfig from kpops.utils.colorify import magentaify from tests.components.test_kafka_connector import ( CONNECTOR_CLEAN_FULL_NAME, @@ -51,17 +50,10 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kafka_connector.log.info") @pytest.fixture() - def connector( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - connector_config: KafkaConnectorConfig, - ) -> KafkaSinkConnector: + def connector(self, connector_config: KafkaConnectorConfig) -> KafkaSinkConnector: return KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, to=ToSection( topics={ @@ -78,7 +70,7 @@ def test_resetter(self, connector: KafkaSinkConnector): assert resetter.full_name == CONNECTOR_CLEAN_FULL_NAME def test_resetter_release_name(self, connector: KafkaSinkConnector): - assert connector.app.name == CONNECTOR_FULL_NAME + assert connector.config.name == CONNECTOR_FULL_NAME assert connector._resetter.helm_release_name == CONNECTOR_CLEAN_RELEASE_NAME def test_resetter_helm_name_override(self, connector: KafkaSinkConnector): @@ -100,41 +92,29 @@ def test_resetter_inheritance(self, connector: KafkaSinkConnector): # check that resetter values are contained in resetter app values assert ( connector.resetter_values.model_dump().items() - <= resetter.app.model_dump().items() + <= resetter.values.model_dump().items() ) - def test_connector_config_parsing( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - connector_config: KafkaConnectorConfig, - ): + def test_connector_config_parsing(self, connector_config: KafkaConnectorConfig): topic_pattern = ".*" connector = KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=KafkaConnectorConfig( + config=KafkaConnectorConfig( **{**connector_config.model_dump(), "topics.regex": topic_pattern} ), resetter_namespace=RESETTER_NAMESPACE, ) - assert connector.app.topics_regex == topic_pattern - assert connector.app.model_dump()["topics.regex"] == topic_pattern + assert connector.config.topics_regex == topic_pattern + assert connector.config.model_dump()["topics.regex"] == topic_pattern def test_from_section_parsing_input_topic( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - connector_config: KafkaConnectorConfig, + self, connector_config: KafkaConnectorConfig ): topic1 = TopicName("connector-topic1") topic2 = TopicName("connector-topic2") connector = KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, from_=FromSection( topics={ @@ -143,7 +123,7 @@ def test_from_section_parsing_input_topic( } ), ) - assert connector.app.topics == [ + assert connector.config.topics == [ KafkaTopic(name=topic1), KafkaTopic(name=topic2), ] @@ -155,32 +135,27 @@ def test_from_section_parsing_input_topic( KafkaTopic(name=topic3), ] ) - assert connector.app.topics == [ + assert connector.config.topics == [ KafkaTopic(name=topic1), KafkaTopic(name=topic2), KafkaTopic(name=topic3), ] - assert connector.app.model_dump()["topics"] == f"{topic1},{topic2},{topic3}" + assert connector.config.model_dump()["topics"] == f"{topic1},{topic2},{topic3}" def test_from_section_parsing_input_pattern( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - connector_config: KafkaConnectorConfig, + self, connector_config: KafkaConnectorConfig ): topic_pattern = TopicName(".*") connector = KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, from_=FromSection( # pyright: ignore[reportGeneralTypeIssues] wrong diagnostic when using TopicName as topics key type topics={topic_pattern: FromTopic(type=InputTopicTypes.PATTERN)} ), ) - assert connector.app.topics_regex == topic_pattern + assert connector.config.topics_regex == topic_pattern @pytest.mark.asyncio() async def test_deploy_order( @@ -189,10 +164,10 @@ async def test_deploy_order( mocker: MockerFixture, ): mock_create_topic = mocker.patch.object( - connector.handlers.topic_handler, "create_topic" + get_handlers().topic_handler, "create_topic" ) mock_create_connector = mocker.patch.object( - connector.handlers.connector_handler, "create_connector" + get_handlers().connector_handler, "create_connector" ) mock = mocker.AsyncMock() @@ -207,7 +182,7 @@ async def test_deploy_order( mocker.call.mock_create_topic(topic, dry_run=dry_run) for topic in connector.to.kafka_topics ), - mocker.call.mock_create_connector(connector.app, dry_run=dry_run), + mocker.call.mock_create_connector(connector.config, dry_run=dry_run), ] @pytest.mark.asyncio() @@ -217,7 +192,7 @@ async def test_destroy( mocker: MockerFixture, ): mock_destroy_connector = mocker.patch.object( - connector.handlers.connector_handler, "destroy_connector" + get_handlers().connector_handler, "destroy_connector" ) await connector.destroy(dry_run=True) @@ -247,10 +222,10 @@ async def test_reset_when_dry_run_is_false( ): mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock_resetter_reset = mocker.spy(connector._resetter, "reset") @@ -335,10 +310,10 @@ async def test_clean_when_dry_run_is_false( mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -416,16 +391,12 @@ async def test_clean_when_dry_run_is_false( @pytest.mark.asyncio() async def test_clean_without_to_when_dry_run_is_true( self, - config: KpopsConfig, - handlers: ComponentHandlers, dry_run_handler_mock: MagicMock, connector_config: KafkaConnectorConfig, ): connector = KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, ) @@ -437,8 +408,6 @@ async def test_clean_without_to_when_dry_run_is_true( @pytest.mark.asyncio() async def test_clean_without_to_when_dry_run_is_false( self, - config: KpopsConfig, - handlers: ComponentHandlers, helm_mock: MagicMock, dry_run_handler_mock: MagicMock, mocker: MockerFixture, @@ -446,19 +415,17 @@ async def test_clean_without_to_when_dry_run_is_false( ): connector = KafkaSinkConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, ) mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.patch.object( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock = mocker.MagicMock() mock.attach_mock(mock_destroy, "destroy_connector") diff --git a/tests/components/test_kafka_source_connector.py b/tests/components/test_kafka_source_connector.py index d6127e9fc..796eb0daf 100644 --- a/tests/components/test_kafka_source_connector.py +++ b/tests/components/test_kafka_source_connector.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture -from kpops.component_handlers import ComponentHandlers +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.model import ( HelmUpgradeInstallFlags, RepoAuthFlags, @@ -25,8 +25,7 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import OutputTopicTypes, TopicConfig -from kpops.config import KpopsConfig +from kpops.components.common.topic import OutputTopicTypes, TopicConfig from kpops.utils.environment import ENV from tests.components.test_kafka_connector import ( CONNECTOR_CLEAN_HELM_NAMEOVERRIDE, @@ -46,15 +45,11 @@ class TestKafkaSourceConnector(TestKafkaConnector): @pytest.fixture() def connector( self, - config: KpopsConfig, - handlers: ComponentHandlers, connector_config: KafkaConnectorConfig, ) -> KafkaSourceConnector: return KafkaSourceConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, to=ToSection( topics={ @@ -67,26 +62,22 @@ def connector( ) def test_resetter_release_name(self, connector: KafkaSourceConnector): - assert connector.app.name == CONNECTOR_FULL_NAME + assert connector.config.name == CONNECTOR_FULL_NAME resetter = connector._resetter assert isinstance(resetter, KafkaConnectorResetter) assert connector._resetter.helm_release_name == CONNECTOR_CLEAN_RELEASE_NAME def test_resetter_offset_topic(self, connector: KafkaSourceConnector): - assert connector._resetter.app.config.offset_topic == OFFSETS_TOPIC + assert connector._resetter.values.config.offset_topic == OFFSETS_TOPIC def test_from_section_raises_exception( self, - config: KpopsConfig, - handlers: ComponentHandlers, connector_config: KafkaConnectorConfig, ): with pytest.raises(NotImplementedError): KafkaSourceConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, from_=FromSection( # pyright: ignore[reportGeneralTypeIssues] wrong diagnostic when using TopicName as topics key type topics={ @@ -104,11 +95,11 @@ async def test_deploy_order( mocker: MockerFixture, ): mock_create_topic = mocker.patch.object( - connector.handlers.topic_handler, "create_topic" + get_handlers().topic_handler, "create_topic" ) mock_create_connector = mocker.patch.object( - connector.handlers.connector_handler, "create_connector" + get_handlers().connector_handler, "create_connector" ) mock = mocker.AsyncMock() @@ -123,7 +114,7 @@ async def test_deploy_order( mocker.call.mock_create_topic(topic, dry_run=dry_run) for topic in connector.to.kafka_topics ), - mocker.call.mock_create_connector(connector.app, dry_run=dry_run), + mocker.call.mock_create_connector(connector.config, dry_run=dry_run), ] @pytest.mark.asyncio() @@ -133,10 +124,10 @@ async def test_destroy( mocker: MockerFixture, ): ENV["KPOPS_KAFKA_CONNECT_RESETTER_OFFSET_TOPIC"] = OFFSETS_TOPIC - assert connector.handlers.connector_handler + assert get_handlers().connector_handler mock_destroy_connector = mocker.patch.object( - connector.handlers.connector_handler, "destroy_connector" + get_handlers().connector_handler, "destroy_connector" ) await connector.destroy(dry_run=True) @@ -151,8 +142,6 @@ async def test_reset_when_dry_run_is_true( connector: KafkaSourceConnector, dry_run_handler_mock: MagicMock, ): - assert connector.handlers.connector_handler - await connector.reset(dry_run=True) dry_run_handler_mock.print_helm_diff.assert_called_once() @@ -165,15 +154,13 @@ async def test_reset_when_dry_run_is_false( helm_mock: MagicMock, mocker: MockerFixture, ): - assert connector.handlers.connector_handler - mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -235,8 +222,6 @@ async def test_clean_when_dry_run_is_true( connector: KafkaSourceConnector, dry_run_handler_mock: MagicMock, ): - assert connector.handlers.connector_handler - await connector.clean(dry_run=True) dry_run_handler_mock.print_helm_diff.assert_called_once() @@ -249,15 +234,13 @@ async def test_clean_when_dry_run_is_false( dry_run_handler_mock: MagicMock, mocker: MockerFixture, ): - assert connector.handlers.connector_handler - mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -322,8 +305,6 @@ async def test_clean_when_dry_run_is_false( @pytest.mark.asyncio() async def test_clean_without_to_when_dry_run_is_false( self, - config: KpopsConfig, - handlers: ComponentHandlers, helm_mock: MagicMock, dry_run_handler_mock: MagicMock, mocker: MockerFixture, @@ -331,23 +312,21 @@ async def test_clean_without_to_when_dry_run_is_false( ): connector = KafkaSourceConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, offset_topic=OFFSETS_TOPIC, ) assert connector.to is None - assert connector.handlers.connector_handler + assert get_handlers().connector_handler mock_destroy = mocker.patch.object(connector, "destroy") mock_delete_topic = mocker.patch.object( - connector.handlers.topic_handler, "delete_topic" + get_handlers().topic_handler, "delete_topic" ) mock_clean_connector = mocker.spy( - connector.handlers.connector_handler, "clean_connector" + get_handlers().connector_handler, "clean_connector" ) mock = mocker.MagicMock() @@ -408,22 +387,18 @@ async def test_clean_without_to_when_dry_run_is_false( @pytest.mark.asyncio() async def test_clean_without_to_when_dry_run_is_true( self, - config: KpopsConfig, - handlers: ComponentHandlers, dry_run_handler_mock: MagicMock, connector_config: KafkaConnectorConfig, ): connector = KafkaSourceConnector( name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, + config=connector_config, resetter_namespace=RESETTER_NAMESPACE, offset_topic=OFFSETS_TOPIC, ) assert connector.to is None - assert connector.handlers.connector_handler + assert get_handlers().connector_handler await connector.clean(dry_run=True) diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 26d68ddd2..d4283cd21 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -1,9 +1,8 @@ -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest from pytest_mock import MockerFixture -from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.helm_wrapper.model import ( HelmRepoConfig, ) @@ -12,10 +11,6 @@ KubernetesApp, KubernetesAppValues, ) -from kpops.config import KpopsConfig -from kpops.pipeline import PIPELINE_PATH -from kpops.utils.environment import ENV -from tests.components import PIPELINE_BASE_DIR, RESOURCES_PATH HELM_RELEASE_NAME = create_helm_release_name("${pipeline.name}-test-kubernetes-app") @@ -25,19 +20,6 @@ class KubernetesTestValues(KubernetesAppValues): class TestKubernetesApp: - @pytest.fixture() - def config(self) -> KpopsConfig: - ENV[PIPELINE_PATH] = str(RESOURCES_PATH / "pipeline.yaml") - return KpopsConfig(pipeline_base_dir=PIPELINE_BASE_DIR) - - @pytest.fixture() - def handlers(self) -> ComponentHandlers: - return ComponentHandlers( - schema_handler=AsyncMock(), - connector_handler=AsyncMock(), - topic_handler=AsyncMock(), - ) - @pytest.fixture() def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kubernetes_app.log.info") @@ -51,34 +33,22 @@ def repo_config(self) -> HelmRepoConfig: return HelmRepoConfig(repository_name="test", url="https://bakdata.com") @pytest.fixture() - def kubernetes_app( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - app_values: KubernetesTestValues, - ) -> KubernetesApp: + def kubernetes_app(self, app_values: KubernetesTestValues) -> KubernetesApp: return KubernetesApp( name="test-kubernetes-app", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", ) def test_should_raise_value_error_when_name_is_not_valid( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - app_values: KubernetesTestValues, + self, app_values: KubernetesTestValues ): with pytest.raises( ValueError, match=r"The component name .* is invalid for Kubernetes." ): KubernetesApp( name="Not-Compatible*", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", ) @@ -87,16 +57,12 @@ def test_should_raise_value_error_when_name_is_not_valid( ): KubernetesApp( name="snake_case*", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", ) assert KubernetesApp( name="valid-name", - config=config, - handlers=handlers, - app=app_values, + values=app_values, namespace="test-namespace", ) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index b0e88bd45..00b0e70d3 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -1,14 +1,14 @@ import logging -from unittest.mock import ANY, AsyncMock, MagicMock +from unittest.mock import ANY, MagicMock import pytest from pytest_mock import MockerFixture -from kpops.component_handlers import ComponentHandlers +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, @@ -17,8 +17,6 @@ ProducerApp, ProducerAppCleaner, ) -from kpops.config import KpopsConfig, TopicNameConfig -from tests.components import PIPELINE_BASE_DIR PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" PRODUCER_APP_FULL_NAME = "${pipeline.name}-" + PRODUCER_APP_NAME @@ -41,35 +39,13 @@ def test_release_name(self): assert PRODUCER_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def handlers(self) -> ComponentHandlers: - return ComponentHandlers( - schema_handler=AsyncMock(), - connector_handler=AsyncMock(), - topic_handler=AsyncMock(), - ) - - @pytest.fixture() - def config(self) -> KpopsConfig: - return KpopsConfig( - topic_name_config=TopicNameConfig( - default_error_topic_name="${component.type}-error-topic", - default_output_topic_name="${component.type}-output-topic", - ), - pipeline_base_dir=PIPELINE_BASE_DIR, - ) - - @pytest.fixture() - def producer_app( - self, config: KpopsConfig, handlers: ComponentHandlers - ) -> ProducerApp: + def producer_app(self) -> ProducerApp: return ProducerApp( name=PRODUCER_APP_NAME, - config=config, - handlers=handlers, **{ "version": "2.4.2", "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "clean_schemas": True, @@ -93,7 +69,7 @@ def test_cleaner(self, producer_app: ProducerApp): assert not hasattr(cleaner, "_cleaner") def test_cleaner_inheritance(self, producer_app: ProducerApp): - assert producer_app._cleaner.app == producer_app.app + assert producer_app._cleaner.values == producer_app.values def test_cleaner_helm_release_name(self, producer_app: ProducerApp): assert ( @@ -107,14 +83,12 @@ def test_cleaner_helm_name_override(self, producer_app: ProducerApp): == PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE ) - def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers): + def test_output_topics(self): producer_app = ProducerApp( name=PRODUCER_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, @@ -132,10 +106,10 @@ def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers): }, ) - assert producer_app.app.streams.output_topic == KafkaTopic( + assert producer_app.values.streams.output_topic == KafkaTopic( name="producer-app-output-topic" ) - assert producer_app.app.streams.extra_output_topics == { + assert producer_app.values.streams.extra_output_topics == { "first-extra-topic": KafkaTopic(name="extra-topic-1") } @@ -146,7 +120,7 @@ async def test_deploy_order_when_dry_run_is_false( mocker: MockerFixture, ): mock_create_topic = mocker.patch.object( - producer_app.handlers.topic_handler, "create_topic" + get_handlers().topic_handler, "create_topic" ) mock_helm_upgrade_install = mocker.patch.object( @@ -351,18 +325,12 @@ async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clea ] ) - def test_get_output_topics( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_get_output_topics(self): producer_app = ProducerApp( name="my-producer", - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, @@ -379,10 +347,10 @@ def test_get_output_topics( }, }, ) - assert producer_app.app.streams.output_topic == KafkaTopic( + assert producer_app.values.streams.output_topic == KafkaTopic( name="producer-app-output-topic" ) - assert producer_app.app.streams.extra_output_topics == { + assert producer_app.values.streams.extra_output_topics == { "first-extra-topic": KafkaTopic(name="extra-topic-1") } assert producer_app.input_topics == [] @@ -393,12 +361,7 @@ def test_get_output_topics( ] @pytest.mark.asyncio() - async def test_should_not_deploy_clean_up_when_rest( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - mocker: MockerFixture, - ): + async def test_should_not_deploy_clean_up_when_rest(self, mocker: MockerFixture): image_tag_in_cluster = "1.1.1" mocker.patch.object( Helm, @@ -417,11 +380,9 @@ async def test_should_not_deploy_clean_up_when_rest( ) producer_app = ProducerApp( name=PRODUCER_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "imageTag": "2.2.2", "streams": {"brokers": "fake-broker:9092"}, }, @@ -449,10 +410,7 @@ async def test_should_not_deploy_clean_up_when_rest( @pytest.mark.asyncio() async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - mocker: MockerFixture, + self, mocker: MockerFixture ): image_tag_in_cluster = "1.1.1" mocker.patch.object( @@ -472,11 +430,9 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( ) producer_app = ProducerApp( name=PRODUCER_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "imageTag": "2.2.2", "streams": {"brokers": "fake-broker:9092"}, }, diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 6b567f2aa..34afc0b21 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -6,10 +6,9 @@ from pytest_mock import MockerFixture from kpops.api.exception import ValidationError -from kpops.component_handlers import ComponentHandlers +from kpops.component_handlers import get_handlers from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import ( - HelmDiffConfig, HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name @@ -17,7 +16,7 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, @@ -30,8 +29,6 @@ StreamsApp, StreamsAppCleaner, ) -from kpops.config import KpopsConfig, TopicNameConfig -from tests.components import PIPELINE_BASE_DIR RESOURCES_PATH = Path(__file__).parent / "resources" @@ -58,37 +55,12 @@ def test_release_name(self): assert STREAMS_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def handlers(self) -> ComponentHandlers: - return ComponentHandlers( - schema_handler=AsyncMock(), - connector_handler=AsyncMock(), - topic_handler=AsyncMock(), - ) - - @pytest.fixture() - def config(self) -> KpopsConfig: - return KpopsConfig( - topic_name_config=TopicNameConfig( - default_error_topic_name="${component.type}-error-topic", - default_output_topic_name="${component.type}-output-topic", - ), - helm_diff_config=HelmDiffConfig(), - pipeline_base_dir=PIPELINE_BASE_DIR, - ) - - @pytest.fixture() - def streams_app( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ) -> StreamsApp: + def streams_app(self) -> StreamsApp: return StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "to": { @@ -102,18 +74,12 @@ def streams_app( ) @pytest.fixture() - def stateful_streams_app( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ) -> StreamsApp: + def stateful_streams_app(self) -> StreamsApp: return StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "statefulSet": True, "persistence": {"enabled": True, "size": "5Gi"}, "streams": { @@ -146,19 +112,19 @@ def test_cleaner(self, streams_app: StreamsApp): assert not hasattr(cleaner, "_cleaner") def test_cleaner_inheritance(self, streams_app: StreamsApp): - streams_app.app.autoscaling = StreamsAppAutoScaling( + streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, consumer_group="foo", lag_threshold=100, idle_replicas=1, ) - assert streams_app._cleaner.app == streams_app.app + assert streams_app._cleaner.values == streams_app.values def test_raise_validation_error_when_autoscaling_enabled_and_mandatory_fields_not_set( self, streams_app: StreamsApp ): with pytest.raises(ValidationError) as error: - streams_app.app.autoscaling = StreamsAppAutoScaling( + streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, ) msg = ( @@ -171,7 +137,7 @@ def test_raise_validation_error_when_autoscaling_enabled_and_only_consumer_group self, streams_app: StreamsApp ): with pytest.raises(ValidationError) as error: - streams_app.app.autoscaling = StreamsAppAutoScaling( + streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, consumer_group="a-test-group" ) msg = ( @@ -184,7 +150,7 @@ def test_raise_validation_error_when_autoscaling_enabled_and_only_lag_threshold_ self, streams_app: StreamsApp ): with pytest.raises(ValidationError) as error: - streams_app.app.autoscaling = StreamsAppAutoScaling( + streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, lag_threshold=1000 ) msg = ( @@ -205,18 +171,12 @@ def test_cleaner_helm_name_override(self, streams_app: StreamsApp): == STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE ) - def test_set_topics( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_set_topics(self): streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "from": { @@ -236,17 +196,17 @@ def test_set_topics( }, }, ) - assert streams_app.app.streams.input_topics == [ + assert streams_app.values.streams.input_topics == [ KafkaTopic(name="example-input"), KafkaTopic(name="b"), KafkaTopic(name="a"), ] - assert streams_app.app.streams.extra_input_topics == { + assert streams_app.values.streams.extra_input_topics == { "role1": [KafkaTopic(name="topic-extra")], "role2": [KafkaTopic(name="topic-extra2"), KafkaTopic(name="topic-extra3")], } - assert streams_app.app.streams.input_pattern == ".*" - assert streams_app.app.streams.extra_input_patterns == { + assert streams_app.values.streams.input_pattern == ".*" + assert streams_app.values.streams.extra_input_patterns == { "another-pattern": "example.*" } @@ -257,18 +217,12 @@ def test_set_topics( assert "inputPattern" in streams_config assert "extraInputPatterns" in streams_config - def test_no_empty_input_topic( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_no_empty_input_topic(self): streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "from": { @@ -278,10 +232,10 @@ def test_no_empty_input_topic( }, }, ) - assert not streams_app.app.streams.extra_input_topics - assert not streams_app.app.streams.input_topics - assert streams_app.app.streams.input_pattern == ".*" - assert not streams_app.app.streams.extra_input_patterns + assert not streams_app.values.streams.extra_input_topics + assert not streams_app.values.streams.input_topics + assert streams_app.values.streams.input_pattern == ".*" + assert not streams_app.values.streams.extra_input_patterns helm_values = streams_app.to_helm_values() streams_config = helm_values["streams"] @@ -290,22 +244,16 @@ def test_no_empty_input_topic( assert "inputPattern" in streams_config assert "extraInputPatterns" not in streams_config - def test_should_validate( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_should_validate(self): # An exception should be raised when both role and type are defined and type is input with pytest.raises( ValueError, match="Define role only if `type` is `pattern` or `None`" ): StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "from": { @@ -325,11 +273,9 @@ def test_should_validate( ): StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "to": { @@ -343,18 +289,12 @@ def test_should_validate( }, ) - def test_set_streams_output_from_to( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_set_streams_output_from_to(self): streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "to": { @@ -377,29 +317,23 @@ def test_set_streams_output_from_to( }, }, ) - assert streams_app.app.streams.extra_output_topics == { + assert streams_app.values.streams.extra_output_topics == { "first-extra-role": KafkaTopic(name="extra-topic-1"), "second-extra-role": KafkaTopic(name="extra-topic-2"), } - assert streams_app.app.streams.output_topic == KafkaTopic( + assert streams_app.values.streams.output_topic == KafkaTopic( name="streams-app-output-topic" ) - assert streams_app.app.streams.error_topic == KafkaTopic( + assert streams_app.values.streams.error_topic == KafkaTopic( name="streams-app-error-topic" ) - def test_weave_inputs_from_prev_component( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + def test_weave_inputs_from_prev_component(self): streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, }, @@ -424,26 +358,19 @@ def test_weave_inputs_from_prev_component( ) ) - assert streams_app.app.streams.input_topics == [ + assert streams_app.values.streams.input_topics == [ KafkaTopic(name="prev-output-topic"), KafkaTopic(name="b"), KafkaTopic(name="a"), ] @pytest.mark.asyncio() - async def test_deploy_order_when_dry_run_is_false( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - mocker: MockerFixture, - ): + async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "to": { @@ -467,7 +394,7 @@ async def test_deploy_order_when_dry_run_is_false( }, ) mock_create_topic = mocker.patch.object( - streams_app.handlers.topic_handler, "create_topic" + get_handlers().topic_handler, "create_topic" ) mock_helm_upgrade_install = mocker.patch.object( streams_app.helm, "upgrade_install" @@ -697,10 +624,7 @@ async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean @pytest.mark.asyncio() async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - mocker: MockerFixture, + self, mocker: MockerFixture ): image_tag_in_cluster = "1.1.1" mocker.patch.object( @@ -723,11 +647,9 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( ) streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "imageTag": "2.2.2", "streams": {"brokers": "fake-broker:9092"}, }, @@ -781,10 +703,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( @pytest.mark.asyncio() async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - mocker: MockerFixture, + self, mocker: MockerFixture ): image_tag_in_cluster = "1.1.1" mocker.patch.object( @@ -807,11 +726,9 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( ) streams_app = StreamsApp( name=STREAMS_APP_NAME, - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "imageTag": "2.2.2", "streams": {"brokers": "fake-broker:9092"}, }, @@ -864,18 +781,12 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( ) @pytest.mark.asyncio() - async def test_get_input_output_topics( - self, - config: KpopsConfig, - handlers: ComponentHandlers, - ): + async def test_get_input_output_topics(self): streams_app = StreamsApp( name="my-app", - config=config, - handlers=handlers, **{ "namespace": "test-namespace", - "app": { + "values": { "streams": {"brokers": "fake-broker:9092"}, }, "from": { @@ -902,12 +813,12 @@ async def test_get_input_output_topics( }, ) - assert streams_app.app.streams.input_topics == [ + assert streams_app.values.streams.input_topics == [ KafkaTopic(name="example-input"), KafkaTopic(name="b"), KafkaTopic(name="a"), ] - assert streams_app.app.streams.extra_input_topics == { + assert streams_app.values.streams.extra_input_topics == { "role1": [KafkaTopic(name="topic-extra")], "role2": [KafkaTopic(name="topic-extra2"), KafkaTopic(name="topic-extra3")], } @@ -932,7 +843,7 @@ def test_raise_validation_error_when_persistence_enabled_and_size_not_set( self, stateful_streams_app: StreamsApp ): with pytest.raises(ValidationError) as error: - stateful_streams_app.app.persistence = PersistenceConfig( + stateful_streams_app.values.persistence = PersistenceConfig( enabled=True, ) msg = ( diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py new file mode 100644 index 000000000..6a55a0f94 --- /dev/null +++ b/tests/components/test_streams_bootstrap.py @@ -0,0 +1,96 @@ +import re + +import pytest +from pydantic import ValidationError +from pytest_mock import MockerFixture + +from kpops.component_handlers.helm_wrapper.model import ( + HelmRepoConfig, + HelmUpgradeInstallFlags, +) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name +from kpops.components.common.streams_bootstrap import ( + StreamsBootstrap, + StreamsBootstrapValues, +) + + +@pytest.mark.usefixtures("mock_env") +class TestStreamsBootstrap: + def test_default_configs(self): + streams_bootstrap = StreamsBootstrap( + name="example-name", + **{ + "namespace": "test-namespace", + "values": {}, + }, + ) + assert streams_bootstrap.repo_config == HelmRepoConfig( + repository_name="bakdata-streams-bootstrap", + url="https://bakdata.github.io/streams-bootstrap/", + ) + assert streams_bootstrap.version == "2.9.0" + assert streams_bootstrap.namespace == "test-namespace" + assert streams_bootstrap.values.image_tag == "latest" + + @pytest.mark.asyncio() + async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): + streams_bootstrap = StreamsBootstrap( + name="example-name", + **{ + "namespace": "test-namespace", + "values": { + "imageTag": "1.0.0", + "streams": { + "outputTopic": "test", + "brokers": "fake-broker:9092", + }, + }, + "version": "1.2.3", + }, + ) + helm_upgrade_install = mocker.patch.object( + streams_bootstrap.helm, "upgrade_install" + ) + print_helm_diff = mocker.patch.object( + streams_bootstrap.dry_run_handler, "print_helm_diff" + ) + mocker.patch.object( + StreamsBootstrap, + "helm_chart", + return_value="test/test-chart", + new_callable=mocker.PropertyMock, + ) + + await streams_bootstrap.deploy(dry_run=True) + + print_helm_diff.assert_called_once() + helm_upgrade_install.assert_called_once_with( + create_helm_release_name("${pipeline.name}-example-name"), + "test/test-chart", + True, + "test-namespace", + { + "nameOverride": "${pipeline.name}-example-name", + "imageTag": "1.0.0", + "streams": { + "brokers": "fake-broker:9092", + "outputTopic": "test", + }, + }, + HelmUpgradeInstallFlags(version="1.2.3"), + ) + + @pytest.mark.asyncio() + async def test_should_raise_validation_error_for_invalid_image_tag(self): + with pytest.raises( + ValidationError, + match=re.escape( + "1 validation error for StreamsBootstrapValues\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" + ), + ): + StreamsBootstrapValues( + **{ + "imageTag": "invalid image tag!", + } + ) diff --git a/tests/components/test_topic.py b/tests/components/test_topic.py index 9193a0d00..78e16c3b9 100644 --- a/tests/components/test_topic.py +++ b/tests/components/test_topic.py @@ -3,7 +3,7 @@ import pydantic import pytest -from kpops.components.base_components.models.topic import ( +from kpops.components.common.topic import ( KafkaTopic, KafkaTopicStr, OutputTopicTypes, diff --git a/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml b/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml index 2141a85c7..8e69fe46c 100644 --- a/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml +++ b/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml @@ -1,7 +1,7 @@ - type: converter - app: + values: labels: - l_1: ${component.app.labels.l_2} - l_2: ${component.app.labels.l_3} - l_3: ${component.app.labels.l_1} - infinite_nesting: ${component.app.labels} + l_1: ${component.values.labels.l_2} + l_2: ${component.values.labels.l_3} + l_3: ${component.values.labels.l_1} + infinite_nesting: ${component.values.labels} diff --git a/tests/pipeline/resources/component-type-substitution/pipeline.yaml b/tests/pipeline/resources/component-type-substitution/pipeline.yaml index 8a4cf60ea..76e2eacb2 100644 --- a/tests/pipeline/resources/component-type-substitution/pipeline.yaml +++ b/tests/pipeline/resources/component-type-substitution/pipeline.yaml @@ -1,14 +1,14 @@ - type: scheduled-producer - app: + values: labels: app_type: "${component.type}" app_name: "${component.name}" - app_schedule: "${component.app.schedule}" + app_schedule: "${component.values.schedule}" commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -18,13 +18,13 @@ memory: 2G - type: filter name: "filter-app" - app: + values: labels: app_type: "${component.type}" app_name: "${component.name}" - app_resources_requests_memory: "${component.app.resources.requests.memory}" - ${component.type}: "${component.app.labels.app_name}-${component.app.labels.app_type}" - test_placeholder_in_placeholder: "${component.app.labels.${component.type}}" + app_resources_requests_memory: "${component.values.resources.requests.memory}" + ${component.type}: "${component.values.labels.app_name}-${component.values.labels.app_type}" + test_placeholder_in_placeholder: "${component.values.labels.${component.type}}" commandLine: TYPE: "nothing" resources: diff --git a/tests/pipeline/resources/custom-config/defaults.yaml b/tests/pipeline/resources/custom-config/defaults.yaml index d08047cf1..40c754e0f 100644 --- a/tests/pipeline/resources/custom-config/defaults.yaml +++ b/tests/pipeline/resources/custom-config/defaults.yaml @@ -1,7 +1,7 @@ kafka-app: version: 2.9.0 namespace: development-namespace - app: + values: streams: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" @@ -14,7 +14,7 @@ producer-app: partitions_count: 3 streams-app: - app: + values: labels: pipeline: ${pipeline.name} to: diff --git a/tests/pipeline/resources/custom-config/pipeline.yaml b/tests/pipeline/resources/custom-config/pipeline.yaml index 25ddbedb9..b50df4f7e 100644 --- a/tests/pipeline/resources/custom-config/pipeline.yaml +++ b/tests/pipeline/resources/custom-config/pipeline.yaml @@ -1,6 +1,6 @@ - type: producer-app name: app1 - app: + values: resources: limits: memory: 2G @@ -9,5 +9,5 @@ - type: streams-app name: app2 - app: + values: image: some-image diff --git a/tests/pipeline/resources/defaults.yaml b/tests/pipeline/resources/defaults.yaml index 64e31d732..32aaf5aeb 100644 --- a/tests/pipeline/resources/defaults.yaml +++ b/tests/pipeline/resources/defaults.yaml @@ -5,7 +5,7 @@ kubernetes-app: namespace: example-namespace kafka-app: - app: + values: streams: brokers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} @@ -14,7 +14,7 @@ kafka-app: producer-app: {} # inherits from kafka-app streams-app: # inherits from kafka-app - app: + values: streams: config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator @@ -28,7 +28,7 @@ streams-app: # inherits from kafka-app cleanup.policy: compact,delete scheduled-producer: - app: + values: image: "example-registry/fake-image" imageTag: "0.0.1" to: @@ -43,7 +43,7 @@ scheduled-producer: "com/bakdata/kafka/fake": 1.0.0 converter: - app: + values: resources: limits: memory: 3G @@ -68,7 +68,7 @@ converter: cleanup.policy: compact,delete filter: - app: + values: image: "fake-registry/filter" imageTag: "2.4.1" autoscaling: @@ -87,7 +87,7 @@ filter: retention.ms: "-1" should-inflate: - app: + values: image: "fake-registry/filter" imageTag: "2.4.1" autoscaling: @@ -106,7 +106,7 @@ should-inflate: retention.ms: "-1" kafka-sink-connector: - app: + config: batch.size: "2000" behavior.on.malformed.documents: "warn" behavior.on.null.values: "delete" diff --git a/tests/pipeline/resources/dotenv/config.yaml b/tests/pipeline/resources/dotenv/config.yaml index 3abfdffd4..324040bf0 100644 --- a/tests/pipeline/resources/dotenv/config.yaml +++ b/tests/pipeline/resources/dotenv/config.yaml @@ -1,4 +1,3 @@ -defaults_path: ../defaults.yaml kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" kafka_connect: url: "http://localhost:8083" diff --git a/tests/pipeline/resources/env-specific-config-only/config_production.yaml b/tests/pipeline/resources/env-specific-config-only/config_production.yaml index 2e40128d4..4e4399fc1 100644 --- a/tests/pipeline/resources/env-specific-config-only/config_production.yaml +++ b/tests/pipeline/resources/env-specific-config-only/config_production.yaml @@ -1,4 +1,3 @@ -defaults_path: ../no-topics-defaults kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" kafka_connect: url: "http://localhost:8083" diff --git a/tests/pipeline/resources/first-pipeline/pipeline.yaml b/tests/pipeline/resources/first-pipeline/pipeline.yaml index bbef841dc..244681b1b 100644 --- a/tests/pipeline/resources/first-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/first-pipeline/pipeline.yaml @@ -1,10 +1,10 @@ - type: scheduled-producer - app: + values: commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -14,7 +14,7 @@ memory: 2G - type: filter name: "a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name" - app: + values: commandLine: TYPE: "nothing" resources: diff --git a/tests/pipeline/resources/kafka-connect-sink-config/config.yaml b/tests/pipeline/resources/kafka-connect-sink-config/config.yaml index 572b695c7..9cf1351f5 100644 --- a/tests/pipeline/resources/kafka-connect-sink-config/config.yaml +++ b/tests/pipeline/resources/kafka-connect-sink-config/config.yaml @@ -1,4 +1,3 @@ -defaults_path: .. kafka_brokers: "broker:9092" topic_name_config: default_error_topic_name: ${component.type}-error-topic diff --git a/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml b/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml index fc012737a..02fd8e087 100644 --- a/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml +++ b/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml @@ -4,7 +4,7 @@ topics: example-topic: type: input - app: + values: image: fake-image to: topics: @@ -13,5 +13,5 @@ - type: kafka-sink-connector name: es-sink-connector - app: + config: connector.class: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector diff --git a/tests/pipeline/resources/multi-config/config.yaml b/tests/pipeline/resources/multi-config/config.yaml index 2e40128d4..4e4399fc1 100644 --- a/tests/pipeline/resources/multi-config/config.yaml +++ b/tests/pipeline/resources/multi-config/config.yaml @@ -1,4 +1,3 @@ -defaults_path: ../no-topics-defaults kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" kafka_connect: url: "http://localhost:8083" diff --git a/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml b/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml index ba25e167a..f1e5f18c4 100644 --- a/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml +++ b/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml @@ -5,7 +5,7 @@ topics: example-topic: type: input - app: + values: image: fake-image to: topics: diff --git a/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml b/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml index ae195fdd2..0edbafc0c 100644 --- a/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml @@ -4,7 +4,7 @@ topics: ".*": type: pattern - app: + values: commandLine: CONVERT_XML: true resources: diff --git a/tests/pipeline/resources/no-topics-defaults/defaults.yaml b/tests/pipeline/resources/no-topics-defaults/defaults.yaml index 7820898a3..f933006ff 100644 --- a/tests/pipeline/resources/no-topics-defaults/defaults.yaml +++ b/tests/pipeline/resources/no-topics-defaults/defaults.yaml @@ -1,5 +1,5 @@ kafka-app: - app: + values: streams: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" @@ -12,7 +12,7 @@ producer-app: partitions_count: 3 streams-app: - app: + values: labels: pipeline: ${pipeline.name} to: diff --git a/tests/pipeline/resources/no-user-defined-components/pipeline.yaml b/tests/pipeline/resources/no-user-defined-components/pipeline.yaml index b5cdfc313..e8c53c87d 100644 --- a/tests/pipeline/resources/no-user-defined-components/pipeline.yaml +++ b/tests/pipeline/resources/no-user-defined-components/pipeline.yaml @@ -3,7 +3,7 @@ topics: example-topic: type: input - app: + values: image: fake-image to: topics: diff --git a/tests/pipeline/resources/parallel-pipeline/config.yaml b/tests/pipeline/resources/parallel-pipeline/config.yaml index 1c3b4443f..e93826df9 100644 --- a/tests/pipeline/resources/parallel-pipeline/config.yaml +++ b/tests/pipeline/resources/parallel-pipeline/config.yaml @@ -11,5 +11,4 @@ kafka_connect: kafka_rest: url: "http://kafka_rest_url:8082" -defaults_path: .. kafka_brokers: "broker:9092" diff --git a/tests/pipeline/resources/parallel-pipeline/defaults.yaml b/tests/pipeline/resources/parallel-pipeline/defaults.yaml index 4b6cd0c91..073e8e877 100644 --- a/tests/pipeline/resources/parallel-pipeline/defaults.yaml +++ b/tests/pipeline/resources/parallel-pipeline/defaults.yaml @@ -4,17 +4,14 @@ pipeline-component: kubernetes-app: namespace: ${NAMESPACE} -kafka-connector: - namespace: ${NAMESPACE} - kafka-app: - app: + values: streams: brokers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} streams-app: - app: + values: labels: pipeline: ${pipeline.name} to: diff --git a/tests/pipeline/resources/parallel-pipeline/pipeline.yaml b/tests/pipeline/resources/parallel-pipeline/pipeline.yaml index 1c461c65d..688d86fb5 100644 --- a/tests/pipeline/resources/parallel-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/parallel-pipeline/pipeline.yaml @@ -41,7 +41,7 @@ topics: account-linker-test-topic: type: input - app: + config: connector.class: io.confluent.connect.s3.S3SinkConnector @@ -51,7 +51,7 @@ topics: account-linker-test-topic: type: input - app: + config: connector.class: io.confluent.connect.s3.S3SinkConnector - type: kafka-sink-connector @@ -60,5 +60,5 @@ topics: account-linker-test-topic: type: input - app: + config: connector.class: io.confluent.connect.s3.S3SinkConnector diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml index 50705aafe..5d3fa4696 100644 --- a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml +++ b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml @@ -6,7 +6,7 @@ kubernetes-app: kafka-app: version: 2.9.0 - app: + values: streams: brokers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml index 6dd30460f..7dcdda440 100644 --- a/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml @@ -1,6 +1,6 @@ - type: producer-app name: account-producer - app: + values: replicaCount: 1 image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 diff --git a/tests/pipeline/resources/pipeline-folders/pipeline-1/pipeline.yaml b/tests/pipeline/resources/pipeline-folders/pipeline-1/pipeline.yaml index 503665e9a..cdffa60c4 100644 --- a/tests/pipeline/resources/pipeline-folders/pipeline-1/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-folders/pipeline-1/pipeline.yaml @@ -1,5 +1,5 @@ - type: scheduled-producer - app: + values: commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" diff --git a/tests/pipeline/resources/pipeline-folders/pipeline-2/pipeline.yaml b/tests/pipeline/resources/pipeline-folders/pipeline-2/pipeline.yaml index 0dfc1da57..f2075dfc9 100644 --- a/tests/pipeline/resources/pipeline-folders/pipeline-2/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-folders/pipeline-2/pipeline.yaml @@ -1,5 +1,5 @@ - type: converter - app: + values: commandLine: CONVERT_XML: true resources: diff --git a/tests/pipeline/resources/pipeline-folders/pipeline-3/pipeline.yaml b/tests/pipeline/resources/pipeline-folders/pipeline-3/pipeline.yaml index 99571e5e6..546c557ba 100644 --- a/tests/pipeline/resources/pipeline-folders/pipeline-3/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-folders/pipeline-3/pipeline.yaml @@ -1,6 +1,6 @@ - type: filter name: "a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name" - app: + values: commandLine: TYPE: "nothing" resources: diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index afc821f3c..c3b71231f 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -2,7 +2,7 @@ kubernetes-app: name: ${component.type} namespace: example-namespace kafka-app: - app: + values: streams: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" @@ -11,7 +11,7 @@ producer-app: {} # inherits from kafka-app streams-app: # inherits from kafka-app version: 2.9.0 - app: + values: streams: config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator @@ -26,7 +26,7 @@ streams-app: # inherits from kafka-app kafka-connector: name: sink-connector - app: + config: batch.size: "2000" behavior.on.malformed.documents: "warn" behavior.on.null.values: "delete" diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml index fc012737a..02fd8e087 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml @@ -4,7 +4,7 @@ topics: example-topic: type: input - app: + values: image: fake-image to: topics: @@ -13,5 +13,5 @@ - type: kafka-sink-connector name: es-sink-connector - app: + config: connector.class: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector diff --git a/tests/pipeline/resources/pipeline-with-envs/pipeline.yaml b/tests/pipeline/resources/pipeline-with-envs/pipeline.yaml index bf343a11b..14446ca4e 100644 --- a/tests/pipeline/resources/pipeline-with-envs/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-envs/pipeline.yaml @@ -1,11 +1,11 @@ - type: scheduled-producer name: input-producer - app: + values: commandLine: FAKE_ARG: fake-arg-value schedule: 30 3/8 * * * - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -14,7 +14,7 @@ requests: memory: 2G - type: filter - app: + values: commandLine: TYPE: nothing resources: diff --git a/tests/pipeline/resources/pipeline-with-envs/pipeline_development.yaml b/tests/pipeline/resources/pipeline-with-envs/pipeline_development.yaml index e1eef2644..0ac326cf8 100644 --- a/tests/pipeline/resources/pipeline-with-envs/pipeline_development.yaml +++ b/tests/pipeline/resources/pipeline-with-envs/pipeline_development.yaml @@ -1,6 +1,6 @@ - type: scheduled-producer name: input-producer - app: + values: commandLine: FAKE_ARG: override-arg schedule: 20 3/8 * * * diff --git a/tests/pipeline/resources/pipeline-with-illegal-kubernetes-name/pipeline.yaml b/tests/pipeline/resources/pipeline-with-illegal-kubernetes-name/pipeline.yaml index 69628b58e..df11a8e63 100644 --- a/tests/pipeline/resources/pipeline-with-illegal-kubernetes-name/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-illegal-kubernetes-name/pipeline.yaml @@ -1,10 +1,10 @@ - type: scheduled-producer - app: + values: commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -14,7 +14,7 @@ memory: 2G - type: filter name: "illegal_name" - app: + values: commandLine: TYPE: "nothing" resources: diff --git a/tests/pipeline/resources/pipeline-with-inflate/pipeline.yaml b/tests/pipeline/resources/pipeline-with-inflate/pipeline.yaml index 51409ad90..c052be05a 100644 --- a/tests/pipeline/resources/pipeline-with-inflate/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-inflate/pipeline.yaml @@ -1,10 +1,10 @@ - type: scheduled-producer - app: + values: commandLine: FAKE_ARG: "fake-arg-value" schedule: "30 3/8 * * *" - type: converter - app: + values: commandLine: CONVERT_XML: true resources: @@ -13,7 +13,7 @@ requests: memory: 2G - type: should-inflate - app: + values: commandLine: TYPE: "nothing" resources: diff --git a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml index 777933d94..cc9b4e585 100644 --- a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml @@ -8,12 +8,12 @@ kafka-connector: namespace: example-namespace kafka-app: - app: + values: streams: brokers: 127.0.0.1:9092 schemaRegistryUrl: 127.0.0.1:8081 streams-app: - app: + values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml b/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml index b8f2866f6..d60e4c36e 100644 --- a/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml @@ -1,6 +1,6 @@ - type: producer-app name: app1 - app: + values: image: producer-image to: topics: @@ -9,7 +9,7 @@ - type: streams-app name: app2 - app: + values: image: app2-image from: topics: @@ -22,7 +22,7 @@ - type: streams-app name: app3 - app: + values: image: app3-image from: topics: diff --git a/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml b/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml index 19ea8e193..ba1bd0000 100644 --- a/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml @@ -1,7 +1,7 @@ - type: producer-app name: account-producer namespace: test - app: + values: streams: brokers: test output_topic: out diff --git a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml index 48cacc6cc..15dc70e01 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml @@ -3,7 +3,7 @@ kubernetes-app: namespace: example-namespace kafka-app: - app: + values: streams: brokers: "${config.kafka_brokers}" schema_registry_url: "${config.schema_registry.url}" @@ -17,7 +17,7 @@ producer-app: streams-app: # inherits from kafka-app - app: + values: streams: brokers: test to: diff --git a/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml b/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml index 79649aeff..f6db75522 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml @@ -14,12 +14,12 @@ name: simple-app from: components: - component-input: - type: null - component-extra: - role: role - component-input-pattern: - type: pattern - component-extra-pattern: - type: pattern - role: role + component-input: + type: null + component-extra: + role: role + component-input-pattern: + type: pattern + component-extra-pattern: + type: pattern + role: role diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/config.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/config.yaml index be0d3fcdf..be2de22e0 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/config.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/config.yaml @@ -1,4 +1,3 @@ - schema_registry: enabled: false url: "http://localhost:8081" @@ -8,5 +7,4 @@ kafka_connect: kafka_rest: url: "http://kafka_rest_url:8082" -defaults_path: .. kafka_brokers: "broker:9092" diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml index 719059611..f45f4ef37 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml @@ -8,13 +8,13 @@ kafka-connector: namespace: example-namespace kafka-app: - app: + values: streams: brokers: 127.0.0.1:9092 schemaRegistryUrl: 127.0.0.1:8081 streams-app: - app: + values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml index 5e578f0a2..1c6511d79 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml @@ -1,6 +1,6 @@ - type: streams-app name: app2-processor - app: + values: image: some-image to: topics: diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/config.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/config.yaml index be0d3fcdf..be2de22e0 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/config.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/config.yaml @@ -1,4 +1,3 @@ - schema_registry: enabled: false url: "http://localhost:8081" @@ -8,5 +7,4 @@ kafka_connect: kafka_rest: url: "http://kafka_rest_url:8082" -defaults_path: .. kafka_brokers: "broker:9092" diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml index 88eebd4c0..8fbb3ea39 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml @@ -8,14 +8,14 @@ kafka-connector: namespace: example-namespace kafka-app: - app: + values: streams: brokers: 127.0.0.1:9092 schemaRegistryUrl: 127.0.0.1:8081 streams-app: - app: + values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/resetter_values/defaults.yaml b/tests/pipeline/resources/resetter_values/defaults.yaml index 950ed4969..d899a5014 100644 --- a/tests/pipeline/resources/resetter_values/defaults.yaml +++ b/tests/pipeline/resources/resetter_values/defaults.yaml @@ -1,13 +1,13 @@ helm-app: name: "${component.type}" namespace: "namespace" - app: + values: label: ${component.name} streams: brokers: ${config.kafka_brokers} kafka-sink-connector: - app: - "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector" + config: + connector.class: "io.confluent.connect.jdbc.JdbcSinkConnector" resetter_values: imageTag: override-default-image-tag diff --git a/tests/pipeline/resources/resetter_values/pipeline_connector_only.yaml b/tests/pipeline/resources/resetter_values/pipeline_connector_only.yaml index 7f66b27cd..af131751d 100644 --- a/tests/pipeline/resources/resetter_values/pipeline_connector_only.yaml +++ b/tests/pipeline/resources/resetter_values/pipeline_connector_only.yaml @@ -1,4 +1,4 @@ - type: kafka-sink-connector name: es-sink-connector - app: + config: connector.class: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector diff --git a/tests/pipeline/resources/simple-pipeline/pipeline.yaml b/tests/pipeline/resources/simple-pipeline/pipeline.yaml index b4116761d..f78d0c385 100644 --- a/tests/pipeline/resources/simple-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/simple-pipeline/pipeline.yaml @@ -3,4 +3,4 @@ - type: streams-app - type: helm-app - app: {} + values: {} diff --git a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml index c895105b7..f777b35e9 100644 --- a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml @@ -2,14 +2,14 @@ kubernetes-app: namespace: example-namespace kafka-app: - app: + values: streams: brokers: "${config.kafka_brokers}" schema_registry_url: "${schema_registry_url}" version: "2.4.2" streams-app: # inherits from kafka-app - app: + values: streams: config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator diff --git a/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml b/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml index d61d6c9ba..4507ff2d3 100644 --- a/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml @@ -1,6 +1,6 @@ - type: streams-app name: in-order-to-have-len-fifty-two-name-should-end--here - app: + values: streams: config: max.poll.records: 100 diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index 528da26bb..76c9b1bf8 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -1,5 +1,16 @@ - _cleaner: - app: + debug: true + name: account-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 labels: @@ -15,34 +26,7 @@ outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: account-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: - image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud - prometheus: - jmx: - enabled: false - replicaCount: 1 - schedule: 0 12 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-producer-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - suspend: true debug: true name: account-producer namespace: ${NAMESPACE} @@ -59,9 +43,36 @@ configs: {} partitions_count: 3 type: producer-app + values: + image: ${DOCKER_REGISTRY}/atm-demo-accountproducer + imageTag: 1.0.0 + labels: + pipeline: atm-fraud + prometheus: + jmx: + enabled: false + replicaCount: 1 + schedule: 0 12 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-producer-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ + suspend: true version: 2.9.0 - _cleaner: - app: + debug: true + name: transaction-avro-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: commandLine: ITERATION: 20 REAL_TX: 19 @@ -80,19 +91,24 @@ outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: transaction-avro-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: + debug: true + name: transaction-avro-producer + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-avro-producer-topic: + configs: {} + partitions_count: 3 + type: producer-app + values: commandLine: ITERATION: 20 REAL_TX: 19 @@ -111,25 +127,20 @@ outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true - debug: true - name: transaction-avro-producer - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-avro-producer-topic: - configs: {} - partitions_count: 3 - type: producer-app version: 2.9.0 - _cleaner: - app: + debug: true + name: transaction-joiner + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic commandLine: @@ -153,19 +164,28 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: transaction-joiner - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: + debug: true + name: transaction-joiner + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-joiner-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-transaction-joiner-topic: + configs: {} + partitions_count: 3 + type: streams-app + values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic commandLine: @@ -189,29 +209,20 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: transaction-joiner - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-joiner-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-transaction-joiner-topic: - configs: {} - partitions_count: 3 - type: streams-app version: 2.9.0 - _cleaner: - app: + debug: true + name: fraud-detector + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic commandLine: @@ -235,19 +246,28 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: fraud-detector - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: + debug: true + name: fraud-detector + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-fraud-detector-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-fraud-detector-topic: + configs: {} + partitions_count: 3 + type: streams-app + values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic commandLine: @@ -271,29 +291,20 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: fraud-detector - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-fraud-detector-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-fraud-detector-topic: - configs: {} - partitions_count: 3 - type: streams-app version: 2.9.0 - _cleaner: - app: + debug: true + name: account-linker + namespace: ${NAMESPACE} + prefix: atm-fraud- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic commandLine: @@ -320,45 +331,7 @@ optimizeLeaveGroupBehavior: false outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: account-linker - namespace: ${NAMESPACE} - prefix: atm-fraud- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - annotations: - consumerGroup: atm-accountlinker-atm-fraud-output-topic - commandLine: - PRODUCTIVE: false - image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: 1.0.0 - labels: - pipeline: atm-fraud - persistence: - enabled: false - prometheus: - jmx: - enabled: false - replicaCount: 1 - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-account-linker-dead-letter-topic - extraInputTopics: - accounts: - - atm-fraud-account-producer-topic - inputTopics: - - atm-fraud-fraud-detector-topic - optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-linker-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true from: components: @@ -386,13 +359,35 @@ configs: {} partitions_count: 3 type: streams-app + values: + annotations: + consumerGroup: atm-accountlinker-atm-fraud-output-topic + commandLine: + PRODUCTIVE: false + image: ${DOCKER_REGISTRY}/atm-demo-accountlinker + imageTag: 1.0.0 + labels: + pipeline: atm-fraud + persistence: + enabled: false + prometheus: + jmx: + enabled: false + replicaCount: 1 + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-account-linker-dead-letter-topic + extraInputTopics: + accounts: + - atm-fraud-account-producer-topic + inputTopics: + - atm-fraud-fraud-detector-topic + optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-linker-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: atm-fraud-postgresql-connector - connectorType: sink name: postgresql-connector namespace: ${NAMESPACE} prefix: atm-fraud- @@ -403,8 +398,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: atm-fraud-postgresql-connector + connectorType: sink version: 1.0.4 - app: + config: auto.create: true connection.ds.pool.size: 5 connection.password: AppPassword diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index 7e7d16fe8..e7afd3398 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -1,5 +1,16 @@ - _cleaner: - app: + debug: true + name: data-producer + namespace: ${NAMESPACE} + prefix: word-count- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: image: bakdata/kpops-demo-sentence-producer imageTag: 1.0.0 labels: @@ -13,32 +24,7 @@ optimizeLeaveGroupBehavior: false outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: data-producer - namespace: ${NAMESPACE} - prefix: word-count- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: - image: bakdata/kpops-demo-sentence-producer - imageTag: 1.0.0 - labels: - pipeline: word-count - prometheus: - jmx: - enabled: false - replicaCount: 1 - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - optimizeLeaveGroupBehavior: false - outputTopic: word-count-data-producer-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true name: data-producer namespace: ${NAMESPACE} @@ -55,9 +41,34 @@ configs: {} partitions_count: 3 type: producer-app + values: + image: bakdata/kpops-demo-sentence-producer + imageTag: 1.0.0 + labels: + pipeline: word-count + prometheus: + jmx: + enabled: false + replicaCount: 1 + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + optimizeLeaveGroupBehavior: false + outputTopic: word-count-data-producer-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: - app: + debug: true + name: word-counter + namespace: ${NAMESPACE} + prefix: word-count- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: commandLine: PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app @@ -79,40 +90,7 @@ optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ - debug: true - name: word-counter - namespace: ${NAMESPACE} - prefix: word-count- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - commandLine: - PRODUCTIVE: false - image: bakdata/kpops-demo-word-count-app - imageTag: 1.0.0 - labels: - pipeline: word-count - persistence: - enabled: false - prometheus: - jmx: - enabled: false - replicaCount: 1 - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: word-count-word-counter-dead-letter-topic - inputTopics: - - word-count-data-producer-topic - optimizeLeaveGroupBehavior: false - outputTopic: word-count-word-counter-topic - schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ debug: true name: word-counter namespace: ${NAMESPACE} @@ -135,13 +113,30 @@ partitions_count: 3 type: output type: streams-app + values: + commandLine: + PRODUCTIVE: false + image: bakdata/kpops-demo-word-count-app + imageTag: 1.0.0 + labels: + pipeline: word-count + persistence: + enabled: false + prometheus: + jmx: + enabled: false + replicaCount: 1 + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: word-count-word-counter-dead-letter-topic + inputTopics: + - word-count-data-producer-topic + optimizeLeaveGroupBehavior: false + outputTopic: word-count-word-counter-topic + schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: word-count-redis-sink-connector - connectorType: sink name: redis-sink-connector namespace: ${NAMESPACE} prefix: word-count- @@ -152,8 +147,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: word-count-redis-sink-connector + connectorType: sink version: 1.0.4 - app: + config: connector.class: com.github.jcustenborder.kafka.connect.redis.RedisSinkConnector key.converter: org.apache.kafka.connect.storage.StringConverter name: word-count-redis-sink-connector diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml index 4549c6209..1db36ed55 100644 --- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml @@ -1,15 +1,4 @@ - _cleaner: - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-custom-config-app1 - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -20,18 +9,18 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-custom-config-app1 + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-custom-config-app1 - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -48,9 +37,30 @@ partitions_count: 3 type: output type: producer-app + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-custom-config-app1 + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - _cleaner: - app: + name: app2 + namespace: development-namespace + prefix: resources-custom-config- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: some-image imageTag: latest labels: @@ -67,34 +77,7 @@ - resources-custom-config-app1 outputTopic: resources-custom-config-app2 schemaRegistryUrl: http://localhost:8081/ - name: app2 - namespace: development-namespace - prefix: resources-custom-config- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - image: some-image - imageTag: latest - labels: - pipeline: resources-custom-config - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-custom-config-app2-error - inputTopics: - - resources-custom-config-app1 - outputTopic: resources-custom-config-app2 - schemaRegistryUrl: http://localhost:8081/ name: app2 namespace: development-namespace prefix: resources-custom-config- @@ -117,5 +100,22 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + image: some-image + imageTag: latest + labels: + pipeline: resources-custom-config + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-custom-config-app2-error + inputTopics: + - resources-custom-config-app1 + outputTopic: resources-custom-config-app2 + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index 1040139c3..20e133c5f 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -1,14 +1,4 @@ - _cleaner: - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-with-inflate-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-with-inflate- @@ -19,17 +9,17 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-with-inflate-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-with-inflate-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-with-inflate- @@ -49,9 +39,29 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-with-inflate-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-with-inflate- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-pipeline-with-inflate-converter cooldownPeriod: 300 @@ -82,18 +92,32 @@ - resources-pipeline-with-inflate-scheduled-producer outputTopic: resources-pipeline-with-inflate-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-with-inflate- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-with-inflate- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-with-inflate-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-with-inflate-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-pipeline-with-inflate-converter cooldownPeriod: 300 @@ -124,33 +148,19 @@ - resources-pipeline-with-inflate-scheduled-producer outputTopic: resources-pipeline-with-inflate-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-with-inflate- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-with-inflate-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-with-inflate-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: should-inflate + namespace: example-namespace + prefix: resources-pipeline-with-inflate- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-pipeline-with-inflate-should-inflate cooldownPeriod: 300 @@ -182,18 +192,31 @@ - resources-pipeline-with-inflate-converter outputTopic: resources-pipeline-with-inflate-should-inflate schemaRegistryUrl: http://localhost:8081/ - name: should-inflate - namespace: example-namespace - prefix: resources-pipeline-with-inflate- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: should-inflate + namespace: example-namespace + prefix: resources-pipeline-with-inflate- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-with-inflate-should-inflate: + configs: + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-with-inflate-should-inflate-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: should-inflate + values: autoscaling: consumerGroup: filter-resources-pipeline-with-inflate-should-inflate cooldownPeriod: 300 @@ -225,36 +248,8 @@ - resources-pipeline-with-inflate-converter outputTopic: resources-pipeline-with-inflate-should-inflate schemaRegistryUrl: http://localhost:8081/ - name: should-inflate - namespace: example-namespace - prefix: resources-pipeline-with-inflate- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-with-inflate-should-inflate: - configs: - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-with-inflate-should-inflate-error: - configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: should-inflate version: 2.4.2 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector - connectorType: sink name: should-inflate-inflated-sink-connector namespace: example-namespace prefix: resources-pipeline-with-inflate- @@ -265,8 +260,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector + connectorType: sink version: 1.0.4 - app: + config: batch.size: '2000' behavior.on.malformed.documents: warn behavior.on.null.values: delete @@ -294,7 +294,17 @@ role: test type: kafka-sink-connector - _cleaner: - app: + name: should-inflate-inflated-streams-app + namespace: example-namespace + prefix: resources-pipeline-with-inflate- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -308,31 +318,7 @@ - kafka-sink-connector outputTopic: resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app schemaRegistryUrl: http://localhost:8081/ - name: should-inflate-inflated-streams-app - namespace: example-namespace - prefix: resources-pipeline-with-inflate- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error - inputTopics: - - kafka-sink-connector - outputTopic: resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app - schemaRegistryUrl: http://localhost:8081/ name: should-inflate-inflated-streams-app namespace: example-namespace prefix: resources-pipeline-with-inflate- @@ -354,5 +340,19 @@ configs: {} type: output type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error + inputTopics: + - kafka-sink-connector + outputTopic: resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index e1dd399cd..2f7fd65ff 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: streams-app + namespace: example-namespace + prefix: resources-kafka-connect-sink- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: fake-image imageTag: latest persistence: @@ -14,32 +24,7 @@ - example-topic outputTopic: example-output schemaRegistryUrl: http://localhost:8081/ - name: streams-app - namespace: example-namespace - prefix: resources-kafka-connect-sink- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - image: fake-image - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-kafka-connect-sink-streams-app-error - inputTopics: - - example-topic - outputTopic: example-output - schemaRegistryUrl: http://localhost:8081/ from: components: {} topics: @@ -66,13 +51,23 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + image: fake-image + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-kafka-connect-sink-streams-app-error + inputTopics: + - example-topic + outputTopic: example-output + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: resources-kafka-connect-sink-es-sink-connector - connectorType: sink name: es-sink-connector namespace: example-namespace prefix: resources-kafka-connect-sink- @@ -83,8 +78,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: resources-kafka-connect-sink-es-sink-connector + connectorType: sink version: 1.0.4 - app: + config: batch.size: '2000' behavior.on.malformed.documents: warn behavior.on.null.values: delete diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml index d49fa57d2..42d5aa38e 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml @@ -1,14 +1,4 @@ - _cleaner: - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-first-pipeline-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-first-pipeline- @@ -19,17 +9,17 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-first-pipeline-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-first-pipeline-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-first-pipeline- @@ -49,9 +39,29 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-first-pipeline-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-first-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-first-pipeline-converter cooldownPeriod: 300 @@ -82,18 +92,32 @@ - resources-first-pipeline-scheduled-producer outputTopic: resources-first-pipeline-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-first-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-first-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-first-pipeline-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-first-pipeline-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-first-pipeline-converter cooldownPeriod: 300 @@ -124,33 +148,19 @@ - resources-first-pipeline-scheduled-producer outputTopic: resources-first-pipeline-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-first-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-first-pipeline-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-first-pipeline-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-first-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -182,18 +192,31 @@ - resources-first-pipeline-converter outputTopic: resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-first-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-first-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + ? resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + : configs: + retention.ms: '-1' + partitions_count: 50 + type: output + ? resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error + : configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: filter + values: autoscaling: consumerGroup: filter-resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -225,28 +248,5 @@ - resources-first-pipeline-converter outputTopic: resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-first-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - ? resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - : configs: - retention.ms: '-1' - partitions_count: 50 - type: output - ? resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error - : configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: filter version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml index 42cd958d5..70e2b424b 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml @@ -1,14 +1,4 @@ - _cleaner: - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-folders-pipeline-1- @@ -19,17 +9,17 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-folders-pipeline-1- @@ -49,10 +39,30 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-2- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-pipeline-folders-pipeline-2-converter cooldownPeriod: 300 @@ -81,18 +91,32 @@ errorTopic: resources-pipeline-folders-pipeline-2-converter-error outputTopic: resources-pipeline-folders-pipeline-2-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-2- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-2- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-folders-pipeline-2-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-folders-pipeline-2-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-pipeline-folders-pipeline-2-converter cooldownPeriod: 300 @@ -121,34 +145,20 @@ errorTopic: resources-pipeline-folders-pipeline-2-converter-error outputTopic: resources-pipeline-folders-pipeline-2-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-2- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-folders-pipeline-2-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-folders-pipeline-2-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -178,18 +188,31 @@ errorTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error outputTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-3- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + : configs: + retention.ms: '-1' + partitions_count: 50 + type: output + ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error + : configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: filter + values: autoscaling: consumerGroup: filter-resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -219,28 +242,5 @@ errorTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error outputTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-3- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - : configs: - retention.ms: '-1' - partitions_count: 50 - type: output - ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error - : configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: filter version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml index 42cd958d5..70e2b424b 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml @@ -1,14 +1,4 @@ - _cleaner: - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-folders-pipeline-1- @@ -19,17 +9,17 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-pipeline-folders-pipeline-1- @@ -49,10 +39,30 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-folders-pipeline-1-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-2- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-pipeline-folders-pipeline-2-converter cooldownPeriod: 300 @@ -81,18 +91,32 @@ errorTopic: resources-pipeline-folders-pipeline-2-converter-error outputTopic: resources-pipeline-folders-pipeline-2-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-2- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-2- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-folders-pipeline-2-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-folders-pipeline-2-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-pipeline-folders-pipeline-2-converter cooldownPeriod: 300 @@ -121,34 +145,20 @@ errorTopic: resources-pipeline-folders-pipeline-2-converter-error outputTopic: resources-pipeline-folders-pipeline-2-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-2- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-folders-pipeline-2-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-folders-pipeline-2-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -178,18 +188,31 @@ errorTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error outputTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-3- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + namespace: example-namespace + prefix: resources-pipeline-folders-pipeline-3- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name + : configs: + retention.ms: '-1' + partitions_count: 50 + type: output + ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error + : configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: filter + values: autoscaling: consumerGroup: filter-resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name cooldownPeriod: 300 @@ -219,28 +242,5 @@ errorTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error outputTopic: resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name schemaRegistryUrl: http://localhost:8081/ - name: a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - namespace: example-namespace - prefix: resources-pipeline-folders-pipeline-3- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name - : configs: - retention.ms: '-1' - partitions_count: 50 - type: output - ? resources-pipeline-folders-pipeline-3-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error - : configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: filter version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml index 9e85d9e01..db123f217 100644 --- a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml @@ -1,10 +1,4 @@ - _cleaner: - app: - imageTag: latest - streams: - brokers: test - outputTopic: out - schemaRegistryUrl: http://localhost:8081/ name: account-producer namespace: test prefix: resources-pipeline-with-paths- @@ -18,13 +12,13 @@ url: masked suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + streams: + brokers: test + outputTopic: out + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - imageTag: latest - streams: - brokers: test - outputTopic: out - schemaRegistryUrl: http://localhost:8081/ name: account-producer namespace: test prefix: resources-pipeline-with-paths- @@ -37,5 +31,11 @@ repository_name: masked url: masked type: producer-app + values: + imageTag: latest + streams: + brokers: test + outputTopic: out + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml index 3c57c8eb1..e4af38635 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: app1 + namespace: example-namespace + prefix: resources-no-input-topic-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: commandLine: CONVERT_XML: true imageTag: latest @@ -19,37 +29,7 @@ inputPattern: .* outputTopic: example-output schemaRegistryUrl: http://localhost:8081/ - name: app1 - namespace: example-namespace - prefix: resources-no-input-topic-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - commandLine: - CONVERT_XML: true - imageTag: latest - persistence: - enabled: false - resources: - limits: - memory: 2G - requests: - memory: 2G - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-no-input-topic-pipeline-app1-error - inputPattern: .* - outputTopic: example-output - schemaRegistryUrl: http://localhost:8081/ from: components: {} topics: @@ -76,9 +56,39 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + commandLine: + CONVERT_XML: true + imageTag: latest + persistence: + enabled: false + resources: + limits: + memory: 2G + requests: + memory: 2G + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-no-input-topic-pipeline-app1-error + inputPattern: .* + outputTopic: example-output + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: app2 + namespace: example-namespace + prefix: resources-no-input-topic-pipeline- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -94,33 +104,7 @@ inputTopics: - example-output schemaRegistryUrl: http://localhost:8081/ - name: app2 - namespace: example-namespace - prefix: resources-no-input-topic-pipeline- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-no-input-topic-pipeline-app2-error - extraOutputTopics: - extra: example-output-extra - test-output: test-output-extra - inputTopics: - - example-output - schemaRegistryUrl: http://localhost:8081/ name: app2 namespace: example-namespace prefix: resources-no-input-topic-pipeline- @@ -145,5 +129,21 @@ configs: {} role: test-output type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-no-input-topic-pipeline-app2-error + extraOutputTopics: + extra: example-output-extra + test-output: test-output-extra + inputTopics: + - example-output + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index 348443bbc..01a8241dc 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: streams-app + namespace: example-namespace + prefix: resources-no-user-defined-components- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: fake-image imageTag: latest persistence: @@ -14,32 +24,7 @@ - example-topic outputTopic: example-output schemaRegistryUrl: http://localhost:8081/ - name: streams-app - namespace: example-namespace - prefix: resources-no-user-defined-components- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - image: fake-image - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-no-user-defined-components-streams-app-error - inputTopics: - - example-topic - outputTopic: example-output - schemaRegistryUrl: http://localhost:8081/ from: components: {} topics: @@ -66,5 +51,20 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + image: fake-image + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-no-user-defined-components-streams-app-error + inputTopics: + - example-topic + outputTopic: example-output + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml index 36d50168f..7641726e9 100644 --- a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml @@ -1,14 +1,4 @@ - _cleaner: - app: - commandLine: - FAKE_ARG: override-arg - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 20 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-with-envs-input-producer - schemaRegistryUrl: http://localhost:8081/ name: input-producer namespace: example-namespace prefix: resources-pipeline-with-envs- @@ -19,17 +9,17 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + commandLine: + FAKE_ARG: override-arg + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 20 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-with-envs-input-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - commandLine: - FAKE_ARG: override-arg - image: example-registry/fake-image - imageTag: 0.0.1 - schedule: 20 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-pipeline-with-envs-input-producer - schemaRegistryUrl: http://localhost:8081/ name: input-producer namespace: example-namespace prefix: resources-pipeline-with-envs- @@ -49,9 +39,29 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: override-arg + image: example-registry/fake-image + imageTag: 0.0.1 + schedule: 20 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-pipeline-with-envs-input-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-with-envs- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-pipeline-with-envs-converter cooldownPeriod: 300 @@ -82,18 +92,32 @@ - resources-pipeline-with-envs-input-producer outputTopic: resources-pipeline-with-envs-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-with-envs- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-pipeline-with-envs- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-with-envs-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-with-envs-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-pipeline-with-envs-converter cooldownPeriod: 300 @@ -124,33 +148,19 @@ - resources-pipeline-with-envs-input-producer outputTopic: resources-pipeline-with-envs-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-pipeline-with-envs- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-with-envs-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-with-envs-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: filter + namespace: example-namespace + prefix: resources-pipeline-with-envs- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-pipeline-with-envs-filter cooldownPeriod: 300 @@ -182,18 +192,31 @@ - resources-pipeline-with-envs-converter outputTopic: resources-pipeline-with-envs-filter schemaRegistryUrl: http://localhost:8081/ - name: filter - namespace: example-namespace - prefix: resources-pipeline-with-envs- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: filter + namespace: example-namespace + prefix: resources-pipeline-with-envs- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-pipeline-with-envs-filter: + configs: + retention.ms: '-1' + partitions_count: 50 + type: output + resources-pipeline-with-envs-filter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: filter + values: autoscaling: consumerGroup: filter-resources-pipeline-with-envs-filter cooldownPeriod: 300 @@ -225,28 +248,5 @@ - resources-pipeline-with-envs-converter outputTopic: resources-pipeline-with-envs-filter schemaRegistryUrl: http://localhost:8081/ - name: filter - namespace: example-namespace - prefix: resources-pipeline-with-envs- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-pipeline-with-envs-filter: - configs: - retention.ms: '-1' - partitions_count: 50 - type: output - resources-pipeline-with-envs-filter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: filter version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml index 096b846d1..7e0b3bcae 100644 --- a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: account-producer + namespace: ${NAMESPACE} + prefix: from-pipeline-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: debug: true image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 @@ -12,18 +22,17 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - name: account-producer - namespace: ${NAMESPACE} - prefix: from-pipeline-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.9.0 - app: + name: account-producer + namespace: ${NAMESPACE} + prefix: from-pipeline-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + type: producer-app + values: debug: true image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 @@ -36,14 +45,5 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - name: account-producer - namespace: ${NAMESPACE} - prefix: from-pipeline-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - type: producer-app version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index ff1dd9e2b..6807b3552 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -1,10 +1,4 @@ - _cleaner: - app: - imageTag: latest - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-read-from-component-producer1 - schemaRegistryUrl: http://localhost:8081/ name: producer1 namespace: example-namespace prefix: resources-read-from-component- @@ -15,13 +9,13 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-read-from-component-producer1 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - imageTag: latest - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-read-from-component-producer1 - schemaRegistryUrl: http://localhost:8081/ name: producer1 namespace: example-namespace prefix: resources-read-from-component- @@ -37,14 +31,14 @@ configs: {} type: output type: producer-app + values: + imageTag: latest + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-read-from-component-producer1 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: - imageTag: latest - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-read-from-component-producer2 - schemaRegistryUrl: http://localhost:8081/ name: producer2 namespace: example-namespace prefix: '' @@ -55,13 +49,13 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-read-from-component-producer2 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - app: - imageTag: latest - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-read-from-component-producer2 - schemaRegistryUrl: http://localhost:8081/ name: producer2 namespace: example-namespace prefix: '' @@ -77,9 +71,25 @@ configs: {} type: output type: producer-app + values: + imageTag: latest + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-read-from-component-producer2 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: inflate-step + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-read-from-component-inflate-step cooldownPeriod: 300 @@ -105,18 +115,31 @@ - resources-read-from-component-producer2 outputTopic: resources-read-from-component-inflate-step schemaRegistryUrl: http://localhost:8081/ - name: inflate-step - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: inflate-step + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-read-from-component-inflate-step: + configs: + retention.ms: '-1' + partitions_count: 50 + type: output + resources-read-from-component-inflate-step-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: should-inflate + values: autoscaling: consumerGroup: filter-resources-read-from-component-inflate-step cooldownPeriod: 300 @@ -142,36 +165,8 @@ - resources-read-from-component-producer2 outputTopic: resources-read-from-component-inflate-step schemaRegistryUrl: http://localhost:8081/ - name: inflate-step - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-read-from-component-inflate-step: - configs: - retention.ms: '-1' - partitions_count: 50 - type: output - resources-read-from-component-inflate-step-error: - configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: should-inflate version: 2.4.2 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: resources-read-from-component-inflate-step-inflated-sink-connector - connectorType: sink name: inflate-step-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- @@ -182,8 +177,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: resources-read-from-component-inflate-step-inflated-sink-connector + connectorType: sink version: 1.0.4 - app: + config: batch.size: '2000' behavior.on.malformed.documents: warn behavior.on.null.values: delete @@ -211,7 +211,17 @@ type: output type: kafka-sink-connector - _cleaner: - app: + name: inflate-step-inflated-streams-app + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -225,31 +235,7 @@ - kafka-sink-connector outputTopic: resources-read-from-component-inflate-step-inflate-step-inflated-streams-app schemaRegistryUrl: http://localhost:8081/ - name: inflate-step-inflated-streams-app - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-inflate-step-inflated-streams-app-error - inputTopics: - - kafka-sink-connector - outputTopic: resources-read-from-component-inflate-step-inflate-step-inflated-streams-app - schemaRegistryUrl: http://localhost:8081/ name: inflate-step-inflated-streams-app namespace: example-namespace prefix: resources-read-from-component- @@ -271,9 +257,33 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-inflate-step-inflated-streams-app-error + inputTopics: + - kafka-sink-connector + outputTopic: resources-read-from-component-inflate-step-inflate-step-inflated-streams-app + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: inflate-step-without-prefix + namespace: example-namespace + prefix: '' + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-read-from-component-inflate-step-without-prefix cooldownPeriod: 300 @@ -299,18 +309,31 @@ - resources-read-from-component-inflate-step-inflate-step-inflated-streams-app outputTopic: resources-read-from-component-inflate-step-without-prefix schemaRegistryUrl: http://localhost:8081/ - name: inflate-step-without-prefix - namespace: example-namespace - prefix: '' - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: inflate-step-without-prefix + namespace: example-namespace + prefix: '' + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-read-from-component-inflate-step-without-prefix: + configs: + retention.ms: '-1' + partitions_count: 50 + type: output + resources-read-from-component-inflate-step-without-prefix-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: should-inflate + values: autoscaling: consumerGroup: filter-resources-read-from-component-inflate-step-without-prefix cooldownPeriod: 300 @@ -336,36 +359,8 @@ - resources-read-from-component-inflate-step-inflate-step-inflated-streams-app outputTopic: resources-read-from-component-inflate-step-without-prefix schemaRegistryUrl: http://localhost:8081/ - name: inflate-step-without-prefix - namespace: example-namespace - prefix: '' - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-read-from-component-inflate-step-without-prefix: - configs: - retention.ms: '-1' - partitions_count: 50 - type: output - resources-read-from-component-inflate-step-without-prefix-error: - configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: should-inflate version: 2.4.2 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector - connectorType: sink name: inflate-step-without-prefix-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- @@ -376,8 +371,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector + connectorType: sink version: 1.0.4 - app: + config: batch.size: '2000' behavior.on.malformed.documents: warn behavior.on.null.values: delete @@ -405,7 +405,17 @@ type: output type: kafka-sink-connector - _cleaner: - app: + name: inflate-step-without-prefix-inflated-streams-app + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -419,31 +429,7 @@ - kafka-sink-connector outputTopic: inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app schemaRegistryUrl: http://localhost:8081/ - name: inflate-step-without-prefix-inflated-streams-app - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error - inputTopics: - - kafka-sink-connector - outputTopic: inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app - schemaRegistryUrl: http://localhost:8081/ name: inflate-step-without-prefix-inflated-streams-app namespace: example-namespace prefix: resources-read-from-component- @@ -465,9 +451,33 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error + inputTopics: + - kafka-sink-connector + outputTopic: inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: consumer1 + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -481,31 +491,7 @@ - resources-read-from-component-producer1 outputTopic: resources-read-from-component-consumer1 schemaRegistryUrl: http://localhost:8081/ - name: consumer1 - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-consumer1-error - inputTopics: - - resources-read-from-component-producer1 - outputTopic: resources-read-from-component-consumer1 - schemaRegistryUrl: http://localhost:8081/ from: components: producer1: @@ -532,9 +518,33 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-consumer1-error + inputTopics: + - resources-read-from-component-producer1 + outputTopic: resources-read-from-component-consumer1 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: consumer2 + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -548,31 +558,7 @@ - resources-read-from-component-producer1 - resources-read-from-component-consumer1 schemaRegistryUrl: http://localhost:8081/ - name: consumer2 - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-consumer2-error - inputTopics: - - resources-read-from-component-producer1 - - resources-read-from-component-consumer1 - schemaRegistryUrl: http://localhost:8081/ from: components: consumer1: @@ -598,9 +584,33 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-consumer2-error + inputTopics: + - resources-read-from-component-producer1 + - resources-read-from-component-consumer1 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: consumer3 + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -614,31 +624,7 @@ - resources-read-from-component-producer1 - resources-read-from-component-producer2 schemaRegistryUrl: http://localhost:8081/ - name: consumer3 - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-consumer3-error - inputTopics: - - resources-read-from-component-producer1 - - resources-read-from-component-producer2 - schemaRegistryUrl: http://localhost:8081/ from: components: producer2: @@ -664,9 +650,33 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-consumer3-error + inputTopics: + - resources-read-from-component-producer1 + - resources-read-from-component-producer2 + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: consumer4 + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -679,30 +689,7 @@ inputTopics: - resources-read-from-component-inflate-step-inflate-step-inflated-streams-app schemaRegistryUrl: http://localhost:8081/ - name: consumer4 - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-consumer4-error - inputTopics: - - resources-read-from-component-inflate-step-inflate-step-inflated-streams-app - schemaRegistryUrl: http://localhost:8081/ from: components: inflate-step: @@ -726,9 +713,32 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-consumer4-error + inputTopics: + - resources-read-from-component-inflate-step-inflate-step-inflated-streams-app + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: consumer5 + namespace: example-namespace + prefix: resources-read-from-component- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: imageTag: latest persistence: enabled: false @@ -741,30 +751,7 @@ inputTopics: - inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app schemaRegistryUrl: http://localhost:8081/ - name: consumer5 - namespace: example-namespace - prefix: resources-read-from-component- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-read-from-component-consumer5-error - inputTopics: - - inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app - schemaRegistryUrl: http://localhost:8081/ from: components: inflate-step-without-prefix: @@ -788,5 +775,18 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-read-from-component-consumer5-error + inputTopics: + - inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml index 8878759fa..ed95cb4fe 100644 --- a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: scheduled-producer + namespace: example-namespace + prefix: resources-component-type-substitution- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: producer-app-cleaner + values: commandLine: FAKE_ARG: fake-arg-value image: example-registry/fake-image @@ -13,31 +23,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-component-type-substitution-scheduled-producer schemaRegistryUrl: http://localhost:8081/ - name: scheduled-producer - namespace: example-namespace - prefix: resources-component-type-substitution- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: producer-app-cleaner version: 2.4.2 - app: - commandLine: - FAKE_ARG: fake-arg-value - image: example-registry/fake-image - imageTag: 0.0.1 - labels: - app_name: scheduled-producer - app_schedule: 30 3/8 * * * - app_type: scheduled-producer - schedule: 30 3/8 * * * - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: resources-component-type-substitution-scheduled-producer - schemaRegistryUrl: http://localhost:8081/ name: scheduled-producer namespace: example-namespace prefix: resources-component-type-substitution- @@ -57,9 +43,33 @@ type: output value_schema: com.bakdata.fake.Produced type: scheduled-producer + values: + commandLine: + FAKE_ARG: fake-arg-value + image: example-registry/fake-image + imageTag: 0.0.1 + labels: + app_name: scheduled-producer + app_schedule: 30 3/8 * * * + app_type: scheduled-producer + schedule: 30 3/8 * * * + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: resources-component-type-substitution-scheduled-producer + schemaRegistryUrl: http://localhost:8081/ version: 2.4.2 - _cleaner: - app: + name: converter + namespace: example-namespace + prefix: resources-component-type-substitution- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: converter-resources-component-type-substitution-converter cooldownPeriod: 300 @@ -90,18 +100,32 @@ - resources-component-type-substitution-scheduled-producer outputTopic: resources-component-type-substitution-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-component-type-substitution- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: converter + namespace: example-namespace + prefix: resources-component-type-substitution- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-component-type-substitution-converter: + configs: + cleanup.policy: compact,delete + retention.ms: '-1' + partitions_count: 50 + type: output + resources-component-type-substitution-converter-error: + configs: + cleanup.policy: compact,delete + partitions_count: 10 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: converter + values: autoscaling: consumerGroup: converter-resources-component-type-substitution-converter cooldownPeriod: 300 @@ -132,33 +156,19 @@ - resources-component-type-substitution-scheduled-producer outputTopic: resources-component-type-substitution-converter schemaRegistryUrl: http://localhost:8081/ - name: converter - namespace: example-namespace - prefix: resources-component-type-substitution- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-component-type-substitution-converter: - configs: - cleanup.policy: compact,delete - retention.ms: '-1' - partitions_count: 50 - type: output - resources-component-type-substitution-converter-error: - configs: - cleanup.policy: compact,delete - partitions_count: 10 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: converter version: 2.4.2 - _cleaner: - app: + name: filter-app + namespace: example-namespace + prefix: resources-component-type-substitution- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: autoscaling: consumerGroup: filter-resources-component-type-substitution-filter-app cooldownPeriod: 300 @@ -196,18 +206,31 @@ - resources-component-type-substitution-converter outputTopic: resources-component-type-substitution-filter-app schemaRegistryUrl: http://localhost:8081/ - name: filter-app - namespace: example-namespace - prefix: resources-component-type-substitution- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.4.2 - app: + name: filter-app + namespace: example-namespace + prefix: resources-component-type-substitution- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + resources-component-type-substitution-filter-app: + configs: + retention.ms: '-1' + partitions_count: 50 + type: output + resources-component-type-substitution-filter-app-error: + configs: + cleanup.policy: compact,delete + partitions_count: 1 + type: error + value_schema: com.bakdata.kafka.DeadLetter + type: filter + values: autoscaling: consumerGroup: filter-resources-component-type-substitution-filter-app cooldownPeriod: 300 @@ -245,28 +268,5 @@ - resources-component-type-substitution-converter outputTopic: resources-component-type-substitution-filter-app schemaRegistryUrl: http://localhost:8081/ - name: filter-app - namespace: example-namespace - prefix: resources-component-type-substitution- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - resources-component-type-substitution-filter-app: - configs: - retention.ms: '-1' - partitions_count: 50 - type: output - resources-component-type-substitution-filter-app-error: - configs: - cleanup.policy: compact,delete - partitions_count: 1 - type: error - value_schema: com.bakdata.kafka.DeadLetter - type: filter version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml index f96db8745..f12f54a12 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml @@ -1,15 +1,4 @@ - _cleaner: - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: app1-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -20,18 +9,18 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: app1-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: app1-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -48,9 +37,30 @@ partitions_count: 3 type: output type: producer-app + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: app1-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - _cleaner: - app: + name: app2 + namespace: development-namespace + prefix: resources-custom-config- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: some-image imageTag: latest labels: @@ -67,34 +77,7 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - name: app2 - namespace: development-namespace - prefix: resources-custom-config- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - image: some-image - imageTag: latest - labels: - pipeline: resources-custom-config - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: app2-dead-letter-topic - inputTopics: - - app1-test-topic - outputTopic: app2-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app2 namespace: development-namespace prefix: resources-custom-config- @@ -117,5 +100,22 @@ partitions_count: 3 type: output type: streams-app + values: + image: some-image + imageTag: latest + labels: + pipeline: resources-custom-config + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: app2-dead-letter-topic + inputTopics: + - app1-test-topic + outputTopic: app2-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml index f96db8745..f12f54a12 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml @@ -1,15 +1,4 @@ - _cleaner: - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: app1-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -20,18 +9,18 @@ url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean type: producer-app-cleaner + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: app1-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - app: - imageTag: latest - resources: - limits: - memory: 2G - requests: - memory: 2G - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - outputTopic: app1-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app1 namespace: development-namespace prefix: resources-custom-config- @@ -48,9 +37,30 @@ partitions_count: 3 type: output type: producer-app + values: + imageTag: latest + resources: + limits: + memory: 2G + requests: + memory: 2G + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + outputTopic: app1-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - _cleaner: - app: + name: app2 + namespace: development-namespace + prefix: resources-custom-config- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: some-image imageTag: latest labels: @@ -67,34 +77,7 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - name: app2 - namespace: development-namespace - prefix: resources-custom-config- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - image: some-image - imageTag: latest - labels: - pipeline: resources-custom-config - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: app2-dead-letter-topic - inputTopics: - - app1-test-topic - outputTopic: app2-test-topic - schemaRegistryUrl: http://localhost:8081/ name: app2 namespace: development-namespace prefix: resources-custom-config- @@ -117,5 +100,22 @@ partitions_count: 3 type: output type: streams-app + values: + image: some-image + imageTag: latest + labels: + pipeline: resources-custom-config + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: app2-dead-letter-topic + inputTopics: + - app1-test-topic + outputTopic: app2-test-topic + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index a37ad0f3d..cdb6f4f3a 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,5 +1,15 @@ - _cleaner: - app: + name: streams-app-development + namespace: development-namespace + prefix: resources-pipeline-with-env-defaults- + repo_config: + repo_auth_flags: + insecure_skip_tls_verify: false + repository_name: bakdata-streams-bootstrap + url: https://bakdata.github.io/streams-bootstrap/ + suffix: -clean + type: streams-app-cleaner + values: image: fake-image imageTag: latest persistence: @@ -14,32 +24,7 @@ - example-topic outputTopic: example-output schemaRegistryUrl: http://localhost:8081/ - name: streams-app-development - namespace: development-namespace - prefix: resources-pipeline-with-env-defaults- - repo_config: - repo_auth_flags: - insecure_skip_tls_verify: false - repository_name: bakdata-streams-bootstrap - url: https://bakdata.github.io/streams-bootstrap/ - suffix: -clean - type: streams-app-cleaner version: 2.9.0 - app: - image: fake-image - imageTag: latest - persistence: - enabled: false - statefulSet: false - streams: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-pipeline-with-env-defaults-streams-app-development-error - inputTopics: - - example-topic - outputTopic: example-output - schemaRegistryUrl: http://localhost:8081/ from: components: {} topics: @@ -66,13 +51,23 @@ type: error value_schema: com.bakdata.kafka.DeadLetter type: streams-app + values: + image: fake-image + imageTag: latest + persistence: + enabled: false + statefulSet: false + streams: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + errorTopic: resources-pipeline-with-env-defaults-streams-app-development-error + inputTopics: + - example-topic + outputTopic: example-output + schemaRegistryUrl: http://localhost:8081/ version: 2.9.0 - _resetter: - app: - config: - brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - connector: resources-pipeline-with-env-defaults-es-sink-connector - connectorType: sink name: es-sink-connector namespace: development-namespace prefix: resources-pipeline-with-env-defaults- @@ -83,8 +78,13 @@ url: https://bakdata.github.io/kafka-connect-resetter/ suffix: -clean type: kafka-connector-resetter + values: + config: + brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + connector: resources-pipeline-with-env-defaults-es-sink-connector + connectorType: sink version: 1.0.4 - app: + config: batch.size: '2000' behavior.on.malformed.documents: warn behavior.on.null.values: delete diff --git a/tests/pipeline/test_clean.py b/tests/pipeline/test_clean.py index 465e7dbc2..1f9325c59 100644 --- a/tests/pipeline/test_clean.py +++ b/tests/pipeline/test_clean.py @@ -32,29 +32,15 @@ def helm_mock(self, mocker: MockerFixture) -> MagicMock: # TODO: test using public Pipeline API # @pytest.fixture() - # def config(self) -> KpopsConfig: - # return KpopsConfig() - - # @pytest.fixture() - # def handlers(self) -> ComponentHandlers: - # return ComponentHandlers( - # schema_handler=AsyncMock(), - # connector_handler=AsyncMock(), - # topic_handler=AsyncMock(), - # ) - - # @pytest.fixture() - # def pipeline(self, config: KpopsConfig, handlers: ComponentHandlers) -> Pipeline: - # kwargs = {"config": config, "handlers": handlers} + # def pipeline(self) -> Pipeline: # pipeline = Pipeline() # pipeline.add( # ProducerApp( # name="producer", # namespace="test-namespace", - # app=ProducerAppValues( - # streams=ProducerStreamsConfig(brokers=config.kafka_brokers) + # values=ProducerAppValues( + # streams=ProducerStreamsConfig(brokers=get_config().kafka_brokers) # ), - # **kwargs, # ) # ) # return pipeline diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 4502ea69e..8557c6b2e 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -10,8 +10,8 @@ from kpops.components.base_components.models.to_section import ( ToSection, ) -from kpops.components.base_components.models.topic import OutputTopicTypes, TopicConfig from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.topic import OutputTopicTypes, TopicConfig from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp @@ -39,9 +39,7 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name=f"{self.name}-inflated-sink-connector", - config=self.config, - handlers=self.handlers, - app={ # type: ignore[reportGeneralTypeIssues], required `connector.class` comes from defaults during enrichment + config={ # type: ignore[reportGeneralTypeIssues], required `connector.class` comes from defaults during enrichment "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", }, @@ -59,8 +57,6 @@ def inflate(self) -> list[PipelineComponent]: inflate_steps.append(kafka_connector) streams_app = StreamsApp( name=f"{self.name}-inflated-streams-app", - config=self.config, - handlers=self.handlers, to=ToSection( # type: ignore[reportGeneralTypeIssues] topics={ TopicName( @@ -94,8 +90,6 @@ class SimpleInflateConnectors(StreamsApp): def inflate(self) -> list[PipelineComponent]: connector = KafkaSinkConnector( name="inflated-connector-name", - config=self.config, - handlers=self.handlers, - app={}, # type: ignore[reportArgumentType] + config={}, # type: ignore[reportArgumentType] ) return [self, connector] diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 9646e569b..5bb2e875a 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -2,8 +2,8 @@ from kpops.component_handlers.kafka_connect.model import KafkaConnectorConfig from kpops.components.base_components.kafka_connector import KafkaSinkConnector -from kpops.components.base_components.models.topic import OutputTopicTypes from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.topic import OutputTopicTypes from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp @@ -23,9 +23,7 @@ def inflate(self) -> list[PipelineComponent]: if topic_config.type == OutputTopicTypes.OUTPUT: kafka_connector = KafkaSinkConnector( name="sink-connector", - config=self.config, - handlers=self.handlers, - app=KafkaConnectorConfig( + config=KafkaConnectorConfig( **{ "topics": topic_name, "transforms.changeTopic.replacement": f"{topic_name}-index-v1", diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 1d2fce41f..c8aac3f0d 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -11,10 +11,10 @@ import kpops.api as kpops from kpops.api.exception import ParsingException, ValidationError -from kpops.api.file_type import PIPELINE_YAML, KpopsFileType from kpops.cli.main import FilterType, app from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.const.file_type import PIPELINE_YAML, KpopsFileType runner = CliRunner() @@ -171,12 +171,12 @@ def test_substitute_in_component(self, snapshot: Snapshot): ) assert enriched_pipeline[0]["name"] == "scheduled-producer" - labels = enriched_pipeline[0]["app"]["labels"] + labels = enriched_pipeline[0]["values"]["labels"] assert labels["app_name"] == "scheduled-producer" assert labels["app_type"] == "scheduled-producer" assert labels["app_schedule"] == "30 3/8 * * *" assert ( - enriched_pipeline[2]["app"]["labels"]["app_resources_requests_memory"] + enriched_pipeline[2]["values"]["labels"]["app_resources_requests_memory"] == "3G" ) assert ( @@ -188,7 +188,7 @@ def test_substitute_in_component(self, snapshot: Snapshot): in enriched_pipeline[1]["to"]["topics"] ) assert ( - enriched_pipeline[2]["app"]["labels"]["test_placeholder_in_placeholder"] + enriched_pipeline[2]["values"]["labels"]["test_placeholder_in_placeholder"] == "filter-app-filter" ) @@ -224,7 +224,7 @@ def test_kafka_connector_config_parsing(self): enriched_pipeline: list = yaml.safe_load(result.stdout) sink_connector = enriched_pipeline[0] assert ( - sink_connector["app"]["errors.deadletterqueue.topic.name"] + sink_connector["config"]["errors.deadletterqueue.topic.name"] == "kafka-sink-connector-error-topic" ) @@ -340,13 +340,13 @@ def test_with_custom_config_with_relative_defaults_path( enriched_pipeline: list = yaml.safe_load(result.stdout) producer_details = enriched_pipeline[0] - output_topic = producer_details["app"]["streams"]["outputTopic"] + output_topic = producer_details["values"]["streams"]["outputTopic"] assert output_topic == "app1-test-topic" streams_app_details = enriched_pipeline[1] - output_topic = streams_app_details["app"]["streams"]["outputTopic"] + output_topic = streams_app_details["values"]["streams"]["outputTopic"] assert output_topic == "app2-test-topic" - error_topic = streams_app_details["app"]["streams"]["errorTopic"] + error_topic = streams_app_details["values"]["streams"]["errorTopic"] assert error_topic == "app2-dead-letter-topic" snapshot.assert_match(result.stdout, PIPELINE_YAML) @@ -383,13 +383,13 @@ def test_with_custom_config_with_absolute_defaults_path( enriched_pipeline: list = yaml.safe_load(result.stdout) producer_details = enriched_pipeline[0] - output_topic = producer_details["app"]["streams"]["outputTopic"] + output_topic = producer_details["values"]["streams"]["outputTopic"] assert output_topic == "app1-test-topic" streams_app_details = enriched_pipeline[1] - output_topic = streams_app_details["app"]["streams"]["outputTopic"] + output_topic = streams_app_details["values"]["streams"]["outputTopic"] assert output_topic == "app2-test-topic" - error_topic = streams_app_details["app"]["streams"]["errorTopic"] + error_topic = streams_app_details["values"]["streams"]["errorTopic"] assert error_topic == "app2-dead-letter-topic" snapshot.assert_match(result.stdout, PIPELINE_YAML) @@ -412,13 +412,13 @@ def test_default_config(self, snapshot: Snapshot): enriched_pipeline: list = yaml.safe_load(result.stdout) producer_details = enriched_pipeline[0] - output_topic = producer_details["app"]["streams"]["outputTopic"] + output_topic = producer_details["values"]["streams"]["outputTopic"] assert output_topic == "resources-custom-config-app1" streams_app_details = enriched_pipeline[1] - output_topic = streams_app_details["app"]["streams"]["outputTopic"] + output_topic = streams_app_details["values"]["streams"]["outputTopic"] assert output_topic == "resources-custom-config-app2" - error_topic = streams_app_details["app"]["streams"]["errorTopic"] + error_topic = streams_app_details["values"]["streams"]["errorTopic"] assert error_topic == "resources-custom-config-app2-error" snapshot.assert_match(result.stdout, PIPELINE_YAML) @@ -440,7 +440,7 @@ def test_env_vars_precedence_over_config(self, monkeypatch: pytest.MonkeyPatch): ) assert result.exit_code == 0, result.stdout enriched_pipeline: list = yaml.safe_load(result.stdout) - assert enriched_pipeline[0]["app"]["streams"]["brokers"] == "env_broker" + assert enriched_pipeline[0]["values"]["streams"]["brokers"] == "env_broker" def test_nested_config_env_vars(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv( @@ -462,7 +462,7 @@ def test_nested_config_env_vars(self, monkeypatch: pytest.MonkeyPatch): assert result.exit_code == 0, result.stdout enriched_pipeline: list = yaml.safe_load(result.stdout) assert ( - enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[0]["values"]["streams"]["schemaRegistryUrl"] == "http://somename:1234/" ) @@ -484,7 +484,7 @@ def test_env_specific_config_env_def_in_env_var( assert result.exit_code == 0, result.stdout enriched_pipeline: list = yaml.safe_load(result.stdout) assert ( - enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[0]["values"]["streams"]["schemaRegistryUrl"] == "http://production:8081/" ) @@ -518,7 +518,8 @@ def test_env_specific_config_env_def_in_cli( assert result.exit_code == 0, result.stdout enriched_pipeline: list = yaml.safe_load(result.stdout) assert ( - enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] == expected_url + enriched_pipeline[0]["values"]["streams"]["schemaRegistryUrl"] + == expected_url ) def test_config_dir_doesnt_exist(self): @@ -570,7 +571,7 @@ def test_dotenv_support(self): enriched_pipeline: list = yaml.safe_load(result.stdout) assert ( - enriched_pipeline[1]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[1]["values"]["streams"]["schemaRegistryUrl"] == "http://notlocalhost:8081/" ) @@ -836,11 +837,11 @@ def test_substitution_in_inflated_component(self): assert result.exit_code == 0, result.stdout enriched_pipeline: list = yaml.safe_load(result.stdout) assert ( - enriched_pipeline[1]["_resetter"]["app"]["label"] + enriched_pipeline[1]["_resetter"]["values"]["label"] == "inflated-connector-name" ) assert ( - enriched_pipeline[1]["_resetter"]["app"]["imageTag"] + enriched_pipeline[1]["_resetter"]["values"]["imageTag"] == "override-default-image-tag" ) @@ -853,14 +854,16 @@ def test_substitution_in_resetter(self): assert isinstance(pipeline.components[0], KafkaSinkConnector) assert pipeline.components[0].name == "es-sink-connector" assert pipeline.components[0]._resetter.name == "es-sink-connector" - assert hasattr(pipeline.components[0]._resetter.app, "label") - assert pipeline.components[0]._resetter.app.label == "es-sink-connector" # type: ignore[reportGeneralTypeIssues] + assert hasattr(pipeline.components[0]._resetter.values, "label") + assert pipeline.components[0]._resetter.values.label == "es-sink-connector" # type: ignore[reportGeneralTypeIssues] enriched_pipeline: list = yaml.safe_load(pipeline.to_yaml()) assert enriched_pipeline[0]["name"] == "es-sink-connector" assert enriched_pipeline[0]["_resetter"]["name"] == "es-sink-connector" - assert enriched_pipeline[0]["_resetter"]["app"]["label"] == "es-sink-connector" assert ( - enriched_pipeline[0]["_resetter"]["app"]["imageTag"] + enriched_pipeline[0]["_resetter"]["values"]["label"] == "es-sink-connector" + ) + assert ( + enriched_pipeline[0]["_resetter"]["values"]["imageTag"] == "override-default-image-tag" ) diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index 61d338dcf..ee32d8cc9 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -1,12 +1,7 @@ -from unittest.mock import MagicMock - import pytest from polyfactory.factories.pydantic_factory import ModelFactory from kpops.api.options import FilterType -from kpops.component_handlers import ( - ComponentHandlers, -) from kpops.components.base_components.models.from_section import FromSection from kpops.components.base_components.models.to_section import ToSection from kpops.components.base_components.pipeline_component import PipelineComponent @@ -20,7 +15,6 @@ class TestComponentFactory(ModelFactory[PipelineComponent]): from_ = FromSection() enrich = False validate = False - handlers = ComponentHandlers(None, MagicMock(), MagicMock()) run_validation = False