From fd995935a0f43deba75ff7291132fe180e3c25f2 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 14 Aug 2024 09:38:39 +0200 Subject: [PATCH 01/19] Remove v3 and suffix old streams boostrap with v2 --- .../dependencies/kpops_structure.yaml | 28 +- docs/docs/schema/defaults.json | 632 +++++++++--------- docs/docs/schema/pipeline.json | 540 +++++++-------- .../components/streams_bootstrap/__init__.py | 10 - .../streams_bootstrap_v2/__init__.py | 10 + .../base.py} | 17 +- .../producer/__init__.py | 0 .../producer/model.py | 6 +- .../producer/producer_app.py | 15 +- .../streams/__init__.py | 0 .../streams/model.py | 12 +- .../streams/streams_app.py | 22 +- .../streams_bootstrap_v3/__init__.py | 8 +- kpops/components/streams_bootstrap_v3/base.py | 10 +- .../components/streams_bootstrap_v3/model.py | 2 +- .../streams_bootstrap_v3/producer/model.py | 4 +- .../producer/producer_app.py | 6 +- .../streams_bootstrap_v3/streams/model.py | 4 +- .../streams/streams_app.py | 6 +- tests/api/test_registry.py | 22 +- .../pipeline-deep/defaults.yaml | 6 +- .../streams_bootstrap_v3/test_producer_app.py | 30 +- .../streams_bootstrap_v3/test_streams_app.py | 48 +- .../test_streams_bootstrap.py | 16 +- tests/components/test_producer_app.py | 34 +- tests/components/test_streams_app.py | 62 +- tests/components/test_streams_bootstrap.py | 16 +- .../resources/custom-config/defaults.yaml | 8 +- .../resources/custom-config/pipeline.yaml | 4 +- tests/pipeline/resources/defaults.yaml | 6 +- .../kafka-connect-sink-config/defaults.yaml | 2 +- .../kafka-connect-sink/pipeline.yaml | 2 +- .../name_prefix_concatenation/pipeline.yaml | 2 +- .../no-input-topic-pipeline/pipeline.yaml | 4 +- .../no-topics-defaults/defaults.yaml | 4 +- .../no-user-defined-components/pipeline.yaml | 2 +- .../resources/parallel-pipeline/defaults.yaml | 2 +- .../resources/parallel-pipeline/pipeline.yaml | 12 +- .../pipeline.yaml | 2 +- .../pipeline-with-env-defaults/defaults.yaml | 4 +- .../pipeline-with-env-defaults/pipeline.yaml | 2 +- .../pipeline-with-loop/defaults.yaml | 2 +- .../pipeline-with-loop/pipeline.yaml | 6 +- .../pipeline-with-paths/pipeline.yaml | 2 +- .../pipeline-with-short-topics/defaults.yaml | 4 +- .../pipeline-with-short-topics/pipeline.yaml | 10 +- .../defaults.yaml | 2 +- .../pipeline.yaml | 2 +- .../simple-pipeline/defaults.yaml | 4 +- .../simple-pipeline/pipeline.yaml | 4 +- .../read-from-component/pipeline.yaml | 14 +- .../resources/simple-pipeline/pipeline.yaml | 4 +- .../streams-bootstrap-v3/defaults.yaml | 6 +- .../temp-trim-release-name/defaults.yaml | 2 +- .../temp-trim-release-name/pipeline.yaml | 2 +- .../test_generate/atm-fraud/pipeline.yaml | 11 +- .../test_generate/word-count/pipeline.yaml | 4 +- .../test_default_config/pipeline.yaml | 20 +- .../test_inflate_pipeline/pipeline.yaml | 2 +- .../pipeline.yaml | 12 +- .../test_model_serialization/pipeline.yaml | 2 +- .../test_no_input_topic/pipeline.yaml | 4 +- .../pipeline.yaml | 12 +- .../pipeline.yaml | 6 +- .../test_read_from_component/pipeline.yaml | 18 +- .../pipeline.yaml | 20 +- .../pipeline.yaml | 20 +- .../test_with_env_defaults/pipeline.yaml | 12 +- .../test_manifest_command/manifest.yaml | 8 +- .../test_manifest/test_python_api/resources | 8 +- tests/pipeline/test_clean.py | 12 +- tests/pipeline/test_components/components.py | 22 +- .../components.py | 10 +- tests/pipeline/test_deploy.py | 14 +- tests/pipeline/test_destroy.py | 14 +- tests/pipeline/test_manifest.py | 8 +- tests/pipeline/test_reset.py | 12 +- 77 files changed, 962 insertions(+), 975 deletions(-) delete mode 100644 kpops/components/streams_bootstrap/__init__.py create mode 100644 kpops/components/streams_bootstrap_v2/__init__.py rename kpops/components/{common/streams_bootstrap.py => streams_bootstrap_v2/base.py} (89%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/producer/__init__.py (100%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/producer/model.py (77%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/producer/producer_app.py (90%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/streams/__init__.py (100%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/streams/model.py (97%) rename kpops/components/{streams_bootstrap => streams_bootstrap_v2}/streams/streams_app.py (89%) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 85cd0a175..265cbf737 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -59,7 +59,7 @@ kpops_components_fields: - values - repo_config - version - producer-app-v3: + producer-app-v2: - name - prefix - from_ @@ -77,7 +77,7 @@ kpops_components_fields: - values - repo_config - version - streams-app-v3: + streams-app-v2: - name - prefix - from_ @@ -95,7 +95,7 @@ kpops_components_fields: - values - repo_config - version - streams-bootstrap-v3: + streams-bootstrap-v2: - name - prefix - from_ @@ -150,20 +150,19 @@ kpops_components_inheritance_ref: - base-defaults-component producer-app: bases: - - kafka-app - streams-bootstrap parents: - - kafka-app - streams-bootstrap + - kafka-app - helm-app - kubernetes-app - pipeline-component - base-defaults-component - producer-app-v3: + producer-app-v2: bases: - - streams-bootstrap-v3 + - streams-bootstrap-v2 parents: - - streams-bootstrap-v3 + - streams-bootstrap-v2 - kafka-app - helm-app - kubernetes-app @@ -171,20 +170,19 @@ kpops_components_inheritance_ref: - base-defaults-component streams-app: bases: - - kafka-app - streams-bootstrap parents: - - kafka-app - streams-bootstrap + - kafka-app - helm-app - kubernetes-app - pipeline-component - base-defaults-component - streams-app-v3: + streams-app-v2: bases: - - streams-bootstrap-v3 + - streams-bootstrap-v2 parents: - - streams-bootstrap-v3 + - streams-bootstrap-v2 - kafka-app - helm-app - kubernetes-app @@ -192,13 +190,15 @@ kpops_components_inheritance_ref: - base-defaults-component streams-bootstrap: bases: + - kafka-app - helm-app parents: + - kafka-app - helm-app - kubernetes-app - pipeline-component - base-defaults-component - streams-bootstrap-v3: + streams-bootstrap-v2: bases: - kafka-app - helm-app diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index d3dbc7905..a91f33dd3 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -896,23 +896,17 @@ "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerAppValues" + "$ref": "#/$defs/ProducerAppValues" } ], "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", + "default": "3.0.0", "description": "Helm chart version", - "title": "Version" + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -924,7 +918,7 @@ "title": "ProducerApp", "type": "object" }, - "ProducerAppV3": { + "ProducerAppV2": { "additionalProperties": true, "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { @@ -982,23 +976,29 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "producer-app-v3", + "const": "producer-app-v2", "title": "Type" }, "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues" + "$ref": "#/$defs/ProducerAppV2Values" } ], "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" + "title": "Version" } }, "required": [ @@ -1007,7 +1007,87 @@ "values", "type" ], - "title": "ProducerAppV3", + "title": "ProducerAppV2", + "type": "object" + }, + "ProducerAppV2Values": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap-v2 app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/ProducerStreamsConfig" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/ProducerConfig" + } + ], + "description": "Kafka Streams settings" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "ProducerAppValues", "type": "object" }, "ProducerConfig": { @@ -1248,23 +1328,17 @@ "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppValues" + "$ref": "#/$defs/StreamsAppValues" } ], "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", + "default": "3.0.0", "description": "Helm chart version", - "title": "Version" + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -1276,9 +1350,9 @@ "title": "StreamsApp", "type": "object" }, - "StreamsAppV3": { + "StreamsAppV2": { "additionalProperties": true, - "description": "StreamsApp component that configures a streams-bootstrap app.", + "description": "StreamsAppV2 component that configures a streams-bootstrap-v2 app.", "properties": { "from": { "anyOf": [ @@ -1341,23 +1415,29 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "streams-app-v3", + "const": "streams-app-v2", "title": "Type" }, "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "description": "streams-bootstrap Helm values" + "description": "streams-bootstrap-v2 Helm values" }, "version": { - "default": "3.0.0", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" + "title": "Version" } }, "required": [ @@ -1366,7 +1446,149 @@ "values", "type" ], - "title": "StreamsAppV3", + "title": "StreamsAppV2", + "type": "object" + }, + "StreamsAppV2Values": { + "additionalProperties": true, + "description": "streams-bootstrap-v2 app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap-v2 app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap-v2 streams section" + } + }, + "required": [ + "streams" + ], + "title": "StreamsAppV2Values", + "type": "object" + }, + "StreamsAppValues": { + "additionalProperties": true, + "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap kafka section" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + } + }, + "required": [ + "kafka" + ], + "title": "StreamsAppValues", "type": "object" }, "StreamsBootstrap": { @@ -1442,17 +1664,11 @@ "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", + "default": "3.0.0", "description": "Helm chart version", - "title": "Version" + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -1462,9 +1678,9 @@ "title": "StreamsBootstrap", "type": "object" }, - "StreamsBootstrapV3": { + "StreamsBootstrapV2": { "additionalProperties": true, - "description": "Base for components with a streams-bootstrap Helm chart.", + "description": "Base for components with a streams-bootstrap-v2 Helm chart.", "properties": { "from": { "anyOf": [ @@ -1529,45 +1745,43 @@ "values": { "allOf": [ { - "$ref": "#/$defs/StreamsBootstrapV3Values" + "$ref": "#/$defs/StreamsBootstrapV2Values" } ], - "description": "streams-bootstrap Helm values" + "description": "streams-bootstrap-v2 Helm values" }, "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, "required": [ "name", "namespace" ], - "title": "StreamsBootstrapV3", + "title": "StreamsBootstrapV2", "type": "object" }, - "StreamsBootstrapV3Values": { + "StreamsBootstrapV2Values": { "additionalProperties": true, - "description": "Base value class for all streams bootstrap related components.", + "description": "Base value class for all streams bootstrap v2 related components.", "properties": { "imageTag": { "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", + "description": "Docker image tag of the streams-bootstrap-v2 app.", "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/KafkaConfig" - } - ], - "description": "Kafka configuration for the streams-bootstrap app." - }, "nameOverride": { "anyOf": [ { @@ -1581,12 +1795,20 @@ "default": null, "description": "Helm chart name override, assigned automatically", "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/KafkaStreamsConfig" + } + ], + "description": "" } }, "required": [ - "kafka" + "streams" ], - "title": "StreamsBootstrapV3Values", + "title": "StreamsBootstrapV2Values", "type": "object" }, "StreamsBootstrapValues": { @@ -1600,6 +1822,14 @@ "title": "Imagetag", "type": "string" }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/KafkaConfig" + } + ], + "description": "Kafka configuration for the streams-bootstrap app." + }, "nameOverride": { "anyOf": [ { @@ -1613,18 +1843,10 @@ "default": null, "description": "Helm chart name override, assigned automatically", "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/KafkaStreamsConfig" - } - ], - "description": "" } }, "required": [ - "streams" + "kafka" ], "title": "StreamsBootstrapValues", "type": "object" @@ -1757,47 +1979,7 @@ "title": "TopicConfig", "type": "object" }, - "kpops__components__streams_bootstrap__producer__model__ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/ProducerStreamsConfig" - } - ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { @@ -1889,78 +2071,7 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap__streams__model__StreamsAppValues": { - "additionalProperties": true, - "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", - "properties": { - "autoscaling": { - "anyOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes event-driven autoscaling config" - }, - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "persistence": { - "allOf": [ - { - "$ref": "#/$defs/PersistenceConfig" - } - ], - "default": { - "enabled": false, - "size": null, - "storage_class": null - }, - "description": "" - }, - "statefulSet": { - "default": false, - "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", - "title": "Statefulset", - "type": "boolean" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" - } - ], - "description": "streams-bootstrap streams section" - } - }, - "required": [ - "streams" - ], - "title": "StreamsAppValues", - "type": "object" - }, - "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { "additionalProperties": true, "description": "Streams Bootstrap streams section.", "properties": { @@ -2084,46 +2195,6 @@ "title": "StreamsConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -2221,77 +2292,6 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues": { - "additionalProperties": true, - "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", - "properties": { - "autoscaling": { - "anyOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes event-driven autoscaling config" - }, - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" - } - ], - "description": "streams-bootstrap kafka section" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "persistence": { - "allOf": [ - { - "$ref": "#/$defs/PersistenceConfig" - } - ], - "default": { - "enabled": false, - "size": null, - "storage_class": null - }, - "description": "" - }, - "statefulSet": { - "default": false, - "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", - "title": "Statefulset", - "type": "boolean" - } - }, - "required": [ - "kafka" - ], - "title": "StreamsAppValues", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { "additionalProperties": true, "description": "Streams Bootstrap kafka section.", @@ -2455,20 +2455,20 @@ "producer-app": { "$ref": "#/$defs/ProducerApp" }, - "producer-app-v3": { - "$ref": "#/$defs/ProducerAppV3" + "producer-app-v2": { + "$ref": "#/$defs/ProducerAppV2" }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, - "streams-app-v3": { - "$ref": "#/$defs/StreamsAppV3" + "streams-app-v2": { + "$ref": "#/$defs/StreamsAppV2" }, "streams-bootstrap": { "$ref": "#/$defs/StreamsBootstrap" }, - "streams-bootstrap-v3": { - "$ref": "#/$defs/StreamsBootstrapV3" + "streams-bootstrap-v2": { + "$ref": "#/$defs/StreamsBootstrapV2" } }, "required": [ @@ -2479,12 +2479,12 @@ "kafka-source-connector", "kubernetes-app", "pipeline-component", + "producer-app-v2", + "streams-app-v2", + "streams-bootstrap-v2", "producer-app", "streams-app", - "streams-bootstrap", - "producer-app-v3", - "streams-app-v3", - "streams-bootstrap-v3" + "streams-bootstrap" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 86edc0b8e..1b9400770 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -556,23 +556,17 @@ "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__producer__model__ProducerAppValues" + "$ref": "#/$defs/ProducerAppValues" } ], "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", + "default": "3.0.0", "description": "Helm chart version", - "title": "Version" + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -584,7 +578,7 @@ "title": "ProducerApp", "type": "object" }, - "ProducerAppV3": { + "ProducerAppV2": { "additionalProperties": true, "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { @@ -642,23 +636,29 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "producer-app-v3", + "const": "producer-app-v2", "title": "Type" }, "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues" + "$ref": "#/$defs/ProducerAppV2Values" } ], "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" + "title": "Version" } }, "required": [ @@ -667,7 +667,87 @@ "values", "type" ], - "title": "ProducerAppV3", + "title": "ProducerAppV2", + "type": "object" + }, + "ProducerAppV2Values": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap-v2 app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/ProducerStreamsConfig" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/ProducerConfig" + } + ], + "description": "Kafka Streams settings" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "ProducerAppValues", "type": "object" }, "ProducerConfig": { @@ -908,23 +988,17 @@ "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppValues" + "$ref": "#/$defs/StreamsAppValues" } ], "description": "streams-bootstrap Helm values" }, "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", + "default": "3.0.0", "description": "Helm chart version", - "title": "Version" + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" } }, "required": [ @@ -936,9 +1010,9 @@ "title": "StreamsApp", "type": "object" }, - "StreamsAppV3": { + "StreamsAppV2": { "additionalProperties": true, - "description": "StreamsApp component that configures a streams-bootstrap app.", + "description": "StreamsAppV2 component that configures a streams-bootstrap-v2 app.", "properties": { "from": { "anyOf": [ @@ -1001,23 +1075,29 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "streams-app-v3", + "const": "streams-app-v2", "title": "Type" }, "values": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "description": "streams-bootstrap Helm values" + "description": "streams-bootstrap-v2 Helm values" }, "version": { - "default": "3.0.0", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" + "title": "Version" } }, "required": [ @@ -1026,7 +1106,149 @@ "values", "type" ], - "title": "StreamsAppV3", + "title": "StreamsAppV2", + "type": "object" + }, + "StreamsAppV2Values": { + "additionalProperties": true, + "description": "streams-bootstrap-v2 app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap-v2 app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap-v2 streams section" + } + }, + "required": [ + "streams" + ], + "title": "StreamsAppV2Values", + "type": "object" + }, + "StreamsAppValues": { + "additionalProperties": true, + "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", + "properties": { + "autoscaling": { + "anyOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kubernetes event-driven autoscaling config" + }, + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" + }, + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + } + ], + "description": "streams-bootstrap kafka section" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + }, + "persistence": { + "allOf": [ + { + "$ref": "#/$defs/PersistenceConfig" + } + ], + "default": { + "enabled": false, + "size": null, + "storage_class": null + }, + "description": "" + }, + "statefulSet": { + "default": false, + "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", + "title": "Statefulset", + "type": "boolean" + } + }, + "required": [ + "kafka" + ], + "title": "StreamsAppValues", "type": "object" }, "ToSection": { @@ -1157,47 +1379,7 @@ "title": "TopicConfig", "type": "object" }, - "kpops__components__streams_bootstrap__producer__model__ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/ProducerStreamsConfig" - } - ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { @@ -1289,78 +1471,7 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap__streams__model__StreamsAppValues": { - "additionalProperties": true, - "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", - "properties": { - "autoscaling": { - "anyOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes event-driven autoscaling config" - }, - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "persistence": { - "allOf": [ - { - "$ref": "#/$defs/PersistenceConfig" - } - ], - "default": { - "enabled": false, - "size": null, - "storage_class": null - }, - "description": "" - }, - "statefulSet": { - "default": false, - "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", - "title": "Statefulset", - "type": "boolean" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" - } - ], - "description": "streams-bootstrap streams section" - } - }, - "required": [ - "streams" - ], - "title": "StreamsAppValues", - "type": "object" - }, - "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { "additionalProperties": true, "description": "Streams Bootstrap streams section.", "properties": { @@ -1484,46 +1595,6 @@ "title": "StreamsConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v3__producer__model__ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", @@ -1621,77 +1692,6 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppValues": { - "additionalProperties": true, - "description": "streams-bootstrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", - "properties": { - "autoscaling": { - "anyOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Kubernetes event-driven autoscaling config" - }, - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" - } - ], - "description": "streams-bootstrap kafka section" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - }, - "persistence": { - "allOf": [ - { - "$ref": "#/$defs/PersistenceConfig" - } - ], - "default": { - "enabled": false, - "size": null, - "storage_class": null - }, - "description": "" - }, - "statefulSet": { - "default": false, - "description": "Whether to use a Statefulset instead of a Deployment to deploy the streams app.", - "title": "Statefulset", - "type": "boolean" - } - }, - "required": [ - "kafka" - ], - "title": "StreamsAppValues", - "type": "object" - }, "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { "additionalProperties": true, "description": "Streams Bootstrap kafka section.", @@ -1837,9 +1837,9 @@ "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", "producer-app": "#/$defs/ProducerApp", - "producer-app-v3": "#/$defs/ProducerAppV3", + "producer-app-v2": "#/$defs/ProducerAppV2", "streams-app": "#/$defs/StreamsApp", - "streams-app-v3": "#/$defs/StreamsAppV3" + "streams-app-v2": "#/$defs/StreamsAppV2" }, "propertyName": "type" }, @@ -1854,16 +1854,16 @@ "$ref": "#/$defs/KafkaSourceConnector" }, { - "$ref": "#/$defs/ProducerApp" + "$ref": "#/$defs/ProducerAppV2" }, { - "$ref": "#/$defs/StreamsApp" + "$ref": "#/$defs/StreamsAppV2" }, { - "$ref": "#/$defs/ProducerAppV3" + "$ref": "#/$defs/ProducerApp" }, { - "$ref": "#/$defs/StreamsAppV3" + "$ref": "#/$defs/StreamsApp" } ] }, diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py deleted file mode 100644 index 19341d74b..000000000 --- a/kpops/components/streams_bootstrap/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from kpops.components.common.streams_bootstrap import StreamsBootstrap - -from .producer.producer_app import ProducerApp -from .streams.streams_app import StreamsApp - -__all__ = ( - "StreamsBootstrap", - "StreamsApp", - "ProducerApp", -) diff --git a/kpops/components/streams_bootstrap_v2/__init__.py b/kpops/components/streams_bootstrap_v2/__init__.py new file mode 100644 index 000000000..6668c707a --- /dev/null +++ b/kpops/components/streams_bootstrap_v2/__init__.py @@ -0,0 +1,10 @@ +from kpops.components.streams_bootstrap_v2.base import StreamsBootstrapV2 + +from .producer.producer_app import ProducerAppV2 +from .streams.streams_app import StreamsAppV2 + +__all__ = ( + "StreamsBootstrapV2", + "StreamsAppV2", + "ProducerAppV2", +) diff --git a/kpops/components/common/streams_bootstrap.py b/kpops/components/streams_bootstrap_v2/base.py similarity index 89% rename from kpops/components/common/streams_bootstrap.py rename to kpops/components/streams_bootstrap_v2/base.py index 0ac0af5f4..92b0ced2f 100644 --- a/kpops/components/common/streams_bootstrap.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -8,6 +8,7 @@ from pydantic import AliasChoices, ConfigDict, Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig +from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.utils.docstring import describe_attr @@ -94,10 +95,10 @@ def serialize_model( ) -class StreamsBootstrapValues(HelmAppValues): - """Base value class for all streams bootstrap related components. +class StreamsBootstrapV2Values(HelmAppValues): + """Base value class for all streams bootstrap v2 related components. - :param image_tag: Docker image tag of the streams-bootstrap app. + :param image_tag: Docker image tag of the streams-bootstrap-v2 app. """ image_tag: str = Field( @@ -111,17 +112,17 @@ class StreamsBootstrapValues(HelmAppValues): ) -class StreamsBootstrap(HelmApp, ABC): - """Base for components with a streams-bootstrap Helm chart. +class StreamsBootstrapV2(KafkaApp, HelmApp, ABC): + """Base for components with a streams-bootstrap-v2 Helm chart. - :param values: streams-bootstrap Helm values + :param values: streams-bootstrap-v2 Helm values :param repo_config: Configuration of the Helm chart repo to be used for deploying the component, defaults to streams-bootstrap Helm repo :param version: Helm chart version, defaults to "2.9.0" """ - values: StreamsBootstrapValues = Field( - default_factory=StreamsBootstrapValues, + values: StreamsBootstrapV2Values = Field( + default_factory=StreamsBootstrapV2Values, description=describe_attr("values", __doc__), ) diff --git a/kpops/components/streams_bootstrap/producer/__init__.py b/kpops/components/streams_bootstrap_v2/producer/__init__.py similarity index 100% rename from kpops/components/streams_bootstrap/producer/__init__.py rename to kpops/components/streams_bootstrap_v2/producer/__init__.py diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap_v2/producer/model.py similarity index 77% rename from kpops/components/streams_bootstrap/producer/model.py rename to kpops/components/streams_bootstrap_v2/producer/model.py index 1e14d2a92..005b7f603 100644 --- a/kpops/components/streams_bootstrap/producer/model.py +++ b/kpops/components/streams_bootstrap_v2/producer/model.py @@ -1,8 +1,8 @@ from pydantic import ConfigDict, Field -from kpops.components.common.streams_bootstrap import ( +from kpops.components.streams_bootstrap_v2.base import ( KafkaStreamsConfig, - StreamsBootstrapValues, + StreamsBootstrapV2Values, ) from kpops.utils.docstring import describe_attr @@ -11,7 +11,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): """Kafka Streams settings specific to Producer.""" -class ProducerAppValues(StreamsBootstrapValues): +class ProducerAppV2Values(StreamsBootstrapV2Values): """Settings specific to producers. :param streams: Kafka Streams settings diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap_v2/producer/producer_app.py similarity index 90% rename from kpops/components/streams_bootstrap/producer/producer_app.py rename to kpops/components/streams_bootstrap_v2/producer/producer_app.py index 6964b0244..1ce87d711 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v2/producer/producer_app.py @@ -5,25 +5,24 @@ from typing_extensions import override from kpops.components.base_components.kafka_app import ( - KafkaApp, KafkaAppCleaner, ) from kpops.components.common.app_type import AppType -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.producer.model import ProducerAppValues +from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 +from kpops.components.streams_bootstrap_v2.producer.model import ProducerAppV2Values from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr -log = logging.getLogger("ProducerApp") +log = logging.getLogger("ProducerAppV2") -class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrap): - values: ProducerAppValues +class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrapV2): + values: ProducerAppV2Values @property @override @@ -33,7 +32,7 @@ def helm_chart(self) -> str: ) -class ProducerApp(KafkaApp, StreamsBootstrap): +class ProducerAppV2(StreamsBootstrapV2): """Producer component. This producer holds configuration to use as values for the streams-bootstrap @@ -45,7 +44,7 @@ class ProducerApp(KafkaApp, StreamsBootstrap): :param from_: Producer doesn't support FromSection, defaults to None """ - values: ProducerAppValues = Field( + values: ProducerAppV2Values = Field( default=..., description=describe_attr("values", __doc__), ) diff --git a/kpops/components/streams_bootstrap/streams/__init__.py b/kpops/components/streams_bootstrap_v2/streams/__init__.py similarity index 100% rename from kpops/components/streams_bootstrap/streams/__init__.py rename to kpops/components/streams_bootstrap_v2/streams/__init__.py diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap_v2/streams/model.py similarity index 97% rename from kpops/components/streams_bootstrap/streams/model.py rename to kpops/components/streams_bootstrap_v2/streams/model.py index b3871d4b0..26e45ccf5 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap_v2/streams/model.py @@ -6,11 +6,11 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from kpops.api.exception import ValidationError -from kpops.components.common.streams_bootstrap import ( +from kpops.components.common.topic import KafkaTopic, KafkaTopicStr +from kpops.components.streams_bootstrap_v2.base import ( KafkaStreamsConfig, - StreamsBootstrapValues, + StreamsBootstrapV2Values, ) -from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, @@ -239,12 +239,12 @@ def validate_mandatory_fields_are_set( return self -class StreamsAppValues(StreamsBootstrapValues): - """streams-bootstrap app configurations. +class StreamsAppV2Values(StreamsBootstrapV2Values): + """streams-bootstrap-v2 app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. - :param streams: streams-bootstrap streams section + :param streams: streams-bootstrap-v2 streams section :param autoscaling: Kubernetes event-driven autoscaling config, defaults to None """ diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap_v2/streams/streams_app.py similarity index 89% rename from kpops/components/streams_bootstrap/streams/streams_app.py rename to kpops/components/streams_bootstrap_v2/streams/streams_app.py index 67a05a88a..3855a86f7 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v2/streams/streams_app.py @@ -6,23 +6,23 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kafka_app import KafkaApp, KafkaAppCleaner +from kpops.components.base_components.kafka_app import KafkaAppCleaner from kpops.components.common.app_type import AppType -from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.common.topic import KafkaTopic -from kpops.components.streams_bootstrap.streams.model import ( - StreamsAppValues, +from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 +from kpops.components.streams_bootstrap_v2.streams.model import ( + StreamsAppV2Values, ) from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr -log = logging.getLogger("StreamsApp") +log = logging.getLogger("StreamsAppV2") -class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrap): +class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrapV2): from_: None = None to: None = None - values: StreamsAppValues + values: StreamsAppV2Values @property @override @@ -53,13 +53,13 @@ async def clean_pvcs(self, dry_run: bool) -> None: await pvc_handler.delete_pvcs() -class StreamsApp(KafkaApp, StreamsBootstrap): - """StreamsApp component that configures a streams-bootstrap app. +class StreamsAppV2(StreamsBootstrapV2): + """StreamsAppV2 component that configures a streams-bootstrap-v2 app. - :param values: streams-bootstrap Helm values + :param values: streams-bootstrap-v2 Helm values """ - values: StreamsAppValues = Field( + values: StreamsAppV2Values = Field( default=..., description=describe_attr("values", __doc__), ) diff --git a/kpops/components/streams_bootstrap_v3/__init__.py b/kpops/components/streams_bootstrap_v3/__init__.py index 07b11bf21..ad2ae6711 100644 --- a/kpops/components/streams_bootstrap_v3/__init__.py +++ b/kpops/components/streams_bootstrap_v3/__init__.py @@ -1,6 +1,6 @@ -from kpops.components.streams_bootstrap_v3.base import StreamsBootstrapV3 +from kpops.components.streams_bootstrap_v3.base import StreamsBootstrap -from .producer.producer_app import ProducerAppV3 -from .streams.streams_app import StreamsAppV3 +from .producer.producer_app import ProducerApp +from .streams.streams_app import StreamsApp -__all__ = ("StreamsBootstrapV3", "StreamsAppV3", "ProducerAppV3") +__all__ = ("StreamsBootstrap", "StreamsApp", "ProducerApp") diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap_v3/base.py index 26ee45d24..fb252f087 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap_v3/base.py @@ -11,7 +11,7 @@ from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp -from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapV3Values +from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapValues from kpops.utils.docstring import describe_attr if TYPE_CHECKING: @@ -30,10 +30,10 @@ STREAMS_BOOTSTRAP_VERSION_PATTERN = r"^(\d+)\.(\d+)\.(\d+)(-[a-zA-Z]+(\.[a-zA-Z]+)?)?$" COMPILED_VERSION_PATTERN = re.compile(STREAMS_BOOTSTRAP_VERSION_PATTERN) -log = logging.getLogger("StreamsBootstrapV3") +log = logging.getLogger("StreamsBootstrap") -class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): +class StreamsBootstrap(KafkaApp, HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. :param values: streams-bootstrap Helm values @@ -42,8 +42,8 @@ class StreamsBootstrapV3(KafkaApp, HelmApp, ABC): :param version: Helm chart version, defaults to "3.0.0" """ - values: StreamsBootstrapV3Values = Field( - default_factory=StreamsBootstrapV3Values, + values: StreamsBootstrapValues = Field( + default_factory=StreamsBootstrapValues, description=describe_attr("values", __doc__), ) diff --git a/kpops/components/streams_bootstrap_v3/model.py b/kpops/components/streams_bootstrap_v3/model.py index 10911a2e0..49b613357 100644 --- a/kpops/components/streams_bootstrap_v3/model.py +++ b/kpops/components/streams_bootstrap_v3/model.py @@ -19,7 +19,7 @@ IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" -class StreamsBootstrapV3Values(HelmAppValues): +class StreamsBootstrapValues(HelmAppValues): """Base value class for all streams bootstrap related components. :param image_tag: Docker image tag of the streams-bootstrap app. diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap_v3/producer/model.py index d4111af1a..87e4fb4d1 100644 --- a/kpops/components/streams_bootstrap_v3/producer/model.py +++ b/kpops/components/streams_bootstrap_v3/producer/model.py @@ -2,7 +2,7 @@ from kpops.components.streams_bootstrap_v3.model import ( KafkaConfig, - StreamsBootstrapV3Values, + StreamsBootstrapValues, ) from kpops.utils.docstring import describe_attr @@ -11,7 +11,7 @@ class ProducerConfig(KafkaConfig): """Kafka Streams settings specific to Producer.""" -class ProducerAppValues(StreamsBootstrapV3Values): +class ProducerAppValues(StreamsBootstrapValues): """Settings specific to producers. :param kafka: Kafka Streams settings diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap_v3/producer/producer_app.py index 97df4a632..e402fb86d 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v3/producer/producer_app.py @@ -12,7 +12,7 @@ TopicConfig, ) from kpops.components.streams_bootstrap_v3.base import ( - StreamsBootstrapV3, + StreamsBootstrap, ) from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr @@ -20,7 +20,7 @@ log = logging.getLogger("ProducerAppV3") -class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): +class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrap): values: ProducerAppValues @property @@ -31,7 +31,7 @@ def helm_chart(self) -> str: ) -class ProducerAppV3(StreamsBootstrapV3): +class ProducerApp(StreamsBootstrap): """Producer component. This producer holds configuration to use as values for the streams-bootstrap diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap_v3/streams/model.py index 2d641660b..30ad4151e 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap_v3/streams/model.py @@ -8,7 +8,7 @@ from kpops.components.common.topic import KafkaTopic, KafkaTopicStr from kpops.components.streams_bootstrap_v3.model import ( KafkaConfig, - StreamsBootstrapV3Values, + StreamsBootstrapValues, ) from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( @@ -224,7 +224,7 @@ class PersistenceConfig(BaseModel): ) -class StreamsAppValues(StreamsBootstrapV3Values): +class StreamsAppValues(StreamsBootstrapValues): """streams-bootstrap app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap_v3/streams/streams_app.py index 56fde5aac..030344140 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v3/streams/streams_app.py @@ -10,7 +10,7 @@ from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap_v3.base import ( - StreamsBootstrapV3, + StreamsBootstrap, ) from kpops.components.streams_bootstrap_v3.streams.model import ( StreamsAppValues, @@ -20,7 +20,7 @@ log = logging.getLogger("StreamsAppV3") -class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrapV3): +class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrap): from_: None = None to: None = None values: StreamsAppValues @@ -54,7 +54,7 @@ async def clean_pvcs(self, dry_run: bool) -> None: await pvc_handler.delete_pvcs() -class StreamsAppV3(StreamsBootstrapV3): +class StreamsApp(StreamsBootstrap): """StreamsApp component that configures a streams-bootstrap app. :param values: streams-bootstrap Helm values diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 9429dab83..e0ebc67a5 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -18,13 +18,13 @@ ) from kpops.components.base_components.kubernetes_app import KubernetesApp from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.common.streams_bootstrap import StreamsBootstrap -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp +from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 +from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 +from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 from kpops.components.streams_bootstrap_v3 import ( - ProducerAppV3, - StreamsAppV3, - StreamsBootstrapV3, + ProducerApp, + StreamsApp, + StreamsBootstrap, ) from tests.cli.resources.custom_module import CustomSchemaProvider @@ -54,7 +54,7 @@ def test_iter_namespace(): assert [module.__name__ for module in _iter_namespace(components_module)] == [ "kpops.components.base_components", "kpops.components.common", - "kpops.components.streams_bootstrap", + "kpops.components.streams_bootstrap_v2", "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] @@ -66,7 +66,7 @@ def test_iter_component_modules(): "kpops.components", "kpops.components.base_components", "kpops.components.common", - "kpops.components.streams_bootstrap", + "kpops.components.streams_bootstrap_v2", "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] @@ -106,12 +106,12 @@ def test_registry(): "kubernetes-app": KubernetesApp, "pipeline-component": PipelineComponent, # TODO: change the old sterams bootstrap to -v2 and remove -v3 + "producer-app-v2": ProducerAppV2, "producer-app": ProducerApp, - "producer-app-v3": ProducerAppV3, + "streams-app-v2": StreamsAppV2, "streams-app": StreamsApp, - "streams-app-v3": StreamsAppV3, + "streams-bootstrap-v2": StreamsBootstrapV2, "streams-bootstrap": StreamsBootstrap, - "streams-bootstrap-v3": StreamsBootstrapV3, } for _type, _class in registry._classes.items(): assert registry[_type] is _class diff --git a/tests/components/resources/pipelines/test-distributed-defaults/pipeline-deep/defaults.yaml b/tests/components/resources/pipelines/test-distributed-defaults/pipeline-deep/defaults.yaml index 5f084bfcf..e19704b3e 100644 --- a/tests/components/resources/pipelines/test-distributed-defaults/pipeline-deep/defaults.yaml +++ b/tests/components/resources/pipelines/test-distributed-defaults/pipeline-deep/defaults.yaml @@ -2,14 +2,14 @@ helm-app: name: ${component.type} namespace: namespace-to-override-it-all -streams-bootstrap: +streams-bootstrap-v2: app: streams: brokers: "${config.kafka_brokers}" -producer-app: {} # inherits from kafka-app +producer-app-v2: {} # inherits from streams-bootstrap-v2 -streams-app: # inherits from kafka-app +streams-app-v2: # inherits from streams-bootstrap-v2 to: topics: ${error_topic_name}: diff --git a/tests/components/streams_bootstrap_v3/test_producer_app.py b/tests/components/streams_bootstrap_v3/test_producer_app.py index 78964033d..1b6d3158d 100644 --- a/tests/components/streams_bootstrap_v3/test_producer_app.py +++ b/tests/components/streams_bootstrap_v3/test_producer_app.py @@ -14,8 +14,8 @@ TopicConfig, ) from kpops.components.streams_bootstrap_v3.producer.producer_app import ( + ProducerApp, ProducerAppCleaner, - ProducerAppV3, ) PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" @@ -39,8 +39,8 @@ def test_release_name(self): assert PRODUCER_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def producer_app(self) -> ProducerAppV3: - return ProducerAppV3( + def producer_app(self) -> ProducerApp: + return ProducerApp( name=PRODUCER_APP_NAME, **{ "version": "3.2.1", @@ -63,28 +63,28 @@ def producer_app(self) -> ProducerAppV3: def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) - def test_cleaner(self, producer_app: ProducerAppV3): + def test_cleaner(self, producer_app: ProducerApp): cleaner = producer_app._cleaner assert isinstance(cleaner, ProducerAppCleaner) assert not hasattr(cleaner, "_cleaner") - def test_cleaner_inheritance(self, producer_app: ProducerAppV3): + def test_cleaner_inheritance(self, producer_app: ProducerApp): assert producer_app._cleaner.values == producer_app.values - def test_cleaner_helm_release_name(self, producer_app: ProducerAppV3): + def test_cleaner_helm_release_name(self, producer_app: ProducerApp): assert ( producer_app._cleaner.helm_release_name == "${pipeline.name}-test-producer-app-with-l-abc43-clean" ) - def test_cleaner_helm_name_override(self, producer_app: ProducerAppV3): + def test_cleaner_helm_name_override(self, producer_app: ProducerApp): assert ( producer_app._cleaner.to_helm_values()["nameOverride"] == PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE ) def test_output_topics(self): - producer_app = ProducerAppV3( + producer_app = ProducerApp( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", @@ -116,7 +116,7 @@ def test_output_topics(self): @pytest.mark.asyncio() async def test_deploy_order_when_dry_run_is_false( self, - producer_app: ProducerAppV3, + producer_app: ProducerApp, mocker: MockerFixture, ): mock_create_topic = mocker.patch.object( @@ -167,7 +167,7 @@ async def test_deploy_order_when_dry_run_is_false( @pytest.mark.asyncio() async def test_destroy( self, - producer_app: ProducerAppV3, + producer_app: ProducerApp, mocker: MockerFixture, ): mock_helm_uninstall = mocker.patch.object(producer_app.helm, "uninstall") @@ -181,7 +181,7 @@ async def test_destroy( @pytest.mark.asyncio() async def test_should_clean_producer_app( self, - producer_app: ProducerAppV3, + producer_app: ProducerApp, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -260,7 +260,7 @@ async def test_should_clean_producer_app( async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_with_dry_run_false( self, mocker: MockerFixture, - producer_app: ProducerAppV3, + producer_app: ProducerApp, empty_helm_get_values: MockerFixture, ): # actual component @@ -326,7 +326,7 @@ async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clea ) def test_get_output_topics(self): - producer_app = ProducerAppV3( + producer_app = ProducerApp( name="my-producer", **{ "namespace": "test-namespace", @@ -378,7 +378,7 @@ async def test_should_not_deploy_clean_up_when_rest(self, mocker: MockerFixture) }, }, ) - producer_app = ProducerAppV3( + producer_app = ProducerApp( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", @@ -428,7 +428,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, }, ) - producer_app = ProducerAppV3( + producer_app = ProducerApp( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", diff --git a/tests/components/streams_bootstrap_v3/test_streams_app.py b/tests/components/streams_bootstrap_v3/test_streams_app.py index 17b3521ea..2fcf49027 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_app.py +++ b/tests/components/streams_bootstrap_v3/test_streams_app.py @@ -20,7 +20,7 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap_v3 import StreamsAppV3 +from kpops.components.streams_bootstrap_v3 import StreamsApp from kpops.components.streams_bootstrap_v3.streams.model import ( StreamsAppAutoScaling, ) @@ -53,8 +53,8 @@ def test_release_name(self): assert STREAMS_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def streams_app(self) -> StreamsAppV3: - return StreamsAppV3( + def streams_app(self) -> StreamsApp: + return StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -72,8 +72,8 @@ def streams_app(self) -> StreamsAppV3: ) @pytest.fixture() - def stateful_streams_app(self) -> StreamsAppV3: - return StreamsAppV3( + def stateful_streams_app(self) -> StreamsApp: + return StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -104,12 +104,12 @@ def dry_run_handler_mock(self, mocker: MockerFixture) -> MagicMock: def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) - def test_cleaner(self, streams_app: StreamsAppV3): + def test_cleaner(self, streams_app: StreamsApp): cleaner = streams_app._cleaner assert isinstance(cleaner, StreamsAppCleaner) assert not hasattr(cleaner, "_cleaner") - def test_cleaner_inheritance(self, streams_app: StreamsAppV3): + def test_cleaner_inheritance(self, streams_app: StreamsApp): streams_app.values.kafka.application_id = "test-application-id" streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, @@ -118,20 +118,20 @@ def test_cleaner_inheritance(self, streams_app: StreamsAppV3): ) assert streams_app._cleaner.values == streams_app.values - def test_cleaner_helm_release_name(self, streams_app: StreamsAppV3): + def test_cleaner_helm_release_name(self, streams_app: StreamsApp): assert ( streams_app._cleaner.helm_release_name == "${pipeline.name}-test-streams-app-with-lo-c98c5-clean" ) - def test_cleaner_helm_name_override(self, streams_app: StreamsAppV3): + def test_cleaner_helm_name_override(self, streams_app: StreamsApp): assert ( streams_app._cleaner.to_helm_values()["nameOverride"] == STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE ) def test_set_topics(self): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -177,7 +177,7 @@ def test_set_topics(self): assert "labeledInputPatterns" in kafka_config def test_no_empty_input_topic(self): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -208,7 +208,7 @@ def test_should_validate(self): with pytest.raises( ValueError, match="Define role only if `type` is `pattern` or `None`" ): - StreamsAppV3( + StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -230,7 +230,7 @@ def test_should_validate(self): with pytest.raises( ValueError, match="Define `role` only if `type` is undefined" ): - StreamsAppV3( + StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -249,7 +249,7 @@ def test_should_validate(self): ) def test_set_streams_output_from_to(self): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -288,7 +288,7 @@ def test_set_streams_output_from_to(self): ) def test_weave_inputs_from_prev_component(self): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -325,7 +325,7 @@ def test_weave_inputs_from_prev_component(self): @pytest.mark.asyncio() async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -437,7 +437,7 @@ async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): @pytest.mark.asyncio() async def test_destroy( self, - streams_app: StreamsAppV3, + streams_app: StreamsApp, mocker: MockerFixture, ): mock_helm_uninstall = mocker.patch.object(streams_app.helm, "uninstall") @@ -451,7 +451,7 @@ async def test_destroy( @pytest.mark.asyncio() async def test_reset_when_dry_run_is_false( self, - streams_app: StreamsAppV3, + streams_app: StreamsApp, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -518,7 +518,7 @@ async def test_reset_when_dry_run_is_false( @pytest.mark.asyncio() async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( self, - streams_app: StreamsAppV3, + streams_app: StreamsApp, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -604,7 +604,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( }, }, ) - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -683,7 +683,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, }, ) - streams_app = StreamsAppV3( + streams_app = StreamsApp( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -741,7 +741,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( @pytest.mark.asyncio() async def test_get_input_output_topics(self): - streams_app = StreamsAppV3( + streams_app = StreamsApp( name="my-app", **{ "namespace": "test-namespace", @@ -801,7 +801,7 @@ async def test_get_input_output_topics(self): @pytest.mark.asyncio() async def test_stateful_clean_with_dry_run_false( self, - stateful_streams_app: StreamsAppV3, + stateful_streams_app: StreamsApp, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -880,7 +880,7 @@ async def test_stateful_clean_with_dry_run_false( @pytest.mark.asyncio() async def test_stateful_clean_with_dry_run_true( self, - stateful_streams_app: StreamsAppV3, + stateful_streams_app: StreamsApp, empty_helm_get_values: MockerFixture, mocker: MockerFixture, caplog: pytest.LogCaptureFixture, diff --git a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py index cdb892119..176ac71b3 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py @@ -9,14 +9,14 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.streams_bootstrap_v3.base import StreamsBootstrapV3 -from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapV3Values +from kpops.components.streams_bootstrap_v3.base import StreamsBootstrap +from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapValues @pytest.mark.usefixtures("mock_env") class TestStreamsBootstrap: def test_default_configs(self): - streams_bootstrap = StreamsBootstrapV3( + streams_bootstrap = StreamsBootstrap( name="example-name", **{ "namespace": "test-namespace", @@ -37,7 +37,7 @@ def test_default_configs(self): @pytest.mark.asyncio() async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): - streams_bootstrap = StreamsBootstrapV3( + streams_bootstrap = StreamsBootstrap( name="example-name", **{ "namespace": "test-namespace", @@ -58,7 +58,7 @@ async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): streams_bootstrap.dry_run_handler, "print_helm_diff" ) mocker.patch.object( - StreamsBootstrapV3, + StreamsBootstrap, "helm_chart", return_value="test/test-chart", new_callable=mocker.PropertyMock, @@ -88,10 +88,10 @@ async def test_should_raise_validation_error_for_invalid_image_tag(self): with pytest.raises( ValidationError, match=re.escape( - "1 validation error for StreamsBootstrapV3Values\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" + "1 validation error for StreamsBootstrapValues\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" ), ): - StreamsBootstrapV3Values( + StreamsBootstrapValues( **{ "imageTag": "invalid image tag!", "kafka": { @@ -108,7 +108,7 @@ async def test_should_raise_validation_error_for_invalid_helm_chart_version(self "When using the streams-bootstrap v3 component your version ('2.1.0') must be at least 3.0.0." ), ): - StreamsBootstrapV3( + StreamsBootstrap( name="example-name", **{ "namespace": "test-namespace", diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index fc49debf8..427498802 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -13,9 +13,9 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.producer.producer_app import ( - ProducerApp, +from kpops.components.streams_bootstrap_v2.producer.producer_app import ( ProducerAppCleaner, + ProducerAppV2, ) PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" @@ -39,8 +39,8 @@ def test_release_name(self): assert PRODUCER_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def producer_app(self) -> ProducerApp: - return ProducerApp( + def producer_app(self) -> ProducerAppV2: + return ProducerAppV2( name=PRODUCER_APP_NAME, **{ "version": "2.4.2", @@ -63,28 +63,28 @@ def producer_app(self) -> ProducerApp: def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) - def test_cleaner(self, producer_app: ProducerApp): + def test_cleaner(self, producer_app: ProducerAppV2): cleaner = producer_app._cleaner assert isinstance(cleaner, ProducerAppCleaner) assert not hasattr(cleaner, "_cleaner") - def test_cleaner_inheritance(self, producer_app: ProducerApp): + def test_cleaner_inheritance(self, producer_app: ProducerAppV2): assert producer_app._cleaner.values == producer_app.values - def test_cleaner_helm_release_name(self, producer_app: ProducerApp): + def test_cleaner_helm_release_name(self, producer_app: ProducerAppV2): assert ( producer_app._cleaner.helm_release_name == "${pipeline.name}-test-producer-app-with-l-abc43-clean" ) - def test_cleaner_helm_name_override(self, producer_app: ProducerApp): + def test_cleaner_helm_name_override(self, producer_app: ProducerAppV2): assert ( producer_app._cleaner.to_helm_values()["nameOverride"] == PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE ) def test_output_topics(self): - producer_app = ProducerApp( + producer_app = ProducerAppV2( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", @@ -116,7 +116,7 @@ def test_output_topics(self): @pytest.mark.asyncio() async def test_deploy_order_when_dry_run_is_false( self, - producer_app: ProducerApp, + producer_app: ProducerAppV2, mocker: MockerFixture, ): mock_create_topic = mocker.patch.object( @@ -167,7 +167,7 @@ async def test_deploy_order_when_dry_run_is_false( @pytest.mark.asyncio() async def test_destroy( self, - producer_app: ProducerApp, + producer_app: ProducerAppV2, mocker: MockerFixture, ): mock_helm_uninstall = mocker.patch.object(producer_app.helm, "uninstall") @@ -181,7 +181,7 @@ async def test_destroy( @pytest.mark.asyncio() async def test_should_clean_producer_app( self, - producer_app: ProducerApp, + producer_app: ProducerAppV2, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -260,7 +260,7 @@ async def test_should_clean_producer_app( async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_with_dry_run_false( self, mocker: MockerFixture, - producer_app: ProducerApp, + producer_app: ProducerAppV2, empty_helm_get_values: MockerFixture, ): # actual component @@ -326,7 +326,7 @@ async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clea ) def test_get_output_topics(self): - producer_app = ProducerApp( + producer_app = ProducerAppV2( name="my-producer", **{ "namespace": "test-namespace", @@ -378,7 +378,7 @@ async def test_should_not_deploy_clean_up_when_rest(self, mocker: MockerFixture) }, }, ) - producer_app = ProducerApp( + producer_app = ProducerAppV2( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", @@ -428,7 +428,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, }, ) - producer_app = ProducerApp( + producer_app = ProducerAppV2( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", @@ -496,7 +496,7 @@ async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster ) # user defined model - producer_app = ProducerApp( + producer_app = ProducerAppV2( name=PRODUCER_APP_NAME, **{ "namespace": "test-namespace", diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 2703ef8ed..0d809c50e 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -19,13 +19,13 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.streams.model import ( +from kpops.components.streams_bootstrap_v2.streams.model import ( PersistenceConfig, StreamsAppAutoScaling, ) -from kpops.components.streams_bootstrap.streams.streams_app import ( - StreamsApp, +from kpops.components.streams_bootstrap_v2.streams.streams_app import ( StreamsAppCleaner, + StreamsAppV2, ) RESOURCES_PATH = Path(__file__).parent / "resources" @@ -53,8 +53,8 @@ def test_release_name(self): assert STREAMS_APP_CLEAN_RELEASE_NAME.endswith("-clean") @pytest.fixture() - def streams_app(self) -> StreamsApp: - return StreamsApp( + def streams_app(self) -> StreamsAppV2: + return StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -72,8 +72,8 @@ def streams_app(self) -> StreamsApp: ) @pytest.fixture() - def stateful_streams_app(self) -> StreamsApp: - return StreamsApp( + def stateful_streams_app(self) -> StreamsAppV2: + return StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -98,12 +98,12 @@ def stateful_streams_app(self) -> StreamsApp: def empty_helm_get_values(self, mocker: MockerFixture) -> MagicMock: return mocker.patch.object(Helm, "get_values", return_value=None) - def test_cleaner(self, streams_app: StreamsApp): + def test_cleaner(self, streams_app: StreamsAppV2): cleaner = streams_app._cleaner assert isinstance(cleaner, StreamsAppCleaner) assert not hasattr(cleaner, "_cleaner") - def test_cleaner_inheritance(self, streams_app: StreamsApp): + def test_cleaner_inheritance(self, streams_app: StreamsAppV2): streams_app.values.autoscaling = StreamsAppAutoScaling( enabled=True, consumer_group="foo", @@ -113,7 +113,7 @@ def test_cleaner_inheritance(self, streams_app: StreamsApp): assert streams_app._cleaner.values == streams_app.values def test_raise_validation_error_when_autoscaling_enabled_and_mandatory_fields_not_set( - self, streams_app: StreamsApp + self, streams_app: StreamsAppV2 ): with pytest.raises(ValidationError) as error: streams_app.values.autoscaling = StreamsAppAutoScaling( @@ -126,7 +126,7 @@ def test_raise_validation_error_when_autoscaling_enabled_and_mandatory_fields_no assert str(error.value) == msg def test_raise_validation_error_when_autoscaling_enabled_and_only_consumer_group_set( - self, streams_app: StreamsApp + self, streams_app: StreamsAppV2 ): with pytest.raises(ValidationError) as error: streams_app.values.autoscaling = StreamsAppAutoScaling( @@ -139,7 +139,7 @@ def test_raise_validation_error_when_autoscaling_enabled_and_only_consumer_group assert str(error.value) == msg def test_raise_validation_error_when_autoscaling_enabled_and_only_lag_threshold_is_set( - self, streams_app: StreamsApp + self, streams_app: StreamsAppV2 ): with pytest.raises(ValidationError) as error: streams_app.values.autoscaling = StreamsAppAutoScaling( @@ -151,20 +151,20 @@ def test_raise_validation_error_when_autoscaling_enabled_and_only_lag_threshold_ ) assert str(error.value) == msg - def test_cleaner_helm_release_name(self, streams_app: StreamsApp): + def test_cleaner_helm_release_name(self, streams_app: StreamsAppV2): assert ( streams_app._cleaner.helm_release_name == "${pipeline.name}-test-streams-app-with-lo-c98c5-clean" ) - def test_cleaner_helm_name_override(self, streams_app: StreamsApp): + def test_cleaner_helm_name_override(self, streams_app: StreamsAppV2): assert ( streams_app._cleaner.to_helm_values()["nameOverride"] == STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE ) def test_set_topics(self): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -210,7 +210,7 @@ def test_set_topics(self): assert "extraInputPatterns" in streams_config def test_no_empty_input_topic(self): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -241,7 +241,7 @@ def test_should_validate(self): with pytest.raises( ValueError, match="Define role only if `type` is `pattern` or `None`" ): - StreamsApp( + StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -263,7 +263,7 @@ def test_should_validate(self): with pytest.raises( ValueError, match="Define `role` only if `type` is undefined" ): - StreamsApp( + StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -282,7 +282,7 @@ def test_should_validate(self): ) def test_set_streams_output_from_to(self): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -321,7 +321,7 @@ def test_set_streams_output_from_to(self): ) def test_weave_inputs_from_prev_component(self): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -358,7 +358,7 @@ def test_weave_inputs_from_prev_component(self): @pytest.mark.asyncio() async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -470,7 +470,7 @@ async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): @pytest.mark.asyncio() async def test_destroy( self, - streams_app: StreamsApp, + streams_app: StreamsAppV2, mocker: MockerFixture, ): mock_helm_uninstall = mocker.patch.object(streams_app.helm, "uninstall") @@ -484,7 +484,7 @@ async def test_destroy( @pytest.mark.asyncio() async def test_reset_when_dry_run_is_false( self, - streams_app: StreamsApp, + streams_app: StreamsAppV2, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -551,7 +551,7 @@ async def test_reset_when_dry_run_is_false( @pytest.mark.asyncio() async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( self, - streams_app: StreamsApp, + streams_app: StreamsAppV2, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -637,7 +637,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( }, }, ) - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -716,7 +716,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, }, ) - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", @@ -774,7 +774,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( @pytest.mark.asyncio() async def test_get_input_output_topics(self): - streams_app = StreamsApp( + streams_app = StreamsAppV2( name="my-app", **{ "namespace": "test-namespace", @@ -832,7 +832,7 @@ async def test_get_input_output_topics(self): ] def test_raise_validation_error_when_persistence_enabled_and_size_not_set( - self, stateful_streams_app: StreamsApp + self, stateful_streams_app: StreamsAppV2 ): with pytest.raises(ValidationError) as error: stateful_streams_app.values.persistence = PersistenceConfig( @@ -847,7 +847,7 @@ def test_raise_validation_error_when_persistence_enabled_and_size_not_set( @pytest.mark.asyncio() async def test_stateful_clean_with_dry_run_false( self, - stateful_streams_app: StreamsApp, + stateful_streams_app: StreamsAppV2, empty_helm_get_values: MockerFixture, mocker: MockerFixture, ): @@ -926,7 +926,7 @@ async def test_stateful_clean_with_dry_run_false( @pytest.mark.asyncio() async def test_stateful_clean_with_dry_run_true( self, - stateful_streams_app: StreamsApp, + stateful_streams_app: StreamsAppV2, empty_helm_get_values: MockerFixture, mocker: MockerFixture, caplog: pytest.LogCaptureFixture, @@ -990,7 +990,7 @@ async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster }, ) - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=STREAMS_APP_NAME, **{ "namespace": "test-namespace", diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index 390ff3228..ecd475f2c 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -9,16 +9,14 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.common.streams_bootstrap import ( - StreamsBootstrap, - StreamsBootstrapValues, -) +from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 +from kpops.components.streams_bootstrap_v2.base import StreamsBootstrapV2Values @pytest.mark.usefixtures("mock_env") class TestStreamsBootstrap: def test_default_configs(self): - streams_bootstrap = StreamsBootstrap( + streams_bootstrap = StreamsBootstrapV2( name="example-name", **{ "namespace": "test-namespace", @@ -39,7 +37,7 @@ def test_default_configs(self): @pytest.mark.asyncio() async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): - streams_bootstrap = StreamsBootstrap( + streams_bootstrap = StreamsBootstrapV2( name="example-name", **{ "namespace": "test-namespace", @@ -60,7 +58,7 @@ async def test_should_deploy_streams_bootstrap_app(self, mocker: MockerFixture): streams_bootstrap.dry_run_handler, "print_helm_diff" ) mocker.patch.object( - StreamsBootstrap, + StreamsBootstrapV2, "helm_chart", return_value="test/test-chart", new_callable=mocker.PropertyMock, @@ -90,10 +88,10 @@ async def test_should_raise_validation_error_for_invalid_image_tag(self): with pytest.raises( ValidationError, match=re.escape( - "1 validation error for StreamsBootstrapValues\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" + "1 validation error for StreamsBootstrapV2Values\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" ), ): - StreamsBootstrapValues( + StreamsBootstrapV2Values( **{ "imageTag": "invalid image tag!", "streams": { diff --git a/tests/pipeline/resources/custom-config/defaults.yaml b/tests/pipeline/resources/custom-config/defaults.yaml index 40c754e0f..044a2af9e 100644 --- a/tests/pipeline/resources/custom-config/defaults.yaml +++ b/tests/pipeline/resources/custom-config/defaults.yaml @@ -1,19 +1,17 @@ -kafka-app: - version: 2.9.0 - namespace: development-namespace +streams-bootstrap-v2: values: streams: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" -producer-app: +producer-app-v2: to: topics: ${output_topic_name}: type: output partitions_count: 3 -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/custom-config/pipeline.yaml b/tests/pipeline/resources/custom-config/pipeline.yaml index b50df4f7e..83c56f136 100644 --- a/tests/pipeline/resources/custom-config/pipeline.yaml +++ b/tests/pipeline/resources/custom-config/pipeline.yaml @@ -1,4 +1,4 @@ -- type: producer-app +- type: producer-app-v2 name: app1 values: resources: @@ -7,7 +7,7 @@ requests: memory: 2G -- type: streams-app +- type: streams-app-v2 name: app2 values: image: some-image diff --git a/tests/pipeline/resources/defaults.yaml b/tests/pipeline/resources/defaults.yaml index b0785a3e3..5aa3f9520 100644 --- a/tests/pipeline/resources/defaults.yaml +++ b/tests/pipeline/resources/defaults.yaml @@ -4,16 +4,16 @@ pipeline-component: kubernetes-app: namespace: example-namespace -streams-bootstrap: +streams-bootstrap-v2: values: streams: brokers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} version: "2.4.2" -producer-app: {} # inherits from kafka-app +producer-app-v2: {} # inherits from streams-bootstrap-v2 -streams-app: # inherits from kafka-app +streams-app-v2: # inherits from streams-bootstrap-v2 values: streams: config: diff --git a/tests/pipeline/resources/kafka-connect-sink-config/defaults.yaml b/tests/pipeline/resources/kafka-connect-sink-config/defaults.yaml index 4fba15fda..bdf22b02e 100644 --- a/tests/pipeline/resources/kafka-connect-sink-config/defaults.yaml +++ b/tests/pipeline/resources/kafka-connect-sink-config/defaults.yaml @@ -1,2 +1,2 @@ -streams-bootstrap: +streams-bootstrap-v2: version: 2.9.0 diff --git a/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml b/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml index 02fd8e087..e88be97e2 100644 --- a/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml +++ b/tests/pipeline/resources/kafka-connect-sink/pipeline.yaml @@ -1,5 +1,5 @@ # Parse Connector topics from previous component output topic -- type: streams-app +- type: streams-app-v2 from: topics: example-topic: diff --git a/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml b/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml index f1e5f18c4..f3fd13980 100644 --- a/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml +++ b/tests/pipeline/resources/name_prefix_concatenation/pipeline.yaml @@ -1,4 +1,4 @@ -- type: streams-app +- type: streams-app-v2 name: my-streams-app prefix: my-fake-prefix- from: diff --git a/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml b/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml index 0edbafc0c..0685e34b1 100644 --- a/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/no-input-topic-pipeline/pipeline.yaml @@ -1,4 +1,4 @@ -- type: streams-app +- type: streams-app-v2 name: app1 from: topics: @@ -16,7 +16,7 @@ topics: example-output: type: output -- type: streams-app +- type: streams-app-v2 name: app2 to: topics: diff --git a/tests/pipeline/resources/no-topics-defaults/defaults.yaml b/tests/pipeline/resources/no-topics-defaults/defaults.yaml index f933006ff..c6267151c 100644 --- a/tests/pipeline/resources/no-topics-defaults/defaults.yaml +++ b/tests/pipeline/resources/no-topics-defaults/defaults.yaml @@ -4,14 +4,14 @@ kafka-app: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" -producer-app: +producer-app-v2: to: topics: ${output_topic_name}: type: output partitions_count: 3 -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/no-user-defined-components/pipeline.yaml b/tests/pipeline/resources/no-user-defined-components/pipeline.yaml index e8c53c87d..0ee4773fd 100644 --- a/tests/pipeline/resources/no-user-defined-components/pipeline.yaml +++ b/tests/pipeline/resources/no-user-defined-components/pipeline.yaml @@ -1,4 +1,4 @@ -- type: streams-app +- type: streams-app-v2 from: topics: example-topic: diff --git a/tests/pipeline/resources/parallel-pipeline/defaults.yaml b/tests/pipeline/resources/parallel-pipeline/defaults.yaml index 073e8e877..2e03b9498 100644 --- a/tests/pipeline/resources/parallel-pipeline/defaults.yaml +++ b/tests/pipeline/resources/parallel-pipeline/defaults.yaml @@ -10,7 +10,7 @@ kafka-app: brokers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/parallel-pipeline/pipeline.yaml b/tests/pipeline/resources/parallel-pipeline/pipeline.yaml index 688d86fb5..ae8785c1e 100644 --- a/tests/pipeline/resources/parallel-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/parallel-pipeline/pipeline.yaml @@ -1,4 +1,4 @@ -- type: producer-app +- type: producer-app-v2 name: transaction-avro-producer-1 to: topics: @@ -6,7 +6,7 @@ type: output partitions_count: 3 -- type: producer-app +- type: producer-app-v2 name: transaction-avro-producer-2 to: topics: @@ -14,7 +14,7 @@ type: output partitions_count: 3 -- type: producer-app +- type: producer-app-v2 name: transaction-avro-producer-3 to: topics: @@ -22,13 +22,13 @@ type: output partitions_count: 3 -- type: streams-app +- type: streams-app-v2 name: transaction-joiner -- type: streams-app +- type: streams-app-v2 name: fraud-detector -- type: streams-app +- type: streams-app-v2 name: account-linker from: components: diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml index 7dcdda440..8b4c558c3 100644 --- a/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-component-should-have-prefix/pipeline.yaml @@ -1,4 +1,4 @@ -- type: producer-app +- type: producer-app-v2 name: account-producer values: replicaCount: 1 diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index c3b71231f..621643b43 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -7,9 +7,9 @@ kafka-app: brokers: "${config.kafka_brokers}" schemaRegistryUrl: "${config.schema_registry.url}" -producer-app: {} # inherits from kafka-app +producer-app-v2: {} # inherits from streams-bootstrap-v2 -streams-app: # inherits from kafka-app +streams-app-v2: # inherits from streams-bootstrap-v2 version: 2.9.0 values: streams: diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml index 02fd8e087..e88be97e2 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/pipeline.yaml @@ -1,5 +1,5 @@ # Parse Connector topics from previous component output topic -- type: streams-app +- type: streams-app-v2 from: topics: example-topic: diff --git a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml index cc9b4e585..9f3d82bef 100644 --- a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml @@ -13,7 +13,7 @@ kafka-app: brokers: 127.0.0.1:9092 schemaRegistryUrl: 127.0.0.1:8081 -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml b/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml index d60e4c36e..634293d69 100644 --- a/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-loop/pipeline.yaml @@ -1,4 +1,4 @@ -- type: producer-app +- type: producer-app-v2 name: app1 values: image: producer-image @@ -7,7 +7,7 @@ my-output-topic: type: output -- type: streams-app +- type: streams-app-v2 name: app2 values: image: app2-image @@ -20,7 +20,7 @@ my-app2-topic: type: output -- type: streams-app +- type: streams-app-v2 name: app3 values: image: app3-image diff --git a/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml b/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml index ba1bd0000..65a6eaab0 100644 --- a/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-paths/pipeline.yaml @@ -1,4 +1,4 @@ -- type: producer-app +- type: producer-app-v2 name: account-producer namespace: test values: diff --git a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml index 15dc70e01..ee1591c72 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml @@ -9,14 +9,14 @@ kafka-app: schema_registry_url: "${config.schema_registry.url}" version: "2.4.2" -producer-app: +producer-app-v2: to: topics: ${output_topic_name}: partitions_count: 3 -streams-app: # inherits from kafka-app +streams-app-v2: # inherits from streams-bootstrap-v2 values: streams: brokers: test diff --git a/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml b/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml index f6db75522..667f6cd43 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/pipeline.yaml @@ -1,16 +1,16 @@ -- type: producer-app +- type: producer-app-v2 name: component-input -- type: producer-app +- type: producer-app-v2 name: component-extra -- type: producer-app +- type: producer-app-v2 name: component-input-pattern -- type: producer-app +- type: producer-app-v2 name: component-extra-pattern -- type: streams-app +- type: streams-app-v2 name: simple-app from: components: diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml index f45f4ef37..a3f2429df 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml @@ -13,7 +13,7 @@ kafka-app: brokers: 127.0.0.1:9092 schemaRegistryUrl: 127.0.0.1:8081 -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml index 1c6511d79..7c3b8ecaa 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/pipeline.yaml @@ -1,4 +1,4 @@ -- type: streams-app +- type: streams-app-v2 name: app2-processor values: image: some-image diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml index 8fbb3ea39..6c50706ab 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml @@ -14,12 +14,12 @@ kafka-app: schemaRegistryUrl: 127.0.0.1:8081 -streams-app: +streams-app-v2: values: labels: pipeline: ${pipeline.name} -producer-app: +producer-app-v2: to: topics: ${output_topic_name}: diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline.yaml index 03d9bee6d..ffb0f8c3f 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/pipeline.yaml @@ -1,6 +1,6 @@ -- type: producer-app +- type: producer-app-v2 name: app1 -- type: streams-app +- type: streams-app-v2 name: app2 diff --git a/tests/pipeline/resources/read-from-component/pipeline.yaml b/tests/pipeline/resources/read-from-component/pipeline.yaml index cc6bf72c7..fab7729d7 100644 --- a/tests/pipeline/resources/read-from-component/pipeline.yaml +++ b/tests/pipeline/resources/read-from-component/pipeline.yaml @@ -1,11 +1,11 @@ -- type: producer-app +- type: producer-app-v2 name: producer1 to: topics: ${output_topic_name}: type: output -- type: producer-app +- type: producer-app-v2 name: producer2 prefix: "" to: @@ -20,7 +20,7 @@ name: inflate-step-without-prefix prefix: "" -- type: streams-app +- type: streams-app-v2 name: consumer1 from: components: @@ -31,7 +31,7 @@ ${output_topic_name}: type: output -- type: streams-app +- type: streams-app-v2 name: consumer2 from: components: @@ -40,7 +40,7 @@ consumer1: type: input -- type: streams-app +- type: streams-app-v2 name: consumer3 from: topics: @@ -50,14 +50,14 @@ producer2: type: input -- type: streams-app +- type: streams-app-v2 name: consumer4 from: components: inflate-step: type: input -- type: streams-app +- type: streams-app-v2 name: consumer5 from: components: diff --git a/tests/pipeline/resources/simple-pipeline/pipeline.yaml b/tests/pipeline/resources/simple-pipeline/pipeline.yaml index f78d0c385..d523398ed 100644 --- a/tests/pipeline/resources/simple-pipeline/pipeline.yaml +++ b/tests/pipeline/resources/simple-pipeline/pipeline.yaml @@ -1,6 +1,6 @@ -- type: producer-app +- type: producer-app-v2 -- type: streams-app +- type: streams-app-v2 - type: helm-app values: {} diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml index 7dc1f296a..f3e181935 100644 --- a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml +++ b/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml @@ -1,4 +1,4 @@ -streams-bootstrap-v3: +streams-bootstrap: values: kafka: bootstrapServers: ${config.kafka_brokers} @@ -6,9 +6,9 @@ streams-bootstrap-v3: # TODO: change to stable version after the PR is merged version: "3.0.0-SNAPSHOT" -producer-app-v3: {} # inherits from streams-bootstrap-v3 +producer-app: {} # inherits from streams-bootstrap -streams-app-v3: # inherits from streams-bootstrap-v3 +streams-app: # inherits from streams-bootstrap values: kafka: config: diff --git a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml index f777b35e9..468bdfde9 100644 --- a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml @@ -8,7 +8,7 @@ kafka-app: schema_registry_url: "${schema_registry_url}" version: "2.4.2" -streams-app: # inherits from kafka-app +streams-app-v2: # inherits from streams-bootstrap-v2 values: streams: config: diff --git a/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml b/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml index 4507ff2d3..4ce0dd534 100644 --- a/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/pipeline.yaml @@ -1,4 +1,4 @@ -- type: streams-app +- type: streams-app-v2 name: in-order-to-have-len-fifty-two-name-should-end--here values: streams: diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index 76c9b1bf8..6594d48bb 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -42,7 +42,7 @@ atm-fraud-account-producer-topic: configs: {} partitions_count: 3 - type: producer-app + type: producer-app-v2 values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer imageTag: 1.0.0 @@ -107,7 +107,7 @@ atm-fraud-transaction-avro-producer-topic: configs: {} partitions_count: 3 - type: producer-app + type: producer-app-v2 values: commandLine: ITERATION: 20 @@ -139,6 +139,7 @@ repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean + # TODO: do we need to version this -> yes type: streams-app-cleaner values: annotations: @@ -184,7 +185,7 @@ atm-fraud-transaction-joiner-topic: configs: {} partitions_count: 3 - type: streams-app + type: streams-app-v2 values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic @@ -266,7 +267,7 @@ atm-fraud-fraud-detector-topic: configs: {} partitions_count: 3 - type: streams-app + type: streams-app-v2 values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic @@ -358,7 +359,7 @@ atm-fraud-account-linker-topic: configs: {} partitions_count: 3 - type: streams-app + type: streams-app-v2 values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index e7afd3398..145184060 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -40,7 +40,7 @@ word-count-data-producer-topic: configs: {} partitions_count: 3 - type: producer-app + type: producer-app-v2 values: image: bakdata/kpops-demo-sentence-producer imageTag: 1.0.0 @@ -112,7 +112,7 @@ cleanup.policy: compact partitions_count: 3 type: output - type: streams-app + type: streams-app-v2 values: commandLine: PRODUCTIVE: false diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml index 1db36ed55..a7cd13ba0 100644 --- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-custom-config-app1 schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -36,7 +36,7 @@ configs: {} partitions_count: 3 type: output - type: producer-app + type: producer-app-v2 values: imageTag: latest resources: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-custom-config-app1 schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 - _cleaner: name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - resources-custom-config-app1 outputTopic: resources-custom-config-app2 schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -99,7 +99,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: image: some-image imageTag: latest @@ -117,5 +117,5 @@ - resources-custom-config-app1 outputTopic: resources-custom-config-app2 schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index 20e133c5f..30b209929 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -339,7 +339,7 @@ resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app: configs: {} type: output - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index 2f7fd65ff..2c600b25d 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: @@ -19,7 +19,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-kafka-connect-sink-streams-app-error + errorTopic: resources-kafka-connect-sink-streams-app-v2-error inputTopics: - example-topic outputTopic: example-output @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: @@ -44,13 +44,13 @@ example-output: configs: {} type: output - resources-kafka-connect-sink-streams-app-error: + resources-kafka-connect-sink-streams-app-v2-error: configs: cleanup.policy: compact,delete partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: image: fake-image imageTag: latest @@ -61,7 +61,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-kafka-connect-sink-streams-app-error + errorTopic: resources-kafka-connect-sink-streams-app-v2-error inputTopics: - example-topic outputTopic: example-output diff --git a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml index db123f217..57a5a4dab 100644 --- a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml @@ -30,7 +30,7 @@ username: masked repository_name: masked url: masked - type: producer-app + type: producer-app-v2 values: imageTag: latest streams: diff --git a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml index e4af38635..46dddafd2 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml @@ -55,7 +55,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: commandLine: CONVERT_XML: true @@ -128,7 +128,7 @@ test-output-extra: configs: {} role: test-output - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index 01a8241dc..82eaf14ca 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: @@ -19,7 +19,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-no-user-defined-components-streams-app-error + errorTopic: resources-no-user-defined-components-streams-app-v2-error inputTopics: - example-topic outputTopic: example-output @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: @@ -44,13 +44,13 @@ example-output: configs: {} type: output - resources-no-user-defined-components-streams-app-error: + resources-no-user-defined-components-streams-app-v2-error: configs: cleanup.policy: compact,delete partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: image: fake-image imageTag: latest @@ -61,7 +61,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-no-user-defined-components-streams-app-error + errorTopic: resources-no-user-defined-components-streams-app-v2-error inputTopics: - example-topic outputTopic: example-output diff --git a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml index 7e0b3bcae..ed8d4724d 100644 --- a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml @@ -22,7 +22,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - version: 2.9.0 + version: 2.4.2 name: account-producer namespace: ${NAMESPACE} prefix: from-pipeline-component- @@ -31,7 +31,7 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - type: producer-app + type: producer-app-v2 values: debug: true image: ${DOCKER_REGISTRY}/atm-demo-accountproducer @@ -45,5 +45,5 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - version: 2.9.0 + version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index 6807b3552..ed59df3fe 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -30,7 +30,7 @@ resources-read-from-component-producer1: configs: {} type: output - type: producer-app + type: producer-app-v2 values: imageTag: latest streams: @@ -70,7 +70,7 @@ resources-read-from-component-producer2: configs: {} type: output - type: producer-app + type: producer-app-v2 values: imageTag: latest streams: @@ -256,7 +256,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -450,7 +450,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -517,7 +517,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -583,7 +583,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -649,7 +649,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -712,7 +712,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: @@ -774,7 +774,7 @@ partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: imageTag: latest persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml index f12f54a12..97c9f8b2d 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -36,7 +36,7 @@ configs: {} partitions_count: 3 type: output - type: producer-app + type: producer-app-v2 values: imageTag: latest resources: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 - _cleaner: name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -99,7 +99,7 @@ configs: {} partitions_count: 3 type: output - type: streams-app + type: streams-app-v2 values: image: some-image imageTag: latest @@ -117,5 +117,5 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml index f12f54a12..97c9f8b2d 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app1 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -36,7 +36,7 @@ configs: {} partitions_count: 3 type: output - type: producer-app + type: producer-app-v2 values: imageTag: latest resources: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 - _cleaner: name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 name: app2 - namespace: development-namespace + namespace: example-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -99,7 +99,7 @@ configs: {} partitions_count: 3 type: output - type: streams-app + type: streams-app-v2 values: image: some-image imageTag: latest @@ -117,5 +117,5 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.9.0 + version: 2.4.2 diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index cdb6f4f3a..e65e5b5b4 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app-development + name: streams-app-v2-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: @@ -19,7 +19,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-pipeline-with-env-defaults-streams-app-development-error + errorTopic: resources-pipeline-with-env-defaults-streams-app-v2-development-error inputTopics: - example-topic outputTopic: example-output @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app-development + name: streams-app-v2-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: @@ -44,13 +44,13 @@ example-output: configs: {} type: output - resources-pipeline-with-env-defaults-streams-app-development-error: + resources-pipeline-with-env-defaults-streams-app-v2-development-error: configs: cleanup.policy: compact,delete partitions_count: 1 type: error value_schema: com.bakdata.kafka.DeadLetter - type: streams-app + type: streams-app-v2 values: image: fake-image imageTag: latest @@ -61,7 +61,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-pipeline-with-env-defaults-streams-app-development-error + errorTopic: resources-pipeline-with-env-defaults-streams-app-v2-development-error inputTopics: - example-topic outputTopic: example-output diff --git a/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml index 4dcabb92d..e676ca8e4 100644 --- a/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml @@ -4,7 +4,7 @@ kind: Job metadata: labels: app: resources-custom-config-app1 - chart: producer-app-2.9.0 + chart: producer-app-2.4.2 release: resources-custom-config-app1 name: resources-custom-config-app1 spec: @@ -52,7 +52,7 @@ kind: ConfigMap metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.9.0 + chart: streams-app-2.4.2 heritage: Helm release: resources-custom-config-app2 name: resources-custom-config-app2-jmx-configmap @@ -63,7 +63,7 @@ kind: Deployment metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.9.0 + chart: streams-app-2.4.2 pipeline: resources-custom-config release: resources-custom-config-app2 name: resources-custom-config-app2 @@ -103,8 +103,6 @@ spec: value: com.bakdata.kafka.MurmurHashIdGenerator - name: KAFKA_JMX_PORT value: '5555' - - name: APP_VOLATILE_GROUP_INSTANCE_ID - value: 'true' - name: APP_BROKERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index eca5af9eb..40637e62e 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -5,7 +5,7 @@ metadata: labels: app: resources-custom-config-app1 - chart: producer-app-2.9.0 + chart: producer-app-2.4.2 release: resources-custom-config-app1 name: resources-custom-config-app1 spec: @@ -54,7 +54,7 @@ metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.9.0 + chart: streams-app-2.4.2 heritage: Helm release: resources-custom-config-app2 name: resources-custom-config-app2-jmx-configmap @@ -65,7 +65,7 @@ metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.9.0 + chart: streams-app-2.4.2 pipeline: resources-custom-config release: resources-custom-config-app2 name: resources-custom-config-app2 @@ -105,8 +105,6 @@ value: com.bakdata.kafka.MurmurHashIdGenerator - name: KAFKA_JMX_PORT value: '5555' - - name: APP_VOLATILE_GROUP_INSTANCE_ID - value: 'true' - name: APP_BROKERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL diff --git a/tests/pipeline/test_clean.py b/tests/pipeline/test_clean.py index 1f9325c59..4381e783b 100644 --- a/tests/pipeline/test_clean.py +++ b/tests/pipeline/test_clean.py @@ -7,13 +7,13 @@ from kpops.cli.main import app from kpops.components.base_components import HelmApp -from kpops.components.streams_bootstrap.producer.producer_app import ( - ProducerApp, +from kpops.components.streams_bootstrap_v2.producer.producer_app import ( ProducerAppCleaner, + ProducerAppV2, ) -from kpops.components.streams_bootstrap.streams.streams_app import ( - StreamsApp, +from kpops.components.streams_bootstrap_v2.streams.streams_app import ( StreamsAppCleaner, + StreamsAppV2, ) runner = CliRunner() @@ -47,8 +47,8 @@ def helm_mock(self, mocker: MockerFixture) -> MagicMock: def test_order(self, mocker: MockerFixture): # destroy - producer_app_mock_destroy = mocker.patch.object(ProducerApp, "destroy") - streams_app_mock_destroy = mocker.patch.object(StreamsApp, "destroy") + producer_app_mock_destroy = mocker.patch.object(ProducerAppV2, "destroy") + streams_app_mock_destroy = mocker.patch.object(StreamsAppV2, "destroy") helm_app_mock_destroy = mocker.patch.object(HelmApp, "destroy") # clean diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index fd3cc08ee..235accdb6 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -12,24 +12,24 @@ ) from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.components.common.topic import OutputTopicTypes, TopicConfig -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp -from kpops.components.streams_bootstrap_v3 import ProducerAppV3, StreamsAppV3 +from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 +from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 +from kpops.components.streams_bootstrap_v3 import ProducerApp, StreamsApp -class MyProducerApp(ProducerAppV3): ... +class MyProducerApp(ProducerApp): ... -class MyStreamsApp(StreamsAppV3): ... +class MyStreamsApp(StreamsApp): ... -class ScheduledProducer(ProducerApp): ... +class ScheduledProducer(ProducerAppV2): ... -class Converter(StreamsApp): ... +class Converter(StreamsAppV2): ... -class SubStreamsApp(StreamsApp): +class SubStreamsApp(StreamsAppV2): """Intermediary subclass of StreamsApp used for Filter.""" @@ -37,7 +37,7 @@ class Filter(SubStreamsApp): """Subsubclass of StreamsApp to test inheritance.""" -class ShouldInflate(StreamsApp): +class ShouldInflate(StreamsAppV2): @override def inflate(self) -> list[PipelineComponent]: inflate_steps = super().inflate() @@ -62,7 +62,7 @@ def inflate(self) -> list[PipelineComponent]: ), ) inflate_steps.append(kafka_connector) - streams_app = StreamsApp( + streams_app = StreamsAppV2( name=f"{self.name}-inflated-streams-app", to=ToSection( # type: ignore[reportGeneralTypeIssues] topics={ @@ -93,7 +93,7 @@ def provide_schema( return AvroSchema(schema) -class SimpleInflateConnectors(StreamsApp): +class SimpleInflateConnectors(StreamsAppV2): def inflate(self) -> list[PipelineComponent]: connector = KafkaSinkConnector( name="inflated-connector-name", diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 5bb2e875a..1a4e6d085 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -4,17 +4,17 @@ from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.components.common.topic import OutputTopicTypes -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp +from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 +from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 -class ScheduledProducer(ProducerApp): ... +class ScheduledProducer(ProducerAppV2): ... -class Converter(StreamsApp): ... +class Converter(StreamsAppV2): ... -class ShouldInflate(StreamsApp): +class ShouldInflate(StreamsAppV2): @override def inflate(self) -> list[PipelineComponent]: inflate_steps = super().inflate() diff --git a/tests/pipeline/test_deploy.py b/tests/pipeline/test_deploy.py index 063b0ddc4..df8d347f5 100644 --- a/tests/pipeline/test_deploy.py +++ b/tests/pipeline/test_deploy.py @@ -6,6 +6,8 @@ from typer.testing import CliRunner from kpops.cli.main import app +from kpops.components.base_components import HelmApp +from kpops.components.streams_bootstrap_v2 import ProducerAppV2, StreamsAppV2 runner = CliRunner() @@ -22,15 +24,9 @@ def mock_helm(self, mocker: MockerFixture) -> MagicMock: ).return_value def test_order(self, mocker: MockerFixture): - producer_app_mock_deploy = mocker.patch( - "kpops.components.streams_bootstrap.producer.producer_app.ProducerApp.deploy", - ) - streams_app_mock_deploy = mocker.patch( - "kpops.components.streams_bootstrap.streams.streams_app.StreamsApp.deploy", - ) - helm_app_mock_deploy = mocker.patch( - "kpops.components.base_components.helm_app.HelmApp.deploy", - ) + producer_app_mock_deploy = mocker.patch.object(ProducerAppV2, "deploy") + streams_app_mock_deploy = mocker.patch.object(StreamsAppV2, "deploy") + helm_app_mock_deploy = mocker.patch.object(HelmApp, "deploy") mock_deploy = mocker.AsyncMock() mock_deploy.attach_mock(producer_app_mock_deploy, "producer_app_mock_deploy") mock_deploy.attach_mock(streams_app_mock_deploy, "streams_app_mock_deploy") diff --git a/tests/pipeline/test_destroy.py b/tests/pipeline/test_destroy.py index 17dd9752c..ce26e64e4 100644 --- a/tests/pipeline/test_destroy.py +++ b/tests/pipeline/test_destroy.py @@ -6,6 +6,8 @@ from typer.testing import CliRunner from kpops.cli.main import app +from kpops.components.base_components import HelmApp +from kpops.components.streams_bootstrap_v2 import ProducerAppV2, StreamsAppV2 runner = CliRunner() @@ -22,15 +24,9 @@ def mock_helm(self, mocker: MockerFixture) -> MagicMock: ).return_value def test_order(self, mocker: MockerFixture): - producer_app_mock_destroy = mocker.patch( - "kpops.components.streams_bootstrap.producer.producer_app.ProducerApp.destroy", - ) - streams_app_mock_destroy = mocker.patch( - "kpops.components.streams_bootstrap.streams.streams_app.StreamsApp.destroy", - ) - helm_app_mock_destroy = mocker.patch( - "kpops.components.base_components.helm_app.HelmApp.destroy", - ) + producer_app_mock_destroy = mocker.patch.object(ProducerAppV2, "destroy") + streams_app_mock_destroy = mocker.patch.object(StreamsAppV2, "destroy") + helm_app_mock_destroy = mocker.patch.object(HelmApp, "destroy") mock_destroy = mocker.AsyncMock() mock_destroy.attach_mock(producer_app_mock_destroy, "producer_app_mock_destroy") mock_destroy.attach_mock(streams_app_mock_destroy, "streams_app_mock_destroy") diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index b58cb6a67..381871a77 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -56,11 +56,11 @@ def test_default_config(self, mock_execute: MagicMock): "resources-custom-config-app2", "bakdata-streams-bootstrap/streams-app", "--namespace", - "development-namespace", + "example-namespace", "--values", ANY, "--version", - "2.9.0", + "2.4.2", "--timeout", "5m0s", "--wait", @@ -88,11 +88,11 @@ def test_custom_config(self, mock_execute: MagicMock): "resources-custom-config-app2", "bakdata-streams-bootstrap/streams-app", "--namespace", - "development-namespace", + "example-namespace", "--values", ANY, "--version", - "2.9.0", + "2.4.2", "--timeout", "5m0s", "--wait", diff --git a/tests/pipeline/test_reset.py b/tests/pipeline/test_reset.py index 989990245..5724f5157 100644 --- a/tests/pipeline/test_reset.py +++ b/tests/pipeline/test_reset.py @@ -7,9 +7,11 @@ from kpops.cli.main import app from kpops.components.base_components import HelmApp -from kpops.components.streams_bootstrap import ProducerApp, StreamsApp -from kpops.components.streams_bootstrap.producer.producer_app import ProducerAppCleaner -from kpops.components.streams_bootstrap.streams.streams_app import StreamsAppCleaner +from kpops.components.streams_bootstrap_v2 import ProducerAppV2, StreamsAppV2 +from kpops.components.streams_bootstrap_v2.producer.producer_app import ( + ProducerAppCleaner, +) +from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppCleaner runner = CliRunner() @@ -27,8 +29,8 @@ def helm_mock(self, mocker: MockerFixture) -> MagicMock: def test_order(self, mocker: MockerFixture): # destroy - producer_app_mock_destroy = mocker.patch.object(ProducerApp, "destroy") - streams_app_mock_destroy = mocker.patch.object(StreamsApp, "destroy") + producer_app_mock_destroy = mocker.patch.object(ProducerAppV2, "destroy") + streams_app_mock_destroy = mocker.patch.object(StreamsAppV2, "destroy") helm_app_mock_destroy = mocker.patch.object(HelmApp, "destroy") # reset From 2fcfdda4c39677ac44427f7c61abdb9a01a42820 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 14 Aug 2024 10:02:34 +0200 Subject: [PATCH 02/19] Remove v3 and suffix old streams boostrap with v2 --- .../test_kafka_connect_sink_weave_from_topics/pipeline.yaml | 4 ++-- .../test_no_user_defined_components/pipeline.yaml | 4 ++-- .../test_generate/test_with_env_defaults/pipeline.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index 2c600b25d..cbca7d8c3 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app-v2 + name: streams-app namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app-v2 + name: streams-app namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index 82eaf14ca..399c94c95 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app-v2 + name: streams-app namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app-v2 + name: streams-app namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index e65e5b5b4..f58577cf3 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app-v2-development + name: streams-app-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app-v2-development + name: streams-app-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: From 230bb958b1137a699a022bba11544bba906a8817 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 14 Aug 2024 10:03:33 +0200 Subject: [PATCH 03/19] Remove v3 and suffix old streams boostrap with v2 --- .../test_kafka_connect_sink_weave_from_topics/pipeline.yaml | 4 ++-- .../test_no_user_defined_components/pipeline.yaml | 4 ++-- .../test_generate/test_with_env_defaults/pipeline.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index cbca7d8c3..2c600b25d 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-kafka-connect-sink- repo_config: diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index 399c94c95..82eaf14ca 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app + name: streams-app-v2 namespace: example-namespace prefix: resources-no-user-defined-components- repo_config: diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index f58577cf3..e65e5b5b4 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,5 +1,5 @@ - _cleaner: - name: streams-app-development + name: streams-app-v2-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: @@ -30,7 +30,7 @@ topics: example-topic: type: input - name: streams-app-development + name: streams-app-v2-development namespace: development-namespace prefix: resources-pipeline-with-env-defaults- repo_config: From 7bb4bcec706a3308ab0a534777140f249560ea23 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 14 Aug 2024 15:59:41 +0200 Subject: [PATCH 04/19] merge main --- examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples b/examples index 608637744..9d1827cad 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 60863774413b2eb08bb5d6020432c7f29a944383 +Subproject commit 9d1827cad2410ff1d0c5ccbc45c8c17395d9fc8e From a8826591d3a2f3c70b47c4bb56acc6e3a82a38ee Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 15 Aug 2024 10:05:53 +0200 Subject: [PATCH 05/19] update snapshot files --- .../test_generate/atm-fraud/pipeline.yaml | 140 ++---------------- .../test_generate/word-count/pipeline.yaml | 41 +---- 2 files changed, 14 insertions(+), 167 deletions(-) diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index a43809c83..cbf54062f 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -12,18 +12,14 @@ type: producer-app-cleaner values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -36,27 +32,17 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-account-producer-topic: - configs: {} - partitions_count: 3 type: producer-app-v2 values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -77,18 +63,14 @@ ITERATION: 20 REAL_TX: 19 image: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -101,30 +83,20 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-avro-producer-topic: - configs: {} - partitions_count: 3 type: producer-app-v2 values: commandLine: ITERATION: 20 REAL_TX: 19 image: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -139,31 +111,21 @@ repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ suffix: -clean - # TODO: do we need to version this -> yes type: streams-app-cleaner values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-transaction-joiner-dead-letter-topic - inputTopics: - - atm-fraud-transaction-avro-producer-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -175,40 +137,21 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-transaction-joiner-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-transaction-joiner-topic: - configs: {} - partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-transaction-joiner-dead-letter-topic - inputTopics: - - atm-fraud-transaction-avro-producer-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -226,26 +169,17 @@ values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-frauddetector - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-fraud-detector-dead-letter-topic - inputTopics: - - atm-fraud-transaction-joiner-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -257,40 +191,21 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-fraud-detector-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-fraud-detector-topic: - configs: {} - partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-frauddetector - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-fraud-detector-dead-letter-topic - inputTopics: - - atm-fraud-transaction-joiner-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -308,29 +223,17 @@ values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-account-linker-dead-letter-topic - extraInputTopics: - accounts: - - atm-fraud-account-producer-topic - inputTopics: - - atm-fraud-fraud-detector-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -349,43 +252,21 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - atm-fraud-account-linker-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error - atm-fraud-account-linker-topic: - configs: {} - partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic - commandLine: - PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: 1.0.0 - labels: - pipeline: atm-fraud + imageTag: latest persistence: enabled: false prometheus: jmx: enabled: false - replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: atm-fraud-account-linker-dead-letter-topic - extraInputTopics: - accounts: - - atm-fraud-account-producer-topic - inputTopics: - - atm-fraud-fraud-detector-topic optimizeLeaveGroupBehavior: false - outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: @@ -423,7 +304,6 @@ pk.mode: record_value table.name.format: fraud_transactions tasks.max: 1 - topics: atm-fraud-account-linker-topic transforms: flatten transforms.flatten.type: org.apache.kafka.connect.transforms.Flatten$Value value.converter: io.confluent.connect.avro.AvroConverter diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index 145184060..d233b180c 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -12,17 +12,13 @@ type: producer-app-cleaner values: image: bakdata/kpops-demo-sentence-producer - imageTag: 1.0.0 - labels: - pipeline: word-count + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -34,26 +30,16 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ - to: - models: {} - topics: - word-count-data-producer-topic: - configs: {} - partitions_count: 3 type: producer-app-v2 values: image: bakdata/kpops-demo-sentence-producer - imageTag: 1.0.0 - labels: - pipeline: word-count + imageTag: latest prometheus: jmx: enabled: false - replicaCount: 1 streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false - outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -69,12 +55,8 @@ suffix: -clean type: streams-app-cleaner values: - commandLine: - PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app - imageTag: 1.0.0 - labels: - pipeline: word-count + imageTag: latest persistence: enabled: false prometheus: @@ -84,9 +66,6 @@ statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: word-count-word-counter-dead-letter-topic - inputTopics: - - word-count-data-producer-topic optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ @@ -103,23 +82,14 @@ to: models: {} topics: - word-count-word-counter-dead-letter-topic: - configs: {} - partitions_count: 1 - type: error word-count-word-counter-topic: configs: cleanup.policy: compact - partitions_count: 3 type: output type: streams-app-v2 values: - commandLine: - PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app - imageTag: 1.0.0 - labels: - pipeline: word-count + imageTag: latest persistence: enabled: false prometheus: @@ -129,9 +99,6 @@ statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - errorTopic: word-count-word-counter-dead-letter-topic - inputTopics: - - word-count-data-producer-topic optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ From bcaaf232b558de021f099027ab9639c8ad5a8c16 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 15 Aug 2024 10:08:07 +0200 Subject: [PATCH 06/19] rename folder --- docs/docs/schema/defaults.json | 240 +++++++++--------- docs/docs/schema/pipeline.json | 240 +++++++++--------- .../__init__.py | 2 +- .../base.py | 2 +- .../model.py | 0 .../producer/__init__.py | 0 .../producer/model.py | 2 +- .../producer/producer_app.py | 4 +- .../streams/__init__.py | 0 .../streams/model.py | 2 +- .../streams/streams_app.py | 4 +- tests/api/test_registry.py | 9 +- .../streams_bootstrap_v3/test_producer_app.py | 2 +- .../streams_bootstrap_v3/test_streams_app.py | 6 +- .../test_streams_bootstrap.py | 4 +- tests/pipeline/test_components/components.py | 2 +- 16 files changed, 259 insertions(+), 260 deletions(-) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/__init__.py (68%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/base.py (97%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/model.py (100%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/producer/__init__.py (100%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/producer/model.py (89%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/producer/producer_app.py (96%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/streams/__init__.py (100%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/streams/model.py (99%) rename kpops/components/{streams_bootstrap_v3 => streams_bootstrap}/streams/streams_app.py (97%) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 6e9c1797b..0cd12864c 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1527,7 +1527,7 @@ "autoscaling": { "anyOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" }, { "type": "null" @@ -1546,7 +1546,7 @@ "kafka": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" } ], "description": "streams-bootstrap kafka section" @@ -1979,22 +1979,18 @@ "title": "TopicConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { - "consumerGroup": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", - "title": "Consumer group" + "additionalTriggers": { + "default": [], + "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", + "items": { + "type": "string" + }, + "title": "Additionaltriggers", + "type": "array" }, "cooldownPeriod": { "default": 300, @@ -2004,7 +2000,7 @@ }, "enabled": { "default": false, - "description": "", + "description": "Whether to enable auto-scaling using KEDA.", "title": "Enabled", "type": "boolean" }, @@ -2021,6 +2017,15 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, + "internalTopics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app", + "items": { + "type": "string" + }, + "title": "Internaltopics", + "type": "array" + }, "lagThreshold": { "anyOf": [ { @@ -2060,7 +2065,7 @@ }, "topics": { "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "description": "List of topics used by the streams app", "items": { "type": "string" }, @@ -2071,13 +2076,26 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { "additionalProperties": true, - "description": "Streams Bootstrap streams section.", + "description": "Streams Bootstrap kafka section.", "properties": { - "brokers": { + "applicationId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique application ID for Kafka Streams. Required for auto-scaling", + "title": "Unique application ID" + }, + "bootstrapServers": { "description": "Brokers", - "title": "Brokers", + "title": "Bootstrapservers", "type": "string" }, "config": { @@ -2111,16 +2129,38 @@ "default": null, "description": "Error topic" }, - "extraInputPatterns": { + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, + "labeledInputPatterns": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra input patterns", - "title": "Extrainputpatterns", + "title": "Labeledinputpatterns", "type": "object" }, - "extraInputTopics": { + "labeledInputTopics": { "additionalProperties": { "items": { "type": "string" @@ -2129,40 +2169,18 @@ }, "default": {}, "description": "Extra input topics", - "title": "Extrainputtopics", + "title": "Labeledinputtopics", "type": "object" }, - "extraOutputTopics": { + "labeledOutputTopics": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra output topics", - "title": "Extraoutputtopics", + "title": "Labeledoutputtopics", "type": "object" }, - "inputPattern": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input pattern", - "title": "Inputpattern" - }, - "inputTopics": { - "default": [], - "description": "Input topics", - "items": { - "type": "string" - }, - "title": "Inputtopics", - "type": "array" - }, "outputTopic": { "anyOf": [ { @@ -2190,23 +2208,27 @@ } }, "required": [ - "brokers" + "bootstrapServers" ], "title": "StreamsConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { - "additionalTriggers": { - "default": [], - "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", - "items": { - "type": "string" - }, - "title": "Additionaltriggers", - "type": "array" + "consumerGroup": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", + "title": "Consumer group" }, "cooldownPeriod": { "default": 300, @@ -2216,7 +2238,7 @@ }, "enabled": { "default": false, - "description": "Whether to enable auto-scaling using KEDA.", + "description": "", "title": "Enabled", "type": "boolean" }, @@ -2233,15 +2255,6 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, - "internalTopics": { - "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app", - "items": { - "type": "string" - }, - "title": "Internaltopics", - "type": "array" - }, "lagThreshold": { "anyOf": [ { @@ -2281,7 +2294,7 @@ }, "topics": { "default": [], - "description": "List of topics used by the streams app", + "description": "List of auto-generated Kafka Streams topics used by the streams app.", "items": { "type": "string" }, @@ -2292,26 +2305,13 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { "additionalProperties": true, - "description": "Streams Bootstrap kafka section.", + "description": "Streams Bootstrap streams section.", "properties": { - "applicationId": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Unique application ID for Kafka Streams. Required for auto-scaling", - "title": "Unique application ID" - }, - "bootstrapServers": { + "brokers": { "description": "Brokers", - "title": "Bootstrapservers", + "title": "Brokers", "type": "string" }, "config": { @@ -2345,38 +2345,16 @@ "default": null, "description": "Error topic" }, - "inputPattern": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input pattern", - "title": "Inputpattern" - }, - "inputTopics": { - "default": [], - "description": "Input topics", - "items": { - "type": "string" - }, - "title": "Inputtopics", - "type": "array" - }, - "labeledInputPatterns": { + "extraInputPatterns": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra input patterns", - "title": "Labeledinputpatterns", + "title": "Extrainputpatterns", "type": "object" }, - "labeledInputTopics": { + "extraInputTopics": { "additionalProperties": { "items": { "type": "string" @@ -2385,18 +2363,40 @@ }, "default": {}, "description": "Extra input topics", - "title": "Labeledinputtopics", + "title": "Extrainputtopics", "type": "object" }, - "labeledOutputTopics": { + "extraOutputTopics": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra output topics", - "title": "Labeledoutputtopics", + "title": "Extraoutputtopics", "type": "object" }, + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, "outputTopic": { "anyOf": [ { @@ -2424,7 +2424,7 @@ } }, "required": [ - "bootstrapServers" + "brokers" ], "title": "StreamsConfig", "type": "object" @@ -2479,12 +2479,12 @@ "kafka-source-connector", "kubernetes-app", "pipeline-component", - "producer-app-v2", - "streams-app-v2", - "streams-bootstrap-v2", "producer-app", "streams-app", - "streams-bootstrap" + "streams-bootstrap", + "producer-app-v2", + "streams-app-v2", + "streams-bootstrap-v2" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index e0c5e39d3..a61cabf87 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1187,7 +1187,7 @@ "autoscaling": { "anyOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling" }, { "type": "null" @@ -1206,7 +1206,7 @@ "kafka": { "allOf": [ { - "$ref": "#/$defs/kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig" + "$ref": "#/$defs/kpops__components__streams_bootstrap__streams__model__StreamsConfig" } ], "description": "streams-bootstrap kafka section" @@ -1379,22 +1379,18 @@ "title": "TopicConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { - "consumerGroup": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", - "title": "Consumer group" + "additionalTriggers": { + "default": [], + "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", + "items": { + "type": "string" + }, + "title": "Additionaltriggers", + "type": "array" }, "cooldownPeriod": { "default": 300, @@ -1404,7 +1400,7 @@ }, "enabled": { "default": false, - "description": "", + "description": "Whether to enable auto-scaling using KEDA.", "title": "Enabled", "type": "boolean" }, @@ -1421,6 +1417,15 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, + "internalTopics": { + "default": [], + "description": "List of auto-generated Kafka Streams topics used by the streams app", + "items": { + "type": "string" + }, + "title": "Internaltopics", + "type": "array" + }, "lagThreshold": { "anyOf": [ { @@ -1460,7 +1465,7 @@ }, "topics": { "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app.", + "description": "List of topics used by the streams app", "items": { "type": "string" }, @@ -1471,13 +1476,26 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap__streams__model__StreamsConfig": { "additionalProperties": true, - "description": "Streams Bootstrap streams section.", + "description": "Streams Bootstrap kafka section.", "properties": { - "brokers": { + "applicationId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique application ID for Kafka Streams. Required for auto-scaling", + "title": "Unique application ID" + }, + "bootstrapServers": { "description": "Brokers", - "title": "Brokers", + "title": "Bootstrapservers", "type": "string" }, "config": { @@ -1511,16 +1529,38 @@ "default": null, "description": "Error topic" }, - "extraInputPatterns": { + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, + "labeledInputPatterns": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra input patterns", - "title": "Extrainputpatterns", + "title": "Labeledinputpatterns", "type": "object" }, - "extraInputTopics": { + "labeledInputTopics": { "additionalProperties": { "items": { "type": "string" @@ -1529,40 +1569,18 @@ }, "default": {}, "description": "Extra input topics", - "title": "Extrainputtopics", + "title": "Labeledinputtopics", "type": "object" }, - "extraOutputTopics": { + "labeledOutputTopics": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra output topics", - "title": "Extraoutputtopics", + "title": "Labeledoutputtopics", "type": "object" }, - "inputPattern": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input pattern", - "title": "Inputpattern" - }, - "inputTopics": { - "default": [], - "description": "Input topics", - "items": { - "type": "string" - }, - "title": "Inputtopics", - "type": "array" - }, "outputTopic": { "anyOf": [ { @@ -1590,23 +1608,27 @@ } }, "required": [ - "brokers" + "bootstrapServers" ], "title": "StreamsConfig", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsAppAutoScaling": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsAppAutoScaling": { "additionalProperties": true, "description": "Kubernetes Event-driven Autoscaling config.", "properties": { - "additionalTriggers": { - "default": [], - "description": "List of additional KEDA triggers, see https://keda.sh/docs/latest/scalers/", - "items": { - "type": "string" - }, - "title": "Additionaltriggers", - "type": "array" + "consumerGroup": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the consumer group used for checking the offset on the topic and processing the related lag. Mandatory to set when auto-scaling is enabled.", + "title": "Consumer group" }, "cooldownPeriod": { "default": 300, @@ -1616,7 +1638,7 @@ }, "enabled": { "default": false, - "description": "Whether to enable auto-scaling using KEDA.", + "description": "", "title": "Enabled", "type": "boolean" }, @@ -1633,15 +1655,6 @@ "description": "If this property is set, KEDA will scale the resource down to this number of replicas. https://keda.sh/docs/2.9/concepts/scaling-deployments/#idlereplicacount", "title": "Idle replica count" }, - "internalTopics": { - "default": [], - "description": "List of auto-generated Kafka Streams topics used by the streams app", - "items": { - "type": "string" - }, - "title": "Internaltopics", - "type": "array" - }, "lagThreshold": { "anyOf": [ { @@ -1681,7 +1694,7 @@ }, "topics": { "default": [], - "description": "List of topics used by the streams app", + "description": "List of auto-generated Kafka Streams topics used by the streams app.", "items": { "type": "string" }, @@ -1692,26 +1705,13 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "kpops__components__streams_bootstrap_v3__streams__model__StreamsConfig": { + "kpops__components__streams_bootstrap_v2__streams__model__StreamsConfig": { "additionalProperties": true, - "description": "Streams Bootstrap kafka section.", + "description": "Streams Bootstrap streams section.", "properties": { - "applicationId": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Unique application ID for Kafka Streams. Required for auto-scaling", - "title": "Unique application ID" - }, - "bootstrapServers": { + "brokers": { "description": "Brokers", - "title": "Bootstrapservers", + "title": "Brokers", "type": "string" }, "config": { @@ -1745,38 +1745,16 @@ "default": null, "description": "Error topic" }, - "inputPattern": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Input pattern", - "title": "Inputpattern" - }, - "inputTopics": { - "default": [], - "description": "Input topics", - "items": { - "type": "string" - }, - "title": "Inputtopics", - "type": "array" - }, - "labeledInputPatterns": { + "extraInputPatterns": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra input patterns", - "title": "Labeledinputpatterns", + "title": "Extrainputpatterns", "type": "object" }, - "labeledInputTopics": { + "extraInputTopics": { "additionalProperties": { "items": { "type": "string" @@ -1785,18 +1763,40 @@ }, "default": {}, "description": "Extra input topics", - "title": "Labeledinputtopics", + "title": "Extrainputtopics", "type": "object" }, - "labeledOutputTopics": { + "extraOutputTopics": { "additionalProperties": { "type": "string" }, "default": {}, "description": "Extra output topics", - "title": "Labeledoutputtopics", + "title": "Extraoutputtopics", "type": "object" }, + "inputPattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Input pattern", + "title": "Inputpattern" + }, + "inputTopics": { + "default": [], + "description": "Input topics", + "items": { + "type": "string" + }, + "title": "Inputtopics", + "type": "array" + }, "outputTopic": { "anyOf": [ { @@ -1824,7 +1824,7 @@ } }, "required": [ - "bootstrapServers" + "brokers" ], "title": "StreamsConfig", "type": "object" @@ -1854,16 +1854,16 @@ "$ref": "#/$defs/KafkaSourceConnector" }, { - "$ref": "#/$defs/ProducerAppV2" + "$ref": "#/$defs/ProducerApp" }, { - "$ref": "#/$defs/StreamsAppV2" + "$ref": "#/$defs/StreamsApp" }, { - "$ref": "#/$defs/ProducerApp" + "$ref": "#/$defs/ProducerAppV2" }, { - "$ref": "#/$defs/StreamsApp" + "$ref": "#/$defs/StreamsAppV2" } ] }, diff --git a/kpops/components/streams_bootstrap_v3/__init__.py b/kpops/components/streams_bootstrap/__init__.py similarity index 68% rename from kpops/components/streams_bootstrap_v3/__init__.py rename to kpops/components/streams_bootstrap/__init__.py index ad2ae6711..f900f809b 100644 --- a/kpops/components/streams_bootstrap_v3/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,4 +1,4 @@ -from kpops.components.streams_bootstrap_v3.base import StreamsBootstrap +from kpops.components.streams_bootstrap.base import StreamsBootstrap from .producer.producer_app import ProducerApp from .streams.streams_app import StreamsApp diff --git a/kpops/components/streams_bootstrap_v3/base.py b/kpops/components/streams_bootstrap/base.py similarity index 97% rename from kpops/components/streams_bootstrap_v3/base.py rename to kpops/components/streams_bootstrap/base.py index fb252f087..f544f5dfb 100644 --- a/kpops/components/streams_bootstrap_v3/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -11,7 +11,7 @@ from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp -from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapValues +from kpops.components.streams_bootstrap.model import StreamsBootstrapValues from kpops.utils.docstring import describe_attr if TYPE_CHECKING: diff --git a/kpops/components/streams_bootstrap_v3/model.py b/kpops/components/streams_bootstrap/model.py similarity index 100% rename from kpops/components/streams_bootstrap_v3/model.py rename to kpops/components/streams_bootstrap/model.py diff --git a/kpops/components/streams_bootstrap_v3/producer/__init__.py b/kpops/components/streams_bootstrap/producer/__init__.py similarity index 100% rename from kpops/components/streams_bootstrap_v3/producer/__init__.py rename to kpops/components/streams_bootstrap/producer/__init__.py diff --git a/kpops/components/streams_bootstrap_v3/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py similarity index 89% rename from kpops/components/streams_bootstrap_v3/producer/model.py rename to kpops/components/streams_bootstrap/producer/model.py index 87e4fb4d1..c6d99c83c 100644 --- a/kpops/components/streams_bootstrap_v3/producer/model.py +++ b/kpops/components/streams_bootstrap/producer/model.py @@ -1,6 +1,6 @@ from pydantic import ConfigDict, Field -from kpops.components.streams_bootstrap_v3.model import ( +from kpops.components.streams_bootstrap.model import ( KafkaConfig, StreamsBootstrapValues, ) diff --git a/kpops/components/streams_bootstrap_v3/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py similarity index 96% rename from kpops/components/streams_bootstrap_v3/producer/producer_app.py rename to kpops/components/streams_bootstrap/producer/producer_app.py index e402fb86d..035ab249e 100644 --- a/kpops/components/streams_bootstrap_v3/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -11,10 +11,10 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap_v3.base import ( +from kpops.components.streams_bootstrap.base import ( StreamsBootstrap, ) -from kpops.components.streams_bootstrap_v3.producer.model import ProducerAppValues +from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerAppV3") diff --git a/kpops/components/streams_bootstrap_v3/streams/__init__.py b/kpops/components/streams_bootstrap/streams/__init__.py similarity index 100% rename from kpops/components/streams_bootstrap_v3/streams/__init__.py rename to kpops/components/streams_bootstrap/streams/__init__.py diff --git a/kpops/components/streams_bootstrap_v3/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py similarity index 99% rename from kpops/components/streams_bootstrap_v3/streams/model.py rename to kpops/components/streams_bootstrap/streams/model.py index 30ad4151e..13c1bb182 100644 --- a/kpops/components/streams_bootstrap_v3/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field from kpops.components.common.topic import KafkaTopic, KafkaTopicStr -from kpops.components.streams_bootstrap_v3.model import ( +from kpops.components.streams_bootstrap.model import ( KafkaConfig, StreamsBootstrapValues, ) diff --git a/kpops/components/streams_bootstrap_v3/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py similarity index 97% rename from kpops/components/streams_bootstrap_v3/streams/streams_app.py rename to kpops/components/streams_bootstrap/streams/streams_app.py index 030344140..68a83c858 100644 --- a/kpops/components/streams_bootstrap_v3/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -9,10 +9,10 @@ from kpops.components.base_components.kafka_app import KafkaAppCleaner from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic -from kpops.components.streams_bootstrap_v3.base import ( +from kpops.components.streams_bootstrap.base import ( StreamsBootstrap, ) -from kpops.components.streams_bootstrap_v3.streams.model import ( +from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, ) from kpops.utils.docstring import describe_attr diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index e0ebc67a5..a925c7334 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -18,14 +18,14 @@ ) from kpops.components.base_components.kubernetes_app import KubernetesApp from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 -from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 -from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 -from kpops.components.streams_bootstrap_v3 import ( +from kpops.components.streams_bootstrap import ( ProducerApp, StreamsApp, StreamsBootstrap, ) +from kpops.components.streams_bootstrap_v2 import StreamsBootstrapV2 +from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 +from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 from tests.cli.resources.custom_module import CustomSchemaProvider @@ -105,7 +105,6 @@ def test_registry(): "kafka-source-connector": KafkaSourceConnector, "kubernetes-app": KubernetesApp, "pipeline-component": PipelineComponent, - # TODO: change the old sterams bootstrap to -v2 and remove -v3 "producer-app-v2": ProducerAppV2, "producer-app": ProducerApp, "streams-app-v2": StreamsAppV2, diff --git a/tests/components/streams_bootstrap_v3/test_producer_app.py b/tests/components/streams_bootstrap_v3/test_producer_app.py index 6837118dc..ff445b27f 100644 --- a/tests/components/streams_bootstrap_v3/test_producer_app.py +++ b/tests/components/streams_bootstrap_v3/test_producer_app.py @@ -13,7 +13,7 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap_v3.producer.producer_app import ( +from kpops.components.streams_bootstrap.producer.producer_app import ( ProducerApp, ProducerAppCleaner, ) diff --git a/tests/components/streams_bootstrap_v3/test_streams_app.py b/tests/components/streams_bootstrap_v3/test_streams_app.py index 1f3a0b015..96276513d 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_app.py +++ b/tests/components/streams_bootstrap_v3/test_streams_app.py @@ -20,11 +20,11 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap_v3 import StreamsApp -from kpops.components.streams_bootstrap_v3.streams.model import ( +from kpops.components.streams_bootstrap import StreamsApp +from kpops.components.streams_bootstrap.streams.model import ( StreamsAppAutoScaling, ) -from kpops.components.streams_bootstrap_v3.streams.streams_app import ( +from kpops.components.streams_bootstrap.streams.streams_app import ( StreamsAppCleaner, ) diff --git a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py index 176ac71b3..11ea644ac 100644 --- a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py @@ -9,8 +9,8 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.streams_bootstrap_v3.base import StreamsBootstrap -from kpops.components.streams_bootstrap_v3.model import StreamsBootstrapValues +from kpops.components.streams_bootstrap.base import StreamsBootstrap +from kpops.components.streams_bootstrap.model import StreamsBootstrapValues @pytest.mark.usefixtures("mock_env") diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index ef46a7229..c6b913ba9 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -12,9 +12,9 @@ ) from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.components.common.topic import OutputTopicTypes, TopicConfig +from kpops.components.streams_bootstrap import ProducerApp, StreamsApp from kpops.components.streams_bootstrap_v2.producer.producer_app import ProducerAppV2 from kpops.components.streams_bootstrap_v2.streams.streams_app import StreamsAppV2 -from kpops.components.streams_bootstrap_v3 import ProducerApp, StreamsApp class MyProducerApp(ProducerApp): ... From 0e05c3f4451201e688adf50237dace60e101f3eb Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 15 Aug 2024 10:18:04 +0200 Subject: [PATCH 07/19] move package --- tests/api/test_registry.py | 4 ++-- .../{streams_bootstrap_v3 => streams_bootstrap}/__init__.py | 0 .../test_producer_app.py | 0 .../test_streams_app.py | 0 .../test_streams_bootstrap.py | 0 tests/components/streams_bootstrap_v2/__init__.py | 0 .../{ => streams_bootstrap_v2}/test_producer_app.py | 0 .../components/{ => streams_bootstrap_v2}/test_streams_app.py | 0 .../{ => streams_bootstrap_v2}/test_streams_bootstrap.py | 0 9 files changed, 2 insertions(+), 2 deletions(-) rename tests/components/{streams_bootstrap_v3 => streams_bootstrap}/__init__.py (100%) rename tests/components/{streams_bootstrap_v3 => streams_bootstrap}/test_producer_app.py (100%) rename tests/components/{streams_bootstrap_v3 => streams_bootstrap}/test_streams_app.py (100%) rename tests/components/{streams_bootstrap_v3 => streams_bootstrap}/test_streams_bootstrap.py (100%) create mode 100644 tests/components/streams_bootstrap_v2/__init__.py rename tests/components/{ => streams_bootstrap_v2}/test_producer_app.py (100%) rename tests/components/{ => streams_bootstrap_v2}/test_streams_app.py (100%) rename tests/components/{ => streams_bootstrap_v2}/test_streams_bootstrap.py (100%) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index a925c7334..64df1f1c8 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -54,8 +54,8 @@ def test_iter_namespace(): assert [module.__name__ for module in _iter_namespace(components_module)] == [ "kpops.components.base_components", "kpops.components.common", + "kpops.components.streams_bootstrap", "kpops.components.streams_bootstrap_v2", - "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] @@ -66,8 +66,8 @@ def test_iter_component_modules(): "kpops.components", "kpops.components.base_components", "kpops.components.common", + "kpops.components.streams_bootstrap", "kpops.components.streams_bootstrap_v2", - "kpops.components.streams_bootstrap_v3", "kpops.components.test_components", ] diff --git a/tests/components/streams_bootstrap_v3/__init__.py b/tests/components/streams_bootstrap/__init__.py similarity index 100% rename from tests/components/streams_bootstrap_v3/__init__.py rename to tests/components/streams_bootstrap/__init__.py diff --git a/tests/components/streams_bootstrap_v3/test_producer_app.py b/tests/components/streams_bootstrap/test_producer_app.py similarity index 100% rename from tests/components/streams_bootstrap_v3/test_producer_app.py rename to tests/components/streams_bootstrap/test_producer_app.py diff --git a/tests/components/streams_bootstrap_v3/test_streams_app.py b/tests/components/streams_bootstrap/test_streams_app.py similarity index 100% rename from tests/components/streams_bootstrap_v3/test_streams_app.py rename to tests/components/streams_bootstrap/test_streams_app.py diff --git a/tests/components/streams_bootstrap_v3/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py similarity index 100% rename from tests/components/streams_bootstrap_v3/test_streams_bootstrap.py rename to tests/components/streams_bootstrap/test_streams_bootstrap.py diff --git a/tests/components/streams_bootstrap_v2/__init__.py b/tests/components/streams_bootstrap_v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/components/test_producer_app.py b/tests/components/streams_bootstrap_v2/test_producer_app.py similarity index 100% rename from tests/components/test_producer_app.py rename to tests/components/streams_bootstrap_v2/test_producer_app.py diff --git a/tests/components/test_streams_app.py b/tests/components/streams_bootstrap_v2/test_streams_app.py similarity index 100% rename from tests/components/test_streams_app.py rename to tests/components/streams_bootstrap_v2/test_streams_app.py diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/streams_bootstrap_v2/test_streams_bootstrap.py similarity index 100% rename from tests/components/test_streams_bootstrap.py rename to tests/components/streams_bootstrap_v2/test_streams_bootstrap.py From c9b03bac6bf2f9c157420c32df07a734be4db0c6 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 15 Aug 2024 10:34:30 +0200 Subject: [PATCH 08/19] move package --- kpops/components/streams_bootstrap/base.py | 2 +- .../streams_bootstrap/producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 2 +- .../streams_bootstrap/test_streams_bootstrap.py | 2 +- .../defaults.yaml | 0 .../pipeline.yaml | 0 .../pipeline.yaml | 14 +++++++------- tests/pipeline/test_generate.py | 4 ++-- tests/pipeline/test_manifest.py | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) rename tests/pipeline/resources/{streams-bootstrap-v3 => streams-bootstrap}/defaults.yaml (100%) rename tests/pipeline/resources/{streams-bootstrap-v3 => streams-bootstrap}/pipeline.yaml (100%) rename tests/pipeline/snapshots/test_generate/{test_streams_bootstrap_v3 => test_streams_bootstrap}/pipeline.yaml (92%) diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index f544f5dfb..f9f4c58f0 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -71,7 +71,7 @@ def version_validator(cls, version: str) -> str: major = int(major) if major != 3: - msg = f"When using the streams-bootstrap v3 component your version ('{version}') must be at least 3.0.0." + msg = f"When using the streams-bootstrap component your version ('{version}') must be at least 3.0.0." raise ValueError(msg) return version diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 035ab249e..9325e35a2 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -17,7 +17,7 @@ from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr -log = logging.getLogger("ProducerAppV3") +log = logging.getLogger("ProducerApp") class ProducerAppCleaner(KafkaAppCleaner, StreamsBootstrap): diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 68a83c858..9675ceae1 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -17,7 +17,7 @@ ) from kpops.utils.docstring import describe_attr -log = logging.getLogger("StreamsAppV3") +log = logging.getLogger("StreamsApp") class StreamsAppCleaner(KafkaAppCleaner, StreamsBootstrap): diff --git a/tests/components/streams_bootstrap/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py index 11ea644ac..0702571fb 100644 --- a/tests/components/streams_bootstrap/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap/test_streams_bootstrap.py @@ -105,7 +105,7 @@ async def test_should_raise_validation_error_for_invalid_helm_chart_version(self with pytest.raises( ValueError, match=re.escape( - "When using the streams-bootstrap v3 component your version ('2.1.0') must be at least 3.0.0." + "When using the streams-bootstrap component your version ('2.1.0') must be at least 3.0.0." ), ): StreamsBootstrap( diff --git a/tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml b/tests/pipeline/resources/streams-bootstrap/defaults.yaml similarity index 100% rename from tests/pipeline/resources/streams-bootstrap-v3/defaults.yaml rename to tests/pipeline/resources/streams-bootstrap/defaults.yaml diff --git a/tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml b/tests/pipeline/resources/streams-bootstrap/pipeline.yaml similarity index 100% rename from tests/pipeline/resources/streams-bootstrap-v3/pipeline.yaml rename to tests/pipeline/resources/streams-bootstrap/pipeline.yaml diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml similarity index 92% rename from tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml rename to tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml index ebf90338f..e93943c0d 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap_v3/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml @@ -1,7 +1,7 @@ - _cleaner: name: my-producer-app namespace: example-namespace - prefix: resources-streams-bootstrap-v3- + prefix: resources-streams-bootstrap- repo_config: repo_auth_flags: insecure_skip_tls_verify: false @@ -23,7 +23,7 @@ version: 3.0.0-SNAPSHOT name: my-producer-app namespace: example-namespace - prefix: resources-streams-bootstrap-v3- + prefix: resources-streams-bootstrap- repo_config: repo_auth_flags: insecure_skip_tls_verify: false @@ -54,7 +54,7 @@ - _cleaner: name: my-streams-app namespace: example-namespace - prefix: resources-streams-bootstrap-v3- + prefix: resources-streams-bootstrap- repo_config: repo_auth_flags: insecure_skip_tls_verify: false @@ -72,7 +72,7 @@ bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-streams-bootstrap-v3-my-streams-app-error + errorTopic: resources-streams-bootstrap-my-streams-app-error inputPattern: my-input-pattern inputTopics: - my-input-topic @@ -103,7 +103,7 @@ label: my-input-topic-label name: my-streams-app namespace: example-namespace - prefix: resources-streams-bootstrap-v3- + prefix: resources-streams-bootstrap- repo_config: repo_auth_flags: insecure_skip_tls_verify: false @@ -121,7 +121,7 @@ my-output-topic: configs: {} type: output - resources-streams-bootstrap-v3-my-streams-app-error: + resources-streams-bootstrap-my-streams-app-error: configs: cleanup.policy: compact,delete partitions_count: 1 @@ -138,7 +138,7 @@ bootstrapServers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 config: large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator - errorTopic: resources-streams-bootstrap-v3-my-streams-app-error + errorTopic: resources-streams-bootstrap-my-streams-app-error inputPattern: my-input-pattern inputTopics: - my-input-topic diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 25266d3b3..a1c9735b0 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -868,12 +868,12 @@ def test_substitution_in_resetter(self): == "override-default-image-tag" ) - def test_streams_bootstrap_v3(self, snapshot: Snapshot): + def test_streams_bootstrap(self, snapshot: Snapshot): result = runner.invoke( app, [ "generate", - str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), ], catch_exceptions=False, ) diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 381871a77..8651b47ce 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -125,12 +125,12 @@ def test_python_api(self, snapshot: Snapshot): assert len(resources) == 2 snapshot.assert_match(yaml.dump_all(resources), "resources") - def test_streams_bootstrap_v3(self, snapshot: Snapshot): + def test_streams_bootstrap(self, snapshot: Snapshot): result = runner.invoke( app, [ "manifest", - str(RESOURCE_PATH / "streams-bootstrap-v3" / PIPELINE_YAML), + str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), ], catch_exceptions=False, ) From d9c2790fb1d704837e79fb75c6ec6d6611bf0d20 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 15 Aug 2024 11:11:00 +0200 Subject: [PATCH 09/19] add manifest --- .../manifest.yaml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) rename tests/pipeline/snapshots/test_manifest/{test_streams_bootstrap_v3 => test_streams_bootstrap}/manifest.yaml (78%) diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml similarity index 78% rename from tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml rename to tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml index 89ce55efb..59689255d 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap_v3/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml @@ -3,17 +3,17 @@ apiVersion: batch/v1 kind: Job metadata: labels: - app: resources-streams-bootstrap-v3-my-producer-app + app: resources-streams-bootstrap-my-producer-app chart: producer-app-3.0.0-SNAPSHOT - release: resources-streams-bootstrap-v3-my-producer-app - name: resources-streams-bootstrap-v3-my-producer-app + release: resources-streams-bootstrap-my-producer-app + name: resources-streams-bootstrap-my-producer-app spec: backoffLimit: 6 template: metadata: labels: - app: resources-streams-bootstrap-v3-my-producer-app - release: resources-streams-bootstrap-v3-my-producer-app + app: resources-streams-bootstrap-my-producer-app + release: resources-streams-bootstrap-my-producer-app spec: containers: - env: @@ -33,7 +33,7 @@ spec: value: '-XX:MaxRAMPercentage=75.0 ' image: my-registry/my-producer-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-v3-my-producer-app + name: resources-streams-bootstrap-my-producer-app resources: limits: cpu: 500m @@ -52,11 +52,11 @@ data: kind: ConfigMap metadata: labels: - app: resources-streams-bootstrap-v3-my-streams-app + app: resources-streams-bootstrap-my-streams-app chart: streams-app-3.0.0-SNAPSHOT heritage: Helm - release: resources-streams-bootstrap-v3-my-streams-app - name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app-jmx-configmap --- apiVersion: apps/v1 @@ -65,24 +65,24 @@ metadata: annotations: consumerGroup: my-streams-app-id labels: - app: resources-streams-bootstrap-v3-my-streams-app + app: resources-streams-bootstrap-my-streams-app chart: streams-app-3.0.0-SNAPSHOT - release: resources-streams-bootstrap-v3-my-streams-app - name: resources-streams-bootstrap-v3-my-streams-app + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app spec: replicas: 1 selector: matchLabels: - app: resources-streams-bootstrap-v3-my-streams-app - release: resources-streams-bootstrap-v3-my-streams-app + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app template: metadata: annotations: prometheus.io/port: '5556' prometheus.io/scrape: 'true' labels: - app: resources-streams-bootstrap-v3-my-streams-app - release: resources-streams-bootstrap-v3-my-streams-app + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app spec: containers: - env: @@ -105,7 +105,7 @@ spec: - name: APP_OUTPUT_TOPIC value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-v3-my-streams-app-error + value: resources-streams-bootstrap-my-streams-app-error - name: APP_LABELED_OUTPUT_TOPICS value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS @@ -121,7 +121,7 @@ spec: -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-v3-my-streams-app + name: resources-streams-bootstrap-my-streams-app ports: - containerPort: 5555 name: jmx @@ -159,6 +159,6 @@ spec: terminationGracePeriodSeconds: 300 volumes: - configMap: - name: resources-streams-bootstrap-v3-my-streams-app-jmx-configmap + name: resources-streams-bootstrap-my-streams-app-jmx-configmap name: jmx-config From 0de2741ea4330a0cded145a79ddd7d326b8cd039 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 19 Aug 2024 08:43:48 +0200 Subject: [PATCH 10/19] gen schema --- .../dependencies/kpops_structure.yaml | 141 --- docs/docs/schema/defaults.json | 997 +++------------- docs/docs/schema/pipeline.json | 1035 +++-------------- 3 files changed, 321 insertions(+), 1852 deletions(-) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 669cf41fb..265cbf737 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -1,22 +1,4 @@ kpops_components_fields: - converter: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - filter: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version helm-app: - name - prefix @@ -63,24 +45,6 @@ kpops_components_fields: - to - namespace - values - my-producer-app: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - my-streams-app: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version pipeline-component: - name - prefix @@ -104,33 +68,6 @@ kpops_components_fields: - values - repo_config - version - scheduled-producer: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - should-inflate: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - simple-inflate-connectors: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version streams-app: - name - prefix @@ -168,29 +105,6 @@ kpops_components_fields: - repo_config - version kpops_components_inheritance_ref: - converter: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - filter: - bases: - - sub-streams-app - parents: - - sub-streams-app - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component helm-app: bases: - kubernetes-app @@ -230,28 +144,6 @@ kpops_components_inheritance_ref: parents: - pipeline-component - base-defaults-component - my-producer-app: - bases: - - producer-app - parents: - - producer-app - - streams-bootstrap - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - my-streams-app: - bases: - - streams-app - parents: - - streams-app - - streams-bootstrap - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component pipeline-component: bases: [] parents: @@ -276,39 +168,6 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component - scheduled-producer: - bases: - - producer-app-v2 - parents: - - producer-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - should-inflate: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - simple-inflate-connectors: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component streams-app: bases: - streams-bootstrap diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 7d7cedcf2..1fe98db24 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1,203 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -935,220 +737,41 @@ "title": "KubernetesAppValues", "type": "object" }, - "MyProducerApp": { - "additionalProperties": true, - "description": "", + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" }, - "repo_config": { - "allOf": [ + "size": { + "anyOf": [ { - "$ref": "#/$defs/HelmRepoConfig" + "type": "string" + }, + { + "type": "null" } ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" }, - "to": { + "storage_class": { "anyOf": [ { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyProducerApp", - "type": "object" - }, - "MyStreamsApp": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" + "type": "string" }, { "type": "null" @@ -1416,393 +1039,138 @@ "allOf": [ { "$ref": "#/$defs/ProducerStreamsConfig" - } - ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppV2Values", - "type": "object" - }, - "ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" }, - "values": { + "kafka": { "allOf": [ { - "$ref": "#/$defs/ProducerAppV2Values" + "$ref": "#/$defs/ProducerConfig" } ], - "description": "streams-bootstrap Helm values" + "description": "Kafka Streams settings" }, - "version": { + "nameOverride": { "anyOf": [ { + "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" } }, "required": [ - "name", - "namespace", - "values", - "type" + "kafka" ], - "title": "ScheduledProducer", + "title": "ProducerAppValues", "type": "object" }, - "ShouldInflate": { + "ProducerConfig": { "additionalProperties": true, - "description": "", + "description": "Kafka Streams settings specific to Producer.", "properties": { - "from": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Output topic" }, - "to": { + "schema_registry_url": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" }, - "type": { - "const": "should-inflate", - "title": "Type" + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" }, - "values": { - "allOf": [ + "outputTopic": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Output topic" }, - "version": { + "schema_registry_url": { "anyOf": [ { "type": "string" @@ -1811,97 +1179,68 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" } }, "required": [ - "name", - "namespace", - "values", - "type" + "brokers" ], - "title": "ShouldInflate", + "title": "ProducerStreamsConfig", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -1910,18 +1249,12 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "RepoAuthFlags", "type": "object" }, "StreamsApp": { @@ -3100,12 +2433,6 @@ } }, "properties": { - "converter": { - "$ref": "#/$defs/Converter" - }, - "filter": { - "$ref": "#/$defs/Filter" - }, "helm-app": { "$ref": "#/$defs/HelmApp" }, @@ -3124,12 +2451,6 @@ "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, - "my-producer-app": { - "$ref": "#/$defs/MyProducerApp" - }, - "my-streams-app": { - "$ref": "#/$defs/MyStreamsApp" - }, "pipeline-component": { "$ref": "#/$defs/PipelineComponent" }, @@ -3139,15 +2460,6 @@ "producer-app-v2": { "$ref": "#/$defs/ProducerAppV2" }, - "scheduled-producer": { - "$ref": "#/$defs/ScheduledProducer" - }, - "should-inflate": { - "$ref": "#/$defs/ShouldInflate" - }, - "simple-inflate-connectors": { - "$ref": "#/$defs/SimpleInflateConnectors" - }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, @@ -3174,14 +2486,7 @@ "streams-bootstrap", "producer-app-v2", "streams-app-v2", - "streams-bootstrap-v2", - "converter", - "filter", - "my-producer-app", - "my-streams-app", - "scheduled-producer", - "should-inflate", - "simple-inflate-connectors" + "streams-bootstrap-v2" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 2235b517e..a61cabf87 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1,203 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -642,9 +444,57 @@ "title": "KafkaSourceConnector", "type": "object" }, - "MyProducerApp": { + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Storage class to use for the persistent volume.", + "title": "Storage Class" + } + }, + "title": "PersistenceConfig", + "type": "object" + }, + "ProducerApp": { "additionalProperties": true, - "description": "", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { "from": { "default": null, @@ -700,7 +550,7 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "my-producer-app", + "const": "producer-app", "title": "Type" }, "values": { @@ -725,239 +575,12 @@ "values", "type" ], - "title": "MyProducerApp", + "title": "ProducerApp", "type": "object" }, - "MyStreamsApp": { + "ProducerAppV2": { "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Storage class to use for the persistent volume.", - "title": "Storage Class" - } - }, - "title": "PersistenceConfig", - "type": "object" - }, - "ProducerApp": { - "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ProducerApp", - "type": "object" - }, - "ProducerAppV2": { - "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { "from": { "default": null, @@ -1083,386 +706,131 @@ }, "required": [ "streams" - ], - "title": "ProducerAppV2Values", - "type": "object" - }, - "ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" }, - "values": { + "kafka": { "allOf": [ { - "$ref": "#/$defs/ProducerAppV2Values" + "$ref": "#/$defs/ProducerConfig" } ], - "description": "streams-bootstrap Helm values" + "description": "Kafka Streams settings" }, - "version": { + "nameOverride": { "anyOf": [ { + "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" } }, "required": [ - "name", - "namespace", - "values", - "type" + "kafka" ], - "title": "ScheduledProducer", + "title": "ProducerAppValues", "type": "object" }, - "ShouldInflate": { + "ProducerConfig": { "additionalProperties": true, - "description": "", + "description": "Kafka Streams settings specific to Producer.", "properties": { - "from": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Output topic" }, - "to": { + "schema_registry_url": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" }, - "type": { - "const": "should-inflate", - "title": "Type" + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" }, - "values": { - "allOf": [ + "outputTopic": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Output topic" }, - "version": { + "schema_registry_url": { "anyOf": [ { "type": "string" @@ -1471,97 +839,68 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" } }, "required": [ - "name", - "namespace", - "values", - "type" + "brokers" ], - "title": "ShouldInflate", + "title": "ProducerStreamsConfig", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -1570,18 +909,12 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "RepoAuthFlags", "type": "object" }, "StreamsApp": { @@ -2500,18 +1833,11 @@ "items": { "discriminator": { "mapping": { - "converter": "#/$defs/Converter", - "filter": "#/$defs/Filter", "helm-app": "#/$defs/HelmApp", "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", - "my-producer-app": "#/$defs/MyProducerApp", - "my-streams-app": "#/$defs/MyStreamsApp", "producer-app": "#/$defs/ProducerApp", "producer-app-v2": "#/$defs/ProducerAppV2", - "scheduled-producer": "#/$defs/ScheduledProducer", - "should-inflate": "#/$defs/ShouldInflate", - "simple-inflate-connectors": "#/$defs/SimpleInflateConnectors", "streams-app": "#/$defs/StreamsApp", "streams-app-v2": "#/$defs/StreamsAppV2" }, @@ -2538,27 +1864,6 @@ }, { "$ref": "#/$defs/StreamsAppV2" - }, - { - "$ref": "#/$defs/Converter" - }, - { - "$ref": "#/$defs/Filter" - }, - { - "$ref": "#/$defs/MyProducerApp" - }, - { - "$ref": "#/$defs/MyStreamsApp" - }, - { - "$ref": "#/$defs/ScheduledProducer" - }, - { - "$ref": "#/$defs/ShouldInflate" - }, - { - "$ref": "#/$defs/SimpleInflateConnectors" } ] }, From ac7a3f48f19c1ce5be82348f3e9df9af19e2ea24 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 19 Aug 2024 10:44:43 +0200 Subject: [PATCH 11/19] add tests and validation check --- .../producer/producer_app.py | 15 +++- .../streams_bootstrap/streams/streams_app.py | 15 +++- .../streams_bootstrap/test_producer_app.py | 71 ++++++++++++++++ .../streams_bootstrap/test_streams_app.py | 84 +++++++++++++++++++ 4 files changed, 179 insertions(+), 6 deletions(-) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 5656c68e7..21cda9ebd 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -1,7 +1,7 @@ import logging from functools import cached_property -from pydantic import Field, computed_field +from pydantic import Field, ValidationError, computed_field from typing_extensions import override from kpops.components.base_components.kafka_app import KafkaAppCleaner @@ -15,6 +15,7 @@ StreamsBootstrap, ) from kpops.components.streams_bootstrap.producer.model import ProducerAppValues +from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerApp") @@ -104,8 +105,16 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.values = self.values.model_validate(cluster_values) - self._cleaner.values.name_override = name_override + try: + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + except ValidationError as validation_error: + warning_msg = f"The values in the cluster are invalid with the current model. Falling back to the enriched values of {PIPELINE_YAML} and {DEFAULTS_YAML}" + log.warning(warning_msg) + debug_msg = f"Cluster values: {cluster_values}" + log.debug(debug_msg) + debug_msg = f"Validation error: {validation_error}" + log.debug(debug_msg) await super().destroy(dry_run) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index b3db4638c..014d80828 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -1,7 +1,7 @@ import logging from functools import cached_property -from pydantic import Field, computed_field +from pydantic import Field, ValidationError, computed_field from typing_extensions import override from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler @@ -15,6 +15,7 @@ from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, ) +from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -132,8 +133,16 @@ async def destroy(self, dry_run: bool) -> None: if cluster_values: log.debug("Fetched Helm chart values from cluster") name_override = self._cleaner.helm_name_override - self._cleaner.values = self.values.model_validate(cluster_values) - self._cleaner.values.name_override = name_override + try: + self._cleaner.values = self.values.model_validate(cluster_values) + self._cleaner.values.name_override = name_override + except ValidationError as validation_error: + warning_msg = f"The values in the cluster are invalid with the current model. Falling back to the enriched values of {PIPELINE_YAML} and {DEFAULTS_YAML}" + log.warning(warning_msg) + debug_msg = f"Cluster values: {cluster_values}" + log.debug(debug_msg) + debug_msg = f"Validation error: {validation_error}" + log.debug(debug_msg) await super().destroy(dry_run) diff --git a/tests/components/streams_bootstrap/test_producer_app.py b/tests/components/streams_bootstrap/test_producer_app.py index ff445b27f..fbf125b45 100644 --- a/tests/components/streams_bootstrap/test_producer_app.py +++ b/tests/components/streams_bootstrap/test_producer_app.py @@ -472,3 +472,74 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( }, HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), ) + + @pytest.mark.asyncio() + async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster_values_fails( + self, mocker: MockerFixture, caplog: pytest.LogCaptureFixture + ): + caplog.set_level(logging.WARNING) + + # invalid model + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": "1.1.1", + "nameOverride": PRODUCER_APP_NAME, + "streams": { + "brokers": "fake-broker:9092", + "outputTopic": "test-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + + # user defined model + producer_app = ProducerApp( + name=PRODUCER_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "image": "registry/producer-app", + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "to": { + "topics": { + "test-output-topic": {"type": "output"}, + } + }, + }, + ) + mocker.patch.object(producer_app.helm, "uninstall") + mocker.patch.object(producer_app._cleaner.dry_run_handler, "print_helm_diff") + mocker.patch.object(producer_app._cleaner.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + producer_app._cleaner.helm, "upgrade_install" + ) + + dry_run = True + await producer_app.clean(dry_run) + assert ( + "The values in the cluster are invalid with the current model. Falling back to the enriched values of pipeline.yaml and defaults.yaml" + in caplog.text + ) + + mock_helm_upgrade_install.assert_called_once_with( + PRODUCER_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/producer-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/producer-app", + "imageTag": "2.2.2", + "nameOverride": PRODUCER_APP_CLEAN_HELM_NAMEOVERRIDE, + "kafka": { + "bootstrapServers": "fake-broker:9092", + "outputTopic": "test-output-topic", + }, + }, + HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + ) diff --git a/tests/components/streams_bootstrap/test_streams_app.py b/tests/components/streams_bootstrap/test_streams_app.py index 96276513d..f87805e64 100644 --- a/tests/components/streams_bootstrap/test_streams_app.py +++ b/tests/components/streams_bootstrap/test_streams_app.py @@ -918,3 +918,87 @@ async def test_stateful_clean_with_dry_run_true( f"Deleting the PVCs {pvc_names} for StatefulSet '{STREAMS_APP_FULL_NAME}'" in caplog.text ) + + @pytest.mark.asyncio() + async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster_values_fails( + self, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ): + caplog.set_level(logging.WARNING) + + # invalid model + mocker.patch.object( + Helm, + "get_values", + return_value={ + "image": "registry/producer-app", + "imageTag": "1.1.1", + "nameOverride": STREAMS_APP_NAME, + "streams": { + "brokers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "schemaRegistryUrl": "http://localhost:8081", + }, + }, + ) + + streams_app = StreamsApp( + name=STREAMS_APP_NAME, + **{ + "namespace": "test-namespace", + "values": { + "image": "registry/streams-app", + "imageTag": "2.2.2", + "kafka": {"bootstrapServers": "fake-broker:9092"}, + }, + "from": { + "topics": { + "test-input-topic": {"type": "input"}, + } + }, + "to": { + "topics": { + "streams-app-output-topic": {"type": "output"}, + } + }, + }, + ) + + mocker.patch.object(streams_app.helm, "uninstall") + + mock_helm_upgrade_install = mocker.patch.object( + streams_app._cleaner.helm, "upgrade_install" + ) + mocker.patch.object(streams_app._cleaner.helm, "uninstall") + + mock = mocker.MagicMock() + mock.attach_mock(mock_helm_upgrade_install, "helm_upgrade_install") + + dry_run = False + await streams_app.clean(dry_run=dry_run) + + assert ( + "The values in the cluster are invalid with the current model. Falling back to the enriched values of pipeline.yaml and defaults.yaml" + in caplog.text + ) + + mock_helm_upgrade_install.assert_called_once_with( + STREAMS_APP_CLEAN_RELEASE_NAME, + "bakdata-streams-bootstrap/streams-app-cleanup-job", + dry_run, + "test-namespace", + { + "image": "registry/streams-app", + "nameOverride": STREAMS_APP_CLEAN_HELM_NAME_OVERRIDE, + "imageTag": "2.2.2", + "kafka": { + "bootstrapServers": "fake-broker:9092", + "inputTopics": ["test-input-topic"], + "outputTopic": "streams-app-output-topic", + "deleteOutput": True, + }, + }, + HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + ) From b765b514fb2a95736c70de56bfc2abf3bdc1ff42 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 19 Aug 2024 11:14:28 +0200 Subject: [PATCH 12/19] add deprication decorator --- .../dependencies/kpops_structure.yaml | 141 +++ docs/docs/schema/defaults.json | 955 +++++++++++++--- docs/docs/schema/pipeline.json | 1015 ++++++++++++++--- kpops/components/streams_bootstrap_v2/base.py | 2 + .../producer/producer_app.py | 3 +- .../streams/streams_app.py | 3 +- 6 files changed, 1827 insertions(+), 292 deletions(-) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 265cbf737..669cf41fb 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -1,4 +1,22 @@ kpops_components_fields: + converter: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version + filter: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version helm-app: - name - prefix @@ -45,6 +63,24 @@ kpops_components_fields: - to - namespace - values + my-producer-app: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version + my-streams-app: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version pipeline-component: - name - prefix @@ -68,6 +104,33 @@ kpops_components_fields: - values - repo_config - version + scheduled-producer: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version + should-inflate: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version + simple-inflate-connectors: + - name + - prefix + - from_ + - to + - namespace + - values + - repo_config + - version streams-app: - name - prefix @@ -105,6 +168,29 @@ kpops_components_fields: - repo_config - version kpops_components_inheritance_ref: + converter: + bases: + - streams-app-v2 + parents: + - streams-app-v2 + - streams-bootstrap-v2 + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component + filter: + bases: + - sub-streams-app + parents: + - sub-streams-app + - streams-app-v2 + - streams-bootstrap-v2 + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component helm-app: bases: - kubernetes-app @@ -144,6 +230,28 @@ kpops_components_inheritance_ref: parents: - pipeline-component - base-defaults-component + my-producer-app: + bases: + - producer-app + parents: + - producer-app + - streams-bootstrap + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component + my-streams-app: + bases: + - streams-app + parents: + - streams-app + - streams-bootstrap + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component pipeline-component: bases: [] parents: @@ -168,6 +276,39 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component + scheduled-producer: + bases: + - producer-app-v2 + parents: + - producer-app-v2 + - streams-bootstrap-v2 + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component + should-inflate: + bases: + - streams-app-v2 + parents: + - streams-app-v2 + - streams-bootstrap-v2 + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component + simple-inflate-connectors: + bases: + - streams-app-v2 + parents: + - streams-app-v2 + - streams-bootstrap-v2 + - kafka-app + - helm-app + - kubernetes-app + - pipeline-component + - base-defaults-component streams-app: bases: - streams-bootstrap diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 1fe98db24..7d7cedcf2 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1,5 +1,203 @@ { "$defs": { + "Converter": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "converter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Converter", + "type": "object" + }, + "Filter": { + "additionalProperties": true, + "description": "Subsubclass of StreamsApp to test inheritance.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "filter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Filter", + "type": "object" + }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -737,41 +935,220 @@ "title": "KubernetesAppValues", "type": "object" }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "MyProducerApp": { + "additionalProperties": true, + "description": "", "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" }, - "size": { - "anyOf": [ - { - "type": "string" - }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/HelmRepoConfig" } ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "storage_class": { + "to": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-producer-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.0", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyProducerApp", + "type": "object" + }, + "MyStreamsApp": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-streams-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.0", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyStreamsApp", + "type": "object" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" }, { "type": "null" @@ -1061,116 +1438,371 @@ "title": "Imagetag", "type": "string" }, - "kafka": { + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/ProducerConfig" + } + ], + "description": "Kafka Streams settings" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "ProducerAppValues", + "type": "object" + }, + "ProducerConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", + "properties": { + "ca_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" + }, + "cert_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" + }, + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" + }, + "password": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password", + "title": "Password" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username", + "title": "Username" + } + }, + "title": "RepoAuthFlags", + "type": "object" + }, + "ScheduledProducer": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "scheduled-producer", + "title": "Type" + }, + "values": { "allOf": [ { - "$ref": "#/$defs/ProducerConfig" + "$ref": "#/$defs/ProducerAppV2Values" } ], - "description": "Kafka Streams settings" + "description": "streams-bootstrap Helm values" }, - "nameOverride": { + "version": { "anyOf": [ { - "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, "required": [ - "kafka" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerAppValues", + "title": "ScheduledProducer", "type": "object" }, - "ProducerConfig": { + "ShouldInflate": { "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", + "description": "", "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { + "from": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/HelmRepoConfig" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "outputTopic": { + "to": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) into which the component will write output" }, - "schema_registry_url": { + "type": { + "const": "should-inflate", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { "anyOf": [ { "type": "string" @@ -1179,68 +1811,97 @@ "type": "null" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, "required": [ - "brokers" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerStreamsConfig", + "title": "ShouldInflate", "type": "object" }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", + "SimpleInflateConnectors": { + "additionalProperties": true, + "description": "", "properties": { - "ca_file": { + "from": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "cert_file": { + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" + "description": "Topic(s) into which the component will write output" }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" + "type": { + "const": "simple-inflate-connectors", + "title": "Type" }, - "password": { - "anyOf": [ - { - "type": "string" - }, + "values": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": null, - "description": "Password", - "title": "Password" + "description": "streams-bootstrap-v2 Helm values" }, - "username": { + "version": { "anyOf": [ { "type": "string" @@ -1249,12 +1910,18 @@ "type": "null" } ], - "default": null, - "description": "Username", - "title": "Username" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "RepoAuthFlags", + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "SimpleInflateConnectors", "type": "object" }, "StreamsApp": { @@ -2433,6 +3100,12 @@ } }, "properties": { + "converter": { + "$ref": "#/$defs/Converter" + }, + "filter": { + "$ref": "#/$defs/Filter" + }, "helm-app": { "$ref": "#/$defs/HelmApp" }, @@ -2451,6 +3124,12 @@ "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, + "my-producer-app": { + "$ref": "#/$defs/MyProducerApp" + }, + "my-streams-app": { + "$ref": "#/$defs/MyStreamsApp" + }, "pipeline-component": { "$ref": "#/$defs/PipelineComponent" }, @@ -2460,6 +3139,15 @@ "producer-app-v2": { "$ref": "#/$defs/ProducerAppV2" }, + "scheduled-producer": { + "$ref": "#/$defs/ScheduledProducer" + }, + "should-inflate": { + "$ref": "#/$defs/ShouldInflate" + }, + "simple-inflate-connectors": { + "$ref": "#/$defs/SimpleInflateConnectors" + }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, @@ -2486,7 +3174,14 @@ "streams-bootstrap", "producer-app-v2", "streams-app-v2", - "streams-bootstrap-v2" + "streams-bootstrap-v2", + "converter", + "filter", + "my-producer-app", + "my-streams-app", + "scheduled-producer", + "should-inflate", + "simple-inflate-connectors" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index a61cabf87..2235b517e 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1,5 +1,203 @@ { "$defs": { + "Converter": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "converter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Converter", + "type": "object" + }, + "Filter": { + "additionalProperties": true, + "description": "Subsubclass of StreamsApp to test inheritance.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "filter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Filter", + "type": "object" + }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -444,57 +642,9 @@ "title": "KafkaSourceConnector", "type": "object" }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Storage class to use for the persistent volume.", - "title": "Storage Class" - } - }, - "title": "PersistenceConfig", - "type": "object" - }, - "ProducerApp": { + "MyProducerApp": { "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "description": "", "properties": { "from": { "default": null, @@ -550,7 +700,7 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "producer-app", + "const": "my-producer-app", "title": "Type" }, "values": { @@ -575,19 +725,246 @@ "values", "type" ], - "title": "ProducerApp", + "title": "MyProducerApp", "type": "object" }, - "ProducerAppV2": { + "MyStreamsApp": { "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "description": "", "properties": { "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-streams-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.0", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyStreamsApp", + "type": "object" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Storage class to use for the persistent volume.", + "title": "Storage Class" + } + }, + "title": "PersistenceConfig", + "type": "object" + }, + "ProducerApp": { + "additionalProperties": true, + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "producer-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.0", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "ProducerApp", + "type": "object" + }, + "ProducerAppV2": { + "additionalProperties": true, + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, "name": { "description": "Component name", "title": "Name", @@ -721,116 +1098,371 @@ "title": "Imagetag", "type": "string" }, - "kafka": { + "kafka": { + "allOf": [ + { + "$ref": "#/$defs/ProducerConfig" + } + ], + "description": "Kafka Streams settings" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "required": [ + "kafka" + ], + "title": "ProducerAppValues", + "type": "object" + }, + "ProducerConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" + }, + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" + }, + "outputTopic": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Output topic" + }, + "schema_registry_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "brokers" + ], + "title": "ProducerStreamsConfig", + "type": "object" + }, + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", + "properties": { + "ca_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" + }, + "cert_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" + }, + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" + }, + "password": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password", + "title": "Password" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username", + "title": "Username" + } + }, + "title": "RepoAuthFlags", + "type": "object" + }, + "ScheduledProducer": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "scheduled-producer", + "title": "Type" + }, + "values": { "allOf": [ { - "$ref": "#/$defs/ProducerConfig" + "$ref": "#/$defs/ProducerAppV2Values" } ], - "description": "Kafka Streams settings" + "description": "streams-bootstrap Helm values" }, - "nameOverride": { + "version": { "anyOf": [ { - "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, "required": [ - "kafka" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerAppValues", + "title": "ScheduledProducer", "type": "object" }, - "ProducerConfig": { + "ShouldInflate": { "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", + "description": "", "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { + "from": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/HelmRepoConfig" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "outputTopic": { + "to": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Output topic" + "description": "Topic(s) into which the component will write output" }, - "schema_registry_url": { + "type": { + "const": "should-inflate", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { "anyOf": [ { "type": "string" @@ -839,68 +1471,97 @@ "type": "null" } ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, "required": [ - "brokers" + "name", + "namespace", + "values", + "type" ], - "title": "ProducerStreamsConfig", + "title": "ShouldInflate", "type": "object" }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", + "SimpleInflateConnectors": { + "additionalProperties": true, + "description": "", "properties": { - "ca_file": { + "from": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "cert_file": { + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" + "description": "Topic(s) into which the component will write output" }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" + "type": { + "const": "simple-inflate-connectors", + "title": "Type" }, - "password": { - "anyOf": [ - { - "type": "string" - }, + "values": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": null, - "description": "Password", - "title": "Password" + "description": "streams-bootstrap-v2 Helm values" }, - "username": { + "version": { "anyOf": [ { "type": "string" @@ -909,12 +1570,18 @@ "type": "null" } ], - "default": null, - "description": "Username", - "title": "Username" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "RepoAuthFlags", + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "SimpleInflateConnectors", "type": "object" }, "StreamsApp": { @@ -1833,11 +2500,18 @@ "items": { "discriminator": { "mapping": { + "converter": "#/$defs/Converter", + "filter": "#/$defs/Filter", "helm-app": "#/$defs/HelmApp", "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", + "my-producer-app": "#/$defs/MyProducerApp", + "my-streams-app": "#/$defs/MyStreamsApp", "producer-app": "#/$defs/ProducerApp", "producer-app-v2": "#/$defs/ProducerAppV2", + "scheduled-producer": "#/$defs/ScheduledProducer", + "should-inflate": "#/$defs/ShouldInflate", + "simple-inflate-connectors": "#/$defs/SimpleInflateConnectors", "streams-app": "#/$defs/StreamsApp", "streams-app-v2": "#/$defs/StreamsAppV2" }, @@ -1864,6 +2538,27 @@ }, { "$ref": "#/$defs/StreamsAppV2" + }, + { + "$ref": "#/$defs/Converter" + }, + { + "$ref": "#/$defs/Filter" + }, + { + "$ref": "#/$defs/MyProducerApp" + }, + { + "$ref": "#/$defs/MyStreamsApp" + }, + { + "$ref": "#/$defs/ScheduledProducer" + }, + { + "$ref": "#/$defs/ShouldInflate" + }, + { + "$ref": "#/$defs/SimpleInflateConnectors" } ] }, diff --git a/kpops/components/streams_bootstrap_v2/base.py b/kpops/components/streams_bootstrap_v2/base.py index ffdf156ae..bf52685c1 100644 --- a/kpops/components/streams_bootstrap_v2/base.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -6,6 +6,7 @@ import pydantic from pydantic import AliasChoices, ConfigDict, Field +from typing_extensions import deprecated from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp @@ -112,6 +113,7 @@ class StreamsBootstrapV2Values(HelmAppValues): ) +@deprecated("StreamsBootstrapV2 component is deprecated, use StreamsBootstrap instead.") class StreamsBootstrapV2(KafkaApp, HelmApp, ABC): """Base for components with a streams-bootstrap-v2 Helm chart. diff --git a/kpops/components/streams_bootstrap_v2/producer/producer_app.py b/kpops/components/streams_bootstrap_v2/producer/producer_app.py index 6eb0e7e00..c36ef2c5b 100644 --- a/kpops/components/streams_bootstrap_v2/producer/producer_app.py +++ b/kpops/components/streams_bootstrap_v2/producer/producer_app.py @@ -2,7 +2,7 @@ from functools import cached_property from pydantic import Field, ValidationError, computed_field -from typing_extensions import override +from typing_extensions import deprecated, override from kpops.components.base_components.kafka_app import ( KafkaAppCleaner, @@ -32,6 +32,7 @@ def helm_chart(self) -> str: ) +@deprecated("ProducerAppV2 component is deprecated, use ProducerApp instead.") class ProducerAppV2(StreamsBootstrapV2): """Producer component. diff --git a/kpops/components/streams_bootstrap_v2/streams/streams_app.py b/kpops/components/streams_bootstrap_v2/streams/streams_app.py index 5ee526cc5..0aa7e9bf0 100644 --- a/kpops/components/streams_bootstrap_v2/streams/streams_app.py +++ b/kpops/components/streams_bootstrap_v2/streams/streams_app.py @@ -2,7 +2,7 @@ from functools import cached_property from pydantic import Field, ValidationError, computed_field -from typing_extensions import override +from typing_extensions import deprecated, override from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp @@ -53,6 +53,7 @@ async def clean_pvcs(self, dry_run: bool) -> None: await pvc_handler.delete_pvcs() +@deprecated("StreamsAppV2 component is deprecated, use StreamsApp instead.") class StreamsAppV2(StreamsBootstrapV2): """StreamsAppV2 component that configures a streams-bootstrap-v2 app. From 9e7bcd1582564fd5e0a9ddcca88eb854846c89c4 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 19 Aug 2024 11:15:26 +0200 Subject: [PATCH 13/19] add deprication decorator --- .../dependencies/kpops_structure.yaml | 141 --- docs/docs/schema/defaults.json | 997 +++------------- docs/docs/schema/pipeline.json | 1035 +++-------------- 3 files changed, 321 insertions(+), 1852 deletions(-) diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml index 669cf41fb..265cbf737 100644 --- a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml +++ b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml @@ -1,22 +1,4 @@ kpops_components_fields: - converter: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - filter: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version helm-app: - name - prefix @@ -63,24 +45,6 @@ kpops_components_fields: - to - namespace - values - my-producer-app: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - my-streams-app: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version pipeline-component: - name - prefix @@ -104,33 +68,6 @@ kpops_components_fields: - values - repo_config - version - scheduled-producer: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - should-inflate: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version - simple-inflate-connectors: - - name - - prefix - - from_ - - to - - namespace - - values - - repo_config - - version streams-app: - name - prefix @@ -168,29 +105,6 @@ kpops_components_fields: - repo_config - version kpops_components_inheritance_ref: - converter: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - filter: - bases: - - sub-streams-app - parents: - - sub-streams-app - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component helm-app: bases: - kubernetes-app @@ -230,28 +144,6 @@ kpops_components_inheritance_ref: parents: - pipeline-component - base-defaults-component - my-producer-app: - bases: - - producer-app - parents: - - producer-app - - streams-bootstrap - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - my-streams-app: - bases: - - streams-app - parents: - - streams-app - - streams-bootstrap - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component pipeline-component: bases: [] parents: @@ -276,39 +168,6 @@ kpops_components_inheritance_ref: - kubernetes-app - pipeline-component - base-defaults-component - scheduled-producer: - bases: - - producer-app-v2 - parents: - - producer-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - should-inflate: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component - simple-inflate-connectors: - bases: - - streams-app-v2 - parents: - - streams-app-v2 - - streams-bootstrap-v2 - - kafka-app - - helm-app - - kubernetes-app - - pipeline-component - - base-defaults-component streams-app: bases: - streams-bootstrap diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 7d7cedcf2..1fe98db24 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1,203 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -935,220 +737,41 @@ "title": "KubernetesAppValues", "type": "object" }, - "MyProducerApp": { - "additionalProperties": true, - "description": "", + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" }, - "repo_config": { - "allOf": [ + "size": { + "anyOf": [ { - "$ref": "#/$defs/HelmRepoConfig" + "type": "string" + }, + { + "type": "null" } ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" }, - "to": { + "storage_class": { "anyOf": [ { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyProducerApp", - "type": "object" - }, - "MyStreamsApp": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" + "type": "string" }, { "type": "null" @@ -1416,393 +1039,138 @@ "allOf": [ { "$ref": "#/$defs/ProducerStreamsConfig" - } - ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerAppV2Values", - "type": "object" - }, - "ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" }, - "values": { + "kafka": { "allOf": [ { - "$ref": "#/$defs/ProducerAppV2Values" + "$ref": "#/$defs/ProducerConfig" } ], - "description": "streams-bootstrap Helm values" + "description": "Kafka Streams settings" }, - "version": { + "nameOverride": { "anyOf": [ { + "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" } }, "required": [ - "name", - "namespace", - "values", - "type" + "kafka" ], - "title": "ScheduledProducer", + "title": "ProducerAppValues", "type": "object" }, - "ShouldInflate": { + "ProducerConfig": { "additionalProperties": true, - "description": "", + "description": "Kafka Streams settings specific to Producer.", "properties": { - "from": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Output topic" }, - "to": { + "schema_registry_url": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" }, - "type": { - "const": "should-inflate", - "title": "Type" + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" }, - "values": { - "allOf": [ + "outputTopic": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Output topic" }, - "version": { + "schema_registry_url": { "anyOf": [ { "type": "string" @@ -1811,97 +1179,68 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" } }, "required": [ - "name", - "namespace", - "values", - "type" + "brokers" ], - "title": "ShouldInflate", + "title": "ProducerStreamsConfig", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -1910,18 +1249,12 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "RepoAuthFlags", "type": "object" }, "StreamsApp": { @@ -3100,12 +2433,6 @@ } }, "properties": { - "converter": { - "$ref": "#/$defs/Converter" - }, - "filter": { - "$ref": "#/$defs/Filter" - }, "helm-app": { "$ref": "#/$defs/HelmApp" }, @@ -3124,12 +2451,6 @@ "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, - "my-producer-app": { - "$ref": "#/$defs/MyProducerApp" - }, - "my-streams-app": { - "$ref": "#/$defs/MyStreamsApp" - }, "pipeline-component": { "$ref": "#/$defs/PipelineComponent" }, @@ -3139,15 +2460,6 @@ "producer-app-v2": { "$ref": "#/$defs/ProducerAppV2" }, - "scheduled-producer": { - "$ref": "#/$defs/ScheduledProducer" - }, - "should-inflate": { - "$ref": "#/$defs/ShouldInflate" - }, - "simple-inflate-connectors": { - "$ref": "#/$defs/SimpleInflateConnectors" - }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, @@ -3174,14 +2486,7 @@ "streams-bootstrap", "producer-app-v2", "streams-app-v2", - "streams-bootstrap-v2", - "converter", - "filter", - "my-producer-app", - "my-streams-app", - "scheduled-producer", - "should-inflate", - "simple-inflate-connectors" + "streams-bootstrap-v2" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 2235b517e..a61cabf87 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1,203 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -642,9 +444,57 @@ "title": "KafkaSourceConnector", "type": "object" }, - "MyProducerApp": { + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Storage class to use for the persistent volume.", + "title": "Storage Class" + } + }, + "title": "PersistenceConfig", + "type": "object" + }, + "ProducerApp": { "additionalProperties": true, - "description": "", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { "from": { "default": null, @@ -700,7 +550,7 @@ "description": "Topic(s) into which the component will write output" }, "type": { - "const": "my-producer-app", + "const": "producer-app", "title": "Type" }, "values": { @@ -725,239 +575,12 @@ "values", "type" ], - "title": "MyProducerApp", + "title": "ProducerApp", "type": "object" }, - "MyStreamsApp": { + "ProducerAppV2": { "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Storage class to use for the persistent volume.", - "title": "Storage Class" - } - }, - "title": "PersistenceConfig", - "type": "object" - }, - "ProducerApp": { - "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.0", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ProducerApp", - "type": "object" - }, - "ProducerAppV2": { - "additionalProperties": true, - "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", + "description": "Producer component.\nThis producer holds configuration to use as values for the streams-bootstrap producer Helm chart. Note that the producer does not support error topics.", "properties": { "from": { "default": null, @@ -1083,386 +706,131 @@ }, "required": [ "streams" - ], - "title": "ProducerAppV2Values", - "type": "object" - }, - "ProducerAppValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Docker image tag of the streams-bootstrap app.", - "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", - "title": "Imagetag", - "type": "string" - }, - "kafka": { - "allOf": [ - { - "$ref": "#/$defs/ProducerConfig" - } - ], - "description": "Kafka Streams settings" - }, - "nameOverride": { - "anyOf": [ - { - "maxLength": 63, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Helm chart name override, assigned automatically", - "title": "Nameoverride" - } - }, - "required": [ - "kafka" - ], - "title": "ProducerAppValues", - "type": "object" - }, - "ProducerConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "bootstrapServers": { - "description": "Brokers", - "title": "Bootstrapservers", - "type": "string" - }, - "labeledOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Labeledoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "bootstrapServers" - ], - "title": "ProducerConfig", - "type": "object" - }, - "ProducerStreamsConfig": { - "additionalProperties": true, - "description": "Kafka Streams settings specific to Producer.", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "extraOutputTopics": { - "additionalProperties": { - "type": "string" - }, - "default": {}, - "description": "Extra output topics", - "title": "Extraoutputtopics", - "type": "object" - }, - "outputTopic": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Output topic" - }, - "schema_registry_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "URL of the schema registry", - "title": "Schema Registry Url" - } - }, - "required": [ - "brokers" - ], - "title": "ProducerStreamsConfig", - "type": "object" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" + ], + "title": "ProducerAppV2Values", + "type": "object" + }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "imageTag": { + "default": "latest", + "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", + "title": "Imagetag", + "type": "string" }, - "values": { + "kafka": { "allOf": [ { - "$ref": "#/$defs/ProducerAppV2Values" + "$ref": "#/$defs/ProducerConfig" } ], - "description": "streams-bootstrap Helm values" + "description": "Kafka Streams settings" }, - "version": { + "nameOverride": { "anyOf": [ { + "maxLength": 63, "type": "string" }, { "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" } }, "required": [ - "name", - "namespace", - "values", - "type" + "kafka" ], - "title": "ScheduledProducer", + "title": "ProducerAppValues", "type": "object" }, - "ShouldInflate": { + "ProducerConfig": { "additionalProperties": true, - "description": "", + "description": "Kafka Streams settings specific to Producer.", "properties": { - "from": { + "bootstrapServers": { + "description": "Brokers", + "title": "Bootstrapservers", + "type": "string" + }, + "labeledOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Labeledoutputtopics", + "type": "object" + }, + "outputTopic": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Output topic" }, - "to": { + "schema_registry_url": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "URL of the schema registry", + "title": "Schema Registry Url" + } + }, + "required": [ + "bootstrapServers" + ], + "title": "ProducerConfig", + "type": "object" + }, + "ProducerStreamsConfig": { + "additionalProperties": true, + "description": "Kafka Streams settings specific to Producer.", + "properties": { + "brokers": { + "description": "Brokers", + "title": "Brokers", + "type": "string" }, - "type": { - "const": "should-inflate", - "title": "Type" + "extraOutputTopics": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Extra output topics", + "title": "Extraoutputtopics", + "type": "object" }, - "values": { - "allOf": [ + "outputTopic": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Output topic" }, - "version": { + "schema_registry_url": { "anyOf": [ { "type": "string" @@ -1471,97 +839,68 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "URL of the schema registry", + "title": "Schema Registry Url" } }, "required": [ - "name", - "namespace", - "values", - "type" + "brokers" ], - "title": "ShouldInflate", + "title": "ProducerStreamsConfig", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -1570,18 +909,12 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "RepoAuthFlags", "type": "object" }, "StreamsApp": { @@ -2500,18 +1833,11 @@ "items": { "discriminator": { "mapping": { - "converter": "#/$defs/Converter", - "filter": "#/$defs/Filter", "helm-app": "#/$defs/HelmApp", "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", - "my-producer-app": "#/$defs/MyProducerApp", - "my-streams-app": "#/$defs/MyStreamsApp", "producer-app": "#/$defs/ProducerApp", "producer-app-v2": "#/$defs/ProducerAppV2", - "scheduled-producer": "#/$defs/ScheduledProducer", - "should-inflate": "#/$defs/ShouldInflate", - "simple-inflate-connectors": "#/$defs/SimpleInflateConnectors", "streams-app": "#/$defs/StreamsApp", "streams-app-v2": "#/$defs/StreamsAppV2" }, @@ -2538,27 +1864,6 @@ }, { "$ref": "#/$defs/StreamsAppV2" - }, - { - "$ref": "#/$defs/Converter" - }, - { - "$ref": "#/$defs/Filter" - }, - { - "$ref": "#/$defs/MyProducerApp" - }, - { - "$ref": "#/$defs/MyStreamsApp" - }, - { - "$ref": "#/$defs/ScheduledProducer" - }, - { - "$ref": "#/$defs/ShouldInflate" - }, - { - "$ref": "#/$defs/SimpleInflateConnectors" } ] }, From ee3f5e973fbca73e1c925d84e14cfd2cb4abe16d Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 19 Aug 2024 11:20:27 +0200 Subject: [PATCH 14/19] add model validator for deprication --- kpops/components/streams_bootstrap_v2/base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kpops/components/streams_bootstrap_v2/base.py b/kpops/components/streams_bootstrap_v2/base.py index bf52685c1..d5e45a4aa 100644 --- a/kpops/components/streams_bootstrap_v2/base.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -143,3 +143,9 @@ def warning_for_latest_image_tag(self) -> Self: f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." ) return self + + @pydantic.model_validator(mode="before") + @classmethod + def deprecation_warning(cls, model: Any) -> Any: + log.warning("StreamsBootstrapV2 is deprecated, use StreamsBootstrap instead.") + return model From 36c4c88ab7a00397a8e36656a117305e373a88f3 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 20 Aug 2024 08:00:47 +0200 Subject: [PATCH 15/19] Update files --- kpops/components/streams_bootstrap_v2/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap_v2/base.py b/kpops/components/streams_bootstrap_v2/base.py index d5e45a4aa..95f3281e2 100644 --- a/kpops/components/streams_bootstrap_v2/base.py +++ b/kpops/components/streams_bootstrap_v2/base.py @@ -147,5 +147,7 @@ def warning_for_latest_image_tag(self) -> Self: @pydantic.model_validator(mode="before") @classmethod def deprecation_warning(cls, model: Any) -> Any: - log.warning("StreamsBootstrapV2 is deprecated, use StreamsBootstrap instead.") + log.warning( + "StreamsBootstrapV2 is deprecated, consider migrating to StreamsBootstrap." + ) return model From 6af9cca950fa2973816dc8efcf6e57322cd3613e Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 20 Aug 2024 09:49:58 +0200 Subject: [PATCH 16/19] fix snapshot tests --- .../resources/custom-config/defaults.yaml | 2 + .../test_generate/atm-fraud/pipeline.yaml | 139 ++++++++++++++++-- .../test_generate/word-count/pipeline.yaml | 41 +++++- .../test_default_config/pipeline.yaml | 16 +- .../pipeline.yaml | 16 +- .../pipeline.yaml | 16 +- .../test_manifest_command/manifest.yaml | 8 +- .../test_manifest/test_python_api/resources | 8 +- 8 files changed, 202 insertions(+), 44 deletions(-) diff --git a/tests/pipeline/resources/custom-config/defaults.yaml b/tests/pipeline/resources/custom-config/defaults.yaml index 044a2af9e..9e39b96f1 100644 --- a/tests/pipeline/resources/custom-config/defaults.yaml +++ b/tests/pipeline/resources/custom-config/defaults.yaml @@ -1,4 +1,6 @@ streams-bootstrap-v2: + version: 2.9.0 + namespace: development-namespace values: streams: brokers: "${config.kafka_brokers}" diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index cbf54062f..78c492b66 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -12,14 +12,18 @@ type: producer-app-cleaner values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud prometheus: jmx: enabled: false + replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -32,17 +36,27 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-account-producer-topic: + configs: {} + partitions_count: 3 type: producer-app-v2 values: image: ${DOCKER_REGISTRY}/atm-demo-accountproducer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud prometheus: jmx: enabled: false + replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -63,14 +77,18 @@ ITERATION: 20 REAL_TX: 19 image: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud prometheus: jmx: enabled: false + replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -83,20 +101,30 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-avro-producer-topic: + configs: {} + partitions_count: 3 type: producer-app-v2 values: commandLine: ITERATION: 20 REAL_TX: 19 image: ${DOCKER_REGISTRY}/atm-demo-transactionavroproducer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud prometheus: jmx: enabled: false + replicaCount: 1 schedule: 0 12 * * * streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-transaction-avro-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ suspend: true version: 2.9.0 @@ -115,17 +143,26 @@ values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-transaction-joiner-dead-letter-topic + inputTopics: + - atm-fraud-transaction-avro-producer-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -137,21 +174,40 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-transaction-joiner-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-transaction-joiner-topic: + configs: {} + partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-transactionjoiner-atm-fraud-joinedtransactions-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-transactionjoiner - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-transaction-joiner-dead-letter-topic + inputTopics: + - atm-fraud-transaction-avro-producer-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-transaction-joiner-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -169,17 +225,26 @@ values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-frauddetector - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-fraud-detector-dead-letter-topic + inputTopics: + - atm-fraud-transaction-joiner-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -191,21 +256,40 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-fraud-detector-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-fraud-detector-topic: + configs: {} + partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-frauddetector-atm-fraud-possiblefraudtransactions-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-frauddetector - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-fraud-detector-dead-letter-topic + inputTopics: + - atm-fraud-transaction-joiner-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-fraud-detector-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -223,17 +307,29 @@ values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-account-linker-dead-letter-topic + extraInputTopics: + accounts: + - atm-fraud-account-producer-topic + inputTopics: + - atm-fraud-fraud-detector-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -252,21 +348,43 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + atm-fraud-account-linker-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error + atm-fraud-account-linker-topic: + configs: {} + partitions_count: 3 type: streams-app-v2 values: annotations: consumerGroup: atm-accountlinker-atm-fraud-output-topic + commandLine: + PRODUCTIVE: false image: ${DOCKER_REGISTRY}/atm-demo-accountlinker - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: atm-fraud persistence: enabled: false prometheus: jmx: enabled: false + replicaCount: 1 statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: atm-fraud-account-linker-dead-letter-topic + extraInputTopics: + accounts: + - atm-fraud-account-producer-topic + inputTopics: + - atm-fraud-fraud-detector-topic optimizeLeaveGroupBehavior: false + outputTopic: atm-fraud-account-linker-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _resetter: @@ -304,6 +422,7 @@ pk.mode: record_value table.name.format: fraud_transactions tasks.max: 1 + topics: atm-fraud-account-linker-topic transforms: flatten transforms.flatten.type: org.apache.kafka.connect.transforms.Flatten$Value value.converter: io.confluent.connect.avro.AvroConverter diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index d233b180c..145184060 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -12,13 +12,17 @@ type: producer-app-cleaner values: image: bakdata/kpops-demo-sentence-producer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: word-count prometheus: jmx: enabled: false + replicaCount: 1 streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 debug: true @@ -30,16 +34,26 @@ insecure_skip_tls_verify: false repository_name: bakdata-streams-bootstrap url: https://bakdata.github.io/streams-bootstrap/ + to: + models: {} + topics: + word-count-data-producer-topic: + configs: {} + partitions_count: 3 type: producer-app-v2 values: image: bakdata/kpops-demo-sentence-producer - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: word-count prometheus: jmx: enabled: false + replicaCount: 1 streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 optimizeLeaveGroupBehavior: false + outputTopic: word-count-data-producer-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ version: 2.9.0 - _cleaner: @@ -55,8 +69,12 @@ suffix: -clean type: streams-app-cleaner values: + commandLine: + PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: word-count persistence: enabled: false prometheus: @@ -66,6 +84,9 @@ statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: word-count-word-counter-dead-letter-topic + inputTopics: + - word-count-data-producer-topic optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ @@ -82,14 +103,23 @@ to: models: {} topics: + word-count-word-counter-dead-letter-topic: + configs: {} + partitions_count: 1 + type: error word-count-word-counter-topic: configs: cleanup.policy: compact + partitions_count: 3 type: output type: streams-app-v2 values: + commandLine: + PRODUCTIVE: false image: bakdata/kpops-demo-word-count-app - imageTag: latest + imageTag: 1.0.0 + labels: + pipeline: word-count persistence: enabled: false prometheus: @@ -99,6 +129,9 @@ statefulSet: false streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + errorTopic: word-count-word-counter-dead-letter-topic + inputTopics: + - word-count-data-producer-topic optimizeLeaveGroupBehavior: false outputTopic: word-count-word-counter-topic schemaRegistryUrl: http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/ diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml index a7cd13ba0..21dadaa27 100644 --- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-custom-config-app1 schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-custom-config-app1 schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 - _cleaner: name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - resources-custom-config-app1 outputTopic: resources-custom-config-app2 schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -117,5 +117,5 @@ - resources-custom-config-app1 outputTopic: resources-custom-config-app2 schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml index 97c9f8b2d..9a9d1d77e 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 - _cleaner: name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -117,5 +117,5 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml index 97c9f8b2d..9a9d1d77e 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml @@ -1,6 +1,6 @@ - _cleaner: name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -20,9 +20,9 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app1 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -48,10 +48,10 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: app1-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 - _cleaner: name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -77,9 +77,9 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 name: app2 - namespace: example-namespace + namespace: development-namespace prefix: resources-custom-config- repo_config: repo_auth_flags: @@ -117,5 +117,5 @@ - app1-test-topic outputTopic: app2-test-topic schemaRegistryUrl: http://localhost:8081/ - version: 2.4.2 + version: 2.9.0 diff --git a/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml index e676ca8e4..4dcabb92d 100644 --- a/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_manifest_command/manifest.yaml @@ -4,7 +4,7 @@ kind: Job metadata: labels: app: resources-custom-config-app1 - chart: producer-app-2.4.2 + chart: producer-app-2.9.0 release: resources-custom-config-app1 name: resources-custom-config-app1 spec: @@ -52,7 +52,7 @@ kind: ConfigMap metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.4.2 + chart: streams-app-2.9.0 heritage: Helm release: resources-custom-config-app2 name: resources-custom-config-app2-jmx-configmap @@ -63,7 +63,7 @@ kind: Deployment metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.4.2 + chart: streams-app-2.9.0 pipeline: resources-custom-config release: resources-custom-config-app2 name: resources-custom-config-app2 @@ -103,6 +103,8 @@ spec: value: com.bakdata.kafka.MurmurHashIdGenerator - name: KAFKA_JMX_PORT value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' - name: APP_BROKERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index 40637e62e..eca5af9eb 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -5,7 +5,7 @@ metadata: labels: app: resources-custom-config-app1 - chart: producer-app-2.4.2 + chart: producer-app-2.9.0 release: resources-custom-config-app1 name: resources-custom-config-app1 spec: @@ -54,7 +54,7 @@ metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.4.2 + chart: streams-app-2.9.0 heritage: Helm release: resources-custom-config-app2 name: resources-custom-config-app2-jmx-configmap @@ -65,7 +65,7 @@ metadata: labels: app: resources-custom-config-app2 - chart: streams-app-2.4.2 + chart: streams-app-2.9.0 pipeline: resources-custom-config release: resources-custom-config-app2 name: resources-custom-config-app2 @@ -105,6 +105,8 @@ value: com.bakdata.kafka.MurmurHashIdGenerator - name: KAFKA_JMX_PORT value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' - name: APP_BROKERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL From 19c00b77af6a152f7f7a46a5f4ac7dcfd67f7a76 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 20 Aug 2024 10:20:25 +0200 Subject: [PATCH 17/19] fix snapshot tests --- examples | 2 +- tests/pipeline/test_manifest.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples b/examples index 9d1827cad..5ea436622 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 9d1827cad2410ff1d0c5ccbc45c8c17395d9fc8e +Subproject commit 5ea436622c3ca857a44762a9b5dd5de678fdc1d4 diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 8651b47ce..84b742ae0 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -56,11 +56,11 @@ def test_default_config(self, mock_execute: MagicMock): "resources-custom-config-app2", "bakdata-streams-bootstrap/streams-app", "--namespace", - "example-namespace", + "development-namespace", "--values", ANY, "--version", - "2.4.2", + "2.9.0", "--timeout", "5m0s", "--wait", @@ -88,11 +88,11 @@ def test_custom_config(self, mock_execute: MagicMock): "resources-custom-config-app2", "bakdata-streams-bootstrap/streams-app", "--namespace", - "example-namespace", + "development-namespace", "--values", ANY, "--version", - "2.4.2", + "2.9.0", "--timeout", "5m0s", "--wait", From 7a1d7e959f7670e8681812da875914ad754b5481 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 20 Aug 2024 10:29:23 +0200 Subject: [PATCH 18/19] replace kafka-app with streams-bootstrap-v2 --- tests/pipeline/resources/no-topics-defaults/defaults.yaml | 2 +- tests/pipeline/resources/parallel-pipeline/defaults.yaml | 2 +- .../pipeline-component-should-have-prefix/defaults.yaml | 2 +- .../resources/pipeline-with-env-defaults/defaults.yaml | 3 ++- tests/pipeline/resources/pipeline-with-loop/defaults.yaml | 2 +- .../resources/pipeline-with-short-topics/defaults.yaml | 2 +- .../same-topic-and-component-name/defaults.yaml | 2 +- .../pipelines-with-graphs/simple-pipeline/defaults.yaml | 2 +- tests/pipeline/resources/temp-trim-release-name/defaults.yaml | 2 +- .../test_prefix_pipeline_component/pipeline.yaml | 4 ++-- 10 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/pipeline/resources/no-topics-defaults/defaults.yaml b/tests/pipeline/resources/no-topics-defaults/defaults.yaml index c6267151c..044a2af9e 100644 --- a/tests/pipeline/resources/no-topics-defaults/defaults.yaml +++ b/tests/pipeline/resources/no-topics-defaults/defaults.yaml @@ -1,4 +1,4 @@ -kafka-app: +streams-bootstrap-v2: values: streams: brokers: "${config.kafka_brokers}" diff --git a/tests/pipeline/resources/parallel-pipeline/defaults.yaml b/tests/pipeline/resources/parallel-pipeline/defaults.yaml index 2e03b9498..7fcd5c154 100644 --- a/tests/pipeline/resources/parallel-pipeline/defaults.yaml +++ b/tests/pipeline/resources/parallel-pipeline/defaults.yaml @@ -4,7 +4,7 @@ pipeline-component: kubernetes-app: namespace: ${NAMESPACE} -kafka-app: +streams-bootstrap-v2: values: streams: brokers: ${config.kafka_brokers} diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml index 5d3fa4696..b122ecf40 100644 --- a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml +++ b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml @@ -4,7 +4,7 @@ pipeline-component: kubernetes-app: namespace: ${NAMESPACE} -kafka-app: +streams-bootstrap-v2: version: 2.9.0 values: streams: diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index 621643b43..3bb6b7c8f 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -1,7 +1,8 @@ kubernetes-app: name: ${component.type} namespace: example-namespace -kafka-app: + +stream-bootstrap-v2: values: streams: brokers: "${config.kafka_brokers}" diff --git a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml index 9f3d82bef..c7da32bc8 100644 --- a/tests/pipeline/resources/pipeline-with-loop/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-loop/defaults.yaml @@ -7,7 +7,7 @@ kubernetes-app: kafka-connector: namespace: example-namespace -kafka-app: +streams-bootstrap-v2: values: streams: brokers: 127.0.0.1:9092 diff --git a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml index 66a5d45bb..f93f0ca40 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml @@ -2,7 +2,7 @@ kubernetes-app: name: "${component_type}" namespace: example-namespace -kafka-app: +streams-bootstrap-v2: values: streams: brokers: "${config.kafka_brokers}" diff --git a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml index a3f2429df..4369bcadb 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/same-topic-and-component-name/defaults.yaml @@ -7,7 +7,7 @@ kubernetes-app: kafka-connector: namespace: example-namespace -kafka-app: +streams-bootstrap-v2: values: streams: brokers: 127.0.0.1:9092 diff --git a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml index 6c50706ab..0a8fb4d11 100644 --- a/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml +++ b/tests/pipeline/resources/pipelines-with-graphs/simple-pipeline/defaults.yaml @@ -7,7 +7,7 @@ kubernetes-app: kafka-connector: namespace: example-namespace -kafka-app: +streams-bootstrap-v2: values: streams: brokers: 127.0.0.1:9092 diff --git a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml index 468bdfde9..08760ac5c 100644 --- a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml @@ -1,7 +1,7 @@ kubernetes-app: namespace: example-namespace -kafka-app: +streams-bootstrap-v2: values: streams: brokers: "${config.kafka_brokers}" diff --git a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml index ed8d4724d..6671e2580 100644 --- a/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_prefix_pipeline_component/pipeline.yaml @@ -22,7 +22,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - version: 2.4.2 + version: 2.9.0 name: account-producer namespace: ${NAMESPACE} prefix: from-pipeline-component- @@ -45,5 +45,5 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 schemaRegistryUrl: http://localhost:8081/ suspend: true - version: 2.4.2 + version: 2.9.0 From 2a06f9f5a8a923e4e57a50b72e48bc8770cfaa3a Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 20 Aug 2024 12:27:20 +0200 Subject: [PATCH 19/19] update streams bootstrap version --- docs/docs/schema/defaults.json | 6 +++--- docs/docs/schema/pipeline.json | 4 ++-- kpops/components/streams_bootstrap/base.py | 5 ++--- .../streams_bootstrap/test_producer_app.py | 4 ++-- .../streams_bootstrap/test_streams_app.py | 14 +++++++------- .../streams_bootstrap/test_streams_bootstrap.py | 4 ++-- .../resources/streams-bootstrap/defaults.yaml | 3 +-- .../test_streams_bootstrap/pipeline.yaml | 8 ++++---- .../test_streams_bootstrap/manifest.yaml | 6 +++--- 9 files changed, 26 insertions(+), 28 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 1fe98db24..f86b8b302 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -902,7 +902,7 @@ "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "default": "3.0.1", "description": "Helm chart version", "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", "title": "Version", @@ -1334,7 +1334,7 @@ "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "default": "3.0.1", "description": "Helm chart version", "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", "title": "Version", @@ -1664,7 +1664,7 @@ "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "default": "3.0.1", "description": "Helm chart version", "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", "title": "Version", diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index a61cabf87..29bcc0dc6 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -562,7 +562,7 @@ "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "default": "3.0.1", "description": "Helm chart version", "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", "title": "Version", @@ -994,7 +994,7 @@ "description": "streams-bootstrap Helm values" }, "version": { - "default": "3.0.0", + "default": "3.0.1", "description": "Helm chart version", "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", "title": "Version", diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index b55e17d05..c7090e830 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -25,8 +25,7 @@ url="https://bakdata.github.io/streams-bootstrap/", ) -# TODO: Update this with the latest stable version release -STREAMS_BOOTSTRAP_VERSION = "3.0.0" +STREAMS_BOOTSTRAP_VERSION = "3.0.1" STREAMS_BOOTSTRAP_VERSION_PATTERN = r"^(\d+)\.(\d+)\.(\d+)(-[a-zA-Z]+(\.[a-zA-Z]+)?)?$" COMPILED_VERSION_PATTERN = re.compile(STREAMS_BOOTSTRAP_VERSION_PATTERN) @@ -70,7 +69,7 @@ def version_validator(cls, version: str) -> str: major = int(major) if major != 3: - msg = f"When using the streams-bootstrap component your version ('{version}') must be at least 3.0.0." + msg = f"When using the streams-bootstrap component your version ('{version}') must be at least 3.0.1." raise ValueError(msg) return version diff --git a/tests/components/streams_bootstrap/test_producer_app.py b/tests/components/streams_bootstrap/test_producer_app.py index fbf125b45..046fa7f08 100644 --- a/tests/components/streams_bootstrap/test_producer_app.py +++ b/tests/components/streams_bootstrap/test_producer_app.py @@ -470,7 +470,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( "schemaRegistryUrl": "http://localhost:8081", }, }, - HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + HelmUpgradeInstallFlags(version="3.0.1", wait=True, wait_for_jobs=True), ) @pytest.mark.asyncio() @@ -541,5 +541,5 @@ async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster "outputTopic": "test-output-topic", }, }, - HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + HelmUpgradeInstallFlags(version="3.0.1", wait=True, wait_for_jobs=True), ) diff --git a/tests/components/streams_bootstrap/test_streams_app.py b/tests/components/streams_bootstrap/test_streams_app.py index f87805e64..09ed3bcbd 100644 --- a/tests/components/streams_bootstrap/test_streams_app.py +++ b/tests/components/streams_bootstrap/test_streams_app.py @@ -427,7 +427,7 @@ async def test_deploy_order_when_dry_run_is_false(self, mocker: MockerFixture): ca_file=None, insecure_skip_tls_verify=False, timeout="5m0s", - version="3.0.0", + version="3.0.1", wait=True, wait_for_jobs=False, ), @@ -504,7 +504,7 @@ async def test_reset_when_dry_run_is_false( }, }, HelmUpgradeInstallFlags( - version="3.0.0", wait=True, wait_for_jobs=True + version="3.0.1", wait=True, wait_for_jobs=True ), ), mocker.call.helm_uninstall( @@ -570,7 +570,7 @@ async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean }, }, HelmUpgradeInstallFlags( - version="3.0.0", wait=True, wait_for_jobs=True + version="3.0.1", wait=True, wait_for_jobs=True ), ), mocker.call.helm_uninstall( @@ -657,7 +657,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_reset( "schemaRegistryUrl": "http://localhost:8081", }, }, - HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + HelmUpgradeInstallFlags(version="3.0.1", wait=True, wait_for_jobs=True), ) @pytest.mark.asyncio() @@ -736,7 +736,7 @@ async def test_should_deploy_clean_up_job_with_values_in_cluster_when_clean( "schemaRegistryUrl": "http://localhost:8081", }, }, - HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + HelmUpgradeInstallFlags(version="3.0.1", wait=True, wait_for_jobs=True), ) @pytest.mark.asyncio() @@ -863,7 +863,7 @@ async def test_stateful_clean_with_dry_run_false( }, }, HelmUpgradeInstallFlags( - version="3.0.0", wait=True, wait_for_jobs=True + version="3.0.1", wait=True, wait_for_jobs=True ), ), mocker.call.helm_uninstall( @@ -1000,5 +1000,5 @@ async def test_clean_should_fall_back_to_local_values_when_validation_of_cluster "deleteOutput": True, }, }, - HelmUpgradeInstallFlags(version="3.0.0", wait=True, wait_for_jobs=True), + HelmUpgradeInstallFlags(version="3.0.1", wait=True, wait_for_jobs=True), ) diff --git a/tests/components/streams_bootstrap/test_streams_bootstrap.py b/tests/components/streams_bootstrap/test_streams_bootstrap.py index 0702571fb..a6533d5b6 100644 --- a/tests/components/streams_bootstrap/test_streams_bootstrap.py +++ b/tests/components/streams_bootstrap/test_streams_bootstrap.py @@ -31,7 +31,7 @@ def test_default_configs(self): repository_name="bakdata-streams-bootstrap", url="https://bakdata.github.io/streams-bootstrap/", ) - assert streams_bootstrap.version == "3.0.0" + assert streams_bootstrap.version == "3.0.1" assert streams_bootstrap.namespace == "test-namespace" assert streams_bootstrap.values.image_tag == "latest" @@ -105,7 +105,7 @@ async def test_should_raise_validation_error_for_invalid_helm_chart_version(self with pytest.raises( ValueError, match=re.escape( - "When using the streams-bootstrap component your version ('2.1.0') must be at least 3.0.0." + "When using the streams-bootstrap component your version ('2.1.0') must be at least 3.0.1." ), ): StreamsBootstrap( diff --git a/tests/pipeline/resources/streams-bootstrap/defaults.yaml b/tests/pipeline/resources/streams-bootstrap/defaults.yaml index f3e181935..e5ed3a48a 100644 --- a/tests/pipeline/resources/streams-bootstrap/defaults.yaml +++ b/tests/pipeline/resources/streams-bootstrap/defaults.yaml @@ -3,8 +3,7 @@ streams-bootstrap: kafka: bootstrapServers: ${config.kafka_brokers} schemaRegistryUrl: ${config.schema_registry.url} - # TODO: change to stable version after the PR is merged - version: "3.0.0-SNAPSHOT" + version: "3.0.1" producer-app: {} # inherits from streams-bootstrap diff --git a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml index e93943c0d..049ec8487 100644 --- a/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_streams_bootstrap/pipeline.yaml @@ -20,7 +20,7 @@ my-producer-app-output-topic-label: my-labeled-producer-app-topic-output outputTopic: my-producer-app-output-topic schemaRegistryUrl: http://localhost:8081/ - version: 3.0.0-SNAPSHOT + version: 3.0.1 name: my-producer-app namespace: example-namespace prefix: resources-streams-bootstrap- @@ -50,7 +50,7 @@ my-producer-app-output-topic-label: my-labeled-producer-app-topic-output outputTopic: my-producer-app-output-topic schemaRegistryUrl: http://localhost:8081/ - version: 3.0.0-SNAPSHOT + version: 3.0.1 - _cleaner: name: my-streams-app namespace: example-namespace @@ -88,7 +88,7 @@ persistence: enabled: false statefulSet: false - version: 3.0.0-SNAPSHOT + version: 3.0.1 from: components: {} topics: @@ -154,5 +154,5 @@ persistence: enabled: false statefulSet: false - version: 3.0.0-SNAPSHOT + version: 3.0.1 diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml index 59689255d..e685eac55 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml @@ -4,7 +4,7 @@ kind: Job metadata: labels: app: resources-streams-bootstrap-my-producer-app - chart: producer-app-3.0.0-SNAPSHOT + chart: producer-app-3.0.1 release: resources-streams-bootstrap-my-producer-app name: resources-streams-bootstrap-my-producer-app spec: @@ -53,7 +53,7 @@ kind: ConfigMap metadata: labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.0-SNAPSHOT + chart: streams-app-3.0.1 heritage: Helm release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app-jmx-configmap @@ -66,7 +66,7 @@ metadata: consumerGroup: my-streams-app-id labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.0-SNAPSHOT + chart: streams-app-3.0.1 release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app spec: