From f33685f2ced5733faab4cd317bd8fbb32f232137 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:14:13 +0200 Subject: [PATCH 01/25] Add image tag field to Kubernetes App values --- docs/docs/schema/defaults.json | 33 ++++++++++++++++++- docs/docs/schema/pipeline.json | 18 ++++++++++ kpops/components/base_components/kafka_app.py | 10 +++++- .../base_components/kafka_connector.py | 9 +++++ .../base_components/kubernetes_app.py | 4 +++ .../test-distributed-defaults/defaults.yaml | 2 ++ tests/components/test_helm_app.py | 4 +-- tests/components/test_kafka_sink_connector.py | 8 ----- tests/components/test_producer_app.py | 3 ++ tests/components/test_streams_bootstrap.py | 1 + .../pipeline-with-env-defaults/defaults.yaml | 2 ++ .../test_generate/atm-fraud/pipeline.yaml | 4 ++- .../test_generate/word-count/pipeline.yaml | 4 ++- .../test_default_config/pipeline.yaml | 4 +++ .../test_inflate_pipeline/pipeline.yaml | 8 ++++- .../pipeline.yaml | 6 +++- .../test_load_pipeline/pipeline.yaml | 2 ++ .../pipeline.yaml | 2 ++ .../pipeline.yaml | 2 ++ .../test_model_serialization/pipeline.yaml | 2 ++ .../test_no_input_topic/pipeline.yaml | 4 +++ .../pipeline.yaml | 2 ++ .../test_pipelines_with_envs/pipeline.yaml | 2 ++ .../test_read_from_component/pipeline.yaml | 26 +++++++++++++-- .../pipeline.yaml | 2 ++ .../pipeline.yaml | 4 +++ .../pipeline.yaml | 4 +++ .../test_with_env_defaults/pipeline.yaml | 6 +++- tests/pipeline/test_generate.py | 1 + 29 files changed, 160 insertions(+), 19 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index ae4af6582..cc7d70207 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -155,6 +155,12 @@ "additionalProperties": true, "description": "Helm app values.", "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { @@ -278,6 +284,12 @@ "additionalProperties": true, "description": "Settings specific to Kafka Apps.", "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { @@ -725,7 +737,14 @@ "KubernetesAppValues": { "additionalProperties": true, "description": "Settings specific to Kubernetes apps.", - "properties": {}, + "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + } + }, "title": "KubernetesAppValues", "type": "object" }, @@ -920,6 +939,12 @@ "additionalProperties": true, "description": "Settings specific to producers.", "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { @@ -1273,6 +1298,12 @@ "default": null, "description": "Kubernetes event-driven autoscaling config" }, + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index ea33470b3..9fc5bdc5b 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -155,6 +155,12 @@ "additionalProperties": true, "description": "Helm app values.", "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { @@ -588,6 +594,12 @@ "additionalProperties": true, "description": "Settings specific to producers.", "properties": { + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { @@ -941,6 +953,12 @@ "default": null, "description": "Kubernetes event-driven autoscaling config" }, + "imageTag": { + "default": "latest", + "description": "", + "title": "Imagetag", + "type": "string" + }, "nameOverride": { "anyOf": [ { diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index f52ca6e78..b3e3248f1 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -3,7 +3,7 @@ import logging from abc import ABC from collections.abc import Callable -from typing import Any +from typing import Any, Self import pydantic from pydantic import AliasChoices, ConfigDict, Field @@ -131,6 +131,14 @@ class KafkaApp(PipelineComponent, ABC): description=describe_attr("app", __doc__), ) + @pydantic.model_validator(mode="after") + def warning_for_latest_image_tag(self) -> Self: + if self.app.image_tag == "latest" and "$" not in self.name: + log.warning( + f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag." + ) + return self + @override async def deploy(self, dry_run: bool) -> None: if self.to: diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 1d9a296b3..134e37567 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -134,6 +134,15 @@ def connector_config_should_have_component_name( app["name"] = component_name return KafkaConnectorConfig(**app) + @pydantic.model_validator(mode="after") + def warning_for_latest_image_tag(self) -> Self: + if self.resetter_values.image_tag == "latest": + log.warning( + f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. " + f"Please, consider providing a stable imageTag." + ) + return self + @cached_property def _resetter(self) -> KafkaConnectorResetter: kwargs: dict[str, Any] = {} diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index e2ad7d39c..3c7968917 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -21,6 +21,10 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): """Settings specific to Kubernetes apps.""" + image_tag: str = Field( + default="latest", description=describe_attr("imageTag", __doc__) + ) + model_config = ConfigDict( extra="allow", ) diff --git a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml index 567c831f3..d835cc16f 100644 --- a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml +++ b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml @@ -1,3 +1,5 @@ helm-app: name: ${component.type} namespace: example-namespace + app: + imageTag: latest diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index cd9a78b66..cf819050f 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -49,7 +49,7 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: @pytest.fixture() def app_values(self) -> HelmAppValues: - return HelmAppValues(**{"foo": "test-value"}) + return HelmAppValues(**{"foo": "test-value", "imageTag": "latest"}) @pytest.fixture() def repo_config(self) -> HelmRepoConfig: @@ -235,7 +235,7 @@ def test_helm_name_override( name="helm-app-name-is-very-long-as-well", config=config, handlers=handlers, - app=HelmAppValues(), + app=HelmAppValues(**{"imageTag": "latest"}), namespace="test-namespace", repo_config=repo_config, ) diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 51b30a61d..1b98e4ac4 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -107,14 +107,6 @@ def test_connector_config_parsing( handlers: ComponentHandlers, connector_config: KafkaConnectorConfig, ): - connector = KafkaSinkConnector( - name=CONNECTOR_NAME, - config=config, - handlers=handlers, - app=connector_config, - resetter_namespace=RESETTER_NAMESPACE, - ) - topic_pattern = ".*" connector = KafkaSinkConnector( name=CONNECTOR_NAME, diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 4f7184ead..514fb1cea 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -67,6 +67,7 @@ def producer_app( "version": "2.4.2", "namespace": "test-namespace", "app": { + "imageTag": "latest", "streams": {"brokers": "fake-broker:9092"}, }, "clean_schemas": True, @@ -108,6 +109,7 @@ def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers): **{ "namespace": "test-namespace", "app": { + "imageTag": "latest", "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, @@ -324,6 +326,7 @@ def test_get_output_topics( **{ "namespace": "test-namespace", "app": { + "imageTag": "latest", "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index a82fca8a9..caef4a904 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -48,6 +48,7 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers) ) assert streams_bootstrap.version == "2.9.0" assert streams_bootstrap.namespace == "test-namespace" + assert streams_bootstrap.app.image_tag == "latest" @pytest.mark.asyncio() async def test_should_deploy_streams_bootstrap_app( diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index afc821f3c..6a77cfd61 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -1,6 +1,8 @@ kubernetes-app: name: ${component.type} namespace: example-namespace + app: + imageTag: "latest" kafka-app: app: streams: diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index 528da26bb..bc1517900 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -393,6 +393,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: atm-fraud-postgresql-connector connectorType: sink + imageTag: latest name: postgresql-connector namespace: ${NAMESPACE} prefix: atm-fraud- @@ -429,5 +430,6 @@ value.converter.schema.registry.url: http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081 name: postgresql-connector prefix: atm-fraud- - resetter_values: {} + resetter_values: + imageTag: latest type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index 7e7d16fe8..1f6571865 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -142,6 +142,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: word-count-redis-sink-connector connectorType: sink + imageTag: latest name: redis-sink-connector namespace: ${NAMESPACE} prefix: word-count- @@ -164,5 +165,6 @@ value.converter: org.apache.kafka.connect.storage.StringConverter name: redis-sink-connector prefix: word-count- - resetter_values: {} + resetter_values: + imageTag: latest type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml index cbd0e251e..4549c6209 100644 --- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml @@ -1,5 +1,6 @@ - _cleaner: app: + imageTag: latest resources: limits: memory: 2G @@ -21,6 +22,7 @@ type: producer-app-cleaner version: 2.9.0 app: + imageTag: latest resources: limits: memory: 2G @@ -50,6 +52,7 @@ - _cleaner: app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: @@ -77,6 +80,7 @@ version: 2.9.0 app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index e5e003376..8d5be21bf 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -64,6 +64,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -105,6 +106,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -253,6 +255,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector connectorType: sink + imageTag: latest name: should-inflate-inflated-sink-connector namespace: example-namespace prefix: resources-pipeline-with-inflate- @@ -280,7 +283,8 @@ transforms.changeTopic.replacement: resources-pipeline-with-inflate-should-inflate-index-v1 name: should-inflate-inflated-sink-connector prefix: resources-pipeline-with-inflate- - resetter_values: {} + resetter_values: + imageTag: latest to: models: {} topics: @@ -293,6 +297,7 @@ type: kafka-sink-connector - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -317,6 +322,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index bb569e772..7a6417d01 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -1,6 +1,7 @@ - _cleaner: app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false @@ -26,6 +27,7 @@ version: 2.4.2 app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false @@ -71,6 +73,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-kafka-connect-sink-es-sink-connector connectorType: sink + imageTag: latest name: es-sink-connector namespace: example-namespace prefix: resources-kafka-connect-sink- @@ -97,6 +100,7 @@ topics: example-output name: es-sink-connector prefix: resources-kafka-connect-sink- - resetter_values: {} + resetter_values: + imageTag: latest type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml index 87a88601c..d49fa57d2 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml @@ -64,6 +64,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -105,6 +106,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml index 2787d7444..42cd958d5 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml @@ -65,6 +65,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -104,6 +105,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml index 2787d7444..42cd958d5 100644 --- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml @@ -65,6 +65,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -104,6 +105,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: diff --git a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml index 02e06ff34..9e85d9e01 100644 --- a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml @@ -1,5 +1,6 @@ - _cleaner: app: + imageTag: latest streams: brokers: test outputTopic: out @@ -19,6 +20,7 @@ type: producer-app-cleaner version: 2.4.2 app: + imageTag: latest streams: brokers: test outputTopic: out diff --git a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml index a77aea9be..3c57c8eb1 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml @@ -2,6 +2,7 @@ app: commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -32,6 +33,7 @@ app: commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -77,6 +79,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -103,6 +106,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml index d8850383e..348443bbc 100644 --- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml @@ -1,6 +1,7 @@ - _cleaner: app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false @@ -26,6 +27,7 @@ version: 2.4.2 app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false diff --git a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml index 344e9c5b1..36d50168f 100644 --- a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml @@ -64,6 +64,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -105,6 +106,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index 761f21e63..0ad5a8dd7 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -1,5 +1,6 @@ - _cleaner: app: + imageTag: latest streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-read-from-component-producer1 @@ -16,6 +17,7 @@ type: producer-app-cleaner version: 2.4.2 app: + imageTag: latest streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-read-from-component-producer1 @@ -38,6 +40,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-read-from-component-producer2 @@ -54,6 +57,7 @@ type: producer-app-cleaner version: 2.4.2 app: + imageTag: latest streams: brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 outputTopic: resources-read-from-component-producer2 @@ -168,6 +172,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-read-from-component-inflate-step-inflated-sink-connector connectorType: sink + imageTag: latest name: inflate-step-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- @@ -195,7 +200,8 @@ transforms.changeTopic.replacement: resources-read-from-component-inflate-step-index-v1 name: inflate-step-inflated-sink-connector prefix: resources-read-from-component- - resetter_values: {} + resetter_values: + imageTag: latest to: models: {} topics: @@ -208,6 +214,7 @@ type: kafka-sink-connector - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -232,6 +239,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -360,6 +368,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector connectorType: sink + imageTag: latest name: inflate-step-without-prefix-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- @@ -387,7 +396,8 @@ transforms.changeTopic.replacement: resources-read-from-component-inflate-step-without-prefix-index-v1 name: inflate-step-without-prefix-inflated-sink-connector prefix: resources-read-from-component- - resetter_values: {} + resetter_values: + imageTag: latest to: models: {} topics: @@ -400,6 +410,7 @@ type: kafka-sink-connector - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -424,6 +435,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -460,6 +472,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -484,6 +497,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -525,6 +539,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -549,6 +564,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -589,6 +605,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -613,6 +630,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -653,6 +671,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -676,6 +695,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -713,6 +733,7 @@ version: 2.4.2 - _cleaner: app: + imageTag: latest persistence: enabled: false statefulSet: false @@ -736,6 +757,7 @@ type: streams-app-cleaner version: 2.4.2 app: + imageTag: latest persistence: enabled: false statefulSet: false diff --git a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml index 8ca686f30..8878759fa 100644 --- a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml @@ -72,6 +72,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: @@ -113,6 +114,7 @@ topics: [] commandLine: CONVERT_XML: true + imageTag: latest persistence: enabled: false resources: diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml index f78e8f0d1..f96db8745 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml @@ -1,5 +1,6 @@ - _cleaner: app: + imageTag: latest resources: limits: memory: 2G @@ -21,6 +22,7 @@ type: producer-app-cleaner version: 2.9.0 app: + imageTag: latest resources: limits: memory: 2G @@ -50,6 +52,7 @@ - _cleaner: app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: @@ -77,6 +80,7 @@ version: 2.9.0 app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml index f78e8f0d1..f96db8745 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml @@ -1,5 +1,6 @@ - _cleaner: app: + imageTag: latest resources: limits: memory: 2G @@ -21,6 +22,7 @@ type: producer-app-cleaner version: 2.9.0 app: + imageTag: latest resources: limits: memory: 2G @@ -50,6 +52,7 @@ - _cleaner: app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: @@ -77,6 +80,7 @@ version: 2.9.0 app: image: some-image + imageTag: latest labels: pipeline: resources-custom-config persistence: diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index cea0b2660..670ee9942 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -1,6 +1,7 @@ - _cleaner: app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false @@ -26,6 +27,7 @@ version: 2.9.0 app: image: fake-image + imageTag: latest persistence: enabled: false statefulSet: false @@ -71,6 +73,7 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-pipeline-with-env-defaults-es-sink-connector connectorType: sink + imageTag: latest name: es-sink-connector namespace: development-namespace prefix: resources-pipeline-with-env-defaults- @@ -97,6 +100,7 @@ topics: example-output name: es-sink-connector prefix: resources-pipeline-with-env-defaults- - resetter_values: {} + resetter_values: + imageTag: latest type: kafka-sink-connector diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 915ae17d7..b0fdd2db4 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -219,6 +219,7 @@ def test_kafka_connector_config_parsing(self): str(RESOURCE_PATH / "kafka-connect-sink-config" / PIPELINE_YAML), "--config", str(RESOURCE_PATH / "kafka-connect-sink-config"), + "--verbose", ], catch_exceptions=False, ) From c7de8c189389d4054ca50b2f0b70afff0d07b344 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:17:21 +0200 Subject: [PATCH 02/25] Add image tag field to Kubernetes App values --- kpops/components/base_components/kubernetes_app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 3c7968917..fef30c35f 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -19,7 +19,10 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): - """Settings specific to Kubernetes apps.""" + """Settings specific to Kubernetes apps. + + :param image_tag: The image tag of the Kubernetes app. + """ image_tag: str = Field( default="latest", description=describe_attr("imageTag", __doc__) From 6fd51edfede0fa042876a6eb0b2b2ec55d0ea554 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:19:31 +0200 Subject: [PATCH 03/25] Update files --- tests/components/test_helm_app.py | 4 ++-- tests/components/test_producer_app.py | 3 --- .../resources/pipeline-with-env-defaults/defaults.yaml | 2 -- tests/pipeline/test_generate.py | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index cf819050f..cd9a78b66 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -49,7 +49,7 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: @pytest.fixture() def app_values(self) -> HelmAppValues: - return HelmAppValues(**{"foo": "test-value", "imageTag": "latest"}) + return HelmAppValues(**{"foo": "test-value"}) @pytest.fixture() def repo_config(self) -> HelmRepoConfig: @@ -235,7 +235,7 @@ def test_helm_name_override( name="helm-app-name-is-very-long-as-well", config=config, handlers=handlers, - app=HelmAppValues(**{"imageTag": "latest"}), + app=HelmAppValues(), namespace="test-namespace", repo_config=repo_config, ) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 514fb1cea..4f7184ead 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -67,7 +67,6 @@ def producer_app( "version": "2.4.2", "namespace": "test-namespace", "app": { - "imageTag": "latest", "streams": {"brokers": "fake-broker:9092"}, }, "clean_schemas": True, @@ -109,7 +108,6 @@ def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers): **{ "namespace": "test-namespace", "app": { - "imageTag": "latest", "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, @@ -326,7 +324,6 @@ def test_get_output_topics( **{ "namespace": "test-namespace", "app": { - "imageTag": "latest", "namespace": "test-namespace", "streams": {"brokers": "fake-broker:9092"}, }, diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index 6a77cfd61..afc821f3c 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -1,8 +1,6 @@ kubernetes-app: name: ${component.type} namespace: example-namespace - app: - imageTag: "latest" kafka-app: app: streams: diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index b0fdd2db4..915ae17d7 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -219,7 +219,6 @@ def test_kafka_connector_config_parsing(self): str(RESOURCE_PATH / "kafka-connect-sink-config" / PIPELINE_YAML), "--config", str(RESOURCE_PATH / "kafka-connect-sink-config"), - "--verbose", ], catch_exceptions=False, ) From f54ef8450dc26f1440468615ec6bfadda44a17b9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:28:35 +0200 Subject: [PATCH 04/25] Update files --- kpops/components/base_components/kubernetes_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index fef30c35f..7c5cdb8ea 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -21,7 +21,7 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): """Settings specific to Kubernetes apps. - :param image_tag: The image tag of the Kubernetes app. + :param image_tag: Specifies the version or specific build of a container image to use. """ image_tag: str = Field( From bd9e48791711036b10463af87601b81e3d60df0d Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:29:11 +0200 Subject: [PATCH 05/25] Update files --- docs/docs/schema/defaults.json | 10 +++++----- docs/docs/schema/pipeline.json | 6 +++--- kpops/components/base_components/kubernetes_app.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index cc7d70207..3dba96dff 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -157,7 +157,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -286,7 +286,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -740,7 +740,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" } @@ -941,7 +941,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -1300,7 +1300,7 @@ }, "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 9fc5bdc5b..cfa63af47 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -157,7 +157,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -596,7 +596,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -955,7 +955,7 @@ }, "imageTag": { "default": "latest", - "description": "", + "description": "Specifies the version or speci\ufb01c build of a container image to use.", "title": "Imagetag", "type": "string" }, diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 7c5cdb8ea..9901a37d1 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -25,7 +25,7 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): """ image_tag: str = Field( - default="latest", description=describe_attr("imageTag", __doc__) + default="latest", description=describe_attr("image_tag", __doc__) ) model_config = ConfigDict( From af35a803a20b881921fcf932f4c571382b911cb6 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 10:29:41 +0200 Subject: [PATCH 06/25] Update files --- docs/docs/schema/defaults.json | 10 +++++----- docs/docs/schema/pipeline.json | 6 +++--- kpops/components/base_components/kubernetes_app.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 3dba96dff..d670aa264 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -157,7 +157,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -286,7 +286,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -740,7 +740,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" } @@ -941,7 +941,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -1300,7 +1300,7 @@ }, "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index cfa63af47..1d66028e5 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -157,7 +157,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -596,7 +596,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -955,7 +955,7 @@ }, "imageTag": { "default": "latest", - "description": "Specifies the version or speci\ufb01c build of a container image to use.", + "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 9901a37d1..27fb1286a 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -21,7 +21,7 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): """Settings specific to Kubernetes apps. - :param image_tag: Specifies the version or specific build of a container image to use. + :param image_tag: Specifies the version or specific build of a container image to use. """ image_tag: str = Field( From 0fd9c1e9f17a07e645b259df076d2a2fcbc3c141 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 14:53:56 +0200 Subject: [PATCH 07/25] Move image tag to streamsbootstrap values --- docs/docs/schema/defaults.json | 57 ++++++++++--------- docs/docs/schema/pipeline.json | 8 --- .../component_handlers/kafka_connect/model.py | 2 + kpops/components/base_components/kafka_app.py | 10 +--- .../base_components/kafka_connector.py | 17 +++--- .../base_components/kubernetes_app.py | 9 +-- .../components/streams_bootstrap/__init__.py | 29 +++++++++- .../streams_bootstrap/producer/model.py | 3 +- .../streams_bootstrap/streams/model.py | 3 +- tests/components/test_streams_bootstrap.py | 1 - .../test_generate/atm-fraud/pipeline.yaml | 3 +- .../test_generate/word-count/pipeline.yaml | 3 +- .../test_inflate_pipeline/pipeline.yaml | 3 +- .../pipeline.yaml | 3 +- .../test_read_from_component/pipeline.yaml | 6 +- .../test_with_env_defaults/pipeline.yaml | 3 +- 16 files changed, 83 insertions(+), 77 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index d670aa264..b6644fa31 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -155,12 +155,6 @@ "additionalProperties": true, "description": "Helm app values.", "properties": { - "imageTag": { - "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", - "title": "Imagetag", - "type": "string" - }, "nameOverride": { "anyOf": [ { @@ -284,12 +278,6 @@ "additionalProperties": true, "description": "Settings specific to Kafka Apps.", "properties": { - "imageTag": { - "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", - "title": "Imagetag", - "type": "string" - }, "nameOverride": { "anyOf": [ { @@ -737,14 +725,7 @@ "KubernetesAppValues": { "additionalProperties": true, "description": "Settings specific to Kubernetes apps.", - "properties": { - "imageTag": { - "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", - "title": "Imagetag", - "type": "string" - } - }, + "properties": {}, "title": "KubernetesAppValues", "type": "object" }, @@ -941,7 +922,6 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -1300,7 +1280,6 @@ }, "imageTag": { "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -1359,10 +1338,10 @@ "app": { "allOf": [ { - "$ref": "#/$defs/HelmAppValues" + "$ref": "#/$defs/StreamsBootstrapValues" } ], - "description": "Helm app values" + "description": "" }, "from": { "anyOf": [ @@ -1440,12 +1419,38 @@ }, "required": [ "name", - "namespace", - "app" + "namespace" ], "title": "StreamsBootstrap", "type": "object" }, + "StreamsBootstrapValues": { + "additionalProperties": true, + "description": "Base value class for all streams bootstrap related components.", + "properties": { + "imageTag": { + "default": "latest", + "title": "Imagetag", + "type": "string" + }, + "nameOverride": { + "anyOf": [ + { + "maxLength": 63, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Helm chart name override, assigned automatically", + "title": "Nameoverride" + } + }, + "title": "StreamsBootstrapValues", + "type": "object" + }, "StreamsConfig": { "additionalProperties": true, "description": "Streams Bootstrap streams section.", diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 1d66028e5..0d98c4e7e 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -155,12 +155,6 @@ "additionalProperties": true, "description": "Helm app values.", "properties": { - "imageTag": { - "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", - "title": "Imagetag", - "type": "string" - }, "nameOverride": { "anyOf": [ { @@ -596,7 +590,6 @@ "properties": { "imageTag": { "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, @@ -955,7 +948,6 @@ }, "imageTag": { "default": "latest", - "description": "Specifies the version or specific build of a container image to use.", "title": "Imagetag", "type": "string" }, diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 840df06c3..c4aeafa31 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -5,6 +5,7 @@ from pydantic import ( BaseModel, ConfigDict, + Field, SerializationInfo, field_validator, model_serializer, @@ -123,3 +124,4 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel): class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] config: KafkaConnectorResetterConfig + image_tag: str = Field(default="latest", alias="imageTag") diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index b3e3248f1..f52ca6e78 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -3,7 +3,7 @@ import logging from abc import ABC from collections.abc import Callable -from typing import Any, Self +from typing import Any import pydantic from pydantic import AliasChoices, ConfigDict, Field @@ -131,14 +131,6 @@ class KafkaApp(PipelineComponent, ABC): description=describe_attr("app", __doc__), ) - @pydantic.model_validator(mode="after") - def warning_for_latest_image_tag(self) -> Self: - if self.app.image_tag == "latest" and "$" not in self.name: - log.warning( - f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag." - ) - return self - @override async def deploy(self, dry_run: bool) -> None: if self.to: diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 134e37567..40e9ad1c8 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -134,14 +134,15 @@ def connector_config_should_have_component_name( app["name"] = component_name return KafkaConnectorConfig(**app) - @pydantic.model_validator(mode="after") - def warning_for_latest_image_tag(self) -> Self: - if self.resetter_values.image_tag == "latest": - log.warning( - f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. " - f"Please, consider providing a stable imageTag." - ) - return self + # @pydantic.model_validator(mode="after") + # def warning_for_latest_image_tag(self) -> Self: + # resetter_image_tag = self.resetter_values.get("imageTag") + # if not resetter_image_tag or resetter_image_tag == "latest": + # log.warning( + # f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. " + # f"Please, consider providing a stable imageTag." + # ) + # return self @cached_property def _resetter(self) -> KafkaConnectorResetter: diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 27fb1286a..e2ad7d39c 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -19,14 +19,7 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): - """Settings specific to Kubernetes apps. - - :param image_tag: Specifies the version or specific build of a container image to use. - """ - - image_tag: str = Field( - default="latest", description=describe_attr("image_tag", __doc__) - ) + """Settings specific to Kubernetes apps.""" model_config = ConfigDict( extra="allow", diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 1b02b091b..2501643ff 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,9 +1,12 @@ +import logging from abc import ABC +from typing import Self +import pydantic from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( @@ -12,6 +15,17 @@ ) STREAMS_BOOTSTRAP_VERSION = "2.9.0" +log = logging.getLogger("StreamsBootstrap") + + +class StreamsBootstrapValues(HelmAppValues): + """Base value class for all streams bootstrap related components. + + :param image_tag: Docker image tag of the Kafka Streams app. + """ + + image_tag: str = Field(default="latest") + class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. @@ -21,6 +35,11 @@ class StreamsBootstrap(HelmApp, ABC): :param version: Helm chart version, defaults to "2.9.0" """ + app: StreamsBootstrapValues = Field( + default_factory=StreamsBootstrapValues, + description=describe_attr("app", __doc__), + ) + repo_config: HelmRepoConfig = Field( default=STREAMS_BOOTSTRAP_HELM_REPO, description=describe_attr("repo_config", __doc__), @@ -29,3 +48,11 @@ class StreamsBootstrap(HelmApp, ABC): default=STREAMS_BOOTSTRAP_VERSION, description=describe_attr("version", __doc__), ) + + @pydantic.model_validator(mode="after") + def warning_for_latest_image_tag(self) -> Self: + if self.app.image_tag == "latest" and "$" not in self.name: + log.warning( + f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag." + ) + return self diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py index caef8d29d..2dc3b5927 100644 --- a/kpops/components/streams_bootstrap/producer/model.py +++ b/kpops/components/streams_bootstrap/producer/model.py @@ -4,6 +4,7 @@ KafkaAppValues, KafkaStreamsConfig, ) +from kpops.components.streams_bootstrap import StreamsBootstrapValues from kpops.utils.docstring import describe_attr @@ -11,7 +12,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): """Kafka Streams settings specific to Producer.""" -class ProducerAppValues(KafkaAppValues): +class ProducerAppValues(StreamsBootstrapValues, KafkaAppValues): """Settings specific to producers. :param streams: Kafka Streams settings diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 04f95b54b..1bffb84c6 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -11,6 +11,7 @@ KafkaStreamsConfig, ) from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr +from kpops.components.streams_bootstrap import StreamsBootstrapValues from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, @@ -237,7 +238,7 @@ def validate_mandatory_fields_are_set( return self -class StreamsAppValues(KafkaAppValues): +class StreamsAppValues(StreamsBootstrapValues, KafkaAppValues): """streams-bootstrap app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index caef4a904..a82fca8a9 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -48,7 +48,6 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers) ) assert streams_bootstrap.version == "2.9.0" assert streams_bootstrap.namespace == "test-namespace" - assert streams_bootstrap.app.image_tag == "latest" @pytest.mark.asyncio() async def test_should_deploy_streams_bootstrap_app( diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index bc1517900..8559e1534 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -430,6 +430,5 @@ value.converter.schema.registry.url: http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081 name: postgresql-connector prefix: atm-fraud- - resetter_values: - imageTag: latest + resetter_values: {} type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index 1f6571865..a10c5b5a3 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -165,6 +165,5 @@ value.converter: org.apache.kafka.connect.storage.StringConverter name: redis-sink-connector prefix: word-count- - resetter_values: - imageTag: latest + resetter_values: {} type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index 8d5be21bf..5ead506c6 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -283,8 +283,7 @@ transforms.changeTopic.replacement: resources-pipeline-with-inflate-should-inflate-index-v1 name: should-inflate-inflated-sink-connector prefix: resources-pipeline-with-inflate- - resetter_values: - imageTag: latest + resetter_values: {} to: models: {} topics: diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index 7a6417d01..180b80ec0 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -100,7 +100,6 @@ topics: example-output name: es-sink-connector prefix: resources-kafka-connect-sink- - resetter_values: - imageTag: latest + resetter_values: {} type: kafka-sink-connector diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index 0ad5a8dd7..63fb42b8e 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -200,8 +200,7 @@ transforms.changeTopic.replacement: resources-read-from-component-inflate-step-index-v1 name: inflate-step-inflated-sink-connector prefix: resources-read-from-component- - resetter_values: - imageTag: latest + resetter_values: {} to: models: {} topics: @@ -396,8 +395,7 @@ transforms.changeTopic.replacement: resources-read-from-component-inflate-step-without-prefix-index-v1 name: inflate-step-without-prefix-inflated-sink-connector prefix: resources-read-from-component- - resetter_values: - imageTag: latest + resetter_values: {} to: models: {} topics: diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index 670ee9942..edb1d4623 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -100,7 +100,6 @@ topics: example-output name: es-sink-connector prefix: resources-pipeline-with-env-defaults- - resetter_values: - imageTag: latest + resetter_values: {} type: kafka-sink-connector From e3ea6af05b8b895fe05c96ffaf2d7b97b23b9ddd Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 14:54:48 +0200 Subject: [PATCH 08/25] Update files --- docs/docs/schema/defaults.json | 2 +- kpops/components/streams_bootstrap/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index b6644fa31..69ca8ffa7 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1341,7 +1341,7 @@ "$ref": "#/$defs/StreamsBootstrapValues" } ], - "description": "" + "description": "Streams bootstrap app values" }, "from": { "anyOf": [ diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 2501643ff..daef43cce 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -30,6 +30,7 @@ class StreamsBootstrapValues(HelmAppValues): class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. + :param app: Streams bootstrap app values :param repo_config: Configuration of the Helm chart repo to be used for deploying the component, defaults to streams-bootstrap Helm repo :param version: Helm chart version, defaults to "2.9.0" From 8b5a5a713ac4a862c8589bacc9e0001d5766d359 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 14:56:32 +0200 Subject: [PATCH 09/25] Update files --- kpops/components/base_components/kafka_connector.py | 10 ---------- .../pipelines/test-distributed-defaults/defaults.yaml | 2 -- 2 files changed, 12 deletions(-) diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 40e9ad1c8..1d9a296b3 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -134,16 +134,6 @@ def connector_config_should_have_component_name( app["name"] = component_name return KafkaConnectorConfig(**app) - # @pydantic.model_validator(mode="after") - # def warning_for_latest_image_tag(self) -> Self: - # resetter_image_tag = self.resetter_values.get("imageTag") - # if not resetter_image_tag or resetter_image_tag == "latest": - # log.warning( - # f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. " - # f"Please, consider providing a stable imageTag." - # ) - # return self - @cached_property def _resetter(self) -> KafkaConnectorResetter: kwargs: dict[str, Any] = {} diff --git a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml index d835cc16f..567c831f3 100644 --- a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml +++ b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml @@ -1,5 +1,3 @@ helm-app: name: ${component.type} namespace: example-namespace - app: - imageTag: latest From 22923463a7b8b9910f14b657905f132de8bb32aa Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 14:57:30 +0200 Subject: [PATCH 10/25] Update files --- kpops/component_handlers/kafka_connect/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index c4aeafa31..840df06c3 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -5,7 +5,6 @@ from pydantic import ( BaseModel, ConfigDict, - Field, SerializationInfo, field_validator, model_serializer, @@ -124,4 +123,3 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel): class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] config: KafkaConnectorResetterConfig - image_tag: str = Field(default="latest", alias="imageTag") From 37743f38d352343b335a1114cf5149ba156b0cd9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 14:58:31 +0200 Subject: [PATCH 11/25] Update files --- kpops/components/streams_bootstrap/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index daef43cce..223fee9e6 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -52,8 +52,11 @@ class StreamsBootstrap(HelmApp, ABC): @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: - if self.app.image_tag == "latest" and "$" not in self.name: + if self.app.image_tag == "latest" and self.is_substituted(): log.warning( f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag." ) return self + + def is_substituted(self): + return "$" not in self.name From 8fa198bec7ae710667274819e8aca4a51c3f6758 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 15:01:26 +0200 Subject: [PATCH 12/25] Update files --- kpops/component_handlers/kafka_connect/model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 840df06c3..005d7c429 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -5,6 +5,7 @@ from pydantic import ( BaseModel, ConfigDict, + Field, SerializationInfo, field_validator, model_serializer, @@ -123,3 +124,4 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel): class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] config: KafkaConnectorResetterConfig + image_tag: str = Field(default="latest") From ce461b9a2a0c398b6ff3bd71dbcf3436ecc6bb15 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 15:55:29 +0200 Subject: [PATCH 13/25] add image tag vaildation --- .../component_handlers/kafka_connect/model.py | 6 +++++ kpops/component_handlers/kubernetes/utils.py | 27 +++++++++++++++++++ .../components/streams_bootstrap/__init__.py | 8 +++++- .../kubernetes/test_utils.py | 20 ++++++++++++++ .../resources/resetter_values/defaults.yaml | 2 ++ tests/pipeline/test_generate.py | 8 ++++++ 6 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/component_handlers/kubernetes/test_utils.py diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 005d7c429..f09a0312c 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -13,6 +13,7 @@ from pydantic.json_schema import SkipJsonSchema from typing_extensions import override +from kpops.component_handlers.kubernetes.utils import validate_image_tag from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.utils.pydantic import ( @@ -125,3 +126,8 @@ class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] config: KafkaConnectorResetterConfig image_tag: str = Field(default="latest") + + @pydantic.field_validator("image_tag", mode="before") + @classmethod + def validate_image_tag_field(cls, image_tag: Any) -> str: + return validate_image_tag(image_tag) diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 4f4599e2b..a9f185162 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -1,5 +1,8 @@ import hashlib import logging +import re + +from kpops.api.exception import ValidationError log = logging.getLogger("K8sUtils") @@ -26,3 +29,27 @@ def trim(max_len: int, name: str, suffix: str) -> str: ) return new_name return name + + +def validate_image_tag(image_tag: str) -> str: + """Validate an image tag. + + Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). + It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} + + :param image_tag: Docker image tag to be validated. + :return: The validated image tag. + """ + if isinstance(image_tag, str) and is_valid_image_tag(image_tag): + return image_tag + msg = ( + "Image tag is not valid. " + "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " + "It can be up to 128 characters long." + ) + raise ValidationError(msg) + + +def is_valid_image_tag(image_tag: str) -> bool: + pattern = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" + return bool(re.match(pattern, image_tag)) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 223fee9e6..f026a77dc 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,11 +1,12 @@ import logging from abc import ABC -from typing import Self +from typing import Any, Self import pydantic from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig +from kpops.component_handlers.kubernetes.utils import validate_image_tag from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr @@ -26,6 +27,11 @@ class StreamsBootstrapValues(HelmAppValues): image_tag: str = Field(default="latest") + @pydantic.field_validator("image_tag", mode="before") + @classmethod + def validate_image_tag_field(cls, image_tag: Any) -> str: + return validate_image_tag(image_tag) + class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py new file mode 100644 index 000000000..0c1bdc3f1 --- /dev/null +++ b/tests/component_handlers/kubernetes/test_utils.py @@ -0,0 +1,20 @@ +import pytest + +from kpops.api.exception import ValidationError +from kpops.component_handlers.kubernetes.utils import validate_image_tag + + +def test_is_valid_image_tag(): + assert validate_image_tag("1.2.3") == "1.2.3" + assert validate_image_tag("123") == "123" + assert validate_image_tag("1_2_3") == "1_2_3" + assert validate_image_tag("1-2-3") == "1-2-3" + assert validate_image_tag("latest") == "latest" + assert ( + validate_image_tag( + "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07" + ) + == "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07" + ) + with pytest.raises(ValidationError): + assert validate_image_tag("la!est") is False diff --git a/tests/pipeline/resources/resetter_values/defaults.yaml b/tests/pipeline/resources/resetter_values/defaults.yaml index 550c9c729..950ed4969 100644 --- a/tests/pipeline/resources/resetter_values/defaults.yaml +++ b/tests/pipeline/resources/resetter_values/defaults.yaml @@ -9,3 +9,5 @@ helm-app: kafka-sink-connector: app: "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector" + resetter_values: + imageTag: override-default-image-tag diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 915ae17d7..71d8946e1 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -840,6 +840,10 @@ def test_substitution_in_inflated_component(self): enriched_pipeline[1]["_resetter"]["app"]["label"] == "inflated-connector-name" ) + assert ( + enriched_pipeline[1]["_resetter"]["app"]["imageTag"] + == "override-default-image-tag" + ) def test_substitution_in_resetter(self): pipeline = kpops.generate( @@ -857,3 +861,7 @@ def test_substitution_in_resetter(self): assert enriched_pipeline[0]["name"] == "es-sink-connector" assert enriched_pipeline[0]["_resetter"]["name"] == "es-sink-connector" assert enriched_pipeline[0]["_resetter"]["app"]["label"] == "es-sink-connector" + assert ( + enriched_pipeline[1]["_resetter"]["app"]["imageTag"] + == "override-default-image-tag" + ) From 3b4830358f2ecb16df407100832bf057f40eeaea Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 26 Jun 2024 16:00:04 +0200 Subject: [PATCH 14/25] Update files --- tests/pipeline/test_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 71d8946e1..ef68aadf9 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -862,6 +862,6 @@ def test_substitution_in_resetter(self): assert enriched_pipeline[0]["_resetter"]["name"] == "es-sink-connector" assert enriched_pipeline[0]["_resetter"]["app"]["label"] == "es-sink-connector" assert ( - enriched_pipeline[1]["_resetter"]["app"]["imageTag"] + enriched_pipeline[0]["_resetter"]["app"]["imageTag"] == "override-default-image-tag" ) From 9ffdce69ddbc993bed083948b602f0396b63ad3f Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 12:34:32 +0200 Subject: [PATCH 15/25] Update files --- docs/docs/schema/defaults.json | 3 +++ docs/docs/schema/pipeline.json | 2 ++ kpops/components/streams_bootstrap/__init__.py | 11 +++++------ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 69ca8ffa7..2c08e0089 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -922,6 +922,7 @@ "properties": { "imageTag": { "default": "latest", + "description": "Docker image tag of the Kafka Streams app.", "title": "Imagetag", "type": "string" }, @@ -1280,6 +1281,7 @@ }, "imageTag": { "default": "latest", + "description": "Docker image tag of the Kafka Streams app.", "title": "Imagetag", "type": "string" }, @@ -1430,6 +1432,7 @@ "properties": { "imageTag": { "default": "latest", + "description": "Docker image tag of the Kafka Streams app.", "title": "Imagetag", "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 0d98c4e7e..ba40f0292 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -590,6 +590,7 @@ "properties": { "imageTag": { "default": "latest", + "description": "Docker image tag of the Kafka Streams app.", "title": "Imagetag", "type": "string" }, @@ -948,6 +949,7 @@ }, "imageTag": { "default": "latest", + "description": "Docker image tag of the Kafka Streams app.", "title": "Imagetag", "type": "string" }, diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index f026a77dc..a28a82b59 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -25,7 +25,9 @@ class StreamsBootstrapValues(HelmAppValues): :param image_tag: Docker image tag of the Kafka Streams app. """ - image_tag: str = Field(default="latest") + image_tag: str = Field( + default="latest", description=describe_attr("image_tag", __doc__) + ) @pydantic.field_validator("image_tag", mode="before") @classmethod @@ -58,11 +60,8 @@ class StreamsBootstrap(HelmApp, ABC): @pydantic.model_validator(mode="after") def warning_for_latest_image_tag(self) -> Self: - if self.app.image_tag == "latest" and self.is_substituted(): + if self.validate_ and self.app.image_tag == "latest": log.warning( - f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag." + f"The imageTag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable imageTag." ) return self - - def is_substituted(self): - return "$" not in self.name From b494f051298cb828bd700c402dd12036043d9e8d Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 12:41:48 +0200 Subject: [PATCH 16/25] Update files --- kpops/components/streams_bootstrap/__init__.py | 2 +- tests/components/test_streams_bootstrap.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index a28a82b59..da24e629d 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -62,6 +62,6 @@ class StreamsBootstrap(HelmApp, ABC): def warning_for_latest_image_tag(self) -> Self: if self.validate_ and self.app.image_tag == "latest": log.warning( - f"The imageTag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable imageTag." + f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." ) return self diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index a82fca8a9..3a153fbde 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -48,6 +48,7 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers) ) assert streams_bootstrap.version == "2.9.0" assert streams_bootstrap.namespace == "test-namespace" + assert streams_bootstrap.app.image_tag == "latest" @pytest.mark.asyncio() async def test_should_deploy_streams_bootstrap_app( @@ -63,6 +64,7 @@ async def test_should_deploy_streams_bootstrap_app( **{ "namespace": "test-namespace", "app": { + "imageTag": "1.0.0", "streams": { "outputTopic": "test", "brokers": "fake-broker:9092", @@ -94,6 +96,7 @@ async def test_should_deploy_streams_bootstrap_app( "test-namespace", { "nameOverride": "${pipeline.name}-example-name", + "imageTag": "1.0.0", "streams": { "brokers": "fake-broker:9092", "outputTopic": "test", From 82b5e809dc3a57e70371fcbef6d264800a2466f8 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 12:44:54 +0200 Subject: [PATCH 17/25] Update files --- kpops/components/streams_bootstrap/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index da24e629d..c11f7557e 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,7 +1,11 @@ import logging from abc import ABC -from typing import Any, Self +from typing import Any +try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] +except ImportError: + from typing_extensions import Self import pydantic from pydantic import Field From 947a22b7e7a608166d6f69805c065040ef2c63b0 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 12:45:20 +0200 Subject: [PATCH 18/25] Update files --- kpops/components/streams_bootstrap/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index c11f7557e..24d951c89 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -2,10 +2,6 @@ from abc import ABC from typing import Any -try: - from typing import Self # pyright: ignore[reportAttributeAccessIssue] -except ImportError: - from typing_extensions import Self import pydantic from pydantic import Field @@ -14,6 +10,12 @@ from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr +try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] +except ImportError: + from typing_extensions import Self + + STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( repository_name="bakdata-streams-bootstrap", url="https://bakdata.github.io/streams-bootstrap/", From 6e51a5fe5adcaa53a2c41c9ddf434aee68fed5da Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 15:18:18 +0200 Subject: [PATCH 19/25] Update files --- docs/docs/schema/defaults.json | 6 +-- docs/docs/schema/pipeline.json | 4 +- kpops/api/exception.py | 5 ++ .../component_handlers/kafka_connect/model.py | 3 +- kpops/component_handlers/kubernetes/utils.py | 25 +++++----- .../components/streams_bootstrap/__init__.py | 18 ++++--- .../kubernetes/test_utils.py | 47 +++++++++++++------ 7 files changed, 68 insertions(+), 40 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 2c08e0089..c66ef3f38 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -922,7 +922,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Docker image tag of the Kafka Streams app.", + "description": "Docker image tag of the streams-bootstrap app.", "title": "Imagetag", "type": "string" }, @@ -1281,7 +1281,7 @@ }, "imageTag": { "default": "latest", - "description": "Docker image tag of the Kafka Streams app.", + "description": "Docker image tag of the streams-bootstrap app.", "title": "Imagetag", "type": "string" }, @@ -1432,7 +1432,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Docker image tag of the Kafka Streams app.", + "description": "Docker image tag of the streams-bootstrap app.", "title": "Imagetag", "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index ba40f0292..8e5f17354 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -590,7 +590,7 @@ "properties": { "imageTag": { "default": "latest", - "description": "Docker image tag of the Kafka Streams app.", + "description": "Docker image tag of the streams-bootstrap app.", "title": "Imagetag", "type": "string" }, @@ -949,7 +949,7 @@ }, "imageTag": { "default": "latest", - "description": "Docker image tag of the Kafka Streams app.", + "description": "Docker image tag of the streams-bootstrap app.", "title": "Imagetag", "type": "string" }, diff --git a/kpops/api/exception.py b/kpops/api/exception.py index 65094fd29..593bdcd5c 100644 --- a/kpops/api/exception.py +++ b/kpops/api/exception.py @@ -11,3 +11,8 @@ class ParsingException(Exception): class ClassNotFoundError(Exception): """Similar to builtin `ModuleNotFoundError`; class doesn't exist inside module.""" + + +class InvalidImageTagError(ValidationError): + def __init__(self, *args): + super().__init__(*args) diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index f09a0312c..1af6b51b5 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -130,4 +130,5 @@ class KafkaConnectorResetterValues(HelmAppValues): @pydantic.field_validator("image_tag", mode="before") @classmethod def validate_image_tag_field(cls, image_tag: Any) -> str: - return validate_image_tag(image_tag) + validate_image_tag(image_tag) + return image_tag diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index a9f185162..1ea5daf67 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -1,11 +1,14 @@ import hashlib import logging import re +from typing import Any from kpops.api.exception import ValidationError log = logging.getLogger("K8sUtils") +IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" + def trim(max_len: int, name: str, suffix: str) -> str: """Shortens long K8s identifiers. @@ -31,25 +34,23 @@ def trim(max_len: int, name: str, suffix: str) -> str: return name -def validate_image_tag(image_tag: str) -> str: +def validate_image_tag(image_tag: Any) -> None: """Validate an image tag. Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} :param image_tag: Docker image tag to be validated. - :return: The validated image tag. """ - if isinstance(image_tag, str) and is_valid_image_tag(image_tag): - return image_tag - msg = ( - "Image tag is not valid. " - "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " - "It can be up to 128 characters long." - ) - raise ValidationError(msg) + if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)): + msg = ( + "Image tag is not valid. " + "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " + "It can be up to 128 characters long." + ) + raise ValidationError(msg) def is_valid_image_tag(image_tag: str) -> bool: - pattern = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" - return bool(re.match(pattern, image_tag)) + """Check if the image tag is valid according to the specified regex pattern.""" + return bool(re.match(IMAGE_TAG_PATTERN, image_tag)) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 24d951c89..3bb4ba59e 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging from abc import ABC -from typing import Any +from typing import TYPE_CHECKING, Any import pydantic from pydantic import Field @@ -10,10 +12,11 @@ from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr -try: - from typing import Self # pyright: ignore[reportAttributeAccessIssue] -except ImportError: - from typing_extensions import Self +if TYPE_CHECKING: + try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] + except ImportError: + from typing_extensions import Self STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( @@ -28,7 +31,7 @@ class StreamsBootstrapValues(HelmAppValues): """Base value class for all streams bootstrap related components. - :param image_tag: Docker image tag of the Kafka Streams app. + :param image_tag: Docker image tag of the streams-bootstrap app. """ image_tag: str = Field( @@ -38,7 +41,8 @@ class StreamsBootstrapValues(HelmAppValues): @pydantic.field_validator("image_tag", mode="before") @classmethod def validate_image_tag_field(cls, image_tag: Any) -> str: - return validate_image_tag(image_tag) + validate_image_tag(image_tag) + return image_tag class StreamsBootstrap(HelmApp, ABC): diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py index 0c1bdc3f1..49885016b 100644 --- a/tests/component_handlers/kubernetes/test_utils.py +++ b/tests/component_handlers/kubernetes/test_utils.py @@ -1,20 +1,37 @@ import pytest from kpops.api.exception import ValidationError -from kpops.component_handlers.kubernetes.utils import validate_image_tag +from kpops.component_handlers.kubernetes.utils import ( + validate_image_tag, +) +MAX_LENGTH = 128 -def test_is_valid_image_tag(): - assert validate_image_tag("1.2.3") == "1.2.3" - assert validate_image_tag("123") == "123" - assert validate_image_tag("1_2_3") == "1_2_3" - assert validate_image_tag("1-2-3") == "1-2-3" - assert validate_image_tag("latest") == "latest" - assert ( - validate_image_tag( - "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07" - ) - == "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07" - ) - with pytest.raises(ValidationError): - assert validate_image_tag("la!est") is False + +def test_validate_image_tag_valid(): + valid_tags = [ + "valid-tag", + "VALID_TAG", + "valid.tag", + "valid-tag_123", + "v" * MAX_LENGTH, + ] + for tag in valid_tags: + validate_image_tag(tag) + + +def test_if_image_tag_is_invalid(): + invalid_tags = [ + "invalid tag!", + "", + " " * (MAX_LENGTH + 1), + "a" * (MAX_LENGTH + 1), + "@invalid", + None, + 123, + {}, + [], + ] + for tag in invalid_tags: + with pytest.raises(ValidationError): + validate_image_tag(tag) From be5e1b009dfca26008f3018b954f57a5d9f26658 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 16:08:57 +0200 Subject: [PATCH 20/25] Remove image tag from connect --- kpops/api/exception.py | 5 ----- .../component_handlers/kafka_connect/model.py | 9 --------- kpops/component_handlers/kubernetes/utils.py | 20 ------------------- .../components/streams_bootstrap/__init__.py | 12 +++++++++-- .../kubernetes/test_utils.py | 16 +++------------ .../test_generate/atm-fraud/pipeline.yaml | 1 - .../test_generate/word-count/pipeline.yaml | 1 - .../test_inflate_pipeline/pipeline.yaml | 1 - .../pipeline.yaml | 1 - .../test_read_from_component/pipeline.yaml | 2 -- .../test_with_env_defaults/pipeline.yaml | 1 - 11 files changed, 13 insertions(+), 56 deletions(-) diff --git a/kpops/api/exception.py b/kpops/api/exception.py index 593bdcd5c..65094fd29 100644 --- a/kpops/api/exception.py +++ b/kpops/api/exception.py @@ -11,8 +11,3 @@ class ParsingException(Exception): class ClassNotFoundError(Exception): """Similar to builtin `ModuleNotFoundError`; class doesn't exist inside module.""" - - -class InvalidImageTagError(ValidationError): - def __init__(self, *args): - super().__init__(*args) diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index 1af6b51b5..840df06c3 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -5,7 +5,6 @@ from pydantic import ( BaseModel, ConfigDict, - Field, SerializationInfo, field_validator, model_serializer, @@ -13,7 +12,6 @@ from pydantic.json_schema import SkipJsonSchema from typing_extensions import override -from kpops.component_handlers.kubernetes.utils import validate_image_tag from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.utils.pydantic import ( @@ -125,10 +123,3 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel): class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] config: KafkaConnectorResetterConfig - image_tag: str = Field(default="latest") - - @pydantic.field_validator("image_tag", mode="before") - @classmethod - def validate_image_tag_field(cls, image_tag: Any) -> str: - validate_image_tag(image_tag) - return image_tag diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 1ea5daf67..5b17dc963 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -1,9 +1,6 @@ import hashlib import logging import re -from typing import Any - -from kpops.api.exception import ValidationError log = logging.getLogger("K8sUtils") @@ -34,23 +31,6 @@ def trim(max_len: int, name: str, suffix: str) -> str: return name -def validate_image_tag(image_tag: Any) -> None: - """Validate an image tag. - - Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). - It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} - - :param image_tag: Docker image tag to be validated. - """ - if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)): - msg = ( - "Image tag is not valid. " - "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " - "It can be up to 128 characters long." - ) - raise ValidationError(msg) - - def is_valid_image_tag(image_tag: str) -> bool: """Check if the image tag is valid according to the specified regex pattern.""" return bool(re.match(IMAGE_TAG_PATTERN, image_tag)) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 3bb4ba59e..fff1e4f1b 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -8,7 +8,9 @@ from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.component_handlers.kubernetes.utils import validate_image_tag +from kpops.component_handlers.kubernetes.utils import ( + is_valid_image_tag, +) from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr @@ -41,7 +43,13 @@ class StreamsBootstrapValues(HelmAppValues): @pydantic.field_validator("image_tag", mode="before") @classmethod def validate_image_tag_field(cls, image_tag: Any) -> str: - validate_image_tag(image_tag) + if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)): + msg = ( + "Image tag is not valid. " + "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " + "It can be up to 128 characters long." + ) + raise pydantic.ValidationError(msg) return image_tag diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py index 49885016b..2648c9a16 100644 --- a/tests/component_handlers/kubernetes/test_utils.py +++ b/tests/component_handlers/kubernetes/test_utils.py @@ -1,9 +1,4 @@ -import pytest - -from kpops.api.exception import ValidationError -from kpops.component_handlers.kubernetes.utils import ( - validate_image_tag, -) +from kpops.component_handlers.kubernetes.utils import is_valid_image_tag MAX_LENGTH = 128 @@ -17,7 +12,7 @@ def test_validate_image_tag_valid(): "v" * MAX_LENGTH, ] for tag in valid_tags: - validate_image_tag(tag) + assert is_valid_image_tag(tag) is True def test_if_image_tag_is_invalid(): @@ -27,11 +22,6 @@ def test_if_image_tag_is_invalid(): " " * (MAX_LENGTH + 1), "a" * (MAX_LENGTH + 1), "@invalid", - None, - 123, - {}, - [], ] for tag in invalid_tags: - with pytest.raises(ValidationError): - validate_image_tag(tag) + assert is_valid_image_tag(tag) is False diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml index 8559e1534..528da26bb 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml @@ -393,7 +393,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: atm-fraud-postgresql-connector connectorType: sink - imageTag: latest name: postgresql-connector namespace: ${NAMESPACE} prefix: atm-fraud- diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml index a10c5b5a3..7e7d16fe8 100644 --- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml +++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml @@ -142,7 +142,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: word-count-redis-sink-connector connectorType: sink - imageTag: latest name: redis-sink-connector namespace: ${NAMESPACE} prefix: word-count- diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml index 5ead506c6..1040139c3 100644 --- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml @@ -255,7 +255,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector connectorType: sink - imageTag: latest name: should-inflate-inflated-sink-connector namespace: example-namespace prefix: resources-pipeline-with-inflate- diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml index 180b80ec0..e1dd399cd 100644 --- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml @@ -73,7 +73,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-kafka-connect-sink-es-sink-connector connectorType: sink - imageTag: latest name: es-sink-connector namespace: example-namespace prefix: resources-kafka-connect-sink- diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml index 63fb42b8e..ff1dd9e2b 100644 --- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml @@ -172,7 +172,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-read-from-component-inflate-step-inflated-sink-connector connectorType: sink - imageTag: latest name: inflate-step-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- @@ -367,7 +366,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector connectorType: sink - imageTag: latest name: inflate-step-without-prefix-inflated-sink-connector namespace: example-namespace prefix: resources-read-from-component- diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml index edb1d4623..a37ad0f3d 100644 --- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml +++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml @@ -73,7 +73,6 @@ brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 connector: resources-pipeline-with-env-defaults-es-sink-connector connectorType: sink - imageTag: latest name: es-sink-connector namespace: development-namespace prefix: resources-pipeline-with-env-defaults- From e1fcdc7d27214998722ec330c63b89599173d671 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 16:11:49 +0200 Subject: [PATCH 21/25] Update files --- kpops/component_handlers/kubernetes/utils.py | 6 ++++- .../kubernetes/test_utils.py | 26 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 5b17dc963..c50b0b617 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -32,5 +32,9 @@ def trim(max_len: int, name: str, suffix: str) -> str: def is_valid_image_tag(image_tag: str) -> bool: - """Check if the image tag is valid according to the specified regex pattern.""" + """Check if the image tag is valid according to the specified regex pattern. + + Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). + It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} + """ return bool(re.match(IMAGE_TAG_PATTERN, image_tag)) diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py index 2648c9a16..c440cf83b 100644 --- a/tests/component_handlers/kubernetes/test_utils.py +++ b/tests/component_handlers/kubernetes/test_utils.py @@ -1,27 +1,33 @@ +import pytest + from kpops.component_handlers.kubernetes.utils import is_valid_image_tag MAX_LENGTH = 128 -def test_validate_image_tag_valid(): - valid_tags = [ +@pytest.mark.parametrize( + "tag", + [ "valid-tag", "VALID_TAG", "valid.tag", "valid-tag_123", "v" * MAX_LENGTH, - ] - for tag in valid_tags: - assert is_valid_image_tag(tag) is True + ], +) +def test_validate_image_tag_valid(tag): + assert is_valid_image_tag(tag) is True -def test_if_image_tag_is_invalid(): - invalid_tags = [ +@pytest.mark.parametrize( + "tag", + [ "invalid tag!", "", " " * (MAX_LENGTH + 1), "a" * (MAX_LENGTH + 1), "@invalid", - ] - for tag in invalid_tags: - assert is_valid_image_tag(tag) is False + ], +) +def test_if_image_tag_is_invalid(tag): + assert is_valid_image_tag(tag) is False From 47d72443d0b843848986978facff1134c3ac0040 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 16:13:13 +0200 Subject: [PATCH 22/25] add type to test --- tests/component_handlers/kubernetes/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py index c440cf83b..982a070e3 100644 --- a/tests/component_handlers/kubernetes/test_utils.py +++ b/tests/component_handlers/kubernetes/test_utils.py @@ -15,7 +15,7 @@ "v" * MAX_LENGTH, ], ) -def test_validate_image_tag_valid(tag): +def test_validate_image_tag_valid(tag: str): assert is_valid_image_tag(tag) is True @@ -29,5 +29,5 @@ def test_validate_image_tag_valid(tag): "@invalid", ], ) -def test_if_image_tag_is_invalid(tag): +def test_if_image_tag_is_invalid(tag: str): assert is_valid_image_tag(tag) is False From 1c3698eabb6f9a8f871d799d7efb2ed11583bac9 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 4 Jul 2024 16:27:56 +0200 Subject: [PATCH 23/25] Update files --- docs/docs/schema/defaults.json | 3 ++ docs/docs/schema/pipeline.json | 2 ++ kpops/component_handlers/kubernetes/utils.py | 11 +------ .../components/streams_bootstrap/__init__.py | 21 +++--------- .../kubernetes/test_utils.py | 33 ------------------- 5 files changed, 11 insertions(+), 59 deletions(-) delete mode 100644 tests/component_handlers/kubernetes/test_utils.py diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index c66ef3f38..d5f479201 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -923,6 +923,7 @@ "imageTag": { "default": "latest", "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, @@ -1282,6 +1283,7 @@ "imageTag": { "default": "latest", "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, @@ -1433,6 +1435,7 @@ "imageTag": { "default": "latest", "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 8e5f17354..d8c953c82 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -591,6 +591,7 @@ "imageTag": { "default": "latest", "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, @@ -950,6 +951,7 @@ "imageTag": { "default": "latest", "description": "Docker image tag of the streams-bootstrap app.", + "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$", "title": "Imagetag", "type": "string" }, diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index c50b0b617..598e1a706 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -1,9 +1,9 @@ import hashlib import logging -import re log = logging.getLogger("K8sUtils") +# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" @@ -29,12 +29,3 @@ def trim(max_len: int, name: str, suffix: str) -> str: ) return new_name return name - - -def is_valid_image_tag(image_tag: str) -> bool: - """Check if the image tag is valid according to the specified regex pattern. - - Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). - It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127} - """ - return bool(re.match(IMAGE_TAG_PATTERN, image_tag)) diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index fff1e4f1b..7ac93c3f8 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -2,14 +2,14 @@ import logging from abc import ABC -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import pydantic from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.component_handlers.kubernetes.utils import ( - is_valid_image_tag, + IMAGE_TAG_PATTERN, ) from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr @@ -20,7 +20,6 @@ except ImportError: from typing_extensions import Self - STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( repository_name="bakdata-streams-bootstrap", url="https://bakdata.github.io/streams-bootstrap/", @@ -37,21 +36,11 @@ class StreamsBootstrapValues(HelmAppValues): """ image_tag: str = Field( - default="latest", description=describe_attr("image_tag", __doc__) + default="latest", + pattern=IMAGE_TAG_PATTERN, + description=describe_attr("image_tag", __doc__), ) - @pydantic.field_validator("image_tag", mode="before") - @classmethod - def validate_image_tag_field(cls, image_tag: Any) -> str: - if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)): - msg = ( - "Image tag is not valid. " - "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). " - "It can be up to 128 characters long." - ) - raise pydantic.ValidationError(msg) - return image_tag - class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py deleted file mode 100644 index 982a070e3..000000000 --- a/tests/component_handlers/kubernetes/test_utils.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest - -from kpops.component_handlers.kubernetes.utils import is_valid_image_tag - -MAX_LENGTH = 128 - - -@pytest.mark.parametrize( - "tag", - [ - "valid-tag", - "VALID_TAG", - "valid.tag", - "valid-tag_123", - "v" * MAX_LENGTH, - ], -) -def test_validate_image_tag_valid(tag: str): - assert is_valid_image_tag(tag) is True - - -@pytest.mark.parametrize( - "tag", - [ - "invalid tag!", - "", - " " * (MAX_LENGTH + 1), - "a" * (MAX_LENGTH + 1), - "@invalid", - ], -) -def test_if_image_tag_is_invalid(tag: str): - assert is_valid_image_tag(tag) is False From 4cd941be00f998e615a095ab9e45922e31a43232 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 8 Jul 2024 13:36:43 +0200 Subject: [PATCH 24/25] Update test and pattern --- kpops/component_handlers/kubernetes/utils.py | 3 --- .../components/streams_bootstrap/__init__.py | 6 ++--- tests/components/test_streams_bootstrap.py | 22 ++++++++++++++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 598e1a706..4f4599e2b 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -3,9 +3,6 @@ log = logging.getLogger("K8sUtils") -# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names -IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" - def trim(max_len: int, name: str, suffix: str) -> str: """Shortens long K8s identifiers. diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 7ac93c3f8..88777a157 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -8,9 +8,6 @@ from pydantic import Field from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.component_handlers.kubernetes.utils import ( - IMAGE_TAG_PATTERN, -) from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr @@ -28,6 +25,9 @@ log = logging.getLogger("StreamsBootstrap") +# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names +IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$" + class StreamsBootstrapValues(HelmAppValues): """Base value class for all streams bootstrap related components. diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index 3a153fbde..3ece4612d 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -1,6 +1,8 @@ +import re from unittest.mock import MagicMock import pytest +from pydantic import ValidationError from pytest_mock import MockerFixture from kpops.component_handlers import ComponentHandlers @@ -10,7 +12,7 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap import StreamsBootstrap, StreamsBootstrapValues from kpops.config import KpopsConfig from tests.components import PIPELINE_BASE_DIR @@ -104,3 +106,21 @@ async def test_should_deploy_streams_bootstrap_app( }, HelmUpgradeInstallFlags(version="1.2.3"), ) + + @pytest.mark.asyncio() + async def test_should_raise_validation_error_for_invalid_image_tag( + self, + config: KpopsConfig, + handlers: ComponentHandlers, + ): + with pytest.raises( + ValidationError, + match=re.escape( + "1 validation error for StreamsBootstrapValues\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'" + ), + ): + StreamsBootstrapValues( + **{ + "imageTag": "invalid image tag!", + } + ) From bb78dcec5d910920226f974a86b664445a85c68a Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 8 Jul 2024 13:38:31 +0200 Subject: [PATCH 25/25] Update files --- docs/docs/schema/defaults.json | 2 +- kpops/components/streams_bootstrap/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index d5f479201..77ceed301 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1345,7 +1345,7 @@ "$ref": "#/$defs/StreamsBootstrapValues" } ], - "description": "Streams bootstrap app values" + "description": "streams-bootstrap app values" }, "from": { "anyOf": [ diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 88777a157..c6c329b85 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -45,7 +45,7 @@ class StreamsBootstrapValues(HelmAppValues): class StreamsBootstrap(HelmApp, ABC): """Base for components with a streams-bootstrap Helm chart. - :param app: Streams bootstrap app values + :param app: streams-bootstrap app values :param repo_config: Configuration of the Helm chart repo to be used for deploying the component, defaults to streams-bootstrap Helm repo :param version: Helm chart version, defaults to "2.9.0"