From c218e0256ce24375d8020cfbf13f276d4b537a52 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 28 Nov 2024 18:02:18 +0100 Subject: [PATCH 01/47] Introduce KPOps operation and manifest resources for deployment --- .../docs/resources/variables/cli_env_vars.env | 2 + docs/docs/resources/variables/cli_env_vars.md | 1 + .../resources/variables/config_env_vars.env | 3 ++ .../resources/variables/config_env_vars.md | 1 + docs/docs/schema/config.json | 18 +++++++ docs/docs/user/references/cli-commands.md | 1 + kpops/api/__init__.py | 28 +++++++++++ kpops/api/operation.py | 9 ++++ kpops/cli/main.py | 49 ++++++++++++++----- kpops/components/base_components/helm_app.py | 6 +++ .../base_components/models/resource.py | 4 +- .../base_components/pipeline_component.py | 16 ++++++ .../streams_bootstrap/streams/streams_app.py | 20 ++++++++ kpops/config/__init__.py | 8 +++ kpops/manifestors/__init__.py | 0 kpops/manifestors/helm_app_manifestor.py | 37 ++++++++++++++ kpops/manifestors/manifestor.py | 27 ++++++++++ kpops/manifestors/streams_app_manifestor.py | 41 ++++++++++++++++ .../test_init_project/config_include_opt.yaml | 3 ++ tests/conftest.py | 3 +- tests/pipeline/test_manifest.py | 28 +++++++++++ 21 files changed, 290 insertions(+), 15 deletions(-) create mode 100644 kpops/api/operation.py create mode 100644 kpops/manifestors/__init__.py create mode 100644 kpops/manifestors/helm_app_manifestor.py create mode 100644 kpops/manifestors/manifestor.py create mode 100644 kpops/manifestors/streams_app_manifestor.py diff --git a/docs/docs/resources/variables/cli_env_vars.env b/docs/docs/resources/variables/cli_env_vars.env index 21436ded7..b47252b5b 100644 --- a/docs/docs/resources/variables/cli_env_vars.env +++ b/docs/docs/resources/variables/cli_env_vars.env @@ -14,6 +14,8 @@ KPOPS_DOTENV_PATH # No default value, not required # Suffix your environment files with this value (e.g. # defaults_development.yaml for environment=development). KPOPS_ENVIRONMENT # No default value, not required +# How KPOps should operate. +KPOPS_OPERATION_MODE=standard # Paths to dir containing 'pipeline.yaml' or files named # 'pipeline.yaml'. KPOPS_PIPELINE_PATHS # No default value, required diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md index da6a2d994..a7d6f0a66 100644 --- a/docs/docs/resources/variables/cli_env_vars.md +++ b/docs/docs/resources/variables/cli_env_vars.md @@ -5,5 +5,6 @@ These variables take precedence over the commands' flags. If a variable is set, |KPOPS_CONFIG_PATH |. |False |Path to the dir containing config.yaml files | |KPOPS_DOTENV_PATH | |False |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one. | |KPOPS_ENVIRONMENT | |False |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).| +|KPOPS_OPERATION_MODE|standard |False |How KPOps should operate. | |KPOPS_PIPELINE_PATHS| |True |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'. | |KPOPS_PIPELINE_STEPS| |False |Comma separated list of steps to apply the command on | diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index f558d4d19..747672702 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -58,3 +58,6 @@ KPOPS_HELM_DIFF_CONFIG__IGNORE # No default value, required # Whether to retain clean up jobs in the cluster or uninstall the, # after completion. KPOPS_RETAIN_CLEAN_JOBS=False +# operation_mode +# The operation mode of KPOps (standard, manifest, argo). +KPOPS_OPERATION_MODE=standard diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 8685acba0..6e5458054 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -19,3 +19,4 @@ These variables take precedence over the settings in `config.yaml`. Variables ma |KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | |KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | |KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs | +|KPOPS_OPERATION_MODE |standard |False |The operation mode of KPOps (standard, manifest, argo). |operation_mode | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index 949c42791..bddbda217 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -118,6 +118,15 @@ "title": "KafkaRestConfig", "type": "object" }, + "OperationMode": { + "enum": [ + "argo", + "manifest", + "standard" + ], + "title": "OperationMode", + "type": "string" + }, "SchemaRegistryConfig": { "additionalProperties": false, "description": "Configuration for Schema Registry.", @@ -239,6 +248,15 @@ }, "description": "Configuration for Kafka REST Proxy." }, + "operation_mode": { + "allOf": [ + { + "$ref": "#/$defs/OperationMode" + } + ], + "default": "standard", + "description": "The operation mode of KPOps (standard, manifest, argo)." + }, "pipeline_base_dir": { "default": ".", "description": "Base directory to the pipelines (default is current working directory)", diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index d10ba44cc..ce75d7b3b 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -74,6 +74,7 @@ $ kpops deploy [OPTIONS] PIPELINE_PATHS... * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] * `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially. [default: no-parallel] +* `--operation-mode [argo|manifest|standard]`: How KPOps should operate. [env var: KPOPS_OPERATION_MODE; default: standard] * `--help`: Show this message and exit. ## `kpops destroy` diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index f376af704..b47491318 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -1,10 +1,12 @@ from __future__ import annotations import asyncio +from collections.abc import Iterator from pathlib import Path from typing import TYPE_CHECKING from kpops.api.logs import log, log_action +from kpops.api.operation import OperationMode from kpops.api.options import FilterType from kpops.api.registry import Registry from kpops.component_handlers import ComponentHandlers @@ -105,6 +107,32 @@ def manifest( return resources +def manifest_deployment( + pipeline_path: Path, + dotenv: list[Path] | None = None, + config: Path = Path(), + steps: set[str] | None = None, + filter_type: FilterType = FilterType.INCLUDE, + environment: str | None = None, + verbose: bool = True, + operation_mode: OperationMode = OperationMode.MANIFEST, +) -> Iterator[Resource]: + pipeline = generate( + pipeline_path=pipeline_path, + dotenv=dotenv, + config=config, + steps=steps, + filter_type=filter_type, + environment=environment, + verbose=verbose, + ) + # TODO: KPOps config is created twice. Once in generate and once here. change it! + KpopsConfig.create(config, dotenv, environment, verbose, operation_mode) + for component in pipeline.components: + resource = component.manifest_deploy() + yield resource + + def deploy( pipeline_path: Path, dotenv: list[Path] | None = None, diff --git a/kpops/api/operation.py b/kpops/api/operation.py new file mode 100644 index 000000000..04c752893 --- /dev/null +++ b/kpops/api/operation.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from enum import Enum + + +class OperationMode(str, Enum): + ARGO = "argo" + MANIFEST = "manifest" + STANDARD = "standard" diff --git a/kpops/cli/main.py b/kpops/cli/main.py index ff0a77338..92000169c 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -5,6 +5,7 @@ import typer import kpops.api as kpops +from kpops.api.operation import OperationMode from kpops.api.options import FilterType from kpops.cli.utils import ( collect_pipeline_paths, @@ -110,6 +111,12 @@ "Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). " ), ) +OPERATION_MODE_OPTION: OperationMode = typer.Option( + default=OperationMode.STANDARD, + envvar=f"{ENV_PREFIX}OPERATION_MODE", + # TODO: better help? + help="How KPOps should operate.", +) def parse_steps(steps: str | None) -> set[str] | None: @@ -219,19 +226,37 @@ def deploy( dry_run: bool = DRY_RUN, verbose: bool = VERBOSE_OPTION, parallel: bool = PARALLEL, + operation_mode: OperationMode = OPERATION_MODE_OPTION, ): - for pipeline_file_path in collect_pipeline_paths(pipeline_paths): - kpops.deploy( - pipeline_path=pipeline_file_path, - dotenv=dotenv, - config=config, - steps=parse_steps(steps), - filter_type=filter_type, - environment=environment, - dry_run=dry_run, - verbose=verbose, - parallel=parallel, - ) + match operation_mode: + case OperationMode.STANDARD: + for pipeline_file_path in collect_pipeline_paths(pipeline_paths): + kpops.deploy( + pipeline_path=pipeline_file_path, + dotenv=dotenv, + config=config, + steps=parse_steps(steps), + filter_type=filter_type, + environment=environment, + dry_run=dry_run, + verbose=verbose, + parallel=parallel, + ) + case _: + for pipeline_file_path in collect_pipeline_paths(pipeline_paths): + resources = kpops.manifest_deployment( + pipeline_file_path, + dotenv, + config, + parse_steps(steps), + filter_type, + environment, + verbose, + operation_mode, + ) + for resource in resources: + for rendered_manifest in resource: + print_yaml(rendered_manifest) @app.command(help="Destroy pipeline steps") diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 3990b3f1c..d139b14d3 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -28,6 +28,7 @@ ) from kpops.components.base_components.models.resource import Resource from kpops.config import get_config +from kpops.manifestors.helm_app_manifestor import HelmAppManifestor from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -151,6 +152,11 @@ def manifest(self) -> Resource: self.template_flags, ) + @override + def manifest_deploy(self) -> Resource: + helm_app_manifestor = HelmAppManifestor() + return helm_app_manifestor.generate_manifest(self) + @property def deploy_flags(self) -> HelmUpgradeInstallFlags: """Return flags for Helm upgrade install command.""" diff --git a/kpops/components/base_components/models/resource.py b/kpops/components/base_components/models/resource.py index 08c01f344..e6867081d 100644 --- a/kpops/components/base_components/models/resource.py +++ b/kpops/components/base_components/models/resource.py @@ -1,5 +1,5 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import Any, TypeAlias # representation of final resource for component, e.g. a list of Kubernetes manifests -Resource: TypeAlias = Sequence[Mapping[str, Any]] +Resource: TypeAlias = list[Mapping[str, Any]] diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 5b6798af6..6e309ab4f 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -233,6 +233,22 @@ def manifest(self) -> Resource: """Render final component resources, e.g. Kubernetes manifests.""" return [] + def manifest_deploy(self) -> Resource: + """Render final component resources for deployment, e.g. Kubernetes manifests.""" + return [] + + def manifest_destroy(self) -> Resource: + """Render final component resources for destroy, e.g. Kubernetes manifests.""" + return [] + + def manifest_reset(self) -> Resource: + """Render final component resources for reset, e.g. Kubernetes manifests.""" + return [] + + def manifest_clean(self) -> Resource: + """Render final component resources for clean, e.g. Kubernetes manifests.""" + return [] + async def deploy(self, dry_run: bool) -> None: """Deploy component, e.g. to Kubernetes cluster. diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index ac3ffae68..393e27fe3 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -4,9 +4,11 @@ from pydantic import Field, ValidationError, computed_field from typing_extensions import override +from kpops.api.operation import OperationMode from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner +from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.base import ( @@ -15,7 +17,9 @@ from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, ) +from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML +from kpops.manifestors.streams_app_manifestor import StreamsAppCleanerManifestor from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -152,3 +156,19 @@ async def clean(self, dry_run: bool) -> None: """Destroy and clean.""" await super().clean(dry_run) await self._cleaner.clean(dry_run) + + @override + def manifest_deploy(self) -> Resource: + manifest = super().manifest_deploy() + operation_mode = get_config().operation_mode + + if operation_mode is OperationMode.ARGO: + clean_manifestor = StreamsAppCleanerManifestor() + clean_manifest = clean_manifestor.generate_manifest(self._cleaner) + manifest.extend(clean_manifest) + + return manifest + + @override + def manifest_clean(self) -> Resource: + return [] diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index a7d0dc281..8cf575819 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -12,6 +12,7 @@ ) from typing_extensions import override +from kpops.api.operation import OperationMode from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig from kpops.utils.docstring import describe_object from kpops.utils.pydantic import YamlConfigSettingsSource @@ -120,6 +121,10 @@ class KpopsConfig(BaseSettings): default=False, description="Whether to retain clean up jobs in the cluster or uninstall the, after completion.", ) + operation_mode: OperationMode = Field( + default=OperationMode.STANDARD, + description="The operation mode of KPOps (standard, manifest, argo).", + ) model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, env_nested_delimiter="__") @@ -130,6 +135,7 @@ def create( dotenv: list[Path] | None = None, environment: str | None = None, verbose: bool = False, + operation_mode: OperationMode | None = None, ) -> KpopsConfig: cls.setup_logging_level(verbose) YamlConfigSettingsSource.config_dir = config_dir @@ -137,6 +143,8 @@ def create( cls._instance = KpopsConfig( _env_file=dotenv # pyright: ignore[reportCallIssue] ) + if operation_mode: + cls._instance.operation_mode = operation_mode return cls._instance @staticmethod diff --git a/kpops/manifestors/__init__.py b/kpops/manifestors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kpops/manifestors/helm_app_manifestor.py b/kpops/manifestors/helm_app_manifestor.py new file mode 100644 index 000000000..747e03aca --- /dev/null +++ b/kpops/manifestors/helm_app_manifestor.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from kpops.api.operation import OperationMode +from kpops.manifestors.manifestor import Manifestor + +if TYPE_CHECKING: + from kpops.components.base_components import HelmApp + from kpops.components.base_components.models.resource import Resource + + +class HelmAppManifestor(Manifestor): + """Manifestor for the HelmApp component.""" + + def generate_annotations(self) -> dict[str, str]: + """Generate annotations for HelmApp based on operation mode.""" + match self.operation_mode: + case OperationMode.ARGO: + return {"argocd.argoproj.io/sync-wave": "1"} + case _: + return {} + + def generate_manifest(self, helm_app: HelmApp) -> Resource: + """Generate the Helm manifest for HelmApp.""" + values = helm_app.to_helm_values() + annotations = self.generate_annotations() + if annotations: + values["annotations"] = annotations + + return helm_app.helm.template( + helm_app.helm_release_name, + helm_app.helm_chart, + helm_app.namespace, + values, + helm_app.template_flags, + ) diff --git a/kpops/manifestors/manifestor.py b/kpops/manifestors/manifestor.py new file mode 100644 index 000000000..64e798f47 --- /dev/null +++ b/kpops/manifestors/manifestor.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from kpops.config import get_config + +if TYPE_CHECKING: + from kpops.components.base_components import HelmApp + from kpops.components.base_components.models.resource import Resource + + +class Manifestor(ABC): + """Base class for generating manifests for different components.""" + + def __init__(self): + self.operation_mode = get_config().operation_mode + + @abstractmethod + def generate_annotations(self) -> dict[str, str]: + """Generate the annotations for the component based on the operation mode.""" + ... + + @abstractmethod + def generate_manifest(self, helm_app: HelmApp) -> Resource: + """Generate the Helm manifest for the component.""" + ... diff --git a/kpops/manifestors/streams_app_manifestor.py b/kpops/manifestors/streams_app_manifestor.py new file mode 100644 index 000000000..eec1e1543 --- /dev/null +++ b/kpops/manifestors/streams_app_manifestor.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from typing_extensions import override + +from kpops.api.operation import OperationMode +from kpops.components.base_components.models.resource import Resource +from kpops.manifestors.helm_app_manifestor import HelmAppManifestor + +if TYPE_CHECKING: + from kpops.components.streams_bootstrap.streams.streams_app import StreamsAppCleaner + + +class StreamsAppCleanerManifestor(HelmAppManifestor): + """Manifestor for the StreamsApp cleaner component.""" + + @override + def generate_annotations(self) -> dict[str, str]: + """Generate annotations for StreamsApp based on operation mode.""" + match self.operation_mode: + case OperationMode.ARGO: + return {"argocd.argoproj.io/hook": "PostDelete"} + case _: + return {} + + @override + def generate_manifest(self, cleaner: StreamsAppCleaner) -> Resource: + """Generate the Helm manifest for StreamsApp.""" + values = cleaner.to_helm_values() + annotations = self.generate_annotations() + if annotations: + values["annotations"] = annotations + + return cleaner.helm.template( + cleaner.helm_release_name, + cleaner.helm_chart, + cleaner.namespace, + values, + cleaner.template_flags, + ) diff --git a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml index 3c86a269a..33143a1a0 100644 --- a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml +++ b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml @@ -16,6 +16,9 @@ kafka_connect: kafka_rest: timeout: 30 url: http://localhost:8082/ +operation_mode: !!python/object/apply:builtins.getattr +- !!python/name:kpops.api.operation.OperationMode '' +- STANDARD pipeline_base_dir: . retain_clean_jobs: false schema_registry: diff --git a/tests/conftest.py b/tests/conftest.py index a819c26af..4983aa325 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ import pytest -from kpops.config import KpopsConfig from kpops.utils.environment import ENV, Environment from kpops.utils.yaml import load_yaml_file @@ -56,6 +55,8 @@ def custom_components() -> Iterator[None]: @pytest.fixture(scope="module") def clear_kpops_config() -> Iterator[None]: + from kpops.config import KpopsConfig + yield KpopsConfig._instance = None diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 84b742ae0..ada22e816 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -136,3 +136,31 @@ def test_streams_bootstrap(self, snapshot: Snapshot): ) assert result.exit_code == 0, result.stdout snapshot.assert_match(result.stdout, MANIFEST_YAML) + + def test_deploy_manifest_mode(self, snapshot: Snapshot): + result = runner.invoke( + app, + [ + "deploy", + str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), + "--operation-mode", + "manifest", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + snapshot.assert_match(result.stdout, MANIFEST_YAML) + + def test_deploy_argo_mode(self, snapshot: Snapshot): + result = runner.invoke( + app, + [ + "deploy", + str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), + "--operation-mode", + "argo", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + snapshot.assert_match(result.stdout, MANIFEST_YAML) From 4a29cbd36f27397f8eed1b788255c58c1bb8f221 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 28 Nov 2024 18:06:45 +0100 Subject: [PATCH 02/47] Introduce KPOps operation and manifest resources for deployment --- .../test_deploy_argo_mode/manifest.yaml | 238 ++++++++++++++++++ .../test_deploy_manifest_mode/manifest.yaml | 171 +++++++++++++ 2 files changed, 409 insertions(+) create mode 100644 tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml create mode 100644 tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml new file mode 100644 index 000000000..c3f2c7bd7 --- /dev/null +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -0,0 +1,238 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + annotations: + argocd.argoproj.io/sync-wave: '1' + labels: + app: resources-streams-bootstrap-my-producer-app + chart: producer-app-3.0.1 + release: resources-streams-bootstrap-my-producer-app + name: resources-streams-bootstrap-my-producer-app +spec: + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-my-producer-app + release: resources-streams-bootstrap-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + schedule: 30 3/8 * * * + successfulJobsHistoryLimit: 1 + suspend: false + +--- +apiVersion: v1 +data: + jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ + lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ + \ - pattern: \".*\"\n" +kind: ConfigMap +metadata: + labels: + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.1 + heritage: Helm + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app-jmx-configmap + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + argocd.argoproj.io/sync-wave: '1' + consumerGroup: my-streams-app-id + labels: + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.1 + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app +spec: + replicas: 1 + selector: + matchLabels: + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app + template: + metadata: + annotations: + prometheus.io/port: '5556' + prometheus.io/scrape: 'true' + labels: + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR + value: com.bakdata.kafka.MurmurHashIdGenerator + - name: KAFKA_JMX_PORT + value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-streams-bootstrap-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-streams-app + ports: + - containerPort: 5555 + name: jmx + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + - command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - '5556' + - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml + image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + name: prometheus-jmx-exporter + ports: + - containerPort: 5556 + resources: + limits: + cpu: 300m + memory: 2G + requests: + cpu: 100m + memory: 500Mi + volumeMounts: + - mountPath: /etc/jmx-streams-app + name: jmx-config + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + name: resources-streams-bootstrap-my-streams-app-jmx-configmap + name: jmx-config + +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: PostDelete + labels: + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-cleanup-job-3.0.1 + release: resources-streams-bootstrap-my-streams-app-clean + name: resources-streams-bootstrap-my-streams-app +spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app-clean + spec: + containers: + - args: + - reset + env: + - name: ENV_PREFIX + value: APP_ + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR + value: com.bakdata.kafka.MurmurHashIdGenerator + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-streams-bootstrap-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=85.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-streams-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + ttlSecondsAfterFinished: 30 + diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml new file mode 100644 index 000000000..7df64af03 --- /dev/null +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -0,0 +1,171 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + labels: + app: resources-streams-bootstrap-my-producer-app + chart: producer-app-3.0.1 + release: resources-streams-bootstrap-my-producer-app + name: resources-streams-bootstrap-my-producer-app +spec: + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-my-producer-app + release: resources-streams-bootstrap-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + schedule: 30 3/8 * * * + successfulJobsHistoryLimit: 1 + suspend: false + +--- +apiVersion: v1 +data: + jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ + lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ + \ - pattern: \".*\"\n" +kind: ConfigMap +metadata: + labels: + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.1 + heritage: Helm + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app-jmx-configmap + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + consumerGroup: my-streams-app-id + labels: + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.1 + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app +spec: + replicas: 1 + selector: + matchLabels: + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app + template: + metadata: + annotations: + prometheus.io/port: '5556' + prometheus.io/scrape: 'true' + labels: + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR + value: com.bakdata.kafka.MurmurHashIdGenerator + - name: KAFKA_JMX_PORT + value: '5555' + - name: APP_VOLATILE_GROUP_INSTANCE_ID + value: 'true' + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_INPUT_TOPICS + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern + - name: APP_OUTPUT_TOPIC + value: my-output-topic + - name: APP_ERROR_TOPIC + value: resources-streams-bootstrap-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' + - name: JAVA_TOOL_OPTIONS + value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + image: my-registry/my-streams-app-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-streams-app + ports: + - containerPort: 5555 + name: jmx + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + - command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - '5556' + - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml + image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + name: prometheus-jmx-exporter + ports: + - containerPort: 5556 + resources: + limits: + cpu: 300m + memory: 2G + requests: + cpu: 100m + memory: 500Mi + volumeMounts: + - mountPath: /etc/jmx-streams-app + name: jmx-config + terminationGracePeriodSeconds: 300 + volumes: + - configMap: + name: resources-streams-bootstrap-my-streams-app-jmx-configmap + name: jmx-config + From 7ccd04fdd861065f1214be35b40089ad7fe4296f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 28 Nov 2024 20:08:44 +0100 Subject: [PATCH 03/47] Propose refactoring for #541 --- kpops/component_handlers/helm_wrapper/helm.py | 8 ++-- kpops/component_handlers/kubernetes/model.py | 30 -------------- kpops/components/base_components/helm_app.py | 10 ++--- .../base_components/models/resource.py | 5 --- .../base_components/pipeline_component.py | 12 +++--- .../streams_bootstrap/streams/streams_app.py | 26 +++++++----- kpops/manifestors/helm_app_manifestor.py | 37 ----------------- kpops/manifestors/manifestor.py | 27 ------------ kpops/manifestors/streams_app_manifestor.py | 41 ------------------- kpops/{manifestors => manifests}/__init__.py | 0 kpops/manifests/argo.py | 16 ++++++++ kpops/manifests/kubernetes.py | 15 +++++++ 12 files changed, 61 insertions(+), 166 deletions(-) delete mode 100644 kpops/component_handlers/kubernetes/model.py delete mode 100644 kpops/components/base_components/models/resource.py delete mode 100644 kpops/manifestors/helm_app_manifestor.py delete mode 100644 kpops/manifestors/manifestor.py delete mode 100644 kpops/manifestors/streams_app_manifestor.py rename kpops/{manifestors => manifests}/__init__.py (100%) create mode 100644 kpops/manifests/argo.py create mode 100644 kpops/manifests/kubernetes.py diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 5ff76fb81..e0634ee4e 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -25,7 +25,6 @@ if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from kpops.components.base_components.models.resource import Resource log = logging.getLogger("Helm") @@ -161,7 +160,7 @@ def template( namespace: str, values: dict[str, Any], flags: HelmTemplateFlags | None = None, - ) -> Resource: + ) -> list[KubernetesManifest]: """From Helm: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -191,8 +190,9 @@ def template( ] command.extend(flags.to_command()) output = self.__execute(command) - manifests = KubernetesManifest.from_yaml(output) - return list(manifests) + # TODO: deserialize Kubernetes model + # manifests = KubernetesManifest.from_yaml(output) + # return list(manifests) def get_manifest(self, release_name: str, namespace: str) -> Iterable[HelmTemplate]: command = [ diff --git a/kpops/component_handlers/kubernetes/model.py b/kpops/component_handlers/kubernetes/model.py deleted file mode 100644 index 5970de1bf..000000000 --- a/kpops/component_handlers/kubernetes/model.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -import json -from collections import UserDict -from collections.abc import Iterator - -import yaml - -from kpops.utils.types import JsonType - -K8S_LABEL_MAX_LEN = 63 - - -class KubernetesManifest(UserDict[str, JsonType]): - """Representation of a Kubernetes API object as YAML/JSON mapping.""" - - @classmethod - def from_yaml( - cls, /, content: str - ) -> Iterator[KubernetesManifest]: # TODO: typing.Self for Python 3.11+ - manifests: Iterator[dict[str, JsonType]] = yaml.load_all(content, yaml.Loader) - for manifest in manifests: - yield cls(manifest) - - @classmethod - def from_json( - cls, /, content: str - ) -> KubernetesManifest: # TODO: typing.Self for Python 3.11+ - manifest: dict[str, JsonType] = json.loads(content) - return cls(manifest) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index d139b14d3..635e657c1 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -26,9 +26,8 @@ KubernetesApp, KubernetesAppValues, ) -from kpops.components.base_components.models.resource import Resource from kpops.config import get_config -from kpops.manifestors.helm_app_manifestor import HelmAppManifestor +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -143,7 +142,7 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest(self) -> Resource: + def manifest(self) -> list[KubernetesManifest]: return self.helm.template( self.helm_release_name, self.helm_chart, @@ -153,9 +152,8 @@ def manifest(self) -> Resource: ) @override - def manifest_deploy(self) -> Resource: - helm_app_manifestor = HelmAppManifestor() - return helm_app_manifestor.generate_manifest(self) + def manifest_deploy(self) -> list[KubernetesManifest]: + return self.manifest() @property def deploy_flags(self) -> HelmUpgradeInstallFlags: diff --git a/kpops/components/base_components/models/resource.py b/kpops/components/base_components/models/resource.py deleted file mode 100644 index e6867081d..000000000 --- a/kpops/components/base_components/models/resource.py +++ /dev/null @@ -1,5 +0,0 @@ -from collections.abc import Mapping -from typing import Any, TypeAlias - -# representation of final resource for component, e.g. a list of Kubernetes manifests -Resource: TypeAlias = list[Mapping[str, Any]] diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 6e309ab4f..f6fcad17a 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -13,7 +13,6 @@ FromTopic, InputTopicTypes, ) -from kpops.components.base_components.models.resource import Resource from kpops.components.base_components.models.to_section import ( ToSection, ) @@ -22,6 +21,7 @@ OutputTopicTypes, TopicConfig, ) +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr @@ -229,23 +229,23 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest(self) -> Resource: + def manifest(self) -> list[KubernetesManifest]: """Render final component resources, e.g. Kubernetes manifests.""" return [] - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: """Render final component resources for deployment, e.g. Kubernetes manifests.""" return [] - def manifest_destroy(self) -> Resource: + def manifest_destroy(self) -> list[KubernetesManifest]: """Render final component resources for destroy, e.g. Kubernetes manifests.""" return [] - def manifest_reset(self) -> Resource: + def manifest_reset(self) -> list[KubernetesManifest]: """Render final component resources for reset, e.g. Kubernetes manifests.""" return [] - def manifest_clean(self) -> Resource: + def manifest_clean(self) -> list[KubernetesManifest]: """Render final component resources for clean, e.g. Kubernetes manifests.""" return [] diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 393e27fe3..a3bbe97a7 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -8,7 +8,6 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner -from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.base import ( @@ -19,7 +18,8 @@ ) from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML -from kpops.manifestors.streams_app_manifestor import StreamsAppCleanerManifestor +from kpops.manifests.argo import ArgoHook +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -48,6 +48,15 @@ async def clean(self, dry_run: bool) -> None: if self.values.stateful_set and self.values.persistence.enabled: await self.clean_pvcs(dry_run) + @override + def manifest(self) -> list[KubernetesManifest]: + resources = super().manifest() + operation_mode = get_config().operation_mode + if operation_mode is OperationMode.ARGO: + # add Argo PostDelete hook + ArgoHook.POST_DELETE.enrich(resources[0]) + return resources + async def clean_pvcs(self, dry_run: bool) -> None: app_full_name = super(HelmApp, self).full_name pvc_handler = PVCHandler(app_full_name, self.namespace) @@ -158,17 +167,14 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> Resource: - manifest = super().manifest_deploy() + def manifest_deploy(self) -> list[KubernetesManifest]: + manifests = super().manifest_deploy() operation_mode = get_config().operation_mode - if operation_mode is OperationMode.ARGO: - clean_manifestor = StreamsAppCleanerManifestor() - clean_manifest = clean_manifestor.generate_manifest(self._cleaner) - manifest.extend(clean_manifest) + manifests.extend(self._cleaner.manifest()) - return manifest + return manifests @override - def manifest_clean(self) -> Resource: + def manifest_clean(self) -> list[KubernetesManifest]: return [] diff --git a/kpops/manifestors/helm_app_manifestor.py b/kpops/manifestors/helm_app_manifestor.py deleted file mode 100644 index 747e03aca..000000000 --- a/kpops/manifestors/helm_app_manifestor.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from kpops.api.operation import OperationMode -from kpops.manifestors.manifestor import Manifestor - -if TYPE_CHECKING: - from kpops.components.base_components import HelmApp - from kpops.components.base_components.models.resource import Resource - - -class HelmAppManifestor(Manifestor): - """Manifestor for the HelmApp component.""" - - def generate_annotations(self) -> dict[str, str]: - """Generate annotations for HelmApp based on operation mode.""" - match self.operation_mode: - case OperationMode.ARGO: - return {"argocd.argoproj.io/sync-wave": "1"} - case _: - return {} - - def generate_manifest(self, helm_app: HelmApp) -> Resource: - """Generate the Helm manifest for HelmApp.""" - values = helm_app.to_helm_values() - annotations = self.generate_annotations() - if annotations: - values["annotations"] = annotations - - return helm_app.helm.template( - helm_app.helm_release_name, - helm_app.helm_chart, - helm_app.namespace, - values, - helm_app.template_flags, - ) diff --git a/kpops/manifestors/manifestor.py b/kpops/manifestors/manifestor.py deleted file mode 100644 index 64e798f47..000000000 --- a/kpops/manifestors/manifestor.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -from kpops.config import get_config - -if TYPE_CHECKING: - from kpops.components.base_components import HelmApp - from kpops.components.base_components.models.resource import Resource - - -class Manifestor(ABC): - """Base class for generating manifests for different components.""" - - def __init__(self): - self.operation_mode = get_config().operation_mode - - @abstractmethod - def generate_annotations(self) -> dict[str, str]: - """Generate the annotations for the component based on the operation mode.""" - ... - - @abstractmethod - def generate_manifest(self, helm_app: HelmApp) -> Resource: - """Generate the Helm manifest for the component.""" - ... diff --git a/kpops/manifestors/streams_app_manifestor.py b/kpops/manifestors/streams_app_manifestor.py deleted file mode 100644 index eec1e1543..000000000 --- a/kpops/manifestors/streams_app_manifestor.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from typing_extensions import override - -from kpops.api.operation import OperationMode -from kpops.components.base_components.models.resource import Resource -from kpops.manifestors.helm_app_manifestor import HelmAppManifestor - -if TYPE_CHECKING: - from kpops.components.streams_bootstrap.streams.streams_app import StreamsAppCleaner - - -class StreamsAppCleanerManifestor(HelmAppManifestor): - """Manifestor for the StreamsApp cleaner component.""" - - @override - def generate_annotations(self) -> dict[str, str]: - """Generate annotations for StreamsApp based on operation mode.""" - match self.operation_mode: - case OperationMode.ARGO: - return {"argocd.argoproj.io/hook": "PostDelete"} - case _: - return {} - - @override - def generate_manifest(self, cleaner: StreamsAppCleaner) -> Resource: - """Generate the Helm manifest for StreamsApp.""" - values = cleaner.to_helm_values() - annotations = self.generate_annotations() - if annotations: - values["annotations"] = annotations - - return cleaner.helm.template( - cleaner.helm_release_name, - cleaner.helm_chart, - cleaner.namespace, - values, - cleaner.template_flags, - ) diff --git a/kpops/manifestors/__init__.py b/kpops/manifests/__init__.py similarity index 100% rename from kpops/manifestors/__init__.py rename to kpops/manifests/__init__.py diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py new file mode 100644 index 000000000..a3e7f92ca --- /dev/null +++ b/kpops/manifests/argo.py @@ -0,0 +1,16 @@ +from enum import Enum + +from kpops.manifests.kubernetes import KubernetesManifest + + +class ArgoHook(str, Enum): + POST_DELETE = "PostDelete" + + @property + def key(self) -> str: + return "argocd.argoproj.io/hook" + + def enrich(self, manifest: KubernetesManifest) -> None: + if manifest.metadata.annotations is None: + manifest.metadata.annotations = {} + manifest.metadata.annotations[self.key] = self.value diff --git a/kpops/manifests/kubernetes.py b/kpops/manifests/kubernetes.py new file mode 100644 index 000000000..43eb2f588 --- /dev/null +++ b/kpops/manifests/kubernetes.py @@ -0,0 +1,15 @@ +import pydantic + + +# TODO: use an existing library for Kubernetes models, e.g. lightkube (dataclass-based) or kubernetes-dynamic (Pydantic-based) +class ObjectMeta(pydantic.BaseModel): + annotations: dict[str, str] | None = None + labels: dict[str, str] | None = None + # ... + + +class KubernetesManifest(pydantic.BaseModel): + group: str + version: str + kind: str + metadata: ObjectMeta From 834cd14b2934f54c2e3006b65ed73ca8616be137 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 28 Nov 2024 20:22:06 +0100 Subject: [PATCH 04/47] Replace `manifest` method with `manifest_deploy` --- kpops/api/__init__.py | 42 ++----------------- kpops/cli/main.py | 30 +------------ kpops/component_handlers/helm_wrapper/helm.py | 1 - .../component_handlers/helm_wrapper/model.py | 2 +- kpops/components/base_components/helm_app.py | 6 +-- .../base_components/pipeline_component.py | 12 ++---- .../streams_bootstrap/streams/streams_app.py | 6 +-- tests/pipeline/test_manifest.py | 2 +- 8 files changed, 14 insertions(+), 87 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index b47491318..c5e5e9d66 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -17,6 +17,7 @@ from kpops.component_handlers.topic.handler import TopicHandler from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper from kpops.config import KpopsConfig +from kpops.manifests.kubernetes import KubernetesManifest from kpops.pipeline import ( Pipeline, PipelineGenerator, @@ -24,7 +25,6 @@ from kpops.utils.cli_commands import init_project if TYPE_CHECKING: - from kpops.components.base_components.models.resource import Resource from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.config import KpopsConfig @@ -71,43 +71,7 @@ def generate( return pipeline -def manifest( - pipeline_path: Path, - dotenv: list[Path] | None = None, - config: Path = Path(), - steps: set[str] | None = None, - filter_type: FilterType = FilterType.INCLUDE, - environment: str | None = None, - verbose: bool = False, -) -> list[Resource]: - """Generate pipeline, return final resource representations for each step. - - :param pipeline_path: Path to pipeline definition yaml file. - :param dotenv: Paths to dotenv files. - :param config: Path to the dir containing config.yaml files. - :param steps: Set of steps (components) to apply the command on. - :param filter_type: Whether `steps` should include/exclude the steps. - :param environment: The environment to generate and deploy the pipeline to. - :param verbose: Enable verbose printing. - :return: Resources. - """ - pipeline = generate( - pipeline_path=pipeline_path, - dotenv=dotenv, - config=config, - steps=steps, - filter_type=filter_type, - environment=environment, - verbose=verbose, - ) - resources: list[Resource] = [] - for component in pipeline.components: - resource = component.manifest() - resources.append(resource) - return resources - - -def manifest_deployment( +def manifest_deploy( pipeline_path: Path, dotenv: list[Path] | None = None, config: Path = Path(), @@ -116,7 +80,7 @@ def manifest_deployment( environment: str | None = None, verbose: bool = True, operation_mode: OperationMode = OperationMode.MANIFEST, -) -> Iterator[Resource]: +) -> Iterator[list[KubernetesManifest]]: pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 92000169c..bc77a8cba 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -187,34 +187,6 @@ def generate( print_yaml(pipeline.to_yaml()) -@app.command( - short_help="Render final resource representation", - help="In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.", -) -def manifest( - pipeline_paths: list[Path] = PIPELINE_PATHS_ARG, - dotenv: list[Path] | None = DOTENV_PATH_OPTION, - config: Path = CONFIG_PATH_OPTION, - steps: str | None = PIPELINE_STEPS, - filter_type: FilterType = FILTER_TYPE, - environment: str | None = ENVIRONMENT, - verbose: bool = VERBOSE_OPTION, -): - for pipeline_file_path in collect_pipeline_paths(pipeline_paths): - resources = kpops.manifest( - pipeline_path=pipeline_file_path, - dotenv=dotenv, - config=config, - steps=parse_steps(steps), - filter_type=filter_type, - environment=environment, - verbose=verbose, - ) - for resource in resources: - for rendered_manifest in resource: - print_yaml(rendered_manifest) - - @app.command(help="Deploy pipeline steps") def deploy( pipeline_paths: list[Path] = PIPELINE_PATHS_ARG, @@ -244,7 +216,7 @@ def deploy( ) case _: for pipeline_file_path in collect_pipeline_paths(pipeline_paths): - resources = kpops.manifest_deployment( + resources = kpops.manifest_deploy( pipeline_file_path, dotenv, config, diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index e0634ee4e..f0dbcd379 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -20,7 +20,6 @@ RepoAuthFlags, Version, ) -from kpops.component_handlers.kubernetes.model import KubernetesManifest if TYPE_CHECKING: from collections.abc import Iterable, Iterator diff --git a/kpops/component_handlers/helm_wrapper/model.py b/kpops/component_handlers/helm_wrapper/model.py index b81328e46..d23f2c299 100644 --- a/kpops/component_handlers/helm_wrapper/model.py +++ b/kpops/component_handlers/helm_wrapper/model.py @@ -6,7 +6,7 @@ from typing_extensions import override from kpops.component_handlers.helm_wrapper.exception import ParseError -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 635e657c1..baafb70fb 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -142,7 +142,7 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> list[KubernetesManifest]: return self.helm.template( self.helm_release_name, self.helm_chart, @@ -151,10 +151,6 @@ def manifest(self) -> list[KubernetesManifest]: self.template_flags, ) - @override - def manifest_deploy(self) -> list[KubernetesManifest]: - return self.manifest() - @property def deploy_flags(self) -> HelmUpgradeInstallFlags: """Return flags for Helm upgrade install command.""" diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index f6fcad17a..96a68f50f 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -229,24 +229,20 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest(self) -> list[KubernetesManifest]: - """Render final component resources, e.g. Kubernetes manifests.""" - return [] - def manifest_deploy(self) -> list[KubernetesManifest]: - """Render final component resources for deployment, e.g. Kubernetes manifests.""" + """Render Kubernetes manifests for deploy.""" return [] def manifest_destroy(self) -> list[KubernetesManifest]: - """Render final component resources for destroy, e.g. Kubernetes manifests.""" + """Render Kubernetes manifests resources for destroy.""" return [] def manifest_reset(self) -> list[KubernetesManifest]: - """Render final component resources for reset, e.g. Kubernetes manifests.""" + """Render Kubernetes manifests resources for reset.""" return [] def manifest_clean(self) -> list[KubernetesManifest]: - """Render final component resources for clean, e.g. Kubernetes manifests.""" + """Render Kubernetes manifests resources for clean.""" return [] async def deploy(self, dry_run: bool) -> None: diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index a3bbe97a7..171e9aae1 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -49,8 +49,8 @@ async def clean(self, dry_run: bool) -> None: await self.clean_pvcs(dry_run) @override - def manifest(self) -> list[KubernetesManifest]: - resources = super().manifest() + def manifest_deploy(self) -> list[KubernetesManifest]: + resources = super().manifest_deploy() operation_mode = get_config().operation_mode if operation_mode is OperationMode.ARGO: # add Argo PostDelete hook @@ -171,7 +171,7 @@ def manifest_deploy(self) -> list[KubernetesManifest]: manifests = super().manifest_deploy() operation_mode = get_config().operation_mode if operation_mode is OperationMode.ARGO: - manifests.extend(self._cleaner.manifest()) + manifests.extend(self._cleaner.manifest_deploy()) return manifests diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index ada22e816..cc5bafcb5 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -117,7 +117,7 @@ def test_manifest_command(self, snapshot: Snapshot): snapshot.assert_match(result.stdout, MANIFEST_YAML) def test_python_api(self, snapshot: Snapshot): - resources = kpops.manifest( + resources = kpops.manifest_deploy( RESOURCE_PATH / "custom-config/pipeline.yaml", environment="development", ) From 3ce5c952539cd84bcc44a8d6514c57a949057cac Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 14:34:46 +0100 Subject: [PATCH 05/47] Introduce KPOps operation and manifest resources for deployment --- docs/docs/user/references/cli-commands.md | 25 --- kpops/api/__init__.py | 4 +- kpops/component_handlers/helm_wrapper/helm.py | 10 +- .../component_handlers/helm_wrapper/model.py | 2 +- .../component_handlers/helm_wrapper/utils.py | 3 +- kpops/component_handlers/kubernetes/model.py | 30 +++ kpops/component_handlers/kubernetes/utils.py | 1 + kpops/components/base_components/helm_app.py | 14 +- .../base_components/models/resource.py | 5 + .../base_components/pipeline_component.py | 10 +- .../streams_bootstrap/streams/streams_app.py | 22 +- kpops/manifests/argo.py | 26 ++- kpops/manifests/kubernetes.py | 15 -- .../test_deploy_argo_mode/manifest.yaml | 76 ++++++- .../test_deploy_manifest_mode/manifest.yaml | 46 +++- .../test_manifest/test_python_api/resources | 199 +++++++++++------- tests/pipeline/test_manifest.py | 24 ++- 17 files changed, 351 insertions(+), 161 deletions(-) create mode 100644 kpops/component_handlers/kubernetes/model.py create mode 100644 kpops/components/base_components/models/resource.py delete mode 100644 kpops/manifests/kubernetes.py diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index ce75d7b3b..a5912d27c 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -20,7 +20,6 @@ $ kpops [OPTIONS] COMMAND [ARGS]... * `destroy`: Destroy pipeline steps * `generate`: Generate enriched pipeline representation * `init`: Initialize a new KPOps project. -* `manifest`: Render final resource representation * `reset`: Reset pipeline steps * `schema`: Generate JSON schema. @@ -146,30 +145,6 @@ $ kpops init [OPTIONS] PATH * `--config-include-opt / --no-config-include-opt`: Whether to include non-required settings in the generated 'config.yaml' [default: no-config-include-opt] * `--help`: Show this message and exit. -## `kpops manifest` - -In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests. - -**Usage**: - -```console -$ kpops manifest [OPTIONS] PIPELINE_PATHS... -``` - -**Arguments**: - -* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'. [env var: KPOPS_PIPELINE_PATHS;required] - -**Options**: - -* `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one. [env var: KPOPS_DOTENV_PATH] -* `--config DIRECTORY`: Path to the dir containing config.yaml files [env var: KPOPS_CONFIG_PATH; default: .] -* `--steps TEXT`: Comma separated list of steps to apply the command on [env var: KPOPS_PIPELINE_STEPS] -* `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps [default: include] -* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development). [env var: KPOPS_ENVIRONMENT] -* `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] -* `--help`: Show this message and exit. - ## `kpops reset` Reset pipeline steps diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index c5e5e9d66..139b89aaf 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -16,8 +16,8 @@ from kpops.component_handlers.schema_handler.schema_handler import SchemaHandler from kpops.component_handlers.topic.handler import TopicHandler from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper +from kpops.components.base_components.models.resource import Resource from kpops.config import KpopsConfig -from kpops.manifests.kubernetes import KubernetesManifest from kpops.pipeline import ( Pipeline, PipelineGenerator, @@ -80,7 +80,7 @@ def manifest_deploy( environment: str | None = None, verbose: bool = True, operation_mode: OperationMode = OperationMode.MANIFEST, -) -> Iterator[list[KubernetesManifest]]: +) -> Iterator[Resource]: pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index f0dbcd379..3fd38df20 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -20,10 +20,13 @@ RepoAuthFlags, Version, ) +from kpops.component_handlers.kubernetes.model import KubernetesManifest if TYPE_CHECKING: from collections.abc import Iterable, Iterator + from kpops.components.base_components.models.resource import Resource + log = logging.getLogger("Helm") @@ -159,7 +162,7 @@ def template( namespace: str, values: dict[str, Any], flags: HelmTemplateFlags | None = None, - ) -> list[KubernetesManifest]: + ) -> Resource: """From Helm: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -189,9 +192,8 @@ def template( ] command.extend(flags.to_command()) output = self.__execute(command) - # TODO: deserialize Kubernetes model - # manifests = KubernetesManifest.from_yaml(output) - # return list(manifests) + manifests = KubernetesManifest.from_yaml(output) + return list(manifests) def get_manifest(self, release_name: str, namespace: str) -> Iterable[HelmTemplate]: command = [ diff --git a/kpops/component_handlers/helm_wrapper/model.py b/kpops/component_handlers/helm_wrapper/model.py index d23f2c299..b81328e46 100644 --- a/kpops/component_handlers/helm_wrapper/model.py +++ b/kpops/component_handlers/helm_wrapper/model.py @@ -6,7 +6,7 @@ from typing_extensions import override from kpops.component_handlers.helm_wrapper.exception import ParseError -from kpops.manifests.kubernetes import KubernetesManifest +from kpops.component_handlers.kubernetes.model import KubernetesManifest from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index aa618f6a1..42a295cf3 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -1,5 +1,4 @@ -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN -from kpops.component_handlers.kubernetes.utils import trim +from kpops.component_handlers.kubernetes.utils import K8S_LABEL_MAX_LEN, trim RELEASE_NAME_MAX_LEN = 53 diff --git a/kpops/component_handlers/kubernetes/model.py b/kpops/component_handlers/kubernetes/model.py new file mode 100644 index 000000000..5970de1bf --- /dev/null +++ b/kpops/component_handlers/kubernetes/model.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import json +from collections import UserDict +from collections.abc import Iterator + +import yaml + +from kpops.utils.types import JsonType + +K8S_LABEL_MAX_LEN = 63 + + +class KubernetesManifest(UserDict[str, JsonType]): + """Representation of a Kubernetes API object as YAML/JSON mapping.""" + + @classmethod + def from_yaml( + cls, /, content: str + ) -> Iterator[KubernetesManifest]: # TODO: typing.Self for Python 3.11+ + manifests: Iterator[dict[str, JsonType]] = yaml.load_all(content, yaml.Loader) + for manifest in manifests: + yield cls(manifest) + + @classmethod + def from_json( + cls, /, content: str + ) -> KubernetesManifest: # TODO: typing.Self for Python 3.11+ + manifest: dict[str, JsonType] = json.loads(content) + return cls(manifest) diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 4f4599e2b..9a0612c26 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -2,6 +2,7 @@ import logging log = logging.getLogger("K8sUtils") +K8S_LABEL_MAX_LEN = 63 def trim(max_len: int, name: str, suffix: str) -> str: diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index baafb70fb..40635529a 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -8,6 +8,7 @@ from pydantic import Field, model_serializer from typing_extensions import override +from kpops.api import OperationMode from kpops.component_handlers.helm_wrapper.dry_run_handler import DryRunHandler from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.helm_diff import HelmDiff @@ -21,13 +22,14 @@ create_helm_name_override, create_helm_release_name, ) -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN +from kpops.component_handlers.kubernetes.utils import K8S_LABEL_MAX_LEN from kpops.components.base_components.kubernetes_app import ( KubernetesApp, KubernetesAppValues, ) +from kpops.components.base_components.models.resource import Resource from kpops.config import get_config -from kpops.manifests.kubernetes import KubernetesManifest +from kpops.manifests.argo import ArgoSyncWave from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -142,12 +144,16 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> Resource: + values = self.to_helm_values() + if get_config().operation_mode is OperationMode.ARGO: + values = ArgoSyncWave.SYNC_WAVE.enrich(values) + return self.helm.template( self.helm_release_name, self.helm_chart, self.namespace, - self.to_helm_values(), + values, self.template_flags, ) diff --git a/kpops/components/base_components/models/resource.py b/kpops/components/base_components/models/resource.py new file mode 100644 index 000000000..e6867081d --- /dev/null +++ b/kpops/components/base_components/models/resource.py @@ -0,0 +1,5 @@ +from collections.abc import Mapping +from typing import Any, TypeAlias + +# representation of final resource for component, e.g. a list of Kubernetes manifests +Resource: TypeAlias = list[Mapping[str, Any]] diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 96a68f50f..28445fe17 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -13,6 +13,7 @@ FromTopic, InputTopicTypes, ) +from kpops.components.base_components.models.resource import Resource from kpops.components.base_components.models.to_section import ( ToSection, ) @@ -21,7 +22,6 @@ OutputTopicTypes, TopicConfig, ) -from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr @@ -229,19 +229,19 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> Resource: """Render Kubernetes manifests for deploy.""" return [] - def manifest_destroy(self) -> list[KubernetesManifest]: + def manifest_destroy(self) -> Resource: """Render Kubernetes manifests resources for destroy.""" return [] - def manifest_reset(self) -> list[KubernetesManifest]: + def manifest_reset(self) -> Resource: """Render Kubernetes manifests resources for reset.""" return [] - def manifest_clean(self) -> list[KubernetesManifest]: + def manifest_clean(self) -> Resource: """Render Kubernetes manifests resources for clean.""" return [] diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 171e9aae1..4552901b2 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -8,6 +8,7 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner +from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.base import ( @@ -19,7 +20,6 @@ from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML from kpops.manifests.argo import ArgoHook -from kpops.manifests.kubernetes import KubernetesManifest from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -49,13 +49,18 @@ async def clean(self, dry_run: bool) -> None: await self.clean_pvcs(dry_run) @override - def manifest_deploy(self) -> list[KubernetesManifest]: - resources = super().manifest_deploy() + def manifest_deploy(self) -> Resource: operation_mode = get_config().operation_mode + values = self.to_helm_values() if operation_mode is OperationMode.ARGO: - # add Argo PostDelete hook - ArgoHook.POST_DELETE.enrich(resources[0]) - return resources + values = ArgoHook.POST_DELETE.enrich(values) + return self.helm.template( + self.helm_release_name, + self.helm_chart, + self.namespace, + values, + self.template_flags, + ) async def clean_pvcs(self, dry_run: bool) -> None: app_full_name = super(HelmApp, self).full_name @@ -167,14 +172,15 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> Resource: manifests = super().manifest_deploy() operation_mode = get_config().operation_mode + if operation_mode is OperationMode.ARGO: manifests.extend(self._cleaner.manifest_deploy()) return manifests @override - def manifest_clean(self) -> list[KubernetesManifest]: + def manifest_clean(self) -> Resource: return [] diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index a3e7f92ca..e3002004d 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -1,16 +1,30 @@ +import enum from enum import Enum +from typing import Any -from kpops.manifests.kubernetes import KubernetesManifest + +class ArgoEnricher(enum.Enum): + @property + def key(self) -> str: + return NotImplemented + + def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: + annotations = helm_values.setdefault("annotations", {}) + annotations[self.key] = self.value + return helm_values -class ArgoHook(str, Enum): +class ArgoHook(ArgoEnricher, str, Enum): POST_DELETE = "PostDelete" @property def key(self) -> str: return "argocd.argoproj.io/hook" - def enrich(self, manifest: KubernetesManifest) -> None: - if manifest.metadata.annotations is None: - manifest.metadata.annotations = {} - manifest.metadata.annotations[self.key] = self.value + +class ArgoSyncWave(ArgoEnricher, str, Enum): + SYNC_WAVE = "1" + + @property + def key(self) -> str: + return "argocd.argoproj.io/sync-wave" diff --git a/kpops/manifests/kubernetes.py b/kpops/manifests/kubernetes.py deleted file mode 100644 index 43eb2f588..000000000 --- a/kpops/manifests/kubernetes.py +++ /dev/null @@ -1,15 +0,0 @@ -import pydantic - - -# TODO: use an existing library for Kubernetes models, e.g. lightkube (dataclass-based) or kubernetes-dynamic (Pydantic-based) -class ObjectMeta(pydantic.BaseModel): - annotations: dict[str, str] | None = None - labels: dict[str, str] | None = None - # ... - - -class KubernetesManifest(pydantic.BaseModel): - group: str - version: str - kind: str - metadata: ObjectMeta diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index c3f2c7bd7..9a2b2e6e4 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -1,3 +1,17 @@ +--- +apiVersion: v1 +data: + log4j2.xml: "\n\n \n\ + \ \n \n \n\ + \ \n \n \n \n \n \n\n" +kind: ConfigMap +metadata: + annotations: + argocd.argoproj.io/sync-wave: '1' + name: resources-streams-bootstrap-my-producer-app + --- apiVersion: batch/v1beta1 kind: CronJob @@ -6,7 +20,7 @@ metadata: argocd.argoproj.io/sync-wave: '1' labels: app: resources-streams-bootstrap-my-producer-app - chart: producer-app-3.0.1 + chart: producer-app-3.0.3 release: resources-streams-bootstrap-my-producer-app name: resources-streams-bootstrap-my-producer-app spec: @@ -47,11 +61,33 @@ spec: requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml restartPolicy: OnFailure + volumes: + - configMap: + name: resources-streams-bootstrap-my-producer-app + name: config schedule: 30 3/8 * * * successfulJobsHistoryLimit: 1 suspend: false +--- +apiVersion: v1 +data: + log4j2.xml: "\n\n \n\ + \ \n \n \n\ + \ \n \n \n \n \n \n\n" +kind: ConfigMap +metadata: + annotations: + argocd.argoproj.io/sync-wave: '1' + name: resources-streams-bootstrap-my-streams-app + --- apiVersion: v1 data: @@ -60,9 +96,11 @@ data: \ - pattern: \".*\"\n" kind: ConfigMap metadata: + annotations: + argocd.argoproj.io/sync-wave: '1' labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.1 + chart: streams-app-3.0.3 heritage: Helm release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app-jmx-configmap @@ -76,7 +114,7 @@ metadata: consumerGroup: my-streams-app-id labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.1 + chart: streams-app-3.0.3 release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app spec: @@ -88,6 +126,7 @@ spec: template: metadata: annotations: + checksum/config: b59b81ef6694f25210cf22fd03d6b6b51582d516bd219ac791985e917ba3f102 prometheus.io/port: '5556' prometheus.io/scrape: 'true' labels: @@ -142,6 +181,10 @@ spec: requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml - command: - java - -XX:+UnlockExperimentalVMOptions @@ -171,6 +214,23 @@ spec: - configMap: name: resources-streams-bootstrap-my-streams-app-jmx-configmap name: jmx-config + - configMap: + name: resources-streams-bootstrap-my-streams-app + name: config + +--- +apiVersion: v1 +data: + log4j2.xml: "\n\n \n\ + \ \n \n \n\ + \ \n \n \n \n \n \n\n" +kind: ConfigMap +metadata: + annotations: + argocd.argoproj.io/hook: PostDelete + name: resources-streams-bootstrap-my-streams-app --- apiVersion: batch/v1 @@ -180,7 +240,7 @@ metadata: argocd.argoproj.io/hook: PostDelete labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-cleanup-job-3.0.1 + chart: streams-app-cleanup-job-3.0.3 release: resources-streams-bootstrap-my-streams-app-clean name: resources-streams-bootstrap-my-streams-app spec: @@ -233,6 +293,14 @@ spec: requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml restartPolicy: OnFailure + volumes: + - configMap: + name: resources-streams-bootstrap-my-streams-app + name: config ttlSecondsAfterFinished: 30 diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml index 7df64af03..ddfd4e2f9 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -1,10 +1,22 @@ +--- +apiVersion: v1 +data: + log4j2.xml: "\n\n \n\ + \ \n \n \n\ + \ \n \n \n \n \n \n\n" +kind: ConfigMap +metadata: + name: resources-streams-bootstrap-my-producer-app + --- apiVersion: batch/v1beta1 kind: CronJob metadata: labels: app: resources-streams-bootstrap-my-producer-app - chart: producer-app-3.0.1 + chart: producer-app-3.0.3 release: resources-streams-bootstrap-my-producer-app name: resources-streams-bootstrap-my-producer-app spec: @@ -45,11 +57,31 @@ spec: requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml restartPolicy: OnFailure + volumes: + - configMap: + name: resources-streams-bootstrap-my-producer-app + name: config schedule: 30 3/8 * * * successfulJobsHistoryLimit: 1 suspend: false +--- +apiVersion: v1 +data: + log4j2.xml: "\n\n \n\ + \ \n \n \n\ + \ \n \n \n \n \n \n\n" +kind: ConfigMap +metadata: + name: resources-streams-bootstrap-my-streams-app + --- apiVersion: v1 data: @@ -60,7 +92,7 @@ kind: ConfigMap metadata: labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.1 + chart: streams-app-3.0.3 heritage: Helm release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app-jmx-configmap @@ -73,7 +105,7 @@ metadata: consumerGroup: my-streams-app-id labels: app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.1 + chart: streams-app-3.0.3 release: resources-streams-bootstrap-my-streams-app name: resources-streams-bootstrap-my-streams-app spec: @@ -85,6 +117,7 @@ spec: template: metadata: annotations: + checksum/config: 7fa55f62bffb69b723535187aa4e5ee14a91be308d56f5a8ca49c3159bf80aef prometheus.io/port: '5556' prometheus.io/scrape: 'true' labels: @@ -139,6 +172,10 @@ spec: requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml - command: - java - -XX:+UnlockExperimentalVMOptions @@ -168,4 +205,7 @@ spec: - configMap: name: resources-streams-bootstrap-my-streams-app-jmx-configmap name: jmx-config + - configMap: + name: resources-streams-bootstrap-my-streams-app + name: config diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index eca5af9eb..0ffee00bf 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -1,48 +1,90 @@ - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: - apiVersion: batch/v1 - kind: Job + apiVersion: v1 + data: + log4j2.xml: "\n\n \ + \ \n \n\ + \ \n \n \n \n \ + \ \n \n \ + \ \n \n\n" + kind: ConfigMap + metadata: + name: resources-streams-bootstrap-my-producer-app +- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest + data: + apiVersion: batch/v1beta1 + kind: CronJob metadata: labels: - app: resources-custom-config-app1 - chart: producer-app-2.9.0 - release: resources-custom-config-app1 - name: resources-custom-config-app1 + app: resources-streams-bootstrap-my-producer-app + chart: producer-app-3.0.3 + release: resources-streams-bootstrap-my-producer-app + name: resources-streams-bootstrap-my-producer-app spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-custom-config-app1 - release: resources-custom-config-app1 + concurrencyPolicy: Replace + failedJobsHistoryLimit: 1 + jobTemplate: spec: - affinity: null - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BROKERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_DEBUG - value: 'false' - - name: APP_OUTPUT_TOPIC - value: resources-custom-config-app1 - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: producerApp:latest - imagePullPolicy: Always - name: resources-custom-config-app1 - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 2G - restartPolicy: OnFailure + backoffLimit: 6 + template: + metadata: + labels: + app: resources-streams-bootstrap-my-producer-app + release: resources-streams-bootstrap-my-producer-app + spec: + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: APP_FAKE_ARG + value: fake-arg-value + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-streams-bootstrap-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml + restartPolicy: OnFailure + volumes: + - configMap: + name: resources-streams-bootstrap-my-producer-app + name: config + schedule: 30 3/8 * * * + successfulJobsHistoryLimit: 1 + suspend: false --- +- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest + data: + apiVersion: v1 + data: + log4j2.xml: "\n\n \ + \ \n \n\ + \ \n \n \n \n \ + \ \n \n \ + \ \n \n\n" + kind: ConfigMap + metadata: + name: resources-streams-bootstrap-my-streams-app - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: apiVersion: v1 @@ -53,78 +95,77 @@ kind: ConfigMap metadata: labels: - app: resources-custom-config-app2 - chart: streams-app-2.9.0 + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.3 heritage: Helm - release: resources-custom-config-app2 - name: resources-custom-config-app2-jmx-configmap + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app-jmx-configmap - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: apiVersion: apps/v1 kind: Deployment metadata: + annotations: + consumerGroup: my-streams-app-id labels: - app: resources-custom-config-app2 - chart: streams-app-2.9.0 - pipeline: resources-custom-config - release: resources-custom-config-app2 - name: resources-custom-config-app2 + app: resources-streams-bootstrap-my-streams-app + chart: streams-app-3.0.3 + release: resources-streams-bootstrap-my-streams-app + name: resources-streams-bootstrap-my-streams-app spec: replicas: 1 selector: matchLabels: - app: resources-custom-config-app2 - release: resources-custom-config-app2 + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app template: metadata: annotations: + checksum/config: 7fa55f62bffb69b723535187aa4e5ee14a91be308d56f5a8ca49c3159bf80aef prometheus.io/port: '5556' prometheus.io/scrape: 'true' labels: - app: resources-custom-config-app2 - pipeline: resources-custom-config - release: resources-custom-config-app2 + app: resources-streams-bootstrap-my-streams-app + release: resources-streams-bootstrap-my-streams-app spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - resources-custom-config-app2 - topologyKey: kubernetes.io/hostname - weight: 1 containers: - env: - name: ENV_PREFIX value: APP_ - - name: STREAMS_LARGE_MESSAGE_ID_GENERATOR + - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR value: com.bakdata.kafka.MurmurHashIdGenerator - name: KAFKA_JMX_PORT value: '5555' - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - - name: APP_BROKERS + - name: APP_BOOTSTRAP_SERVERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL value: http://localhost:8081/ - - name: APP_DEBUG - value: 'false' - name: APP_INPUT_TOPICS - value: resources-custom-config-app1 + value: my-input-topic + - name: APP_INPUT_PATTERN + value: my-input-pattern - name: APP_OUTPUT_TOPIC - value: resources-custom-config-app2 + value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-custom-config-app2-error + value: resources-streams-bootstrap-my-streams-app-error + - name: APP_LABELED_OUTPUT_TOPICS + value: my-output-topic-label=my-labeled-topic-output, + - name: APP_LABELED_INPUT_TOPICS + value: my-input-topic-label=my-labeled-input-topic, + - name: APP_LABELED_INPUT_PATTERNS + value: my-input-topic-labeled-pattern=my-labeled-input-pattern, + - name: APP_APPLICATION_ID + value: my-streams-app-id + - name: APP_CONVERT_XML + value: 'true' - name: JAVA_TOOL_OPTIONS value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' - image: some-image:latest + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-custom-config-app2 + name: resources-streams-bootstrap-my-streams-app ports: - containerPort: 5555 name: jmx @@ -135,6 +176,10 @@ requests: cpu: 200m memory: 300Mi + volumeMounts: + - mountPath: app/resources/log4j2.xml + name: config + subPath: log4j2.xml - command: - java - -XX:+UnlockExperimentalVMOptions @@ -159,7 +204,11 @@ volumeMounts: - mountPath: /etc/jmx-streams-app name: jmx-config + terminationGracePeriodSeconds: 300 volumes: - configMap: - name: resources-custom-config-app2-jmx-configmap + name: resources-streams-bootstrap-my-streams-app-jmx-configmap name: jmx-config + - configMap: + name: resources-streams-bootstrap-my-streams-app + name: config diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index cc5bafcb5..49799d214 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -1,3 +1,4 @@ +from collections.abc import Iterator from pathlib import Path from unittest.mock import ANY, MagicMock @@ -42,10 +43,12 @@ def test_default_config(self, mock_execute: MagicMock): result = runner.invoke( app, [ - "manifest", + "deploy", str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--environment", "development", + "--operation-mode", + "manifest", ], catch_exceptions=False, ) @@ -72,12 +75,14 @@ def test_custom_config(self, mock_execute: MagicMock): result = runner.invoke( app, [ - "manifest", + "deploy", str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--config", str(RESOURCE_PATH / "custom-config"), "--environment", "development", + "--operation-mode", + "manifest", ], catch_exceptions=False, ) @@ -106,10 +111,12 @@ def test_manifest_command(self, snapshot: Snapshot): result = runner.invoke( app, [ - "manifest", + "deploy", str(RESOURCE_PATH / "custom-config/pipeline.yaml"), "--environment", "development", + "--operation-mode", + "manifest", ], catch_exceptions=False, ) @@ -117,11 +124,12 @@ def test_manifest_command(self, snapshot: Snapshot): snapshot.assert_match(result.stdout, MANIFEST_YAML) def test_python_api(self, snapshot: Snapshot): - resources = kpops.manifest_deploy( - RESOURCE_PATH / "custom-config/pipeline.yaml", + generator = kpops.manifest_deploy( + RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML, environment="development", ) - assert isinstance(resources, list) + assert isinstance(generator, Iterator) + resources = list(generator) assert len(resources) == 2 snapshot.assert_match(yaml.dump_all(resources), "resources") @@ -129,8 +137,10 @@ def test_streams_bootstrap(self, snapshot: Snapshot): result = runner.invoke( app, [ - "manifest", + "deploy", str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), + "--operation-mode", + "manifest", ], catch_exceptions=False, ) From 56f72a102457753d7cdde05b60fd1bef9cfb9b10 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 14:53:11 +0100 Subject: [PATCH 06/47] Introduce KPOps operation and manifest resources for deployment --- .../component_handlers/helm_wrapper/utils.py | 3 +- kpops/component_handlers/kubernetes/utils.py | 1 - kpops/components/base_components/helm_app.py | 2 +- .../producer/producer_app.py | 27 ++ .../resources/manifest-pipeline/defaults.yaml | 25 ++ .../resources/manifest-pipeline/pipeline.yaml | 40 +++ .../test_deploy_argo_mode/manifest.yaml | 278 ++++++------------ .../test_deploy_manifest_mode/manifest.yaml | 194 +++--------- tests/pipeline/test_manifest.py | 4 +- 9 files changed, 240 insertions(+), 334 deletions(-) create mode 100644 tests/pipeline/resources/manifest-pipeline/defaults.yaml create mode 100644 tests/pipeline/resources/manifest-pipeline/pipeline.yaml diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index 42a295cf3..aa618f6a1 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -1,4 +1,5 @@ -from kpops.component_handlers.kubernetes.utils import K8S_LABEL_MAX_LEN, trim +from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN +from kpops.component_handlers.kubernetes.utils import trim RELEASE_NAME_MAX_LEN = 53 diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py index 9a0612c26..4f4599e2b 100644 --- a/kpops/component_handlers/kubernetes/utils.py +++ b/kpops/component_handlers/kubernetes/utils.py @@ -2,7 +2,6 @@ import logging log = logging.getLogger("K8sUtils") -K8S_LABEL_MAX_LEN = 63 def trim(max_len: int, name: str, suffix: str) -> str: diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 40635529a..f64ba1981 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -22,7 +22,7 @@ create_helm_name_override, create_helm_release_name, ) -from kpops.component_handlers.kubernetes.utils import K8S_LABEL_MAX_LEN +from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.kubernetes_app import ( KubernetesApp, KubernetesAppValues, diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 21cda9ebd..89ab96112 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -4,7 +4,9 @@ from pydantic import Field, ValidationError, computed_field from typing_extensions import override +from kpops.api import OperationMode from kpops.components.base_components.kafka_app import KafkaAppCleaner +from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType from kpops.components.common.topic import ( KafkaTopic, @@ -15,7 +17,9 @@ StreamsBootstrap, ) from kpops.components.streams_bootstrap.producer.model import ProducerAppValues +from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML +from kpops.manifests.argo import ArgoHook from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerApp") @@ -31,6 +35,20 @@ def helm_chart(self) -> str: f"{self.repo_config.repository_name}/{AppType.CLEANUP_PRODUCER_APP.value}" ) + @override + def manifest_deploy(self) -> Resource: + operation_mode = get_config().operation_mode + values = self.values.model_dump() + if operation_mode is OperationMode.ARGO: + values = ArgoHook.POST_DELETE.enrich(values) + return self.helm.template( + self.helm_release_name, + self.helm_chart, + self.namespace, + values, + self.template_flags, + ) + class ProducerApp(StreamsBootstrap): """Producer component. @@ -123,3 +141,12 @@ async def clean(self, dry_run: bool) -> None: """Destroy and clean.""" await super().clean(dry_run) await self._cleaner.clean(dry_run) + + def manifest_deploy(self) -> Resource: + manifests = super().manifest_deploy() + operation_mode = get_config().operation_mode + + if operation_mode is OperationMode.ARGO: + manifests.extend(self._cleaner.manifest_deploy()) + + return manifests diff --git a/tests/pipeline/resources/manifest-pipeline/defaults.yaml b/tests/pipeline/resources/manifest-pipeline/defaults.yaml new file mode 100644 index 000000000..4f1c2ea2a --- /dev/null +++ b/tests/pipeline/resources/manifest-pipeline/defaults.yaml @@ -0,0 +1,25 @@ +streams-bootstrap: + version: "3.0.3" + values: + kafka: + bootstrapServers: ${config.kafka_brokers} + schemaRegistryUrl: ${config.schema_registry.url} + +producer-app: {} # inherits from streams-bootstrap + +streams-app: # inherits from streams-bootstrap + values: + prometheus: + jmx: + enabled: false + kafka: + config: + large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator + to: + topics: + ${error_topic_name}: + type: error + value_schema: com.bakdata.kafka.DeadLetter + partitions_count: 1 + configs: + cleanup.policy: compact,delete diff --git a/tests/pipeline/resources/manifest-pipeline/pipeline.yaml b/tests/pipeline/resources/manifest-pipeline/pipeline.yaml new file mode 100644 index 000000000..cf5f7342c --- /dev/null +++ b/tests/pipeline/resources/manifest-pipeline/pipeline.yaml @@ -0,0 +1,40 @@ +- type: my-producer-app + values: + image: "my-registry/my-producer-image" + imageTag: "1.0.0" + + to: + topics: + my-producer-app-output-topic: + type: output + my-labeled-producer-app-topic-output: + label: my-producer-app-output-topic-label + + +- type: my-streams-app + values: + image: "my-registry/my-streams-app-image" + imageTag: "1.0.0" + kafka: + applicationId: "my-streams-app-id" + + from: + topics: + my-input-topic: + type: input + my-labeled-input-topic: + label: my-input-topic-label + my-input-pattern: + type: pattern + my-labeled-input-pattern: + type: pattern + label: my-input-topic-labeled-pattern + + to: + topics: + my-output-topic: + type: output + my-error-topic: + type: error + my-labeled-topic-output: + label: my-output-topic-label diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index 9a2b2e6e4..abf6c7c63 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -1,109 +1,95 @@ --- -apiVersion: v1 -data: - log4j2.xml: "\n\n \n\ - \ \n \n \n\ - \ \n \n \n \n \n \n\n" -kind: ConfigMap -metadata: - annotations: - argocd.argoproj.io/sync-wave: '1' - name: resources-streams-bootstrap-my-producer-app - ---- -apiVersion: batch/v1beta1 -kind: CronJob +apiVersion: batch/v1 +kind: Job metadata: annotations: argocd.argoproj.io/sync-wave: '1' labels: - app: resources-streams-bootstrap-my-producer-app + app: resources-manifest-pipeline-my-producer-app chart: producer-app-3.0.3 - release: resources-streams-bootstrap-my-producer-app - name: resources-streams-bootstrap-my-producer-app + release: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app spec: - concurrencyPolicy: Replace - failedJobsHistoryLimit: 1 - jobTemplate: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + release: resources-manifest-pipeline-my-producer-app spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-streams-bootstrap-my-producer-app - release: resources-streams-bootstrap-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: APP_FAKE_ARG - value: fake-arg-value - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - restartPolicy: OnFailure - volumes: - - configMap: - name: resources-streams-bootstrap-my-producer-app - name: config - schedule: 30 3/8 * * * - successfulJobsHistoryLimit: 1 - suspend: false - ---- -apiVersion: v1 -data: - log4j2.xml: "\n\n \n\ - \ \n \n \n\ - \ \n \n \n \n \n \n\n" -kind: ConfigMap -metadata: - annotations: - argocd.argoproj.io/sync-wave: '1' - name: resources-streams-bootstrap-my-streams-app + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure --- -apiVersion: v1 -data: - jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ - lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ - \ - pattern: \".*\"\n" -kind: ConfigMap +apiVersion: batch/v1 +kind: Job metadata: annotations: - argocd.argoproj.io/sync-wave: '1' + argocd.argoproj.io/hook: PostDelete labels: - app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.3 - heritage: Helm - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app-jmx-configmap + app: resources-manifest-pipeline-my-producer-app + chart: producer-app-cleanup-job-3.0.3 + release: resources-manifest-pipeline-my-producer-app-clean + name: resources-manifest-pipeline-my-producer-app +spec: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + release: resources-manifest-pipeline-my-producer-app-clean + spec: + containers: + - args: + - clean + env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure + ttlSecondsAfterFinished: 30 --- apiVersion: apps/v1 @@ -113,25 +99,21 @@ metadata: argocd.argoproj.io/sync-wave: '1' consumerGroup: my-streams-app-id labels: - app: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app chart: streams-app-3.0.3 - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app + release: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app spec: replicas: 1 selector: matchLabels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app template: metadata: - annotations: - checksum/config: b59b81ef6694f25210cf22fd03d6b6b51582d516bd219ac791985e917ba3f102 - prometheus.io/port: '5556' - prometheus.io/scrape: 'true' labels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app spec: containers: - env: @@ -139,8 +121,6 @@ spec: value: APP_ - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR value: com.bakdata.kafka.MurmurHashIdGenerator - - name: KAFKA_JMX_PORT - value: '5555' - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS @@ -154,7 +134,7 @@ spec: - name: APP_OUTPUT_TOPIC value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-my-streams-app-error + value: resources-manifest-pipeline-my-streams-app-error - name: APP_LABELED_OUTPUT_TOPICS value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS @@ -163,17 +143,12 @@ spec: value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - name: APP_APPLICATION_ID value: my-streams-app-id - - name: APP_CONVERT_XML - value: 'true' - name: JAVA_TOOL_OPTIONS value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-my-streams-app - ports: - - containerPort: 5555 - name: jmx + name: resources-manifest-pipeline-my-streams-app resources: limits: cpu: 500m @@ -181,56 +156,7 @@ spec: requests: cpu: 200m memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - '5556' - - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml - image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - name: prometheus-jmx-exporter - ports: - - containerPort: 5556 - resources: - limits: - cpu: 300m - memory: 2G - requests: - cpu: 100m - memory: 500Mi - volumeMounts: - - mountPath: /etc/jmx-streams-app - name: jmx-config terminationGracePeriodSeconds: 300 - volumes: - - configMap: - name: resources-streams-bootstrap-my-streams-app-jmx-configmap - name: jmx-config - - configMap: - name: resources-streams-bootstrap-my-streams-app - name: config - ---- -apiVersion: v1 -data: - log4j2.xml: "\n\n \n\ - \ \n \n \n\ - \ \n \n \n \n \n \n\n" -kind: ConfigMap -metadata: - annotations: - argocd.argoproj.io/hook: PostDelete - name: resources-streams-bootstrap-my-streams-app --- apiVersion: batch/v1 @@ -239,17 +165,17 @@ metadata: annotations: argocd.argoproj.io/hook: PostDelete labels: - app: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app chart: streams-app-cleanup-job-3.0.3 - release: resources-streams-bootstrap-my-streams-app-clean - name: resources-streams-bootstrap-my-streams-app + release: resources-manifest-pipeline-my-streams-app-clean + name: resources-manifest-pipeline-my-streams-app spec: backoffLimit: 6 template: metadata: labels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app-clean + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app-clean spec: containers: - args: @@ -270,7 +196,7 @@ spec: - name: APP_OUTPUT_TOPIC value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-my-streams-app-error + value: resources-manifest-pipeline-my-streams-app-error - name: APP_LABELED_OUTPUT_TOPICS value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS @@ -279,13 +205,11 @@ spec: value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - name: APP_APPLICATION_ID value: my-streams-app-id - - name: APP_CONVERT_XML - value: 'true' - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=85.0 ' + value: '-XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-my-streams-app + name: resources-manifest-pipeline-my-streams-app resources: limits: cpu: 500m @@ -293,14 +217,6 @@ spec: requests: cpu: 200m memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml restartPolicy: OnFailure - volumes: - - configMap: - name: resources-streams-bootstrap-my-streams-app - name: config ttlSecondsAfterFinished: 30 diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml index ddfd4e2f9..62fe9eca2 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -1,101 +1,45 @@ --- -apiVersion: v1 -data: - log4j2.xml: "\n\n \n\ - \ \n \n \n\ - \ \n \n \n \n \n \n\n" -kind: ConfigMap -metadata: - name: resources-streams-bootstrap-my-producer-app - ---- -apiVersion: batch/v1beta1 -kind: CronJob +apiVersion: batch/v1 +kind: Job metadata: labels: - app: resources-streams-bootstrap-my-producer-app + app: resources-manifest-pipeline-my-producer-app chart: producer-app-3.0.3 - release: resources-streams-bootstrap-my-producer-app - name: resources-streams-bootstrap-my-producer-app + release: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app spec: - concurrencyPolicy: Replace - failedJobsHistoryLimit: 1 - jobTemplate: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + release: resources-manifest-pipeline-my-producer-app spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-streams-bootstrap-my-producer-app - release: resources-streams-bootstrap-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: APP_FAKE_ARG - value: fake-arg-value - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - restartPolicy: OnFailure - volumes: - - configMap: - name: resources-streams-bootstrap-my-producer-app - name: config - schedule: 30 3/8 * * * - successfulJobsHistoryLimit: 1 - suspend: false - ---- -apiVersion: v1 -data: - log4j2.xml: "\n\n \n\ - \ \n \n \n\ - \ \n \n \n \n \n \n\n" -kind: ConfigMap -metadata: - name: resources-streams-bootstrap-my-streams-app - ---- -apiVersion: v1 -data: - jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ - lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ - \ - pattern: \".*\"\n" -kind: ConfigMap -metadata: - labels: - app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.3 - heritage: Helm - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app-jmx-configmap + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure --- apiVersion: apps/v1 @@ -104,25 +48,21 @@ metadata: annotations: consumerGroup: my-streams-app-id labels: - app: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app chart: streams-app-3.0.3 - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app + release: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app spec: replicas: 1 selector: matchLabels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app template: metadata: - annotations: - checksum/config: 7fa55f62bffb69b723535187aa4e5ee14a91be308d56f5a8ca49c3159bf80aef - prometheus.io/port: '5556' - prometheus.io/scrape: 'true' labels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app spec: containers: - env: @@ -130,8 +70,6 @@ spec: value: APP_ - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR value: com.bakdata.kafka.MurmurHashIdGenerator - - name: KAFKA_JMX_PORT - value: '5555' - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS @@ -145,7 +83,7 @@ spec: - name: APP_OUTPUT_TOPIC value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-my-streams-app-error + value: resources-manifest-pipeline-my-streams-app-error - name: APP_LABELED_OUTPUT_TOPICS value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS @@ -154,17 +92,12 @@ spec: value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - name: APP_APPLICATION_ID value: my-streams-app-id - - name: APP_CONVERT_XML - value: 'true' - name: JAVA_TOOL_OPTIONS value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-my-streams-app - ports: - - containerPort: 5555 - name: jmx + name: resources-manifest-pipeline-my-streams-app resources: limits: cpu: 500m @@ -172,40 +105,5 @@ spec: requests: cpu: 200m memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - '5556' - - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml - image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - name: prometheus-jmx-exporter - ports: - - containerPort: 5556 - resources: - limits: - cpu: 300m - memory: 2G - requests: - cpu: 100m - memory: 500Mi - volumeMounts: - - mountPath: /etc/jmx-streams-app - name: jmx-config terminationGracePeriodSeconds: 300 - volumes: - - configMap: - name: resources-streams-bootstrap-my-streams-app-jmx-configmap - name: jmx-config - - configMap: - name: resources-streams-bootstrap-my-streams-app - name: config diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 49799d214..17075be9b 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -152,7 +152,7 @@ def test_deploy_manifest_mode(self, snapshot: Snapshot): app, [ "deploy", - str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), + str(RESOURCE_PATH / "manifest-pipeline" / PIPELINE_YAML), "--operation-mode", "manifest", ], @@ -166,7 +166,7 @@ def test_deploy_argo_mode(self, snapshot: Snapshot): app, [ "deploy", - str(RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML), + str(RESOURCE_PATH / "manifest-pipeline" / PIPELINE_YAML), "--operation-mode", "argo", ], From af9b23b13324cd7738d80604ba668fbb5401a016 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 14:59:17 +0100 Subject: [PATCH 07/47] Update files --- kpops/config/__init__.py | 4 +- .../test_manifest/test_python_api/resources | 196 ++++-------------- tests/pipeline/test_manifest.py | 2 +- 3 files changed, 50 insertions(+), 152 deletions(-) diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 8cf575819..6668f3bb4 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -126,7 +126,9 @@ class KpopsConfig(BaseSettings): description="The operation mode of KPOps (standard, manifest, argo).", ) - model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, env_nested_delimiter="__") + model_config = SettingsConfigDict( + env_prefix=ENV_PREFIX, env_nested_delimiter="__", use_enum_values=True + ) @classmethod def create( diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index 0ffee00bf..b7b09b725 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -1,105 +1,47 @@ - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: - apiVersion: v1 - data: - log4j2.xml: "\n\n \ - \ \n \n\ - \ \n \n \n \n \ - \ \n \n \ - \ \n \n\n" - kind: ConfigMap - metadata: - name: resources-streams-bootstrap-my-producer-app -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: batch/v1beta1 - kind: CronJob + apiVersion: batch/v1 + kind: Job metadata: labels: - app: resources-streams-bootstrap-my-producer-app + app: resources-manifest-pipeline-my-producer-app chart: producer-app-3.0.3 - release: resources-streams-bootstrap-my-producer-app - name: resources-streams-bootstrap-my-producer-app + release: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app spec: - concurrencyPolicy: Replace - failedJobsHistoryLimit: 1 - jobTemplate: + backoffLimit: 6 + template: + metadata: + labels: + app: resources-manifest-pipeline-my-producer-app + release: resources-manifest-pipeline-my-producer-app spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-streams-bootstrap-my-producer-app - release: resources-streams-bootstrap-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: APP_FAKE_ARG - value: fake-arg-value - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-streams-bootstrap-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - restartPolicy: OnFailure - volumes: - - configMap: - name: resources-streams-bootstrap-my-producer-app - name: config - schedule: 30 3/8 * * * - successfulJobsHistoryLimit: 1 - suspend: false + containers: + - env: + - name: ENV_PREFIX + value: APP_ + - name: APP_BOOTSTRAP_SERVERS + value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + - name: APP_SCHEMA_REGISTRY_URL + value: http://localhost:8081/ + - name: APP_OUTPUT_TOPIC + value: my-producer-app-output-topic + - name: APP_LABELED_OUTPUT_TOPICS + value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, + - name: JAVA_TOOL_OPTIONS + value: '-XX:MaxRAMPercentage=75.0 ' + image: my-registry/my-producer-image:1.0.0 + imagePullPolicy: Always + name: resources-manifest-pipeline-my-producer-app + resources: + limits: + cpu: 500m + memory: 2G + requests: + cpu: 200m + memory: 300Mi + restartPolicy: OnFailure --- -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: v1 - data: - log4j2.xml: "\n\n \ - \ \n \n\ - \ \n \n \n \n \ - \ \n \n \ - \ \n \n\n" - kind: ConfigMap - metadata: - name: resources-streams-bootstrap-my-streams-app -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: v1 - data: - jmx-kafka-streams-app-prometheus.yml: "jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi\n\ - lowercaseOutputName: true\nlowercaseOutputLabelNames: true\nssl: false\nrules:\n\ - \ - pattern: \".*\"\n" - kind: ConfigMap - metadata: - labels: - app: resources-streams-bootstrap-my-streams-app - chart: streams-app-3.0.3 - heritage: Helm - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app-jmx-configmap - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: apiVersion: apps/v1 @@ -108,25 +50,21 @@ annotations: consumerGroup: my-streams-app-id labels: - app: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app chart: streams-app-3.0.3 - release: resources-streams-bootstrap-my-streams-app - name: resources-streams-bootstrap-my-streams-app + release: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app spec: replicas: 1 selector: matchLabels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app template: metadata: - annotations: - checksum/config: 7fa55f62bffb69b723535187aa4e5ee14a91be308d56f5a8ca49c3159bf80aef - prometheus.io/port: '5556' - prometheus.io/scrape: 'true' labels: - app: resources-streams-bootstrap-my-streams-app - release: resources-streams-bootstrap-my-streams-app + app: resources-manifest-pipeline-my-streams-app + release: resources-manifest-pipeline-my-streams-app spec: containers: - env: @@ -134,8 +72,6 @@ value: APP_ - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR value: com.bakdata.kafka.MurmurHashIdGenerator - - name: KAFKA_JMX_PORT - value: '5555' - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS @@ -149,7 +85,7 @@ - name: APP_OUTPUT_TOPIC value: my-output-topic - name: APP_ERROR_TOPIC - value: resources-streams-bootstrap-my-streams-app-error + value: resources-manifest-pipeline-my-streams-app-error - name: APP_LABELED_OUTPUT_TOPICS value: my-output-topic-label=my-labeled-topic-output, - name: APP_LABELED_INPUT_TOPICS @@ -158,17 +94,12 @@ value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - name: APP_APPLICATION_ID value: my-streams-app-id - - name: APP_CONVERT_XML - value: 'true' - name: JAVA_TOOL_OPTIONS value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=85.0 ' + -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-streams-bootstrap-my-streams-app - ports: - - containerPort: 5555 - name: jmx + name: resources-manifest-pipeline-my-streams-app resources: limits: cpu: 500m @@ -176,39 +107,4 @@ requests: cpu: 200m memory: 300Mi - volumeMounts: - - mountPath: app/resources/log4j2.xml - name: config - subPath: log4j2.xml - - command: - - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - - -jar - - jmx_prometheus_httpserver.jar - - '5556' - - /etc/jmx-streams-app/jmx-kafka-streams-app-prometheus.yml - image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 - name: prometheus-jmx-exporter - ports: - - containerPort: 5556 - resources: - limits: - cpu: 300m - memory: 2G - requests: - cpu: 100m - memory: 500Mi - volumeMounts: - - mountPath: /etc/jmx-streams-app - name: jmx-config terminationGracePeriodSeconds: 300 - volumes: - - configMap: - name: resources-streams-bootstrap-my-streams-app-jmx-configmap - name: jmx-config - - configMap: - name: resources-streams-bootstrap-my-streams-app - name: config diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 17075be9b..983859a44 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -125,7 +125,7 @@ def test_manifest_command(self, snapshot: Snapshot): def test_python_api(self, snapshot: Snapshot): generator = kpops.manifest_deploy( - RESOURCE_PATH / "streams-bootstrap" / PIPELINE_YAML, + RESOURCE_PATH / "manifest-pipeline" / PIPELINE_YAML, environment="development", ) assert isinstance(generator, Iterator) From 4d1cc1eaa2914660adfe6515cdcc1bede4d850c6 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 15:14:53 +0100 Subject: [PATCH 08/47] Update files --- .../components/streams_bootstrap/producer/producer_app.py | 6 +++--- kpops/components/streams_bootstrap/streams/streams_app.py | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 89ab96112..7cc071b39 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -37,9 +37,9 @@ def helm_chart(self) -> str: @override def manifest_deploy(self) -> Resource: - operation_mode = get_config().operation_mode - values = self.values.model_dump() - if operation_mode is OperationMode.ARGO: + self.values.name_override = None + values = self.to_helm_values() + if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) return self.helm.template( self.helm_release_name, diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 4552901b2..c7afb0a20 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -50,9 +50,9 @@ async def clean(self, dry_run: bool) -> None: @override def manifest_deploy(self) -> Resource: - operation_mode = get_config().operation_mode + self.values.name_override = None values = self.to_helm_values() - if operation_mode is OperationMode.ARGO: + if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) return self.helm.template( self.helm_release_name, @@ -174,9 +174,7 @@ async def clean(self, dry_run: bool) -> None: @override def manifest_deploy(self) -> Resource: manifests = super().manifest_deploy() - operation_mode = get_config().operation_mode - - if operation_mode is OperationMode.ARGO: + if get_config().operation_mode is OperationMode.ARGO: manifests.extend(self._cleaner.manifest_deploy()) return manifests From c7170025f7abddb3719bbf7d14df9b28f0e9f2ef Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 15:16:49 +0100 Subject: [PATCH 09/47] Update files --- kpops/components/streams_bootstrap/producer/producer_app.py | 2 +- kpops/components/streams_bootstrap/streams/streams_app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 7cc071b39..8cf228db9 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -37,7 +37,7 @@ def helm_chart(self) -> str: @override def manifest_deploy(self) -> Resource: - self.values.name_override = None + self.values.name_override = self.helm_release_name values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index c7afb0a20..7f877c306 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -50,7 +50,7 @@ async def clean(self, dry_run: bool) -> None: @override def manifest_deploy(self) -> Resource: - self.values.name_override = None + self.values.name_override = self.helm_release_name values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) From 975b10ea46d4b709c5c3996959f2ed3e6b3d3059 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 16:35:23 +0100 Subject: [PATCH 10/47] fix init command --- kpops/api/__init__.py | 6 ++--- kpops/api/operation.py | 4 +-- kpops/cli/main.py | 2 +- .../producer/producer_app.py | 1 + kpops/config/__init__.py | 5 ++-- kpops/utils/cli_commands.py | 12 ++++++--- .../config.yaml} | 0 .../defaults.yaml | 0 .../pipeline.yaml | 0 .../config.yaml} | 7 +++-- .../defaults.yaml | 0 .../pipeline.yaml | 0 tests/cli/test_init.py | 26 ++++++++++--------- .../test_deploy_argo_mode/manifest.yaml | 16 ++++++------ 14 files changed, 44 insertions(+), 35 deletions(-) rename tests/cli/snapshots/test_init/{test_init_project/config_exclude_opt.yaml => test_init_project_exclude_optional/config.yaml} (100%) rename tests/cli/snapshots/test_init/{test_init_project => test_init_project_exclude_optional}/defaults.yaml (100%) rename tests/cli/snapshots/test_init/{test_init_project => test_init_project_exclude_optional}/pipeline.yaml (100%) rename tests/cli/snapshots/test_init/{test_init_project/config_include_opt.yaml => test_init_project_include_optional/config.yaml} (80%) create mode 100644 tests/cli/snapshots/test_init/test_init_project_include_optional/defaults.yaml create mode 100644 tests/cli/snapshots/test_init/test_init_project_include_optional/pipeline.yaml diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 139b89aaf..59b3cd5be 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -293,12 +293,12 @@ async def async_clean(): def init( path: Path, - config_include_opt: bool = False, + config_include_optional: bool = False, ): """Initiate a default empty project. :param path: Directory in which the project should be initiated. - :param conf_incl_opt: Whether to include non-required settings + :param config_include_optional: Whether to include non-required settings in the generated config file. """ if not path.exists(): @@ -306,7 +306,7 @@ def init( elif next(path.iterdir(), False): log.warning("Please provide a path to an empty directory.") return - init_project(path, config_include_opt) + init_project(path, config_include_optional) def _create_pipeline( diff --git a/kpops/api/operation.py b/kpops/api/operation.py index 04c752893..cd33b3cae 100644 --- a/kpops/api/operation.py +++ b/kpops/api/operation.py @@ -1,9 +1,9 @@ from __future__ import annotations -from enum import Enum +from enum import StrEnum -class OperationMode(str, Enum): +class OperationMode(StrEnum): ARGO = "argo" MANIFEST = "manifest" STANDARD = "standard" diff --git a/kpops/cli/main.py b/kpops/cli/main.py index bc77a8cba..3360723ca 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -128,7 +128,7 @@ def init( path: Path = PROJECT_PATH, config_include_opt: bool = CONFIG_INCLUDE_OPTIONAL, ): - kpops.init(path, config_include_opt=config_include_opt) + kpops.init(path, config_include_optional=config_include_opt) @app.command( diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 8cf228db9..3ba0412c8 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -37,6 +37,7 @@ def helm_chart(self) -> str: @override def manifest_deploy(self) -> Resource: + # TODO: check were name_override is set... self.values.name_override = self.helm_release_name values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 6668f3bb4..bc5175e68 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -73,7 +73,7 @@ class KafkaConnectConfig(BaseSettings): ) -class KpopsConfig(BaseSettings): +class KpopsConfig(BaseSettings, use_enum_values=True): """Global configuration for KPOps project.""" _instance: ClassVar[KpopsConfig | None] = PrivateAttr(None) @@ -127,7 +127,8 @@ class KpopsConfig(BaseSettings): ) model_config = SettingsConfigDict( - env_prefix=ENV_PREFIX, env_nested_delimiter="__", use_enum_values=True + env_prefix=ENV_PREFIX, + env_nested_delimiter="__", ) @classmethod diff --git a/kpops/utils/cli_commands.py b/kpops/utils/cli_commands.py index ba1b295ad..2f994b494 100644 --- a/kpops/utils/cli_commands.py +++ b/kpops/utils/cli_commands.py @@ -52,7 +52,7 @@ def create_config(file_name: str, dir_path: Path, include_optional: bool) -> Non file_path = Path(dir_path / (file_name + ".yaml")) file_path.touch(exist_ok=False) with file_path.open(mode="w") as conf: - conf.write("# " + describe_object(KpopsConfig.__doc__)) # Write title + conf.write(f"# {describe_object(KpopsConfig.__doc__)}") # Write title non_required = extract_config_fields_for_yaml( collect_fields(KpopsConfig), False ) @@ -60,10 +60,16 @@ def create_config(file_name: str, dir_path: Path, include_optional: bool) -> Non for k in non_required: required.pop(k, None) conf.write("\n\n# Required fields\n") - conf.write(yaml.dump(required)) + conf.write(yaml.safe_dump(required)) + if include_optional: + dump = KpopsConfig(**non_required).model_dump( + mode="json", exclude_none=False + ) + for k in required: + dump.pop(k, None) conf.write("\n# Non-required fields\n") - conf.write(yaml.dump(non_required)) + conf.write(yaml.safe_dump(dump)) def init_project(path: Path, conf_incl_opt: bool): diff --git a/tests/cli/snapshots/test_init/test_init_project/config_exclude_opt.yaml b/tests/cli/snapshots/test_init/test_init_project_exclude_optional/config.yaml similarity index 100% rename from tests/cli/snapshots/test_init/test_init_project/config_exclude_opt.yaml rename to tests/cli/snapshots/test_init/test_init_project_exclude_optional/config.yaml diff --git a/tests/cli/snapshots/test_init/test_init_project/defaults.yaml b/tests/cli/snapshots/test_init/test_init_project_exclude_optional/defaults.yaml similarity index 100% rename from tests/cli/snapshots/test_init/test_init_project/defaults.yaml rename to tests/cli/snapshots/test_init/test_init_project_exclude_optional/defaults.yaml diff --git a/tests/cli/snapshots/test_init/test_init_project/pipeline.yaml b/tests/cli/snapshots/test_init/test_init_project_exclude_optional/pipeline.yaml similarity index 100% rename from tests/cli/snapshots/test_init/test_init_project/pipeline.yaml rename to tests/cli/snapshots/test_init/test_init_project_exclude_optional/pipeline.yaml diff --git a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml similarity index 80% rename from tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml rename to tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml index 33143a1a0..3543823d8 100644 --- a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml +++ b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml @@ -9,16 +9,15 @@ helm_config: api_version: null context: null debug: false -helm_diff_config: {} +helm_diff_config: + ignore: [] kafka_connect: timeout: 30 url: http://localhost:8083/ kafka_rest: timeout: 30 url: http://localhost:8082/ -operation_mode: !!python/object/apply:builtins.getattr -- !!python/name:kpops.api.operation.OperationMode '' -- STANDARD +operation_mode: standard pipeline_base_dir: . retain_clean_jobs: false schema_registry: diff --git a/tests/cli/snapshots/test_init/test_init_project_include_optional/defaults.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/defaults.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/snapshots/test_init/test_init_project_include_optional/pipeline.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/pipeline.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 3109d16fc..6a5c61303 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -1,5 +1,6 @@ from pathlib import Path +import pytest from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner @@ -22,23 +23,24 @@ def test_create_config(tmp_path: Path): assert len(opt_conf.read_text()) > len(req_conf.read_text()) -def test_init_project(tmp_path: Path, snapshot: Snapshot): +@pytest.mark.usefixtures("mock_env", "load_yaml_file_clear_cache", "clear_kpops_config") +def test_init_project_exclude_optional(tmp_path: Path, snapshot: Snapshot): + req_path = tmp_path / "req" + req_path.mkdir() + kpops.init(req_path, config_include_optional=False) + snapshot.assert_match(Path(req_path / "config.yaml").read_text(), "config.yaml") + snapshot.assert_match(Path(req_path / "pipeline.yaml").read_text(), "pipeline.yaml") + snapshot.assert_match(Path(req_path / "defaults.yaml").read_text(), "defaults.yaml") + + +def test_init_project_include_optional(tmp_path: Path, snapshot: Snapshot): opt_path = tmp_path / "opt" opt_path.mkdir() - kpops.init(opt_path, config_include_opt=False) - snapshot.assert_match( - Path(opt_path / "config.yaml").read_text(), "config_exclude_opt.yaml" - ) + kpops.init(opt_path, config_include_optional=True) + snapshot.assert_match(Path(opt_path / "config.yaml").read_text(), "config.yaml") snapshot.assert_match(Path(opt_path / "pipeline.yaml").read_text(), "pipeline.yaml") snapshot.assert_match(Path(opt_path / "defaults.yaml").read_text(), "defaults.yaml") - req_path = tmp_path / "req" - req_path.mkdir() - kpops.init(req_path, config_include_opt=True) - snapshot.assert_match( - Path(req_path / "config.yaml").read_text(), "config_include_opt.yaml" - ) - def test_init_project_from_cli_with_bad_path(tmp_path: Path): bad_path = Path(tmp_path / "random_file.yaml") diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index abf6c7c63..1263b18d1 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -50,16 +50,16 @@ metadata: annotations: argocd.argoproj.io/hook: PostDelete labels: - app: resources-manifest-pipeline-my-producer-app + app: resources-manifest-pipeline-my-producer-app-clean chart: producer-app-cleanup-job-3.0.3 release: resources-manifest-pipeline-my-producer-app-clean - name: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app-clean spec: backoffLimit: 6 template: metadata: labels: - app: resources-manifest-pipeline-my-producer-app + app: resources-manifest-pipeline-my-producer-app-clean release: resources-manifest-pipeline-my-producer-app-clean spec: containers: @@ -80,7 +80,7 @@ spec: value: '-XX:MaxRAMPercentage=75.0 ' image: my-registry/my-producer-image:1.0.0 imagePullPolicy: Always - name: resources-manifest-pipeline-my-producer-app + name: resources-manifest-pipeline-my-producer-app-clean resources: limits: cpu: 500m @@ -165,16 +165,16 @@ metadata: annotations: argocd.argoproj.io/hook: PostDelete labels: - app: resources-manifest-pipeline-my-streams-app + app: resources-manifest-pipeline-my-streams-app-clean chart: streams-app-cleanup-job-3.0.3 release: resources-manifest-pipeline-my-streams-app-clean - name: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app-clean spec: backoffLimit: 6 template: metadata: labels: - app: resources-manifest-pipeline-my-streams-app + app: resources-manifest-pipeline-my-streams-app-clean release: resources-manifest-pipeline-my-streams-app-clean spec: containers: @@ -209,7 +209,7 @@ spec: value: '-XX:MaxRAMPercentage=75.0 ' image: my-registry/my-streams-app-image:1.0.0 imagePullPolicy: Always - name: resources-manifest-pipeline-my-streams-app + name: resources-manifest-pipeline-my-streams-app-clean resources: limits: cpu: 500m From 7b47919179752bd4e14ee5ce963bb216b547d06c Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 16:42:47 +0100 Subject: [PATCH 11/47] clean ups --- docs/docs/resources/variables/cli_env_vars.env | 2 +- docs/docs/resources/variables/cli_env_vars.md | 2 +- docs/docs/resources/variables/config_env_vars.env | 4 ++-- docs/docs/resources/variables/config_env_vars.md | 2 +- docs/docs/schema/config.json | 6 +++--- docs/docs/user/references/cli-commands.md | 2 +- kpops/api/__init__.py | 9 +++------ kpops/api/operation.py | 2 +- kpops/cli/main.py | 5 ++--- kpops/config/__init__.py | 4 ++-- .../test_init_project_include_optional/config.yaml | 2 +- 11 files changed, 18 insertions(+), 22 deletions(-) diff --git a/docs/docs/resources/variables/cli_env_vars.env b/docs/docs/resources/variables/cli_env_vars.env index b47252b5b..2aca45807 100644 --- a/docs/docs/resources/variables/cli_env_vars.env +++ b/docs/docs/resources/variables/cli_env_vars.env @@ -15,7 +15,7 @@ KPOPS_DOTENV_PATH # No default value, not required # defaults_development.yaml for environment=development). KPOPS_ENVIRONMENT # No default value, not required # How KPOps should operate. -KPOPS_OPERATION_MODE=standard +KPOPS_OPERATION_MODE=managed # Paths to dir containing 'pipeline.yaml' or files named # 'pipeline.yaml'. KPOPS_PIPELINE_PATHS # No default value, required diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md index a7d6f0a66..b4cead3ce 100644 --- a/docs/docs/resources/variables/cli_env_vars.md +++ b/docs/docs/resources/variables/cli_env_vars.md @@ -5,6 +5,6 @@ These variables take precedence over the commands' flags. If a variable is set, |KPOPS_CONFIG_PATH |. |False |Path to the dir containing config.yaml files | |KPOPS_DOTENV_PATH | |False |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one. | |KPOPS_ENVIRONMENT | |False |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).| -|KPOPS_OPERATION_MODE|standard |False |How KPOps should operate. | +|KPOPS_OPERATION_MODE|managed |False |How KPOps should operate. | |KPOPS_PIPELINE_PATHS| |True |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'. | |KPOPS_PIPELINE_STEPS| |False |Comma separated list of steps to apply the command on | diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index 747672702..b64bac2e0 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -59,5 +59,5 @@ KPOPS_HELM_DIFF_CONFIG__IGNORE # No default value, required # after completion. KPOPS_RETAIN_CLEAN_JOBS=False # operation_mode -# The operation mode of KPOps (standard, manifest, argo). -KPOPS_OPERATION_MODE=standard +# The operation mode of KPOps (managed, manifest, argo). +KPOPS_OPERATION_MODE=managed diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 6e5458054..6b79a743c 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -19,4 +19,4 @@ These variables take precedence over the settings in `config.yaml`. Variables ma |KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | |KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | |KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs | -|KPOPS_OPERATION_MODE |standard |False |The operation mode of KPOps (standard, manifest, argo). |operation_mode | +|KPOPS_OPERATION_MODE |managed |False |The operation mode of KPOps (managed, manifest, argo). |operation_mode | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index bddbda217..02d04ea6e 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -122,7 +122,7 @@ "enum": [ "argo", "manifest", - "standard" + "managed" ], "title": "OperationMode", "type": "string" @@ -254,8 +254,8 @@ "$ref": "#/$defs/OperationMode" } ], - "default": "standard", - "description": "The operation mode of KPOps (standard, manifest, argo)." + "default": "managed", + "description": "The operation mode of KPOps (managed, manifest, argo)." }, "pipeline_base_dir": { "default": ".", diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index a5912d27c..abfc43a08 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -73,7 +73,7 @@ $ kpops deploy [OPTIONS] PIPELINE_PATHS... * `--dry-run / --execute`: Whether to dry run the command or execute it [default: dry-run] * `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially. [default: no-parallel] -* `--operation-mode [argo|manifest|standard]`: How KPOps should operate. [env var: KPOPS_OPERATION_MODE; default: standard] +* `--operation-mode [argo|manifest|managed]`: How KPOps should operate. [env var: KPOPS_OPERATION_MODE; default: managed] * `--help`: Show this message and exit. ## `kpops destroy` diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 59b3cd5be..39fc63198 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -37,6 +37,7 @@ def generate( filter_type: FilterType = FilterType.INCLUDE, environment: str | None = None, verbose: bool = False, + operation_mode: OperationMode = OperationMode.MANAGED, ) -> Pipeline: """Generate enriched pipeline representation. @@ -50,10 +51,7 @@ def generate( :return: Generated `Pipeline` object. """ kpops_config = KpopsConfig.create( - config, - dotenv, - environment, - verbose, + config, dotenv, environment, verbose, operation_mode ) pipeline = _create_pipeline(pipeline_path, kpops_config, environment) log.info(f"Picked up pipeline '{pipeline_path.parent.name}'") @@ -89,9 +87,8 @@ def manifest_deploy( filter_type=filter_type, environment=environment, verbose=verbose, + operation_mode=operation_mode, ) - # TODO: KPOps config is created twice. Once in generate and once here. change it! - KpopsConfig.create(config, dotenv, environment, verbose, operation_mode) for component in pipeline.components: resource = component.manifest_deploy() yield resource diff --git a/kpops/api/operation.py b/kpops/api/operation.py index cd33b3cae..7005008ae 100644 --- a/kpops/api/operation.py +++ b/kpops/api/operation.py @@ -6,4 +6,4 @@ class OperationMode(StrEnum): ARGO = "argo" MANIFEST = "manifest" - STANDARD = "standard" + MANAGED = "managed" diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 3360723ca..81267ff8e 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -112,9 +112,8 @@ ), ) OPERATION_MODE_OPTION: OperationMode = typer.Option( - default=OperationMode.STANDARD, + default=OperationMode.MANAGED, envvar=f"{ENV_PREFIX}OPERATION_MODE", - # TODO: better help? help="How KPOps should operate.", ) @@ -201,7 +200,7 @@ def deploy( operation_mode: OperationMode = OPERATION_MODE_OPTION, ): match operation_mode: - case OperationMode.STANDARD: + case OperationMode.MANAGED: for pipeline_file_path in collect_pipeline_paths(pipeline_paths): kpops.deploy( pipeline_path=pipeline_file_path, diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index bc5175e68..ad8457213 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -122,8 +122,8 @@ class KpopsConfig(BaseSettings, use_enum_values=True): description="Whether to retain clean up jobs in the cluster or uninstall the, after completion.", ) operation_mode: OperationMode = Field( - default=OperationMode.STANDARD, - description="The operation mode of KPOps (standard, manifest, argo).", + default=OperationMode.MANAGED, + description="The operation mode of KPOps (managed, manifest, argo).", ) model_config = SettingsConfigDict( diff --git a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml index 3543823d8..19b7f5d95 100644 --- a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml +++ b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml @@ -17,7 +17,7 @@ kafka_connect: kafka_rest: timeout: 30 url: http://localhost:8082/ -operation_mode: standard +operation_mode: managed pipeline_base_dir: . retain_clean_jobs: false schema_registry: From 9fcd55fd1048b8414ac254e9b7d3f0e189f681dc Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 16:52:54 +0100 Subject: [PATCH 12/47] clean ups --- kpops/components/streams_bootstrap/producer/producer_app.py | 6 +++--- kpops/components/streams_bootstrap/streams/streams_app.py | 5 +++-- kpops/manifests/argo.py | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 3ba0412c8..cc8dd4448 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -37,8 +37,6 @@ def helm_chart(self) -> str: @override def manifest_deploy(self) -> Resource: - # TODO: check were name_override is set... - self.values.name_override = self.helm_release_name values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) @@ -76,9 +74,11 @@ class ProducerApp(StreamsBootstrap): @computed_field @cached_property def _cleaner(self) -> ProducerAppCleaner: - return ProducerAppCleaner( + cleaner = ProducerAppCleaner( **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) ) + cleaner.values.name_override = None + return cleaner @override def apply_to_outputs(self, name: str, topic: TopicConfig) -> None: diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 7f877c306..213134445 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -50,7 +50,6 @@ async def clean(self, dry_run: bool) -> None: @override def manifest_deploy(self) -> Resource: - self.values.name_override = self.helm_release_name values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: values = ArgoHook.POST_DELETE.enrich(values) @@ -81,9 +80,11 @@ class StreamsApp(StreamsBootstrap): @computed_field @cached_property def _cleaner(self) -> StreamsAppCleaner: - return StreamsAppCleaner( + cleaner = StreamsAppCleaner( **self.model_dump(by_alias=True, exclude={"_cleaner", "from_", "to"}) ) + cleaner.values.name_override = None + return cleaner @property @override diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index e3002004d..152e6dda3 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -2,6 +2,8 @@ from enum import Enum from typing import Any +from typing_extensions import override + class ArgoEnricher(enum.Enum): @property @@ -18,6 +20,7 @@ class ArgoHook(ArgoEnricher, str, Enum): POST_DELETE = "PostDelete" @property + @override def key(self) -> str: return "argocd.argoproj.io/hook" @@ -26,5 +29,6 @@ class ArgoSyncWave(ArgoEnricher, str, Enum): SYNC_WAVE = "1" @property + @override def key(self) -> str: return "argocd.argoproj.io/sync-wave" From aaed48d5f63c0f2b27ff37eaa56180540ef233ec Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 16:54:08 +0100 Subject: [PATCH 13/47] clean ups --- kpops/manifests/argo.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index 152e6dda3..5a808e3fd 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -1,5 +1,4 @@ import enum -from enum import Enum from typing import Any from typing_extensions import override @@ -16,7 +15,7 @@ def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: return helm_values -class ArgoHook(ArgoEnricher, str, Enum): +class ArgoHook(ArgoEnricher, enum.StrEnum): POST_DELETE = "PostDelete" @property @@ -25,7 +24,7 @@ def key(self) -> str: return "argocd.argoproj.io/hook" -class ArgoSyncWave(ArgoEnricher, str, Enum): +class ArgoSyncWave(ArgoEnricher, enum.StrEnum): SYNC_WAVE = "1" @property From ece6e9f272136c0094be8d7f1f9eb25be88c2348 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 16:54:57 +0100 Subject: [PATCH 14/47] clean ups --- docs/docs/schema/defaults.json | 1009 +++++++++++++++++++++++++++----- docs/docs/schema/pipeline.json | 1001 ++++++++++++++++++++++++++----- 2 files changed, 1700 insertions(+), 310 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index ad8d833e1..05ee439be 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1,5 +1,104 @@ { "$defs": { + "Converter": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "converter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Converter", + "type": "object" + }, "Effects": { "enum": [ "NoExecute", @@ -9,6 +108,105 @@ "title": "Effects", "type": "string" }, + "Filter": { + "additionalProperties": true, + "description": "Subsubclass of StreamsApp to test inheritance.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "filter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Filter", + "type": "object" + }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -801,52 +999,231 @@ "title": "KubernetesAppValues", "type": "object" }, - "Operation": { - "enum": [ - "Exists", - "Equal" - ], - "title": "Operation", - "type": "string" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "MyProducerApp": { + "additionalProperties": true, + "description": "", "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "from": { "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-producer-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.1", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyProducerApp", + "type": "object" + }, + "MyStreamsApp": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-streams-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.1", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyStreamsApp", + "type": "object" + }, + "Operation": { + "enum": [ + "Exists", + "Equal" + ], + "title": "Operation", + "type": "string" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" } ], "default": null, @@ -1611,64 +1988,355 @@ "ProtocolSchema": { "description": "Represents the different Kubernetes protocols.\n\nhttps://kubernetes.io/docs/reference/networking/service-protocols/", "enum": [ - "TCP", - "UDP", - "SCTP" + "TCP", + "UDP", + "SCTP" + ], + "title": "ProtocolSchema", + "type": "string" + }, + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", + "properties": { + "ca_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" + }, + "cert_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" + }, + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" + }, + "password": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password", + "title": "Password" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username", + "title": "Username" + } + }, + "title": "RepoAuthFlags", + "type": "object" + }, + "ResourceDefinition": { + "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", + "properties": { + "cpu": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", + "title": "Cpu" + }, + "memory": { + "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", + "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", + "title": "Memory", + "type": "string" + } + }, + "required": [ + "cpu", + "memory" + ], + "title": "ResourceDefinition", + "type": "object" + }, + "Resources": { + "description": "Model representing the resource specifications for a Kubernetes container.", + "properties": { + "limits": { + "allOf": [ + { + "$ref": "#/$defs/ResourceDefinition" + } + ], + "description": "The maximum resource limits for the container." + }, + "requests": { + "allOf": [ + { + "$ref": "#/$defs/ResourceDefinition" + } + ], + "description": "The minimum resource requirements for the container." + } + }, + "required": [ + "requests", + "limits" + ], + "title": "Resources", + "type": "object" + }, + "RestartPolicy": { + "enum": [ + "Always", + "OnFailure", + "Never" + ], + "title": "RestartPolicy", + "type": "string" + }, + "ScheduledProducer": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "scheduled-producer", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppV2Values" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "ScheduledProducer", + "type": "object" + }, + "ServiceConfig": { + "description": "Base model for configuring a service for the Kafka Streams application.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to create a service.", + "title": "Enabled", + "type": "boolean" + }, + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Additional service labels.", + "title": "Labels", + "type": "object" + }, + "type": { + "allOf": [ + { + "$ref": "#/$defs/ServiceType" + } + ], + "default": "ClusterIP", + "description": "Service type." + } + }, + "title": "ServiceConfig", + "type": "object" + }, + "ServiceType": { + "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer", + "ExternalName" ], - "title": "ProtocolSchema", + "title": "ServiceType", "type": "string" }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", + "ShouldInflate": { + "additionalProperties": true, + "description": "", "properties": { - "ca_file": { + "from": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "cert_file": { + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" + "description": "Topic(s) into which the component will write output" }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" + "type": { + "const": "should-inflate", + "title": "Type" }, - "password": { - "anyOf": [ - { - "type": "string" - }, + "values": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": null, - "description": "Password", - "title": "Password" + "description": "streams-bootstrap-v2 Helm values" }, - "username": { + "version": { "anyOf": [ { "type": "string" @@ -1677,119 +2345,118 @@ "type": "null" } ], - "default": null, - "description": "Username", - "title": "Username" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "RepoAuthFlags", + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "ShouldInflate", "type": "object" }, - "ResourceDefinition": { - "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", + "SimpleInflateConnectors": { + "additionalProperties": true, + "description": "", "properties": { - "cpu": { + "from": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/FromSection" }, { - "type": "integer" + "type": "null" } ], - "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", - "title": "Cpu" + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "memory": { - "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", - "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", - "title": "Memory", + "name": { + "description": "Component name", + "title": "Name", "type": "string" - } - }, - "required": [ - "cpu", - "memory" - ], - "title": "ResourceDefinition", - "type": "object" - }, - "Resources": { - "description": "Model representing the resource specifications for a Kubernetes container.", - "properties": { - "limits": { + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { "allOf": [ { - "$ref": "#/$defs/ResourceDefinition" + "$ref": "#/$defs/HelmRepoConfig" } ], - "description": "The maximum resource limits for the container." + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "requests": { - "allOf": [ + "to": { + "anyOf": [ { - "$ref": "#/$defs/ResourceDefinition" + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" } ], - "description": "The minimum resource requirements for the container." - } - }, - "required": [ - "requests", - "limits" - ], - "title": "Resources", - "type": "object" - }, - "RestartPolicy": { - "enum": [ - "Always", - "OnFailure", - "Never" - ], - "title": "RestartPolicy", - "type": "string" - }, - "ServiceConfig": { - "description": "Base model for configuring a service for the Kafka Streams application.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to create a service.", - "title": "Enabled", - "type": "boolean" - }, - "labels": { - "additionalProperties": { - "type": "string" - }, - "description": "Additional service labels.", - "title": "Labels", - "type": "object" + "default": null, + "description": "Topic(s) into which the component will write output" }, "type": { + "const": "simple-inflate-connectors", + "title": "Type" + }, + "values": { "allOf": [ { - "$ref": "#/$defs/ServiceType" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": "ClusterIP", - "description": "Service type." + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "ServiceConfig", - "type": "object" - }, - "ServiceType": { - "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", - "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer", - "ExternalName" + "required": [ + "name", + "namespace", + "values", + "type" ], - "title": "ServiceType", - "type": "string" + "title": "SimpleInflateConnectors", + "type": "object" }, "StreamsApp": { "additionalProperties": true, @@ -3378,6 +4045,12 @@ } }, "properties": { + "converter": { + "$ref": "#/$defs/Converter" + }, + "filter": { + "$ref": "#/$defs/Filter" + }, "helm-app": { "$ref": "#/$defs/HelmApp" }, @@ -3396,6 +4069,12 @@ "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, + "my-producer-app": { + "$ref": "#/$defs/MyProducerApp" + }, + "my-streams-app": { + "$ref": "#/$defs/MyStreamsApp" + }, "pipeline-component": { "$ref": "#/$defs/PipelineComponent" }, @@ -3405,6 +4084,15 @@ "producer-app-v2": { "$ref": "#/$defs/ProducerAppV2" }, + "scheduled-producer": { + "$ref": "#/$defs/ScheduledProducer" + }, + "should-inflate": { + "$ref": "#/$defs/ShouldInflate" + }, + "simple-inflate-connectors": { + "$ref": "#/$defs/SimpleInflateConnectors" + }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, @@ -3431,7 +4119,14 @@ "streams-bootstrap", "producer-app-v2", "streams-app-v2", - "streams-bootstrap-v2" + "streams-bootstrap-v2", + "converter", + "filter", + "my-producer-app", + "my-streams-app", + "scheduled-producer", + "should-inflate", + "simple-inflate-connectors" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 0b466584f..b494ae478 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1,5 +1,104 @@ { "$defs": { + "Converter": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "converter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Converter", + "type": "object" + }, "Effects": { "enum": [ "NoExecute", @@ -9,6 +108,105 @@ "title": "Effects", "type": "string" }, + "Filter": { + "additionalProperties": true, + "description": "Subsubclass of StreamsApp to test inheritance.", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "filter", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppV2Values" + } + ], + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "Filter", + "type": "object" + }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -508,52 +706,231 @@ "title": "KafkaSourceConnector", "type": "object" }, - "Operation": { - "enum": [ - "Exists", - "Equal" - ], - "title": "Operation", - "type": "string" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "MyProducerApp": { + "additionalProperties": true, + "description": "", "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "from": { "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-producer-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.1", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyProducerApp", + "type": "object" + }, + "MyStreamsApp": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "anyOf": [ + { + "$ref": "#/$defs/FromSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "my-streams-app", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/StreamsAppValues" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "default": "3.0.1", + "description": "Helm chart version", + "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "MyStreamsApp", + "type": "object" + }, + "Operation": { + "enum": [ + "Exists", + "Equal" + ], + "title": "Operation", + "type": "string" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" + }, + "storage_class": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" } ], "default": null, @@ -1275,60 +1652,351 @@ "UDP", "SCTP" ], - "title": "ProtocolSchema", + "title": "ProtocolSchema", + "type": "string" + }, + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", + "properties": { + "ca_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" + }, + "cert_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" + }, + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" + }, + "password": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password", + "title": "Password" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username", + "title": "Username" + } + }, + "title": "RepoAuthFlags", + "type": "object" + }, + "ResourceDefinition": { + "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", + "properties": { + "cpu": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", + "title": "Cpu" + }, + "memory": { + "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", + "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", + "title": "Memory", + "type": "string" + } + }, + "required": [ + "cpu", + "memory" + ], + "title": "ResourceDefinition", + "type": "object" + }, + "Resources": { + "description": "Model representing the resource specifications for a Kubernetes container.", + "properties": { + "limits": { + "allOf": [ + { + "$ref": "#/$defs/ResourceDefinition" + } + ], + "description": "The maximum resource limits for the container." + }, + "requests": { + "allOf": [ + { + "$ref": "#/$defs/ResourceDefinition" + } + ], + "description": "The minimum resource requirements for the container." + } + }, + "required": [ + "requests", + "limits" + ], + "title": "Resources", + "type": "object" + }, + "RestartPolicy": { + "enum": [ + "Always", + "OnFailure", + "Never" + ], + "title": "RestartPolicy", + "type": "string" + }, + "ScheduledProducer": { + "additionalProperties": true, + "description": "", + "properties": { + "from": { + "default": null, + "description": "Producer doesn't support FromSection", + "title": "From", + "type": "null" + }, + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { + "anyOf": [ + { + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Topic(s) into which the component will write output" + }, + "type": { + "const": "scheduled-producer", + "title": "Type" + }, + "values": { + "allOf": [ + { + "$ref": "#/$defs/ProducerAppV2Values" + } + ], + "description": "streams-bootstrap Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" + } + }, + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "ScheduledProducer", + "type": "object" + }, + "ServiceConfig": { + "description": "Base model for configuring a service for the Kafka Streams application.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to create a service.", + "title": "Enabled", + "type": "boolean" + }, + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Additional service labels.", + "title": "Labels", + "type": "object" + }, + "type": { + "allOf": [ + { + "$ref": "#/$defs/ServiceType" + } + ], + "default": "ClusterIP", + "description": "Service type." + } + }, + "title": "ServiceConfig", + "type": "object" + }, + "ServiceType": { + "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer", + "ExternalName" + ], + "title": "ServiceType", "type": "string" }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", + "ShouldInflate": { + "additionalProperties": true, + "description": "", "properties": { - "ca_file": { + "from": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/FromSection" }, { "type": "null" } ], "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "cert_file": { + "name": { + "description": "Component name", + "title": "Name", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { + "allOf": [ + { + "$ref": "#/$defs/HelmRepoConfig" + } + ], + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" + }, + "to": { "anyOf": [ { - "format": "path", - "type": "string" + "$ref": "#/$defs/ToSection" }, { "type": "null" } ], "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" + "description": "Topic(s) into which the component will write output" }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" + "type": { + "const": "should-inflate", + "title": "Type" }, - "password": { - "anyOf": [ - { - "type": "string" - }, + "values": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": null, - "description": "Password", - "title": "Password" + "description": "streams-bootstrap-v2 Helm values" }, - "username": { + "version": { "anyOf": [ { "type": "string" @@ -1337,119 +2005,118 @@ "type": "null" } ], - "default": null, - "description": "Username", - "title": "Username" + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "RepoAuthFlags", + "required": [ + "name", + "namespace", + "values", + "type" + ], + "title": "ShouldInflate", "type": "object" }, - "ResourceDefinition": { - "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", + "SimpleInflateConnectors": { + "additionalProperties": true, + "description": "", "properties": { - "cpu": { + "from": { "anyOf": [ { - "type": "string" + "$ref": "#/$defs/FromSection" }, { - "type": "integer" + "type": "null" } ], - "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", - "title": "Cpu" + "default": null, + "description": "Topic(s) and/or components from which the component will read input", + "title": "From" }, - "memory": { - "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", - "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", - "title": "Memory", + "name": { + "description": "Component name", + "title": "Name", "type": "string" - } - }, - "required": [ - "cpu", - "memory" - ], - "title": "ResourceDefinition", - "type": "object" - }, - "Resources": { - "description": "Model representing the resource specifications for a Kubernetes container.", - "properties": { - "limits": { + }, + "namespace": { + "description": "Kubernetes namespace in which the component shall be deployed", + "title": "Namespace", + "type": "string" + }, + "prefix": { + "default": "${pipeline.name}-", + "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", + "title": "Prefix", + "type": "string" + }, + "repo_config": { "allOf": [ { - "$ref": "#/$defs/ResourceDefinition" + "$ref": "#/$defs/HelmRepoConfig" } ], - "description": "The maximum resource limits for the container." + "default": { + "repo_auth_flags": { + "ca_file": null, + "cert_file": null, + "insecure_skip_tls_verify": false, + "password": null, + "username": null + }, + "repository_name": "bakdata-streams-bootstrap", + "url": "https://bakdata.github.io/streams-bootstrap/" + }, + "description": "Configuration of the Helm chart repo to be used for deploying the component" }, - "requests": { - "allOf": [ + "to": { + "anyOf": [ { - "$ref": "#/$defs/ResourceDefinition" + "$ref": "#/$defs/ToSection" + }, + { + "type": "null" } ], - "description": "The minimum resource requirements for the container." - } - }, - "required": [ - "requests", - "limits" - ], - "title": "Resources", - "type": "object" - }, - "RestartPolicy": { - "enum": [ - "Always", - "OnFailure", - "Never" - ], - "title": "RestartPolicy", - "type": "string" - }, - "ServiceConfig": { - "description": "Base model for configuring a service for the Kafka Streams application.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to create a service.", - "title": "Enabled", - "type": "boolean" - }, - "labels": { - "additionalProperties": { - "type": "string" - }, - "description": "Additional service labels.", - "title": "Labels", - "type": "object" + "default": null, + "description": "Topic(s) into which the component will write output" }, "type": { + "const": "simple-inflate-connectors", + "title": "Type" + }, + "values": { "allOf": [ { - "$ref": "#/$defs/ServiceType" + "$ref": "#/$defs/StreamsAppV2Values" } ], - "default": "ClusterIP", - "description": "Service type." + "description": "streams-bootstrap-v2 Helm values" + }, + "version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "2.9.0", + "description": "Helm chart version", + "title": "Version" } }, - "title": "ServiceConfig", - "type": "object" - }, - "ServiceType": { - "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", - "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer", - "ExternalName" + "required": [ + "name", + "namespace", + "values", + "type" ], - "title": "ServiceType", - "type": "string" + "title": "SimpleInflateConnectors", + "type": "object" }, "StreamsApp": { "additionalProperties": true, @@ -2613,11 +3280,18 @@ "items": { "discriminator": { "mapping": { + "converter": "#/$defs/Converter", + "filter": "#/$defs/Filter", "helm-app": "#/$defs/HelmApp", "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", + "my-producer-app": "#/$defs/MyProducerApp", + "my-streams-app": "#/$defs/MyStreamsApp", "producer-app": "#/$defs/ProducerApp", "producer-app-v2": "#/$defs/ProducerAppV2", + "scheduled-producer": "#/$defs/ScheduledProducer", + "should-inflate": "#/$defs/ShouldInflate", + "simple-inflate-connectors": "#/$defs/SimpleInflateConnectors", "streams-app": "#/$defs/StreamsApp", "streams-app-v2": "#/$defs/StreamsAppV2" }, @@ -2644,6 +3318,27 @@ }, { "$ref": "#/$defs/StreamsAppV2" + }, + { + "$ref": "#/$defs/Converter" + }, + { + "$ref": "#/$defs/Filter" + }, + { + "$ref": "#/$defs/MyProducerApp" + }, + { + "$ref": "#/$defs/MyStreamsApp" + }, + { + "$ref": "#/$defs/ScheduledProducer" + }, + { + "$ref": "#/$defs/ShouldInflate" + }, + { + "$ref": "#/$defs/SimpleInflateConnectors" } ] }, From 3cc771eda9a76f7677323645fafc8baad198c1a8 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 17:02:28 +0100 Subject: [PATCH 15/47] Update files --- docs/docs/schema/defaults.json | 1015 +++++--------------------------- docs/docs/schema/pipeline.json | 1007 +++++-------------------------- kpops/config/__init__.py | 3 +- 3 files changed, 318 insertions(+), 1707 deletions(-) diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 05ee439be..ad8d833e1 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -1,104 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, "Effects": { "enum": [ "NoExecute", @@ -108,105 +9,6 @@ "title": "Effects", "type": "string" }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -999,231 +801,52 @@ "title": "KubernetesAppValues", "type": "object" }, - "MyProducerApp": { - "additionalProperties": true, - "description": "", + "Operation": { + "enum": [ + "Exists", + "Equal" + ], + "title": "Operation", + "type": "string" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" }, - "repo_config": { - "allOf": [ + "storage_class": { + "anyOf": [ { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.1", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyProducerApp", - "type": "object" - }, - "MyStreamsApp": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.1", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "Operation": { - "enum": [ - "Exists", - "Equal" - ], - "title": "Operation", - "type": "string" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" + "type": "string" + }, + { + "type": "null" } ], "default": null, @@ -1984,359 +1607,68 @@ }, "title": "PrometheusJMXExporterConfig", "type": "object" - }, - "ProtocolSchema": { - "description": "Represents the different Kubernetes protocols.\n\nhttps://kubernetes.io/docs/reference/networking/service-protocols/", - "enum": [ - "TCP", - "UDP", - "SCTP" - ], - "title": "ProtocolSchema", - "type": "string" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ResourceDefinition": { - "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", - "properties": { - "cpu": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ], - "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", - "title": "Cpu" - }, - "memory": { - "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", - "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", - "title": "Memory", - "type": "string" - } - }, - "required": [ - "cpu", - "memory" - ], - "title": "ResourceDefinition", - "type": "object" - }, - "Resources": { - "description": "Model representing the resource specifications for a Kubernetes container.", - "properties": { - "limits": { - "allOf": [ - { - "$ref": "#/$defs/ResourceDefinition" - } - ], - "description": "The maximum resource limits for the container." - }, - "requests": { - "allOf": [ - { - "$ref": "#/$defs/ResourceDefinition" - } - ], - "description": "The minimum resource requirements for the container." - } - }, - "required": [ - "requests", - "limits" - ], - "title": "Resources", - "type": "object" - }, - "RestartPolicy": { - "enum": [ - "Always", - "OnFailure", - "Never" - ], - "title": "RestartPolicy", - "type": "string" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppV2Values" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ScheduledProducer", - "type": "object" - }, - "ServiceConfig": { - "description": "Base model for configuring a service for the Kafka Streams application.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to create a service.", - "title": "Enabled", - "type": "boolean" - }, - "labels": { - "additionalProperties": { - "type": "string" - }, - "description": "Additional service labels.", - "title": "Labels", - "type": "object" - }, - "type": { - "allOf": [ - { - "$ref": "#/$defs/ServiceType" - } - ], - "default": "ClusterIP", - "description": "Service type." - } - }, - "title": "ServiceConfig", - "type": "object" - }, - "ServiceType": { - "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + }, + "ProtocolSchema": { + "description": "Represents the different Kubernetes protocols.\n\nhttps://kubernetes.io/docs/reference/networking/service-protocols/", "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer", - "ExternalName" + "TCP", + "UDP", + "SCTP" ], - "title": "ServiceType", + "title": "ProtocolSchema", "type": "string" }, - "ShouldInflate": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "should-inflate", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -2345,119 +1677,120 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ShouldInflate", + "title": "RepoAuthFlags", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "ResourceDefinition": { + "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", "properties": { - "from": { + "cpu": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { - "type": "null" + "type": "integer" } ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" + "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", + "title": "Cpu" }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", + "memory": { + "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", + "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", + "title": "Memory", "type": "string" - }, - "repo_config": { + } + }, + "required": [ + "cpu", + "memory" + ], + "title": "ResourceDefinition", + "type": "object" + }, + "Resources": { + "description": "Model representing the resource specifications for a Kubernetes container.", + "properties": { + "limits": { "allOf": [ { - "$ref": "#/$defs/HelmRepoConfig" + "$ref": "#/$defs/ResourceDefinition" } ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "The maximum resource limits for the container." }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, + "requests": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/ResourceDefinition" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "The minimum resource requirements for the container." + } + }, + "required": [ + "requests", + "limits" + ], + "title": "Resources", + "type": "object" + }, + "RestartPolicy": { + "enum": [ + "Always", + "OnFailure", + "Never" + ], + "title": "RestartPolicy", + "type": "string" + }, + "ServiceConfig": { + "description": "Base model for configuring a service for the Kafka Streams application.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to create a service.", + "title": "Enabled", + "type": "boolean" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Additional service labels.", + "title": "Labels", + "type": "object" }, - "values": { + "type": { "allOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" + "$ref": "#/$defs/ServiceType" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": "ClusterIP", + "description": "Service type." } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "ServiceConfig", "type": "object" }, + "ServiceType": { + "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer", + "ExternalName" + ], + "title": "ServiceType", + "type": "string" + }, "StreamsApp": { "additionalProperties": true, "description": "StreamsApp component that configures a streams-bootstrap app.", @@ -4045,12 +3378,6 @@ } }, "properties": { - "converter": { - "$ref": "#/$defs/Converter" - }, - "filter": { - "$ref": "#/$defs/Filter" - }, "helm-app": { "$ref": "#/$defs/HelmApp" }, @@ -4069,12 +3396,6 @@ "kubernetes-app": { "$ref": "#/$defs/KubernetesApp" }, - "my-producer-app": { - "$ref": "#/$defs/MyProducerApp" - }, - "my-streams-app": { - "$ref": "#/$defs/MyStreamsApp" - }, "pipeline-component": { "$ref": "#/$defs/PipelineComponent" }, @@ -4084,15 +3405,6 @@ "producer-app-v2": { "$ref": "#/$defs/ProducerAppV2" }, - "scheduled-producer": { - "$ref": "#/$defs/ScheduledProducer" - }, - "should-inflate": { - "$ref": "#/$defs/ShouldInflate" - }, - "simple-inflate-connectors": { - "$ref": "#/$defs/SimpleInflateConnectors" - }, "streams-app": { "$ref": "#/$defs/StreamsApp" }, @@ -4119,14 +3431,7 @@ "streams-bootstrap", "producer-app-v2", "streams-app-v2", - "streams-bootstrap-v2", - "converter", - "filter", - "my-producer-app", - "my-streams-app", - "scheduled-producer", - "should-inflate", - "simple-inflate-connectors" + "streams-bootstrap-v2" ], "title": "DefaultsSchema", "type": "object" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index b494ae478..0b466584f 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -1,104 +1,5 @@ { "$defs": { - "Converter": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "converter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Converter", - "type": "object" - }, "Effects": { "enum": [ "NoExecute", @@ -108,105 +9,6 @@ "title": "Effects", "type": "string" }, - "Filter": { - "additionalProperties": true, - "description": "Subsubclass of StreamsApp to test inheritance.", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "filter", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "Filter", - "type": "object" - }, "FromSection": { "additionalProperties": false, "description": "Holds multiple input topics.", @@ -706,231 +508,52 @@ "title": "KafkaSourceConnector", "type": "object" }, - "MyProducerApp": { - "additionalProperties": true, - "description": "", + "Operation": { + "enum": [ + "Exists", + "Equal" + ], + "title": "Operation", + "type": "string" + }, + "OutputTopicTypes": { + "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", + "enum": [ + "output", + "error" + ], + "title": "OutputTopicTypes", + "type": "string" + }, + "PersistenceConfig": { + "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" + "enabled": { + "default": false, + "description": "Whether to use a persistent volume to store the state of the streams app.\t", + "title": "Enabled", + "type": "boolean" }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", + "title": "Size" }, - "repo_config": { - "allOf": [ + "storage_class": { + "anyOf": [ { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-producer-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.1", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyProducerApp", - "type": "object" - }, - "MyStreamsApp": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "anyOf": [ - { - "$ref": "#/$defs/FromSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "my-streams-app", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/StreamsAppValues" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "default": "3.0.1", - "description": "Helm chart version", - "pattern": "^(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z]+(\\.[a-zA-Z]+)?)?$", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "MyStreamsApp", - "type": "object" - }, - "Operation": { - "enum": [ - "Exists", - "Equal" - ], - "title": "Operation", - "type": "string" - }, - "OutputTopicTypes": { - "description": "Types of output topic.\n\n- OUTPUT: output topic\n- ERROR: error topic", - "enum": [ - "output", - "error" - ], - "title": "OutputTopicTypes", - "type": "string" - }, - "PersistenceConfig": { - "description": "streams-bootstrap persistence configurations.\n\n:param enabled: Whether to use a persistent volume to store the state of the streams app.\n:param size: The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.\n:param storage_class: Storage class to use for the persistent volume.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to use a persistent volume to store the state of the streams app.\t", - "title": "Enabled", - "type": "boolean" - }, - "size": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The size of the PersistentVolume to allocate to each streams pod in the StatefulSet.", - "title": "Size" - }, - "storage_class": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" + "type": "string" + }, + { + "type": "null" } ], "default": null, @@ -1648,355 +1271,64 @@ "ProtocolSchema": { "description": "Represents the different Kubernetes protocols.\n\nhttps://kubernetes.io/docs/reference/networking/service-protocols/", "enum": [ - "TCP", - "UDP", - "SCTP" - ], - "title": "ProtocolSchema", - "type": "string" - }, - "RepoAuthFlags": { - "description": "Authorisation-related flags for `helm repo`.", - "properties": { - "ca_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", - "title": "Ca File" - }, - "cert_file": { - "anyOf": [ - { - "format": "path", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Path to SSL certificate file to identify HTTPS client", - "title": "Cert File" - }, - "insecure_skip_tls_verify": { - "default": false, - "description": "If true, Kubernetes API server's certificate will not be checked for validity", - "title": "Insecure Skip Tls Verify", - "type": "boolean" - }, - "password": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Password", - "title": "Password" - }, - "username": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Username", - "title": "Username" - } - }, - "title": "RepoAuthFlags", - "type": "object" - }, - "ResourceDefinition": { - "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", - "properties": { - "cpu": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ], - "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", - "title": "Cpu" - }, - "memory": { - "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", - "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", - "title": "Memory", - "type": "string" - } - }, - "required": [ - "cpu", - "memory" - ], - "title": "ResourceDefinition", - "type": "object" - }, - "Resources": { - "description": "Model representing the resource specifications for a Kubernetes container.", - "properties": { - "limits": { - "allOf": [ - { - "$ref": "#/$defs/ResourceDefinition" - } - ], - "description": "The maximum resource limits for the container." - }, - "requests": { - "allOf": [ - { - "$ref": "#/$defs/ResourceDefinition" - } - ], - "description": "The minimum resource requirements for the container." - } - }, - "required": [ - "requests", - "limits" - ], - "title": "Resources", - "type": "object" - }, - "RestartPolicy": { - "enum": [ - "Always", - "OnFailure", - "Never" - ], - "title": "RestartPolicy", - "type": "string" - }, - "ScheduledProducer": { - "additionalProperties": true, - "description": "", - "properties": { - "from": { - "default": null, - "description": "Producer doesn't support FromSection", - "title": "From", - "type": "null" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" - }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Topic(s) into which the component will write output" - }, - "type": { - "const": "scheduled-producer", - "title": "Type" - }, - "values": { - "allOf": [ - { - "$ref": "#/$defs/ProducerAppV2Values" - } - ], - "description": "streams-bootstrap Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" - } - }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ScheduledProducer", - "type": "object" - }, - "ServiceConfig": { - "description": "Base model for configuring a service for the Kafka Streams application.", - "properties": { - "enabled": { - "default": false, - "description": "Whether to create a service.", - "title": "Enabled", - "type": "boolean" - }, - "labels": { - "additionalProperties": { - "type": "string" - }, - "description": "Additional service labels.", - "title": "Labels", - "type": "object" - }, - "type": { - "allOf": [ - { - "$ref": "#/$defs/ServiceType" - } - ], - "default": "ClusterIP", - "description": "Service type." - } - }, - "title": "ServiceConfig", - "type": "object" - }, - "ServiceType": { - "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", - "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer", - "ExternalName" + "TCP", + "UDP", + "SCTP" ], - "title": "ServiceType", + "title": "ProtocolSchema", "type": "string" }, - "ShouldInflate": { - "additionalProperties": true, - "description": "", + "RepoAuthFlags": { + "description": "Authorisation-related flags for `helm repo`.", "properties": { - "from": { + "ca_file": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/$defs/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "Path to CA bundle file to verify certificates of HTTPS-enabled servers", + "title": "Ca File" }, - "to": { + "cert_file": { "anyOf": [ { - "$ref": "#/$defs/ToSection" + "format": "path", + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Topic(s) into which the component will write output" + "description": "Path to SSL certificate file to identify HTTPS client", + "title": "Cert File" }, - "type": { - "const": "should-inflate", - "title": "Type" + "insecure_skip_tls_verify": { + "default": false, + "description": "If true, Kubernetes API server's certificate will not be checked for validity", + "title": "Insecure Skip Tls Verify", + "type": "boolean" }, - "values": { - "allOf": [ + "password": { + "anyOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" + "type": "string" + }, + { + "type": "null" } ], - "description": "streams-bootstrap-v2 Helm values" + "default": null, + "description": "Password", + "title": "Password" }, - "version": { + "username": { "anyOf": [ { "type": "string" @@ -2005,119 +1337,120 @@ "type": "null" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": null, + "description": "Username", + "title": "Username" } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "ShouldInflate", + "title": "RepoAuthFlags", "type": "object" }, - "SimpleInflateConnectors": { - "additionalProperties": true, - "description": "", + "ResourceDefinition": { + "description": "Model representing the 'limits' or `request` section of Kubernetes resource specifications.", "properties": { - "from": { + "cpu": { "anyOf": [ { - "$ref": "#/$defs/FromSection" + "type": "string" }, { - "type": "null" + "type": "integer" } ], - "default": null, - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Kubernetes namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" + "description": "The maximum amount of CPU a container can use, expressed in milli CPUs (e.g., '300m').", + "title": "Cpu" }, - "prefix": { - "default": "${pipeline.name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", + "memory": { + "description": "The maximum amount of memory a container can use, with valid units such as 'Mi' or 'Gi' (e.g., '2G').", + "pattern": "^\\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$", + "title": "Memory", "type": "string" - }, - "repo_config": { + } + }, + "required": [ + "cpu", + "memory" + ], + "title": "ResourceDefinition", + "type": "object" + }, + "Resources": { + "description": "Model representing the resource specifications for a Kubernetes container.", + "properties": { + "limits": { "allOf": [ { - "$ref": "#/$defs/HelmRepoConfig" + "$ref": "#/$defs/ResourceDefinition" } ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component" + "description": "The maximum resource limits for the container." }, - "to": { - "anyOf": [ - { - "$ref": "#/$defs/ToSection" - }, + "requests": { + "allOf": [ { - "type": "null" + "$ref": "#/$defs/ResourceDefinition" } ], - "default": null, - "description": "Topic(s) into which the component will write output" + "description": "The minimum resource requirements for the container." + } + }, + "required": [ + "requests", + "limits" + ], + "title": "Resources", + "type": "object" + }, + "RestartPolicy": { + "enum": [ + "Always", + "OnFailure", + "Never" + ], + "title": "RestartPolicy", + "type": "string" + }, + "ServiceConfig": { + "description": "Base model for configuring a service for the Kafka Streams application.", + "properties": { + "enabled": { + "default": false, + "description": "Whether to create a service.", + "title": "Enabled", + "type": "boolean" }, - "type": { - "const": "simple-inflate-connectors", - "title": "Type" + "labels": { + "additionalProperties": { + "type": "string" + }, + "description": "Additional service labels.", + "title": "Labels", + "type": "object" }, - "values": { + "type": { "allOf": [ { - "$ref": "#/$defs/StreamsAppV2Values" - } - ], - "description": "streams-bootstrap-v2 Helm values" - }, - "version": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" + "$ref": "#/$defs/ServiceType" } ], - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version" + "default": "ClusterIP", + "description": "Service type." } }, - "required": [ - "name", - "namespace", - "values", - "type" - ], - "title": "SimpleInflateConnectors", + "title": "ServiceConfig", "type": "object" }, + "ServiceType": { + "description": "Represents the different Kubernetes service types.\n\nhttps://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer", + "ExternalName" + ], + "title": "ServiceType", + "type": "string" + }, "StreamsApp": { "additionalProperties": true, "description": "StreamsApp component that configures a streams-bootstrap app.", @@ -3280,18 +2613,11 @@ "items": { "discriminator": { "mapping": { - "converter": "#/$defs/Converter", - "filter": "#/$defs/Filter", "helm-app": "#/$defs/HelmApp", "kafka-sink-connector": "#/$defs/KafkaSinkConnector", "kafka-source-connector": "#/$defs/KafkaSourceConnector", - "my-producer-app": "#/$defs/MyProducerApp", - "my-streams-app": "#/$defs/MyStreamsApp", "producer-app": "#/$defs/ProducerApp", "producer-app-v2": "#/$defs/ProducerAppV2", - "scheduled-producer": "#/$defs/ScheduledProducer", - "should-inflate": "#/$defs/ShouldInflate", - "simple-inflate-connectors": "#/$defs/SimpleInflateConnectors", "streams-app": "#/$defs/StreamsApp", "streams-app-v2": "#/$defs/StreamsAppV2" }, @@ -3318,27 +2644,6 @@ }, { "$ref": "#/$defs/StreamsAppV2" - }, - { - "$ref": "#/$defs/Converter" - }, - { - "$ref": "#/$defs/Filter" - }, - { - "$ref": "#/$defs/MyProducerApp" - }, - { - "$ref": "#/$defs/MyStreamsApp" - }, - { - "$ref": "#/$defs/ScheduledProducer" - }, - { - "$ref": "#/$defs/ShouldInflate" - }, - { - "$ref": "#/$defs/SimpleInflateConnectors" } ] }, diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index ad8457213..7398d5e7c 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -73,7 +73,7 @@ class KafkaConnectConfig(BaseSettings): ) -class KpopsConfig(BaseSettings, use_enum_values=True): +class KpopsConfig(BaseSettings): """Global configuration for KPOps project.""" _instance: ClassVar[KpopsConfig | None] = PrivateAttr(None) @@ -129,6 +129,7 @@ class KpopsConfig(BaseSettings, use_enum_values=True): model_config = SettingsConfigDict( env_prefix=ENV_PREFIX, env_nested_delimiter="__", + use_enum_values=True, ) @classmethod From a7e3c0034ef6b028901bf00fdf50bdf4e988934d Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 17:07:48 +0100 Subject: [PATCH 16/47] Update files --- kpops/api/__init__.py | 1 + kpops/api/operation.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 39fc63198..5fd6c856d 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -48,6 +48,7 @@ def generate( :param filter_type: Whether `steps` should include/exclude the steps. :param environment: The environment to generate and deploy the pipeline to. :param verbose: Enable verbose printing. + :param operation_mode: How KPOps should operate. :return: Generated `Pipeline` object. """ kpops_config = KpopsConfig.create( diff --git a/kpops/api/operation.py b/kpops/api/operation.py index 7005008ae..6b7558398 100644 --- a/kpops/api/operation.py +++ b/kpops/api/operation.py @@ -1,9 +1,9 @@ from __future__ import annotations -from enum import StrEnum +import enum -class OperationMode(StrEnum): +class OperationMode(str, enum.Enum): ARGO = "argo" MANIFEST = "manifest" MANAGED = "managed" From 0ec82e2efb3a408e90c43f8932f446b4420f0e0a Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 2 Dec 2024 17:09:39 +0100 Subject: [PATCH 17/47] Update files --- kpops/manifests/argo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index 5a808e3fd..c1b114d65 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -15,7 +15,7 @@ def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: return helm_values -class ArgoHook(ArgoEnricher, enum.StrEnum): +class ArgoHook(ArgoEnricher, str, enum.Enum): POST_DELETE = "PostDelete" @property @@ -24,7 +24,7 @@ def key(self) -> str: return "argocd.argoproj.io/hook" -class ArgoSyncWave(ArgoEnricher, enum.StrEnum): +class ArgoSyncWave(ArgoEnricher, str, enum.Enum): SYNC_WAVE = "1" @property From af25f92ead299f0b94652fed730abe2dca9ac0a6 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 3 Dec 2024 10:33:46 +0100 Subject: [PATCH 18/47] address reviews --- docs/docs/user/references/cli-commands.md | 2 +- kpops/cli/main.py | 4 ++-- kpops/components/base_components/helm_app.py | 2 +- kpops/manifests/argo.py | 23 ++++++++++++++++--- .../resources/manifest-pipeline/defaults.yaml | 3 --- .../test_deploy_argo_mode/manifest.yaml | 4 ---- .../test_deploy_manifest_mode/manifest.yaml | 2 -- .../test_manifest/test_python_api/resources | 2 -- 8 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index abfc43a08..bbbd7cd3f 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -142,7 +142,7 @@ $ kpops init [OPTIONS] PATH **Options**: -* `--config-include-opt / --no-config-include-opt`: Whether to include non-required settings in the generated 'config.yaml' [default: no-config-include-opt] +* `--config-include-optional / --no-config-include-optional`: Whether to include non-required settings in the generated 'config.yaml' [default: no-config-include-optional] * `--help`: Show this message and exit. ## `kpops reset` diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 81267ff8e..0d6af160e 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -125,9 +125,9 @@ def parse_steps(steps: str | None) -> set[str] | None: @app.command(help="Initialize a new KPOps project.") def init( path: Path = PROJECT_PATH, - config_include_opt: bool = CONFIG_INCLUDE_OPTIONAL, + config_include_optional: bool = CONFIG_INCLUDE_OPTIONAL, ): - kpops.init(path, config_include_optional=config_include_opt) + kpops.init(path, config_include_optional=config_include_optional) @app.command( diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index f64ba1981..677cd57fb 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -147,7 +147,7 @@ def template_flags(self) -> HelmTemplateFlags: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoSyncWave.SYNC_WAVE.enrich(values) + values = ArgoSyncWave(1).enrich(values) return self.helm.template( self.helm_release_name, diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index c1b114d65..2f2ff2928 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -4,11 +4,15 @@ from typing_extensions import override -class ArgoEnricher(enum.Enum): +class ArgoEnricher: @property def key(self) -> str: return NotImplemented + @property + def value(self) -> str: + return NotImplemented + def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: annotations = helm_values.setdefault("annotations", {}) annotations[self.key] = self.value @@ -23,11 +27,24 @@ class ArgoHook(ArgoEnricher, str, enum.Enum): def key(self) -> str: return "argocd.argoproj.io/hook" + @property + @override + def value(self) -> str: + return self.POST_DELETE._value_ + + +class ArgoSyncWave(ArgoEnricher): + sync_wave: int -class ArgoSyncWave(ArgoEnricher, str, enum.Enum): - SYNC_WAVE = "1" + def __init__(self, sync_wave: int) -> None: + self.sync_wave = sync_wave @property @override def key(self) -> str: return "argocd.argoproj.io/sync-wave" + + @property + @override + def value(self) -> str: + return str(self.sync_wave) diff --git a/tests/pipeline/resources/manifest-pipeline/defaults.yaml b/tests/pipeline/resources/manifest-pipeline/defaults.yaml index 4f1c2ea2a..d24ae3b07 100644 --- a/tests/pipeline/resources/manifest-pipeline/defaults.yaml +++ b/tests/pipeline/resources/manifest-pipeline/defaults.yaml @@ -12,9 +12,6 @@ streams-app: # inherits from streams-bootstrap prometheus: jmx: enabled: false - kafka: - config: - large.message.id.generator: com.bakdata.kafka.MurmurHashIdGenerator to: topics: ${error_topic_name}: diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index 1263b18d1..e64bce0b3 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -119,8 +119,6 @@ spec: - env: - name: ENV_PREFIX value: APP_ - - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR - value: com.bakdata.kafka.MurmurHashIdGenerator - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS @@ -183,8 +181,6 @@ spec: env: - name: ENV_PREFIX value: APP_ - - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR - value: com.bakdata.kafka.MurmurHashIdGenerator - name: APP_BOOTSTRAP_SERVERS value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - name: APP_SCHEMA_REGISTRY_URL diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml index 62fe9eca2..f7086850d 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -68,8 +68,6 @@ spec: - env: - name: ENV_PREFIX value: APP_ - - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR - value: com.bakdata.kafka.MurmurHashIdGenerator - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index b7b09b725..723353c93 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -70,8 +70,6 @@ - env: - name: ENV_PREFIX value: APP_ - - name: KAFKA_LARGE_MESSAGE_ID_GENERATOR - value: com.bakdata.kafka.MurmurHashIdGenerator - name: APP_VOLATILE_GROUP_INSTANCE_ID value: 'true' - name: APP_BOOTSTRAP_SERVERS From 9e1938e19f1f1164f173ed47c0d41af0d0547bec Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 3 Dec 2024 10:39:32 +0100 Subject: [PATCH 19/47] add tests --- tests/manifest/__init__.py | 0 tests/manifest/test_argo_enricher.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tests/manifest/__init__.py create mode 100644 tests/manifest/test_argo_enricher.py diff --git a/tests/manifest/__init__.py b/tests/manifest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/manifest/test_argo_enricher.py b/tests/manifest/test_argo_enricher.py new file mode 100644 index 000000000..75f3b6ff3 --- /dev/null +++ b/tests/manifest/test_argo_enricher.py @@ -0,0 +1,56 @@ +from typing import Any + +import pytest + +from kpops.manifests.argo import ArgoHook, ArgoSyncWave + + +@pytest.fixture +def empty_manifest() -> dict[str, Any]: + return {} + + +@pytest.fixture +def manifest_with_annotations() -> dict[str, Any]: + return {"annotations": {"existing-annotation": "annotation-value"}} + + +def test_argo_hook_enrich_empty_manifest(empty_manifest: dict[str, Any]): + hook = ArgoHook.POST_DELETE + enriched_manifest = hook.enrich(empty_manifest) + assert enriched_manifest["annotations"][hook.key] == hook.value + assert len(enriched_manifest["annotations"]) == 1 + + +def test_argo_hook_enrich_existing_annotations( + manifest_with_annotations: dict[str, Any], +): + hook = ArgoHook.POST_DELETE + enriched_manifest = hook.enrich(manifest_with_annotations) + assert enriched_manifest["annotations"][hook.key] == hook.value + assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" + + +def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): + sync_wave = ArgoSyncWave(sync_wave=1) + enriched_manifest = sync_wave.enrich(empty_manifest) + assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value + assert len(enriched_manifest["annotations"]) == 1 + + +def test_argo_sync_wave_enrich_existing_annotations( + manifest_with_annotations: dict[str, Any], +): + sync_wave = ArgoSyncWave(sync_wave=2) + enriched_manifest = sync_wave.enrich(manifest_with_annotations) + assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value + assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" + + +def test_argo_sync_wave_multiple_enrichments(empty_manifest: dict[str, Any]): + sync_wave_1 = ArgoSyncWave(sync_wave=1) + sync_wave_2 = ArgoSyncWave(sync_wave=2) + enriched_manifest = sync_wave_1.enrich(empty_manifest) + enriched_manifest = sync_wave_2.enrich(enriched_manifest) + assert enriched_manifest["annotations"][sync_wave_1.key] == sync_wave_2.value + assert len(enriched_manifest["annotations"]) == 1 From 9a4dc3503d8b7fa2fde5f8b03311615d4f7bcbea Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 09:57:24 +0100 Subject: [PATCH 20/47] Update files --- kpops/components/base_components/helm_app.py | 2 +- kpops/manifests/argo.py | 12 +++++------- tests/manifest/test_argo_enricher.py | 8 ++++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 677cd57fb..8c875a1a0 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -147,7 +147,7 @@ def template_flags(self) -> HelmTemplateFlags: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoSyncWave(1).enrich(values) + values = ArgoSyncWave(**{"SyncWave": 1}).enrich(values) return self.helm.template( self.helm_release_name, diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index 2f2ff2928..70fcdb392 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -1,17 +1,18 @@ import enum from typing import Any +from pydantic import BaseModel, Field from typing_extensions import override class ArgoEnricher: @property def key(self) -> str: - return NotImplemented + raise NotImplementedError @property def value(self) -> str: - return NotImplemented + raise NotImplementedError def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: annotations = helm_values.setdefault("annotations", {}) @@ -33,11 +34,8 @@ def value(self) -> str: return self.POST_DELETE._value_ -class ArgoSyncWave(ArgoEnricher): - sync_wave: int - - def __init__(self, sync_wave: int) -> None: - self.sync_wave = sync_wave +class ArgoSyncWave(BaseModel, ArgoEnricher): + sync_wave: int = Field(default=0, alias="SyncWave") @property @override diff --git a/tests/manifest/test_argo_enricher.py b/tests/manifest/test_argo_enricher.py index 75f3b6ff3..8f9f4399d 100644 --- a/tests/manifest/test_argo_enricher.py +++ b/tests/manifest/test_argo_enricher.py @@ -32,7 +32,7 @@ def test_argo_hook_enrich_existing_annotations( def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): - sync_wave = ArgoSyncWave(sync_wave=1) + sync_wave = ArgoSyncWave(**{"SyncWave": 1}) enriched_manifest = sync_wave.enrich(empty_manifest) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert len(enriched_manifest["annotations"]) == 1 @@ -41,15 +41,15 @@ def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): def test_argo_sync_wave_enrich_existing_annotations( manifest_with_annotations: dict[str, Any], ): - sync_wave = ArgoSyncWave(sync_wave=2) + sync_wave = ArgoSyncWave(**{"SyncWave": 2}) enriched_manifest = sync_wave.enrich(manifest_with_annotations) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" def test_argo_sync_wave_multiple_enrichments(empty_manifest: dict[str, Any]): - sync_wave_1 = ArgoSyncWave(sync_wave=1) - sync_wave_2 = ArgoSyncWave(sync_wave=2) + sync_wave_1 = ArgoSyncWave(**{"SyncWave": 1}) + sync_wave_2 = ArgoSyncWave(**{"SyncWave": 2}) enriched_manifest = sync_wave_1.enrich(empty_manifest) enriched_manifest = sync_wave_2.enrich(enriched_manifest) assert enriched_manifest["annotations"][sync_wave_1.key] == sync_wave_2.value From 64cc384834d00d2da460bed74f98a957cb386eae Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 10:02:26 +0100 Subject: [PATCH 21/47] refactor argo sync wave --- kpops/components/base_components/helm_app.py | 2 +- kpops/manifests/argo.py | 12 +++++++++++- tests/manifest/test_argo_enricher.py | 8 ++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 8c875a1a0..231ff38a9 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -147,7 +147,7 @@ def template_flags(self) -> HelmTemplateFlags: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoSyncWave(**{"SyncWave": 1}).enrich(values) + values = ArgoSyncWave.create(1).enrich(values) return self.helm.template( self.helm_release_name, diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index 70fcdb392..d58cc5f05 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -1,9 +1,15 @@ +from __future__ import annotations + import enum -from typing import Any from pydantic import BaseModel, Field from typing_extensions import override +try: + from typing import Any, Self # pyright: ignore[reportAttributeAccessIssue] +except ImportError: + from typing_extensions import Self + class ArgoEnricher: @property @@ -46,3 +52,7 @@ def key(self) -> str: @override def value(self) -> str: return str(self.sync_wave) + + @classmethod + def create(cls, sync_wave: int) -> Self: + return cls(**{"SyncWave": sync_wave}) diff --git a/tests/manifest/test_argo_enricher.py b/tests/manifest/test_argo_enricher.py index 8f9f4399d..2c7c01ff0 100644 --- a/tests/manifest/test_argo_enricher.py +++ b/tests/manifest/test_argo_enricher.py @@ -32,7 +32,7 @@ def test_argo_hook_enrich_existing_annotations( def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): - sync_wave = ArgoSyncWave(**{"SyncWave": 1}) + sync_wave = ArgoSyncWave().create(1) enriched_manifest = sync_wave.enrich(empty_manifest) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert len(enriched_manifest["annotations"]) == 1 @@ -41,15 +41,15 @@ def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): def test_argo_sync_wave_enrich_existing_annotations( manifest_with_annotations: dict[str, Any], ): - sync_wave = ArgoSyncWave(**{"SyncWave": 2}) + sync_wave = ArgoSyncWave.create(2) enriched_manifest = sync_wave.enrich(manifest_with_annotations) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" def test_argo_sync_wave_multiple_enrichments(empty_manifest: dict[str, Any]): - sync_wave_1 = ArgoSyncWave(**{"SyncWave": 1}) - sync_wave_2 = ArgoSyncWave(**{"SyncWave": 2}) + sync_wave_1 = ArgoSyncWave().create(1) + sync_wave_2 = ArgoSyncWave.create(2) enriched_manifest = sync_wave_1.enrich(empty_manifest) enriched_manifest = sync_wave_2.enrich(enriched_manifest) assert enriched_manifest["annotations"][sync_wave_1.key] == sync_wave_2.value From 15fe7177117e037df7d33a74368790321962d70d Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 10:10:22 +0100 Subject: [PATCH 22/47] Manifest toSection with Strimzi Kafka topic --- kpops/components/streams_bootstrap/base.py | 21 ++++++ .../streams_bootstrap/strimzi/__init__.py | 0 .../streams_bootstrap/strimzi/kafka_topic.py | 64 ++++++++++++++++ .../test_deploy_argo_mode/manifest.yaml | 73 +++++++++++++++++++ .../test_deploy_manifest_mode/manifest.yaml | 73 +++++++++++++++++++ .../test_manifest/test_python_api/resources | 61 ++++++++++++++++ .../test_streams_bootstrap/manifest.yaml | 73 +++++++++++++++++++ 7 files changed, 365 insertions(+) create mode 100644 kpops/components/streams_bootstrap/strimzi/__init__.py create mode 100644 kpops/components/streams_bootstrap/strimzi/kafka_topic.py diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index c7090e830..44dc514fd 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -7,11 +7,15 @@ import pydantic from pydantic import Field +from typing_extensions import override from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.models.resource import Resource +from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.model import StreamsBootstrapValues +from kpops.components.streams_bootstrap.strimzi.kafka_topic import StrimziKafkaTopic from kpops.utils.docstring import describe_attr if TYPE_CHECKING: @@ -81,3 +85,20 @@ def warning_for_latest_image_tag(self) -> Self: f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag." ) return self + + @override + def manifest_deploy(self) -> Resource: + resource = super().manifest_deploy() + if self.to: + self.extend_with_topics(self.to.kafka_topics, resource) + + return resource + + def extend_with_topics(self, kafka_topics: list[KafkaTopic], resource: Resource): + topics = [] + for topic in kafka_topics: + strimzi_topic = StrimziKafkaTopic.create_strimzi_topic( + topic, self.values.kafka.bootstrap_servers + ) + topics.append(strimzi_topic.model_dump()) + resource.extend(topics) diff --git a/kpops/components/streams_bootstrap/strimzi/__init__.py b/kpops/components/streams_bootstrap/strimzi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kpops/components/streams_bootstrap/strimzi/kafka_topic.py b/kpops/components/streams_bootstrap/strimzi/kafka_topic.py new file mode 100644 index 000000000..f0d92ff95 --- /dev/null +++ b/kpops/components/streams_bootstrap/strimzi/kafka_topic.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic import BaseModel, Field, model_validator +from typing_extensions import Any, override + +from kpops.components.common.topic import KafkaTopic + +if TYPE_CHECKING: + try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] + except ImportError: + from typing_extensions import Self + + +# Define the Pydantic model for the spec and metadata +class TopicSpec(BaseModel): + partitions: int = Field(default=1) + replicas: int = Field(default=1) + config: dict[str, str | int] = Field(default_factory=dict) + + @model_validator(mode="before") + @classmethod + def set_defaults_if_none(cls, values: Any) -> Any: + if values.get("partitions") is None: + values["partitions"] = 1 + if values.get("replicas") is None: + values["replicas"] = 1 + return values + + +class Metadata(BaseModel): + name: str + namespace: str | None = None + labels: dict[str, str] + + +class StrimziKafkaTopic(BaseModel): + api_version: str = Field(default="kafka.strimzi.io/v1beta2", alias="apiVersion") + kind: str = Field(default="KafkaTopic") + metadata: Metadata + spec: TopicSpec + + @override + def model_dump(self, **_: Any) -> dict[str, Any]: + return super().model_dump( + by_alias=True, exclude_none=True, exclude_defaults=False + ) + + @classmethod + def create_strimzi_topic(cls, topic: KafkaTopic, bootstrap_servers: str) -> Self: + metadata = { + "name": topic.name, + "labels": { + "strimzi.io/cluster": bootstrap_servers, + }, + } + spec = { + "partitions": topic.config.partitions_count, + "replicas": topic.config.replication_factor, + "config": topic.config.configs, + } + return cls(metadata=Metadata(**metadata), spec=TopicSpec(**spec)) diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index e64bce0b3..c892993ba 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -43,6 +43,30 @@ spec: memory: 300Mi restartPolicy: OnFailure +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-producer-app-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-producer-app-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + --- apiVersion: batch/v1 kind: Job @@ -156,6 +180,55 @@ spec: memory: 300Mi terminationGracePeriodSeconds: 300 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-error-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: resources-manifest-pipeline-my-streams-app-error +spec: + config: + cleanup.policy: compact,delete + partitions: 1 + replicas: 1 + --- apiVersion: batch/v1 kind: Job diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml index f7086850d..7041aa03a 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -41,6 +41,30 @@ spec: memory: 300Mi restartPolicy: OnFailure +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-producer-app-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-producer-app-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + --- apiVersion: apps/v1 kind: Deployment @@ -105,3 +129,52 @@ spec: memory: 300Mi terminationGracePeriodSeconds: 300 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-error-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: resources-manifest-pipeline-my-streams-app-error +spec: + config: + cleanup.policy: compact,delete + partitions: 1 + replicas: 1 + diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index 723353c93..936c7e07f 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -41,6 +41,26 @@ cpu: 200m memory: 300Mi restartPolicy: OnFailure +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-producer-app-output-topic + spec: + config: {} + partitions: 1 + replicas: 1 +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-producer-app-topic-output + spec: + config: {} + partitions: 1 + replicas: 1 --- - !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest data: @@ -106,3 +126,44 @@ cpu: 200m memory: 300Mi terminationGracePeriodSeconds: 300 +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-output-topic + spec: + config: {} + partitions: 1 + replicas: 1 +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-error-topic + spec: + config: {} + partitions: 1 + replicas: 1 +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-topic-output + spec: + config: {} + partitions: 1 + replicas: 1 +- apiVersion: kafka.strimzi.io/v1beta2 + kind: KafkaTopic + metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: resources-manifest-pipeline-my-streams-app-error + spec: + config: + cleanup.policy: compact,delete + partitions: 1 + replicas: 1 diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml index 075777bf9..04e89d82b 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml @@ -70,6 +70,30 @@ spec: successfulJobsHistoryLimit: 1 suspend: false +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-producer-app-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-producer-app-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + --- apiVersion: v1 data: @@ -209,3 +233,52 @@ spec: name: resources-streams-bootstrap-my-streams-app name: config +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-error-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: resources-streams-bootstrap-my-streams-app-error +spec: + config: + cleanup.policy: compact,delete + partitions: 1 + replicas: 1 + From 0c62a53d1c9e39ef88c0dc0b3fb3672da115203b Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 11:36:20 +0100 Subject: [PATCH 23/47] Address reviews --- kpops/components/base_components/helm_app.py | 5 +-- .../producer/producer_app.py | 5 +-- .../streams_bootstrap/streams/streams_app.py | 5 +-- kpops/manifests/argo.py | 34 +++++-------------- tests/manifest/test_argo_enricher.py | 32 +++++++++++------ 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 231ff38a9..a2658b0e3 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -29,7 +29,7 @@ ) from kpops.components.base_components.models.resource import Resource from kpops.config import get_config -from kpops.manifests.argo import ArgoSyncWave +from kpops.manifests.argo import ArgoSyncWave, enrich_annotations from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -147,7 +147,8 @@ def template_flags(self) -> HelmTemplateFlags: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoSyncWave.create(1).enrich(values) + sync_wave = ArgoSyncWave(sync_wave=1) + values = enrich_annotations(values, sync_wave.key, sync_wave.value) return self.helm.template( self.helm_release_name, diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index cc8dd4448..c31393a19 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -19,7 +19,7 @@ from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML -from kpops.manifests.argo import ArgoHook +from kpops.manifests.argo import ArgoHook, enrich_annotations from kpops.utils.docstring import describe_attr log = logging.getLogger("ProducerApp") @@ -39,7 +39,8 @@ def helm_chart(self) -> str: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoHook.POST_DELETE.enrich(values) + post_delete = ArgoHook.POST_DELETE + values = enrich_annotations(values, post_delete.key, post_delete.value) return self.helm.template( self.helm_release_name, self.helm_chart, diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 213134445..39677bfe2 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -19,7 +19,7 @@ ) from kpops.config import get_config from kpops.const.file_type import DEFAULTS_YAML, PIPELINE_YAML -from kpops.manifests.argo import ArgoHook +from kpops.manifests.argo import ArgoHook, enrich_annotations from kpops.utils.docstring import describe_attr log = logging.getLogger("StreamsApp") @@ -52,7 +52,8 @@ async def clean(self, dry_run: bool) -> None: def manifest_deploy(self) -> Resource: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: - values = ArgoHook.POST_DELETE.enrich(values) + post_delete = ArgoHook.POST_DELETE + values = enrich_annotations(values, post_delete.key, post_delete.value) return self.helm.template( self.helm_release_name, self.helm_chart, diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index d58cc5f05..2ddeec7d4 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -3,7 +3,6 @@ import enum from pydantic import BaseModel, Field -from typing_extensions import override try: from typing import Any, Self # pyright: ignore[reportAttributeAccessIssue] @@ -11,45 +10,30 @@ from typing_extensions import Self -class ArgoEnricher: - @property - def key(self) -> str: - raise NotImplementedError - - @property - def value(self) -> str: - raise NotImplementedError +def enrich_annotations( + helm_values: dict[str, Any], key: str, value: str +) -> dict[str, Any]: + annotations = helm_values.setdefault("annotations", {}) + annotations[key] = value + return helm_values - def enrich(self, helm_values: dict[str, Any]) -> dict[str, Any]: - annotations = helm_values.setdefault("annotations", {}) - annotations[self.key] = self.value - return helm_values - -class ArgoHook(ArgoEnricher, str, enum.Enum): +class ArgoHook(str, enum.Enum): POST_DELETE = "PostDelete" @property - @override def key(self) -> str: return "argocd.argoproj.io/hook" - @property - @override - def value(self) -> str: - return self.POST_DELETE._value_ - -class ArgoSyncWave(BaseModel, ArgoEnricher): - sync_wave: int = Field(default=0, alias="SyncWave") +class ArgoSyncWave(BaseModel): + sync_wave: int = Field(default=0, serialization_alias="SyncWave") @property - @override def key(self) -> str: return "argocd.argoproj.io/sync-wave" @property - @override def value(self) -> str: return str(self.sync_wave) diff --git a/tests/manifest/test_argo_enricher.py b/tests/manifest/test_argo_enricher.py index 2c7c01ff0..0ee4165a1 100644 --- a/tests/manifest/test_argo_enricher.py +++ b/tests/manifest/test_argo_enricher.py @@ -2,7 +2,7 @@ import pytest -from kpops.manifests.argo import ArgoHook, ArgoSyncWave +from kpops.manifests.argo import ArgoHook, ArgoSyncWave, enrich_annotations @pytest.fixture @@ -17,7 +17,7 @@ def manifest_with_annotations() -> dict[str, Any]: def test_argo_hook_enrich_empty_manifest(empty_manifest: dict[str, Any]): hook = ArgoHook.POST_DELETE - enriched_manifest = hook.enrich(empty_manifest) + enriched_manifest = enrich_annotations(empty_manifest, hook.key, hook.value) assert enriched_manifest["annotations"][hook.key] == hook.value assert len(enriched_manifest["annotations"]) == 1 @@ -26,14 +26,18 @@ def test_argo_hook_enrich_existing_annotations( manifest_with_annotations: dict[str, Any], ): hook = ArgoHook.POST_DELETE - enriched_manifest = hook.enrich(manifest_with_annotations) + enriched_manifest = enrich_annotations( + manifest_with_annotations, hook.key, hook.value + ) assert enriched_manifest["annotations"][hook.key] == hook.value assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): - sync_wave = ArgoSyncWave().create(1) - enriched_manifest = sync_wave.enrich(empty_manifest) + sync_wave = ArgoSyncWave(sync_wave=1) + enriched_manifest = enrich_annotations( + empty_manifest, sync_wave.key, sync_wave.value + ) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert len(enriched_manifest["annotations"]) == 1 @@ -41,16 +45,22 @@ def test_argo_sync_wave_enrich_empty_manifest(empty_manifest): def test_argo_sync_wave_enrich_existing_annotations( manifest_with_annotations: dict[str, Any], ): - sync_wave = ArgoSyncWave.create(2) - enriched_manifest = sync_wave.enrich(manifest_with_annotations) + sync_wave = ArgoSyncWave(sync_wave=2) + enriched_manifest = enrich_annotations( + manifest_with_annotations, sync_wave.key, sync_wave.value + ) assert enriched_manifest["annotations"][sync_wave.key] == sync_wave.value assert enriched_manifest["annotations"]["existing-annotation"] == "annotation-value" def test_argo_sync_wave_multiple_enrichments(empty_manifest: dict[str, Any]): - sync_wave_1 = ArgoSyncWave().create(1) - sync_wave_2 = ArgoSyncWave.create(2) - enriched_manifest = sync_wave_1.enrich(empty_manifest) - enriched_manifest = sync_wave_2.enrich(enriched_manifest) + sync_wave_1 = ArgoSyncWave(sync_wave=1) + sync_wave_2 = ArgoSyncWave(sync_wave=2) + enriched_manifest = enrich_annotations( + empty_manifest, sync_wave_1.key, sync_wave_1.value + ) + enriched_manifest = enrich_annotations( + enriched_manifest, sync_wave_2.key, sync_wave_2.value + ) assert enriched_manifest["annotations"][sync_wave_1.key] == sync_wave_2.value assert len(enriched_manifest["annotations"]) == 1 From ab5b74e01d9a83a4cfee361d50b63e3fb71823d1 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 13:27:24 +0100 Subject: [PATCH 24/47] Address reviews --- kpops/manifests/argo.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/kpops/manifests/argo.py b/kpops/manifests/argo.py index 2ddeec7d4..3ecabdaf9 100644 --- a/kpops/manifests/argo.py +++ b/kpops/manifests/argo.py @@ -1,13 +1,9 @@ from __future__ import annotations import enum +from typing import Any -from pydantic import BaseModel, Field - -try: - from typing import Any, Self # pyright: ignore[reportAttributeAccessIssue] -except ImportError: - from typing_extensions import Self +from pydantic import BaseModel def enrich_annotations( @@ -27,7 +23,7 @@ def key(self) -> str: class ArgoSyncWave(BaseModel): - sync_wave: int = Field(default=0, serialization_alias="SyncWave") + sync_wave: int = 0 @property def key(self) -> str: @@ -36,7 +32,3 @@ def key(self) -> str: @property def value(self) -> str: return str(self.sync_wave) - - @classmethod - def create(cls, sync_wave: int) -> Self: - return cls(**{"SyncWave": sync_wave}) From 8838bfabb6364b5acf83c365e10bf687c54381d5 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 14:46:01 +0100 Subject: [PATCH 25/47] Define pydantic model to representig Kubernetes manifest --- kpops/api/__init__.py | 4 +- kpops/cli/main.py | 2 +- kpops/component_handlers/helm_wrapper/helm.py | 8 +- .../helm_wrapper/helm_diff.py | 17 ++- .../component_handlers/helm_wrapper/model.py | 2 +- .../component_handlers/helm_wrapper/utils.py | 2 +- kpops/component_handlers/kubernetes/model.py | 30 ---- kpops/components/base_components/helm_app.py | 8 +- .../base_components/models/resource.py | 5 - .../base_components/pipeline_component.py | 10 +- kpops/components/common/kubernetes_model.py | 81 ++++++++++- .../producer/producer_app.py | 6 +- .../streams_bootstrap/streams/streams_app.py | 8 +- .../helm_wrapper/test_dry_run_handler.py | 27 +++- .../helm_wrapper/test_helm_diff.py | 137 ++++++++++++++++-- .../helm_wrapper/test_helm_wrapper.py | 67 ++++++--- tests/components/test_helm_app.py | 2 +- .../test_manifest/test_python_api/resources | 92 +++++++++--- 18 files changed, 383 insertions(+), 125 deletions(-) delete mode 100644 kpops/component_handlers/kubernetes/model.py delete mode 100644 kpops/components/base_components/models/resource.py diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 5fd6c856d..7c6f22a20 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -16,7 +16,7 @@ from kpops.component_handlers.schema_handler.schema_handler import SchemaHandler from kpops.component_handlers.topic.handler import TopicHandler from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper -from kpops.components.base_components.models.resource import Resource +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.config import KpopsConfig from kpops.pipeline import ( Pipeline, @@ -79,7 +79,7 @@ def manifest_deploy( environment: str | None = None, verbose: bool = True, operation_mode: OperationMode = OperationMode.MANIFEST, -) -> Iterator[Resource]: +) -> Iterator[list[KubernetesManifest]]: pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 0d6af160e..27afb1d94 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -227,7 +227,7 @@ def deploy( ) for resource in resources: for rendered_manifest in resource: - print_yaml(rendered_manifest) + print_yaml(rendered_manifest.model_dump()) @app.command(help="Destroy pipeline steps") diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 3fd38df20..1968a0c71 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -20,13 +20,11 @@ RepoAuthFlags, Version, ) -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.components.common.kubernetes_model import KubernetesManifest if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from kpops.components.base_components.models.resource import Resource - log = logging.getLogger("Helm") @@ -162,7 +160,7 @@ def template( namespace: str, values: dict[str, Any], flags: HelmTemplateFlags | None = None, - ) -> Resource: + ) -> list[KubernetesManifest]: """From Helm: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -192,6 +190,8 @@ def template( ] command.extend(flags.to_command()) output = self.__execute(command) + if output == "": + return [] manifests = KubernetesManifest.from_yaml(output) return list(manifests) diff --git a/kpops/component_handlers/helm_wrapper/helm_diff.py b/kpops/component_handlers/helm_wrapper/helm_diff.py index e90edc433..dd6d93cef 100644 --- a/kpops/component_handlers/helm_wrapper/helm_diff.py +++ b/kpops/component_handlers/helm_wrapper/helm_diff.py @@ -1,8 +1,8 @@ import logging from collections.abc import Iterable, Iterator +from typing import Any from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest from kpops.utils.dict_differ import Change, render_diff log = logging.getLogger("HelmDiff") @@ -16,7 +16,7 @@ def __init__(self, config: HelmDiffConfig) -> None: def calculate_changes( current_release: Iterable[HelmTemplate], new_release: Iterable[HelmTemplate], - ) -> Iterator[Change[KubernetesManifest, KubernetesManifest]]: + ) -> Iterator[Change[dict[str, Any], dict[str, Any]]]: """Compare 2 releases and generate a Change object for each difference. :param current_release: Iterable containing HelmTemplate objects for the current release @@ -32,13 +32,16 @@ def calculate_changes( # get corresponding dry-run release new_resource = new_release_index.pop(current_resource.filepath, None) yield Change( - current_resource.manifest, - new_resource.manifest if new_resource else KubernetesManifest(), + current_resource.manifest.model_dump(), + new_resource.manifest.model_dump() if new_resource else {}, ) # collect added files for new_resource in new_release_index.values(): - yield Change(KubernetesManifest(), new_resource.manifest) + yield Change( + {}, + new_resource.manifest.model_dump(mode="json"), + ) def log_helm_diff( self, @@ -48,8 +51,8 @@ def log_helm_diff( ) -> None: for change in self.calculate_changes(current_release, new_release): if diff := render_diff( - change.old_value.data, - change.new_value.data, + change.old_value, + change.new_value, ignore=self.config.ignore, ): logger.info("\n" + diff) diff --git a/kpops/component_handlers/helm_wrapper/model.py b/kpops/component_handlers/helm_wrapper/model.py index b81328e46..b61258b59 100644 --- a/kpops/component_handlers/helm_wrapper/model.py +++ b/kpops/component_handlers/helm_wrapper/model.py @@ -6,7 +6,7 @@ from typing_extensions import override from kpops.component_handlers.helm_wrapper.exception import ParseError -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index aa618f6a1..244242258 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -1,5 +1,5 @@ -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.component_handlers.kubernetes.utils import trim +from kpops.components.common.kubernetes_model import K8S_LABEL_MAX_LEN RELEASE_NAME_MAX_LEN = 53 diff --git a/kpops/component_handlers/kubernetes/model.py b/kpops/component_handlers/kubernetes/model.py deleted file mode 100644 index 5970de1bf..000000000 --- a/kpops/component_handlers/kubernetes/model.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -import json -from collections import UserDict -from collections.abc import Iterator - -import yaml - -from kpops.utils.types import JsonType - -K8S_LABEL_MAX_LEN = 63 - - -class KubernetesManifest(UserDict[str, JsonType]): - """Representation of a Kubernetes API object as YAML/JSON mapping.""" - - @classmethod - def from_yaml( - cls, /, content: str - ) -> Iterator[KubernetesManifest]: # TODO: typing.Self for Python 3.11+ - manifests: Iterator[dict[str, JsonType]] = yaml.load_all(content, yaml.Loader) - for manifest in manifests: - yield cls(manifest) - - @classmethod - def from_json( - cls, /, content: str - ) -> KubernetesManifest: # TODO: typing.Self for Python 3.11+ - manifest: dict[str, JsonType] = json.loads(content) - return cls(manifest) diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index a2658b0e3..9b229bd9b 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -22,12 +22,14 @@ create_helm_name_override, create_helm_release_name, ) -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.kubernetes_app import ( KubernetesApp, KubernetesAppValues, ) -from kpops.components.base_components.models.resource import Resource +from kpops.components.common.kubernetes_model import ( + K8S_LABEL_MAX_LEN, + KubernetesManifest, +) from kpops.config import get_config from kpops.manifests.argo import ArgoSyncWave, enrich_annotations from kpops.utils.colorify import magentaify @@ -144,7 +146,7 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: sync_wave = ArgoSyncWave(sync_wave=1) diff --git a/kpops/components/base_components/models/resource.py b/kpops/components/base_components/models/resource.py deleted file mode 100644 index e6867081d..000000000 --- a/kpops/components/base_components/models/resource.py +++ /dev/null @@ -1,5 +0,0 @@ -from collections.abc import Mapping -from typing import Any, TypeAlias - -# representation of final resource for component, e.g. a list of Kubernetes manifests -Resource: TypeAlias = list[Mapping[str, Any]] diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 28445fe17..6ae4e71e5 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -13,10 +13,10 @@ FromTopic, InputTopicTypes, ) -from kpops.components.base_components.models.resource import Resource from kpops.components.base_components.models.to_section import ( ToSection, ) +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, @@ -229,19 +229,19 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: """Render Kubernetes manifests for deploy.""" return [] - def manifest_destroy(self) -> Resource: + def manifest_destroy(self) -> list[KubernetesManifest]: """Render Kubernetes manifests resources for destroy.""" return [] - def manifest_reset(self) -> Resource: + def manifest_reset(self) -> list[KubernetesManifest]: """Render Kubernetes manifests resources for reset.""" return [] - def manifest_clean(self) -> Resource: + def manifest_clean(self) -> list[KubernetesManifest]: """Render Kubernetes manifests resources for clean.""" return [] diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index d78a197d0..bb9d35941 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -1,12 +1,91 @@ import enum +from collections.abc import Iterator +from typing import Any -from pydantic import Field +import yaml +from pydantic import ConfigDict, Field +from typing_extensions import override from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel +try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] +except ImportError: + from typing_extensions import Self # Matches plain integer or numbers with valid suffixes: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory MEMORY_PATTERN = r"^\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$" +K8S_LABEL_MAX_LEN = 63 + + +class ManagedFieldsEntry(DescConfigModel): + # Define this class based on its actual structure + fields: dict[str, Any] | None = None + + +class OwnerReference(DescConfigModel): + # Define this class based on its actual structure + apiVersion: str + kind: str + name: str + uid: str + controller: bool | None = None + blockOwnerDeletion: bool | None = None + + +class ObjectMeta(DescConfigModel): + """Metadata for all Kubernetes objects.""" + + annotations: dict[str, str] | None = None + creationTimestamp: str | None = Field( + None, description="Timestamp in RFC3339 format" + ) + deletionGracePeriodSeconds: int | None = None + deletionTimestamp: str | None = Field( + None, description="Timestamp in RFC3339 format" + ) + finalizers: list[str] | None = None + generateName: str | None = None + generation: int | None = None + labels: dict[str, str] | None = None + managedFields: list[ManagedFieldsEntry] | None = None + name: str | None = None + namespace: str | None = None + ownerReferences: list[OwnerReference] | None = None + resourceVersion: str | None = None + selfLink: str | None = Field( + None, + description="Deprecated field, not populated by Kubernetes in modern versions", + ) + uid: str | None = None + model_config = ConfigDict(extra="allow") + + +class KubernetesManifest(DescConfigModel): + # TODO: this is not working without the 'alias' + api_version: str = Field(alias="apiVersion") + kind: str + metadata: ObjectMeta + + model_config = ConfigDict(extra="allow") + + @classmethod + def from_yaml( + cls, /, content: str + ) -> Iterator[Self]: # TODO: typing.Self for Python 3.11+ + manifests: Iterator[dict[str, Any]] = yaml.load_all(content, yaml.Loader) + for manifest in manifests: + yield cls(**manifest) + + @override + def model_dump(self, **_: Any) -> dict[str, Any]: + return super().model_dump( + mode="json", + by_alias=True, + exclude_none=True, + exclude_defaults=True, + exclude_unset=True, + ) class ServiceType(str, enum.Enum): diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index c31393a19..4e772f5a6 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -6,8 +6,8 @@ from kpops.api import OperationMode from kpops.components.base_components.kafka_app import KafkaAppCleaner -from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.components.common.topic import ( KafkaTopic, OutputTopicTypes, @@ -36,7 +36,7 @@ def helm_chart(self) -> str: ) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE @@ -144,7 +144,7 @@ async def clean(self, dry_run: bool) -> None: await super().clean(dry_run) await self._cleaner.clean(dry_run) - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: manifests = super().manifest_deploy() operation_mode = get_config().operation_mode diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 39677bfe2..be5999a0a 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -8,8 +8,8 @@ from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler from kpops.components.base_components.helm_app import HelmApp from kpops.components.base_components.kafka_app import KafkaAppCleaner -from kpops.components.base_components.models.resource import Resource from kpops.components.common.app_type import AppType +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.base import ( StreamsBootstrap, @@ -49,7 +49,7 @@ async def clean(self, dry_run: bool) -> None: await self.clean_pvcs(dry_run) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE @@ -174,7 +174,7 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> Resource: + def manifest_deploy(self) -> list[KubernetesManifest]: manifests = super().manifest_deploy() if get_config().operation_mode is OperationMode.ARGO: manifests.extend(self._cleaner.manifest_deploy()) @@ -182,5 +182,5 @@ def manifest_deploy(self) -> Resource: return manifests @override - def manifest_clean(self) -> Resource: + def manifest_clean(self) -> list[KubernetesManifest]: return [] diff --git a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py index 0e05ad09f..a72be0bad 100644 --- a/tests/component_handlers/helm_wrapper/test_dry_run_handler.py +++ b/tests/component_handlers/helm_wrapper/test_dry_run_handler.py @@ -8,7 +8,7 @@ from kpops.component_handlers.helm_wrapper.dry_run_handler import DryRunHandler from kpops.component_handlers.helm_wrapper.model import HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.components.common.kubernetes_model import KubernetesManifest log = Logger("TestLogger") @@ -35,7 +35,14 @@ def test_should_print_helm_diff_when_release_is_new( ): helm_mock.get_manifest.return_value = iter(()) new_release = iter( - [HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1}))] + [ + HelmTemplate( + Path("path.yaml"), + KubernetesManifest( + **{"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) + ] ) mock_load_manifest = mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.Helm.load_manifest", @@ -61,12 +68,24 @@ def test_should_print_helm_diff_when_release_exists( caplog: LogCaptureFixture, ): current_release = [ - HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1})) + HelmTemplate( + Path("path.yaml"), + KubernetesManifest( + **{"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) ] helm_mock.get_manifest.return_value = iter(current_release) new_release = iter( - [HelmTemplate(Path("path.yaml"), KubernetesManifest({"a": 1}))] + [ + HelmTemplate( + Path("path.yaml"), + KubernetesManifest( + **{"apiVersion": "v1", "kind": "Deployment", "metadata": {}} + ), + ) + ] ) mock_load_manifest = mocker.patch( "kpops.component_handlers.helm_wrapper.dry_run_handler.Helm.load_manifest", diff --git a/tests/component_handlers/helm_wrapper/test_helm_diff.py b/tests/component_handlers/helm_wrapper/test_helm_diff.py index ce64ec4ae..9e8d686a1 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_diff.py +++ b/tests/component_handlers/helm_wrapper/test_helm_diff.py @@ -6,7 +6,7 @@ from kpops.component_handlers.helm_wrapper.helm_diff import HelmDiff from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.utils.dict_differ import Change logger = logging.getLogger("TestHelmDiff") @@ -18,11 +18,30 @@ def helm_diff(self) -> HelmDiff: return HelmDiff(HelmDiffConfig()) def test_calculate_changes_unchanged(self, helm_diff: HelmDiff): - templates = [HelmTemplate(Path("a.yaml"), KubernetesManifest())] + templates = [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {}, + } + ), + ) + ] assert list(helm_diff.calculate_changes(templates, templates)) == [ Change( - old_value={}, - new_value={}, + old_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {}, + }, + new_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {}, + }, ), ] @@ -31,26 +50,78 @@ def test_calculate_changes_matching(self, helm_diff: HelmDiff): assert list( helm_diff.calculate_changes( [ - HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1})), - HelmTemplate(Path("b.yaml"), KubernetesManifest({"b": 1})), + HelmTemplate( + Path("a.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + } + ), + ), + HelmTemplate( + Path("b.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"b": "1"}, + } + ), + ), ], [ - HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 2})), - HelmTemplate(Path("c.yaml"), KubernetesManifest({"c": 1})), + HelmTemplate( + Path("a.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "2"}, + } + ), + ), + HelmTemplate( + Path("c.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"c": "1"}, + } + ), + ), ], ) ) == [ Change( - old_value={"a": 1}, - new_value={"a": 2}, + old_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + }, + new_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "2"}, + }, ), Change( - old_value={"b": 1}, + old_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"b": "1"}, + }, new_value={}, ), Change( old_value={}, - new_value={"c": 1}, + new_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"c": "1"}, + }, ), ] @@ -58,12 +129,28 @@ def test_calculate_changes_new_release(self, helm_diff: HelmDiff): # test no current release assert list( helm_diff.calculate_changes( - (), [HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1}))] + (), + [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + } + ), + ) + ], ) ) == [ Change( old_value={}, - new_value={"a": 1}, + new_value={ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + }, ), ] @@ -71,6 +158,24 @@ def test_log_helm_diff(self, helm_diff: HelmDiff, caplog: LogCaptureFixture): helm_diff.log_helm_diff( logger, (), - [HelmTemplate(Path("a.yaml"), KubernetesManifest({"a": 1}))], + [ + HelmTemplate( + Path("a.yaml"), + KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + } + ), + ) + ], ) - assert caplog.messages == ["\n\x1b[32m+ a: 1\n\x1b[0m"] + assert caplog.messages == [ + "\n" + "\x1b[32m+ apiVersion: v1\n" + "\x1b[0m\x1b[32m+ kind: Deployment\n" + "\x1b[0m\x1b[32m+ metadata:\n" + "\x1b[0m\x1b[32m+ a: '1'\n" + "\x1b[0m" + ] diff --git a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py index 23c855251..4e5ceec8e 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_wrapper.py +++ b/tests/component_handlers/helm_wrapper/test_helm_wrapper.py @@ -293,7 +293,7 @@ def test_validate_console_output(self): def test_helm_template(self): path = Path("test2.yaml") manifest = KubernetesManifest( - { + **{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"labels": {"foo": "bar"}}, @@ -309,12 +309,16 @@ def test_load_manifest_with_no_notes(self): MANIFEST: --- # Source: chart/templates/test3a.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: test-3a --- # Source: chart/templates/test3b.yaml - foo: bar + apiVersion: v1 + kind: Pod + metadata: + name: test-3b """ ) helm_templates = list(Helm.load_manifest(stdout)) @@ -324,10 +328,20 @@ def test_load_manifest_with_no_notes(self): ) assert helm_templates[0].filepath == Path("chart/templates/test3a.yaml") assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + **{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3a"}, + } ) assert helm_templates[1].filepath == Path("chart/templates/test3b.yaml") - assert helm_templates[1].manifest == KubernetesManifest({"foo": "bar"}) + assert helm_templates[1].manifest == KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3b"}, + } + ) def test_raise_parse_error_when_helm_content_is_invalid(self): stdout = dedent( @@ -372,12 +386,16 @@ def test_load_manifest(self): MANIFEST: --- # Source: chart/templates/test3a.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: test-3a --- # Source: chart/templates/test3b.yaml - foo: bar + apiVersion: v1 + kind: Pod + metadata: + name: test-3b NOTES: 1. Get the application URL by running these commands: @@ -394,19 +412,30 @@ def test_load_manifest(self): ) assert helm_templates[0].filepath == Path("chart/templates/test3a.yaml") assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + **{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3a"}, + } ) assert helm_templates[1].filepath == Path("chart/templates/test3b.yaml") - assert helm_templates[1].manifest == KubernetesManifest({"foo": "bar"}) + assert helm_templates[1].manifest == KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "test-3b"}, + } + ) def test_helm_get_manifest(self, helm: Helm, mock_execute: MagicMock): mock_execute.return_value = dedent( """ --- # Source: chart/templates/test.yaml - data: - - a: 1 - - b: 2 + apiVersion: v1 + kind: Pod + metadata: + name: my-pod """ ) helm_templates = list(helm.get_manifest("test-release", "test-namespace")) @@ -423,7 +452,11 @@ def test_helm_get_manifest(self, helm: Helm, mock_execute: MagicMock): assert len(helm_templates) == 1 assert helm_templates[0].filepath == Path("chart/templates/test.yaml") assert helm_templates[0].manifest == KubernetesManifest( - {"data": [{"a": 1}, {"b": 2}]} + **{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "my-pod"}, + } ) mock_execute.side_effect = ReleaseNotFoundException() diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index 0d0649747..fbd0b9e58 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -9,8 +9,8 @@ HelmUpgradeInstallFlags, RepoAuthFlags, ) -from kpops.component_handlers.kubernetes.model import K8S_LABEL_MAX_LEN from kpops.components.base_components.helm_app import HelmApp, HelmAppValues +from kpops.components.common.kubernetes_model import K8S_LABEL_MAX_LEN from kpops.utils.colorify import magentaify diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources index 723353c93..a2064b9f8 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/resources @@ -1,13 +1,33 @@ -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: batch/v1 +- !!python/object:kpops.components.common.kubernetes_model.KubernetesManifest + __dict__: + api_version: batch/v1 kind: Job - metadata: - labels: - app: resources-manifest-pipeline-my-producer-app - chart: producer-app-3.0.3 - release: resources-manifest-pipeline-my-producer-app - name: resources-manifest-pipeline-my-producer-app + metadata: !!python/object:kpops.components.common.kubernetes_model.ObjectMeta + __dict__: + annotations: null + creationTimestamp: null + deletionGracePeriodSeconds: null + deletionTimestamp: null + finalizers: null + generateName: null + generation: null + labels: + app: resources-manifest-pipeline-my-producer-app + chart: producer-app-3.0.3 + release: resources-manifest-pipeline-my-producer-app + managedFields: null + name: resources-manifest-pipeline-my-producer-app + namespace: null + ownerReferences: null + resourceVersion: null + selfLink: null + uid: null + __pydantic_extra__: {} + __pydantic_fields_set__: !!set + labels: null + name: null + __pydantic_private__: null + __pydantic_extra__: spec: backoffLimit: 6 template: @@ -41,19 +61,45 @@ cpu: 200m memory: 300Mi restartPolicy: OnFailure + __pydantic_fields_set__: !!set + api_version: null + kind: null + metadata: null + spec: null + __pydantic_private__: null --- -- !!python/object:kpops.component_handlers.kubernetes.model.KubernetesManifest - data: - apiVersion: apps/v1 +- !!python/object:kpops.components.common.kubernetes_model.KubernetesManifest + __dict__: + api_version: apps/v1 kind: Deployment - metadata: - annotations: - consumerGroup: my-streams-app-id - labels: - app: resources-manifest-pipeline-my-streams-app - chart: streams-app-3.0.3 - release: resources-manifest-pipeline-my-streams-app - name: resources-manifest-pipeline-my-streams-app + metadata: !!python/object:kpops.components.common.kubernetes_model.ObjectMeta + __dict__: + annotations: + consumerGroup: my-streams-app-id + creationTimestamp: null + deletionGracePeriodSeconds: null + deletionTimestamp: null + finalizers: null + generateName: null + generation: null + labels: + app: resources-manifest-pipeline-my-streams-app + chart: streams-app-3.0.3 + release: resources-manifest-pipeline-my-streams-app + managedFields: null + name: resources-manifest-pipeline-my-streams-app + namespace: null + ownerReferences: null + resourceVersion: null + selfLink: null + uid: null + __pydantic_extra__: {} + __pydantic_fields_set__: !!set + annotations: null + labels: null + name: null + __pydantic_private__: null + __pydantic_extra__: spec: replicas: 1 selector: @@ -106,3 +152,9 @@ cpu: 200m memory: 300Mi terminationGracePeriodSeconds: 300 + __pydantic_fields_set__: !!set + api_version: null + kind: null + metadata: null + spec: null + __pydantic_private__: null From a24de230cc8be70fe60ceb2b9e11ab621abceb92 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 14:57:01 +0100 Subject: [PATCH 26/47] Update files --- kpops/components/common/kubernetes_model.py | 6 +----- tests/component_handlers/kubernetes/model.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index bb9d35941..ce7108b80 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -9,10 +9,6 @@ from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import DescConfigModel -try: - from typing import Self # pyright: ignore[reportAttributeAccessIssue] -except ImportError: - from typing_extensions import Self # Matches plain integer or numbers with valid suffixes: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory MEMORY_PATTERN = r"^\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$" K8S_LABEL_MAX_LEN = 63 @@ -72,7 +68,7 @@ class KubernetesManifest(DescConfigModel): @classmethod def from_yaml( cls, /, content: str - ) -> Iterator[Self]: # TODO: typing.Self for Python 3.11+ + ) -> Iterator["KubernetesManifest"]: # TODO: typing.Self for Python 3.11+ manifests: Iterator[dict[str, Any]] = yaml.load_all(content, yaml.Loader) for manifest in manifests: yield cls(**manifest) diff --git a/tests/component_handlers/kubernetes/model.py b/tests/component_handlers/kubernetes/model.py index 334c1f937..26e79245b 100644 --- a/tests/component_handlers/kubernetes/model.py +++ b/tests/component_handlers/kubernetes/model.py @@ -2,7 +2,7 @@ import pytest -from kpops.component_handlers.kubernetes.model import KubernetesManifest +from kpops.components.common.kubernetes_model import KubernetesManifest class TestKubernetesManifest: @@ -23,7 +23,7 @@ class TestKubernetesManifest: ), [ KubernetesManifest( - { + **{ "apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"labels": {"foo": "bar"}}, From 32ea4dc5100e03cfce3085a77e20721237b6e1c0 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 16:58:28 +0100 Subject: [PATCH 27/47] Update files --- kpops/api/__init__.py | 2 +- kpops/component_handlers/helm_wrapper/helm.py | 6 +- kpops/components/base_components/helm_app.py | 2 +- .../base_components/pipeline_component.py | 16 +- .../producer/producer_app.py | 7 +- .../streams_bootstrap/streams/streams_app.py | 10 +- .../test_manifest/test_python_api/resources | 160 ------------------ tests/pipeline/test_manifest.py | 10 +- 8 files changed, 29 insertions(+), 184 deletions(-) delete mode 100644 tests/pipeline/snapshots/test_manifest/test_python_api/resources diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 7c6f22a20..b1141c145 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -79,7 +79,7 @@ def manifest_deploy( environment: str | None = None, verbose: bool = True, operation_mode: OperationMode = OperationMode.MANIFEST, -) -> Iterator[list[KubernetesManifest]]: +) -> Iterator[tuple[KubernetesManifest, ...]]: pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 1968a0c71..5c0b405e0 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -160,7 +160,7 @@ def template( namespace: str, values: dict[str, Any], flags: HelmTemplateFlags | None = None, - ) -> list[KubernetesManifest]: + ) -> tuple[KubernetesManifest, ...]: """From Helm: Render chart templates locally and display the output. Any values that would normally be looked up or retrieved in-cluster will @@ -191,9 +191,9 @@ def template( command.extend(flags.to_command()) output = self.__execute(command) if output == "": - return [] + return () manifests = KubernetesManifest.from_yaml(output) - return list(manifests) + return tuple(manifests) def get_manifest(self, release_name: str, namespace: str) -> Iterable[HelmTemplate]: command = [ diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 9b229bd9b..20bb3f8c5 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -146,7 +146,7 @@ def template_flags(self) -> HelmTemplateFlags: ) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: sync_wave = ArgoSyncWave(sync_wave=1) diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 6ae4e71e5..9e59e36c7 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -229,21 +229,21 @@ def inflate(self) -> list[PipelineComponent]: """ return [self] - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests for deploy.""" - return [] + return () - def manifest_destroy(self) -> list[KubernetesManifest]: + def manifest_destroy(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for destroy.""" - return [] + return () - def manifest_reset(self) -> list[KubernetesManifest]: + def manifest_reset(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for reset.""" - return [] + return () - def manifest_clean(self) -> list[KubernetesManifest]: + def manifest_clean(self) -> tuple[KubernetesManifest, ...]: """Render Kubernetes manifests resources for clean.""" - return [] + return () async def deploy(self, dry_run: bool) -> None: """Deploy component, e.g. to Kubernetes cluster. diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 7159c1f65..c25d7db19 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -36,11 +36,12 @@ def helm_chart(self) -> str: ) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE values = enrich_annotations(values, post_delete.key, post_delete.value) + return self.helm.template( self.helm_release_name, self.helm_chart, @@ -145,11 +146,11 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: manifests = super().manifest_deploy() operation_mode = get_config().operation_mode if operation_mode is OperationMode.ARGO: - manifests.extend(self._cleaner.manifest_deploy()) + manifests = manifests + self._cleaner.manifest_deploy() return manifests diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index be5999a0a..5a2b16226 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -49,7 +49,7 @@ async def clean(self, dry_run: bool) -> None: await self.clean_pvcs(dry_run) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: values = self.to_helm_values() if get_config().operation_mode is OperationMode.ARGO: post_delete = ArgoHook.POST_DELETE @@ -174,13 +174,13 @@ async def clean(self, dry_run: bool) -> None: await self._cleaner.clean(dry_run) @override - def manifest_deploy(self) -> list[KubernetesManifest]: + def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: manifests = super().manifest_deploy() if get_config().operation_mode is OperationMode.ARGO: - manifests.extend(self._cleaner.manifest_deploy()) + manifests = manifests + self._cleaner.manifest_deploy() return manifests @override - def manifest_clean(self) -> list[KubernetesManifest]: - return [] + def manifest_clean(self) -> tuple[KubernetesManifest, ...]: + return () diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/resources b/tests/pipeline/snapshots/test_manifest/test_python_api/resources deleted file mode 100644 index a2064b9f8..000000000 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/resources +++ /dev/null @@ -1,160 +0,0 @@ -- !!python/object:kpops.components.common.kubernetes_model.KubernetesManifest - __dict__: - api_version: batch/v1 - kind: Job - metadata: !!python/object:kpops.components.common.kubernetes_model.ObjectMeta - __dict__: - annotations: null - creationTimestamp: null - deletionGracePeriodSeconds: null - deletionTimestamp: null - finalizers: null - generateName: null - generation: null - labels: - app: resources-manifest-pipeline-my-producer-app - chart: producer-app-3.0.3 - release: resources-manifest-pipeline-my-producer-app - managedFields: null - name: resources-manifest-pipeline-my-producer-app - namespace: null - ownerReferences: null - resourceVersion: null - selfLink: null - uid: null - __pydantic_extra__: {} - __pydantic_fields_set__: !!set - labels: null - name: null - __pydantic_private__: null - __pydantic_extra__: - spec: - backoffLimit: 6 - template: - metadata: - labels: - app: resources-manifest-pipeline-my-producer-app - release: resources-manifest-pipeline-my-producer-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_OUTPUT_TOPIC - value: my-producer-app-output-topic - - name: APP_LABELED_OUTPUT_TOPICS - value: my-producer-app-output-topic-label=my-labeled-producer-app-topic-output, - - name: JAVA_TOOL_OPTIONS - value: '-XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-producer-image:1.0.0 - imagePullPolicy: Always - name: resources-manifest-pipeline-my-producer-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - restartPolicy: OnFailure - __pydantic_fields_set__: !!set - api_version: null - kind: null - metadata: null - spec: null - __pydantic_private__: null ---- -- !!python/object:kpops.components.common.kubernetes_model.KubernetesManifest - __dict__: - api_version: apps/v1 - kind: Deployment - metadata: !!python/object:kpops.components.common.kubernetes_model.ObjectMeta - __dict__: - annotations: - consumerGroup: my-streams-app-id - creationTimestamp: null - deletionGracePeriodSeconds: null - deletionTimestamp: null - finalizers: null - generateName: null - generation: null - labels: - app: resources-manifest-pipeline-my-streams-app - chart: streams-app-3.0.3 - release: resources-manifest-pipeline-my-streams-app - managedFields: null - name: resources-manifest-pipeline-my-streams-app - namespace: null - ownerReferences: null - resourceVersion: null - selfLink: null - uid: null - __pydantic_extra__: {} - __pydantic_fields_set__: !!set - annotations: null - labels: null - name: null - __pydantic_private__: null - __pydantic_extra__: - spec: - replicas: 1 - selector: - matchLabels: - app: resources-manifest-pipeline-my-streams-app - release: resources-manifest-pipeline-my-streams-app - template: - metadata: - labels: - app: resources-manifest-pipeline-my-streams-app - release: resources-manifest-pipeline-my-streams-app - spec: - containers: - - env: - - name: ENV_PREFIX - value: APP_ - - name: APP_VOLATILE_GROUP_INSTANCE_ID - value: 'true' - - name: APP_BOOTSTRAP_SERVERS - value: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 - - name: APP_SCHEMA_REGISTRY_URL - value: http://localhost:8081/ - - name: APP_INPUT_TOPICS - value: my-input-topic - - name: APP_INPUT_PATTERN - value: my-input-pattern - - name: APP_OUTPUT_TOPIC - value: my-output-topic - - name: APP_ERROR_TOPIC - value: resources-manifest-pipeline-my-streams-app-error - - name: APP_LABELED_OUTPUT_TOPICS - value: my-output-topic-label=my-labeled-topic-output, - - name: APP_LABELED_INPUT_TOPICS - value: my-input-topic-label=my-labeled-input-topic, - - name: APP_LABELED_INPUT_PATTERNS - value: my-input-topic-labeled-pattern=my-labeled-input-pattern, - - name: APP_APPLICATION_ID - value: my-streams-app-id - - name: JAVA_TOOL_OPTIONS - value: '-Dcom.sun.management.jmxremote.port=5555 -Dcom.sun.management.jmxremote.authenticate=false - -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage=75.0 ' - image: my-registry/my-streams-app-image:1.0.0 - imagePullPolicy: Always - name: resources-manifest-pipeline-my-streams-app - resources: - limits: - cpu: 500m - memory: 2G - requests: - cpu: 200m - memory: 300Mi - terminationGracePeriodSeconds: 300 - __pydantic_fields_set__: !!set - api_version: null - kind: null - metadata: null - spec: null - __pydantic_private__: null diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 983859a44..22d769521 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -3,7 +3,6 @@ from unittest.mock import ANY, MagicMock import pytest -import yaml from pytest_mock import MockerFixture from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner @@ -12,6 +11,7 @@ from kpops.cli.main import app from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmConfig, Version +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.const.file_type import PIPELINE_YAML MANIFEST_YAML = "manifest.yaml" @@ -123,7 +123,7 @@ def test_manifest_command(self, snapshot: Snapshot): assert result.exit_code == 0, result.stdout snapshot.assert_match(result.stdout, MANIFEST_YAML) - def test_python_api(self, snapshot: Snapshot): + def test_python_api(self): generator = kpops.manifest_deploy( RESOURCE_PATH / "manifest-pipeline" / PIPELINE_YAML, environment="development", @@ -131,7 +131,11 @@ def test_python_api(self, snapshot: Snapshot): assert isinstance(generator, Iterator) resources = list(generator) assert len(resources) == 2 - snapshot.assert_match(yaml.dump_all(resources), "resources") + assert all( + isinstance(manifest, KubernetesManifest) + for resource in resources + for manifest in resource + ) def test_streams_bootstrap(self, snapshot: Snapshot): result = runner.invoke( From 79e1af4a9d5575145b1b9e81d1ebcaa7948085a4 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 17:12:02 +0100 Subject: [PATCH 28/47] Update files --- kpops/components/common/kubernetes_model.py | 32 ++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index ce7108b80..a638324e1 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -7,59 +7,59 @@ from typing_extensions import override from kpops.utils.docstring import describe_attr -from kpops.utils.pydantic import DescConfigModel +from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel # Matches plain integer or numbers with valid suffixes: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory MEMORY_PATTERN = r"^\d+([EPTGMk]|Ei|Pi|Ti|Gi|Mi|Ki)?$" K8S_LABEL_MAX_LEN = 63 -class ManagedFieldsEntry(DescConfigModel): +class ManagedFieldsEntry(CamelCaseConfigModel): # Define this class based on its actual structure fields: dict[str, Any] | None = None -class OwnerReference(DescConfigModel): +class OwnerReference(CamelCaseConfigModel): # Define this class based on its actual structure apiVersion: str kind: str name: str uid: str controller: bool | None = None - blockOwnerDeletion: bool | None = None + block_owner_deletion: bool | None = None -class ObjectMeta(DescConfigModel): +class ObjectMeta(CamelCaseConfigModel): """Metadata for all Kubernetes objects.""" annotations: dict[str, str] | None = None - creationTimestamp: str | None = Field( + creation_timestamp: str | None = Field( None, description="Timestamp in RFC3339 format" ) - deletionGracePeriodSeconds: int | None = None - deletionTimestamp: str | None = Field( + deletion_grace_period_seconds: int | None = None + deletion_timestamp: str | None = Field( None, description="Timestamp in RFC3339 format" ) finalizers: list[str] | None = None - generateName: str | None = None + generate_name: str | None = None generation: int | None = None labels: dict[str, str] | None = None - managedFields: list[ManagedFieldsEntry] | None = None + managed_fields: list[ManagedFieldsEntry] | None = None name: str | None = None namespace: str | None = None - ownerReferences: list[OwnerReference] | None = None - resourceVersion: str | None = None - selfLink: str | None = Field( + owner_references: list[OwnerReference] | None = None + resource_version: str | None = None + self_link: str | None = Field( None, description="Deprecated field, not populated by Kubernetes in modern versions", ) uid: str | None = None + model_config = ConfigDict(extra="allow") -class KubernetesManifest(DescConfigModel): - # TODO: this is not working without the 'alias' - api_version: str = Field(alias="apiVersion") +class KubernetesManifest(CamelCaseConfigModel): + api_version: str kind: str metadata: ObjectMeta From b2b2983c0faef7dfd3261413a9fd25a020499da1 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 17:13:37 +0100 Subject: [PATCH 29/47] Update files --- kpops/component_handlers/helm_wrapper/helm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index 5c0b405e0..2b923aaad 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -190,8 +190,6 @@ def template( ] command.extend(flags.to_command()) output = self.__execute(command) - if output == "": - return () manifests = KubernetesManifest.from_yaml(output) return tuple(manifests) From d894e57417d71590f32c85e99a3335c639c65e97 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 17:47:05 +0100 Subject: [PATCH 30/47] Update files --- .../component_handlers/helm_wrapper/helm_diff.py | 16 ++++++++-------- .../helm_wrapper/test_helm_diff.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/kpops/component_handlers/helm_wrapper/helm_diff.py b/kpops/component_handlers/helm_wrapper/helm_diff.py index dd6d93cef..f6214f4f8 100644 --- a/kpops/component_handlers/helm_wrapper/helm_diff.py +++ b/kpops/component_handlers/helm_wrapper/helm_diff.py @@ -1,8 +1,8 @@ import logging from collections.abc import Iterable, Iterator -from typing import Any from kpops.component_handlers.helm_wrapper.model import HelmDiffConfig, HelmTemplate +from kpops.components.common.kubernetes_model import KubernetesManifest from kpops.utils.dict_differ import Change, render_diff log = logging.getLogger("HelmDiff") @@ -16,7 +16,7 @@ def __init__(self, config: HelmDiffConfig) -> None: def calculate_changes( current_release: Iterable[HelmTemplate], new_release: Iterable[HelmTemplate], - ) -> Iterator[Change[dict[str, Any], dict[str, Any]]]: + ) -> Iterator[Change[KubernetesManifest | None, KubernetesManifest | None]]: """Compare 2 releases and generate a Change object for each difference. :param current_release: Iterable containing HelmTemplate objects for the current release @@ -32,15 +32,15 @@ def calculate_changes( # get corresponding dry-run release new_resource = new_release_index.pop(current_resource.filepath, None) yield Change( - current_resource.manifest.model_dump(), - new_resource.manifest.model_dump() if new_resource else {}, + current_resource.manifest, + new_resource.manifest if new_resource else None, ) # collect added files for new_resource in new_release_index.values(): yield Change( - {}, - new_resource.manifest.model_dump(mode="json"), + None, + new_resource.manifest, ) def log_helm_diff( @@ -51,8 +51,8 @@ def log_helm_diff( ) -> None: for change in self.calculate_changes(current_release, new_release): if diff := render_diff( - change.old_value, - change.new_value, + change.old_value.model_dump() if change.old_value else {}, + change.new_value.model_dump() if change.new_value else {}, ignore=self.config.ignore, ): logger.info("\n" + diff) diff --git a/tests/component_handlers/helm_wrapper/test_helm_diff.py b/tests/component_handlers/helm_wrapper/test_helm_diff.py index 9e8d686a1..8277d6ead 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_diff.py +++ b/tests/component_handlers/helm_wrapper/test_helm_diff.py @@ -145,12 +145,14 @@ def test_calculate_changes_new_release(self, helm_diff: HelmDiff): ) ) == [ Change( - old_value={}, - new_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {"a": "1"}, - }, + old_value=None, + new_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + } + ), ), ] From e67908e0d89ff9e9bab1d8b3059e741cfcefdcbd Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 17:50:59 +0100 Subject: [PATCH 31/47] Update files --- kpops/components/common/kubernetes_model.py | 2 - .../helm_wrapper/test_helm_diff.py | 76 +++++++++++-------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index a638324e1..d396e7ef3 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -78,8 +78,6 @@ def model_dump(self, **_: Any) -> dict[str, Any]: return super().model_dump( mode="json", by_alias=True, - exclude_none=True, - exclude_defaults=True, exclude_unset=True, ) diff --git a/tests/component_handlers/helm_wrapper/test_helm_diff.py b/tests/component_handlers/helm_wrapper/test_helm_diff.py index 8277d6ead..916fcb2f0 100644 --- a/tests/component_handlers/helm_wrapper/test_helm_diff.py +++ b/tests/component_handlers/helm_wrapper/test_helm_diff.py @@ -32,16 +32,20 @@ def test_calculate_changes_unchanged(self, helm_diff: HelmDiff): ] assert list(helm_diff.calculate_changes(templates, templates)) == [ Change( - old_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {}, - }, - new_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {}, - }, + old_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {}, + } + ), + new_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {}, + } + ), ), ] @@ -96,32 +100,40 @@ def test_calculate_changes_matching(self, helm_diff: HelmDiff): ) ) == [ Change( - old_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {"a": "1"}, - }, - new_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {"a": "2"}, - }, + old_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "1"}, + } + ), + new_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"a": "2"}, + } + ), ), Change( - old_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {"b": "1"}, - }, - new_value={}, + old_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"b": "1"}, + } + ), + new_value=None, ), Change( - old_value={}, - new_value={ - "apiVersion": "v1", - "kind": "Deployment", - "metadata": {"c": "1"}, - }, + old_value=None, + new_value=KubernetesManifest( + **{ + "apiVersion": "v1", + "kind": "Deployment", + "metadata": {"c": "1"}, + } + ), ), ] From 0a2e7c802e38b59360b21a37cee5ed1dfacf2b76 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 4 Dec 2024 17:51:38 +0100 Subject: [PATCH 32/47] Update files --- kpops/components/common/kubernetes_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kpops/components/common/kubernetes_model.py b/kpops/components/common/kubernetes_model.py index d396e7ef3..29d031eed 100644 --- a/kpops/components/common/kubernetes_model.py +++ b/kpops/components/common/kubernetes_model.py @@ -34,11 +34,11 @@ class ObjectMeta(CamelCaseConfigModel): annotations: dict[str, str] | None = None creation_timestamp: str | None = Field( - None, description="Timestamp in RFC3339 format" + default=None, description="Timestamp in RFC3339 format" ) deletion_grace_period_seconds: int | None = None deletion_timestamp: str | None = Field( - None, description="Timestamp in RFC3339 format" + default=None, description="Timestamp in RFC3339 format" ) finalizers: list[str] | None = None generate_name: str | None = None @@ -50,7 +50,7 @@ class ObjectMeta(CamelCaseConfigModel): owner_references: list[OwnerReference] | None = None resource_version: str | None = None self_link: str | None = Field( - None, + default=None, description="Deprecated field, not populated by Kubernetes in modern versions", ) uid: str | None = None From 1070fde6d0294de34582e3e0915ccf63e3a8ac17 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 5 Dec 2024 17:04:32 +0100 Subject: [PATCH 33/47] clean ups --- kpops/components/streams_bootstrap/base.py | 11 ++- .../streams_bootstrap/strimzi/kafka_topic.py | 53 ------------- .../strimzi/__init__.py | 0 kpops/manifests/strimzi/kafka_topic.py | 78 +++++++++++++++++++ .../test_python_api/manifest.yaml | 73 +++++++++++++++++ 5 files changed, 156 insertions(+), 59 deletions(-) delete mode 100644 kpops/components/streams_bootstrap/strimzi/kafka_topic.py rename kpops/{components/streams_bootstrap => manifests}/strimzi/__init__.py (100%) create mode 100644 kpops/manifests/strimzi/kafka_topic.py diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index d607e2a7c..55e3daaa6 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -14,8 +14,8 @@ from kpops.components.base_components.helm_app import HelmApp from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.model import StreamsBootstrapValues -from kpops.components.streams_bootstrap.strimzi.kafka_topic import StrimziKafkaTopic from kpops.manifests.kubernetes import KubernetesManifest +from kpops.manifests.strimzi.kafka_topic import StrimziKafkaTopic from kpops.utils.docstring import describe_attr if TYPE_CHECKING: @@ -97,10 +97,9 @@ def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: def manifest_strimzi_topics( self, kafka_topics: list[KafkaTopic] ) -> tuple[StrimziKafkaTopic, ...]: - topics = [] - for topic in kafka_topics: - strimzi_topic = StrimziKafkaTopic.create_strimzi_topic( + return tuple( + StrimziKafkaTopic.create_strimzi_topic( topic, self.values.kafka.bootstrap_servers ) - topics.append(strimzi_topic) - return tuple(topics) + for topic in kafka_topics + ) diff --git a/kpops/components/streams_bootstrap/strimzi/kafka_topic.py b/kpops/components/streams_bootstrap/strimzi/kafka_topic.py deleted file mode 100644 index e6df53ef7..000000000 --- a/kpops/components/streams_bootstrap/strimzi/kafka_topic.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from pydantic import BaseModel, Field, model_validator -from typing_extensions import Any - -from kpops.components.common.topic import KafkaTopic -from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta - -if TYPE_CHECKING: - try: - from typing import Self # pyright: ignore[reportAttributeAccessIssue] - except ImportError: - from typing_extensions import Self - - -# Define the Pydantic model for the spec and metadata -class TopicSpec(BaseModel): - partitions: int = Field(default=1) - replicas: int = Field(default=1) - config: dict[str, str | int] = Field(default_factory=dict) - - @model_validator(mode="before") - @classmethod - def set_defaults_if_none(cls, values: Any) -> Any: - if values.get("partitions") is None: - values["partitions"] = 1 - if values.get("replicas") is None: - values["replicas"] = 1 - return values - - -class StrimziKafkaTopic(KubernetesManifest): - api_version: str = "kafka.strimzi.io/v1beta2" - kind: str = "KafkaTopic" - metadata: ObjectMeta - spec: TopicSpec - - @classmethod - def create_strimzi_topic(cls, topic: KafkaTopic, bootstrap_servers: str) -> Self: - metadata = { - "name": topic.name, - "labels": { - "strimzi.io/cluster": bootstrap_servers, - }, - } - spec = { - "partitions": topic.config.partitions_count, - "replicas": topic.config.replication_factor, - "config": topic.config.configs, - } - return cls(metadata=ObjectMeta(**metadata), spec=TopicSpec(**spec)) diff --git a/kpops/components/streams_bootstrap/strimzi/__init__.py b/kpops/manifests/strimzi/__init__.py similarity index 100% rename from kpops/components/streams_bootstrap/strimzi/__init__.py rename to kpops/manifests/strimzi/__init__.py diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py new file mode 100644 index 000000000..8f9e7eb94 --- /dev/null +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic import ConfigDict, Field, model_validator +from typing_extensions import Any + +from kpops.components.common.topic import KafkaTopic +from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta +from kpops.utils.pydantic import CamelCaseConfigModel + +if TYPE_CHECKING: + try: + from typing import Self # pyright: ignore[reportAttributeAccessIssue] + except ImportError: + from typing_extensions import Self + + +# Define the Pydantic model for the spec and metadata +class TopicSpec(CamelCaseConfigModel): + """Specification of a Kafka topic. + + :param partitions: The number of partitions the topic should have. This cannot be decreased after topic creation. It can be increased after topic creation, but it is important to understand the consequences that has, especially for topics with semantic partitioning. When absent this will default to the broker configuration for `num.partitions`. + :param replicas: The number of replicas the topic should have. When absent this will default to the broker configuration for `default.replication.factor`. + :param config: The topic configuration. + + """ + + partitions: int = Field(default=1, ge=1) + replicas: int = Field(default=1, ge=1, le=32767) + config: dict[str, str | int] = Field(default_factory=dict) + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="before") + @classmethod + def set_defaults_if_none(cls, values: Any) -> Any: + if values.get("partitions") is None: + values["partitions"] = 1 + if values.get("replicas") is None: + values["replicas"] = 1 + return values + + +class StrimziKafkaTopic(KubernetesManifest): + """Represents a Strimzi Kafka Topic CRD. + + CRD definition: https://github.com/strimzi/strimzi-kafka-operator/blob/main/install/cluster-operator/043-Crd-kafkatopic.yaml + example: https://github.com/strimzi/strimzi-kafka-operator/blob/main/examples/topic/kafka-topic.yaml + """ + + api_version: str = "kafka.strimzi.io/v1beta2" + kind: str = "KafkaTopic" + metadata: ObjectMeta + spec: TopicSpec + status: dict[str, Any] | None = None + + @classmethod + def create_strimzi_topic(cls, topic: KafkaTopic, bootstrap_servers: str) -> Self: + metadata = ObjectMeta.model_validate( + { + "name": topic.name, + "labels": { + "strimzi.io/cluster": bootstrap_servers, + }, + } + ) + spec = TopicSpec.model_validate( + { + "partitions": topic.config.partitions_count, + "replicas": topic.config.replication_factor, + "config": topic.config.configs, + } + ) + return cls( + metadata=metadata, + spec=spec, + ) diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml index f7086850d..7041aa03a 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml @@ -41,6 +41,30 @@ spec: memory: 300Mi restartPolicy: OnFailure +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-producer-app-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-producer-app-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + --- apiVersion: apps/v1 kind: Deployment @@ -105,3 +129,52 @@ spec: memory: 300Mi terminationGracePeriodSeconds: 300 +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-output-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-error-topic +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: my-labeled-topic-output +spec: + config: {} + partitions: 1 + replicas: 1 + +--- +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaTopic +metadata: + labels: + strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + name: resources-manifest-pipeline-my-streams-app-error +spec: + config: + cleanup.policy: compact,delete + partitions: 1 + replicas: 1 + From 50e81df11d6725211a87b176b2a85b3dd2fbceae Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Thu, 5 Dec 2024 17:08:53 +0100 Subject: [PATCH 34/47] Update files --- kpops/manifests/strimzi/kafka_topic.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 8f9e7eb94..098ef564f 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -7,6 +7,7 @@ from kpops.components.common.topic import KafkaTopic from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta +from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel if TYPE_CHECKING: @@ -16,7 +17,6 @@ from typing_extensions import Self -# Define the Pydantic model for the spec and metadata class TopicSpec(CamelCaseConfigModel): """Specification of a Kafka topic. @@ -26,9 +26,15 @@ class TopicSpec(CamelCaseConfigModel): """ - partitions: int = Field(default=1, ge=1) - replicas: int = Field(default=1, ge=1, le=32767) - config: dict[str, str | int] = Field(default_factory=dict) + partitions: int = Field( + default=1, ge=1, description=describe_attr("partitions", __doc__) + ) + replicas: int = Field( + default=1, ge=1, le=32767, description=describe_attr("replicas", __doc__) + ) + config: dict[str, str | int] = Field( + default_factory=dict, description=describe_attr("config", __doc__) + ) model_config = ConfigDict(extra="allow") From 4b187bb2965a25d8af56882307be2e29aeed7d89 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 6 Dec 2024 09:32:49 +0100 Subject: [PATCH 35/47] clean ups --- kpops/components/streams_bootstrap/base.py | 17 +++++------------ kpops/manifests/strimzi/kafka_topic.py | 4 ++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index 55e3daaa6..9fc35e526 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -12,7 +12,6 @@ from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig from kpops.components.base_components import KafkaApp from kpops.components.base_components.helm_app import HelmApp -from kpops.components.common.topic import KafkaTopic from kpops.components.streams_bootstrap.model import StreamsBootstrapValues from kpops.manifests.kubernetes import KubernetesManifest from kpops.manifests.strimzi.kafka_topic import StrimziKafkaTopic @@ -90,16 +89,10 @@ def warning_for_latest_image_tag(self) -> Self: def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: resource = super().manifest_deploy() if self.to: - resource = resource + self.manifest_strimzi_topics(self.to.kafka_topics) + resource = resource + tuple( + # TODO: change to cluster_name + StrimziKafkaTopic.from_topic(topic, self.values.kafka.bootstrap_servers) + for topic in self.to.kafka_topics + ) return resource - - def manifest_strimzi_topics( - self, kafka_topics: list[KafkaTopic] - ) -> tuple[StrimziKafkaTopic, ...]: - return tuple( - StrimziKafkaTopic.create_strimzi_topic( - topic, self.values.kafka.bootstrap_servers - ) - for topic in kafka_topics - ) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 098ef564f..543e0be2b 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -62,12 +62,12 @@ class StrimziKafkaTopic(KubernetesManifest): status: dict[str, Any] | None = None @classmethod - def create_strimzi_topic(cls, topic: KafkaTopic, bootstrap_servers: str) -> Self: + def from_topic(cls, topic: KafkaTopic, cluster_name: str) -> Self: metadata = ObjectMeta.model_validate( { "name": topic.name, "labels": { - "strimzi.io/cluster": bootstrap_servers, + "strimzi.io/cluster": cluster_name, }, } ) From c25a883071acd48c002e1108b605621b9bb45ae4 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 6 Dec 2024 10:13:21 +0100 Subject: [PATCH 36/47] add cluster name label --- config.yaml | 1 + .../resources/variables/config_env_vars.env | 7 ++++ .../resources/variables/config_env_vars.md | 41 ++++++++++--------- docs/docs/schema/config.json | 13 ++++++ kpops/components/streams_bootstrap/base.py | 4 +- kpops/config/__init__.py | 4 ++ kpops/manifests/strimzi/kafka_topic.py | 11 ++++- .../config.yaml | 1 + .../test_deploy_argo_mode/manifest.yaml | 12 +++--- .../test_deploy_manifest_mode/manifest.yaml | 12 +++--- .../test_python_api/manifest.yaml | 12 +++--- .../test_streams_bootstrap/manifest.yaml | 12 +++--- 12 files changed, 81 insertions(+), 49 deletions(-) diff --git a/config.yaml b/config.yaml index 359b51a21..3c388de5c 100644 --- a/config.yaml +++ b/config.yaml @@ -1,2 +1,3 @@ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" pipeline_base_dir: tests/pipeline +kafka_topic_resource_label: "my-cluster" diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index b64bac2e0..554450f8b 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -61,3 +61,10 @@ KPOPS_RETAIN_CLEAN_JOBS=False # operation_mode # The operation mode of KPOps (managed, manifest, argo). KPOPS_OPERATION_MODE=managed +# kafka_topic_resource_label +# The label to identify the KafkaTopic resources managed by the Topic +# Operator. This does not have to be the name of the Kafka cluster. It +# can be the label assigned to the KafkaTopic resource. If you deploy +# more than one Topic Operator, the labels must be unique for each. +# That is, the operators cannot manage the same resources. +KPOPS_KAFKA_TOPIC_RESOURCE_LABEL # No default value, not required diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 6b79a743c..86231771f 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -1,22 +1,23 @@ These variables take precedence over the settings in `config.yaml`. Variables marked as required can instead be set in the global config. -| Name | Default Value |Required| Description | Setting name | -|--------------------------------------------------|----------------------------------------|--------|----------------------------------------------------------------------------------|-------------------------------------------| -|KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | -|KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | -|KPOPS_SCHEMA_REGISTRY__ENABLED |False |False |Whether the Schema Registry handler should be initialized. |schema_registry.enabled | -|KPOPS_SCHEMA_REGISTRY__URL |http://localhost:8081/ |False |Address of the Schema Registry. |schema_registry.url | -|KPOPS_SCHEMA_REGISTRY__TIMEOUT |30 |False |Operation timeout in seconds. |schema_registry.timeout | -|KPOPS_KAFKA_REST__URL |http://localhost:8082/ |False |Address of the Kafka REST Proxy. |kafka_rest.url | -|KPOPS_KAFKA_REST__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_rest.timeout | -|KPOPS_KAFKA_CONNECT__URL |http://localhost:8083/ |False |Address of Kafka Connect. |kafka_connect.url | -|KPOPS_KAFKA_CONNECT__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_connect.timeout | -|KPOPS_CREATE_NAMESPACE |False |False |Flag for `helm upgrade --install`. Create the release namespace if not present. |create_namespace | -|KPOPS_HELM_CONFIG__CONTEXT | |False |Name of kubeconfig context (`--kube-context`) |helm_config.context | -|KPOPS_HELM_CONFIG__DEBUG |False |False |Run Helm in Debug mode |helm_config.debug | -|KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | -|KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | -|KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs | -|KPOPS_OPERATION_MODE |managed |False |The operation mode of KPOps (managed, manifest, argo). |operation_mode | +| Name | Default Value |Required| Description | Setting name | +|--------------------------------------------------|----------------------------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| +|KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | +|KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | +|KPOPS_SCHEMA_REGISTRY__ENABLED |False |False |Whether the Schema Registry handler should be initialized. |schema_registry.enabled | +|KPOPS_SCHEMA_REGISTRY__URL |http://localhost:8081/ |False |Address of the Schema Registry. |schema_registry.url | +|KPOPS_SCHEMA_REGISTRY__TIMEOUT |30 |False |Operation timeout in seconds. |schema_registry.timeout | +|KPOPS_KAFKA_REST__URL |http://localhost:8082/ |False |Address of the Kafka REST Proxy. |kafka_rest.url | +|KPOPS_KAFKA_REST__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_rest.timeout | +|KPOPS_KAFKA_CONNECT__URL |http://localhost:8083/ |False |Address of Kafka Connect. |kafka_connect.url | +|KPOPS_KAFKA_CONNECT__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_connect.timeout | +|KPOPS_CREATE_NAMESPACE |False |False |Flag for `helm upgrade --install`. Create the release namespace if not present. |create_namespace | +|KPOPS_HELM_CONFIG__CONTEXT | |False |Name of kubeconfig context (`--kube-context`) |helm_config.context | +|KPOPS_HELM_CONFIG__DEBUG |False |False |Run Helm in Debug mode |helm_config.debug | +|KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | +|KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | +|KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion. |retain_clean_jobs | +|KPOPS_OPERATION_MODE |managed |False |The operation mode of KPOps (managed, manifest, argo). |operation_mode | +|KPOPS_KAFKA_TOPIC_RESOURCE_LABEL | |False |The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.|kafka_topic_resource_label | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index 02d04ea6e..068c32921 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -248,6 +248,19 @@ }, "description": "Configuration for Kafka REST Proxy." }, + "kafka_topic_resource_label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + "title": "Kafka Topic Resource Label" + }, "operation_mode": { "allOf": [ { diff --git a/kpops/components/streams_bootstrap/base.py b/kpops/components/streams_bootstrap/base.py index 9fc35e526..f81d23b32 100644 --- a/kpops/components/streams_bootstrap/base.py +++ b/kpops/components/streams_bootstrap/base.py @@ -90,9 +90,7 @@ def manifest_deploy(self) -> tuple[KubernetesManifest, ...]: resource = super().manifest_deploy() if self.to: resource = resource + tuple( - # TODO: change to cluster_name - StrimziKafkaTopic.from_topic(topic, self.values.kafka.bootstrap_servers) - for topic in self.to.kafka_topics + StrimziKafkaTopic.from_topic(topic) for topic in self.to.kafka_topics ) return resource diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 7398d5e7c..89543df82 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -125,6 +125,10 @@ class KpopsConfig(BaseSettings): default=OperationMode.MANAGED, description="The operation mode of KPOps (managed, manifest, argo).", ) + kafka_topic_resource_label: str | None = Field( + default=None, + description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + ) model_config = SettingsConfigDict( env_prefix=ENV_PREFIX, diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 543e0be2b..81f1f3382 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -5,7 +5,9 @@ from pydantic import ConfigDict, Field, model_validator from typing_extensions import Any +from kpops.api.exception import ValidationError from kpops.components.common.topic import KafkaTopic +from kpops.config import get_config from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel @@ -62,12 +64,17 @@ class StrimziKafkaTopic(KubernetesManifest): status: dict[str, Any] | None = None @classmethod - def from_topic(cls, topic: KafkaTopic, cluster_name: str) -> Self: + def from_topic(cls, topic: KafkaTopic) -> Self: + topic_resource_label = get_config().kafka_topic_resource_label + if not topic_resource_label: + msg = "When manifesting KafkaTopic you must define 'kafka_topic_resource_label' in the config.yaml" + raise ValidationError(msg) + metadata = ObjectMeta.model_validate( { "name": topic.name, "labels": { - "strimzi.io/cluster": cluster_name, + "strimzi.io/cluster": topic_resource_label, }, } ) diff --git a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml index 19b7f5d95..8f04208f6 100644 --- a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml +++ b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml @@ -17,6 +17,7 @@ kafka_connect: kafka_rest: timeout: 30 url: http://localhost:8082/ +kafka_topic_resource_label: null operation_mode: managed pipeline_base_dir: . retain_clean_jobs: false diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml index c892993ba..336d3d726 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_argo_mode/manifest.yaml @@ -48,7 +48,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-producer-app-output-topic spec: config: {} @@ -60,7 +60,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-producer-app-topic-output spec: config: {} @@ -185,7 +185,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-output-topic spec: config: {} @@ -197,7 +197,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-error-topic spec: config: {} @@ -209,7 +209,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-topic-output spec: config: {} @@ -221,7 +221,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: resources-manifest-pipeline-my-streams-app-error spec: config: diff --git a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml index 7041aa03a..7b6d5b8aa 100644 --- a/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_deploy_manifest_mode/manifest.yaml @@ -46,7 +46,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-producer-app-output-topic spec: config: {} @@ -58,7 +58,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-producer-app-topic-output spec: config: {} @@ -134,7 +134,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-output-topic spec: config: {} @@ -146,7 +146,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-error-topic spec: config: {} @@ -158,7 +158,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-topic-output spec: config: {} @@ -170,7 +170,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: resources-manifest-pipeline-my-streams-app-error spec: config: diff --git a/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml index 7041aa03a..7b6d5b8aa 100644 --- a/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_python_api/manifest.yaml @@ -46,7 +46,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-producer-app-output-topic spec: config: {} @@ -58,7 +58,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-producer-app-topic-output spec: config: {} @@ -134,7 +134,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-output-topic spec: config: {} @@ -146,7 +146,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-error-topic spec: config: {} @@ -158,7 +158,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-topic-output spec: config: {} @@ -170,7 +170,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: resources-manifest-pipeline-my-streams-app-error spec: config: diff --git a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml index 04e89d82b..a1dd22945 100644 --- a/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml +++ b/tests/pipeline/snapshots/test_manifest/test_streams_bootstrap/manifest.yaml @@ -75,7 +75,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-producer-app-output-topic spec: config: {} @@ -87,7 +87,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-producer-app-topic-output spec: config: {} @@ -238,7 +238,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-output-topic spec: config: {} @@ -250,7 +250,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-error-topic spec: config: {} @@ -262,7 +262,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: my-labeled-topic-output spec: config: {} @@ -274,7 +274,7 @@ apiVersion: kafka.strimzi.io/v1beta2 kind: KafkaTopic metadata: labels: - strimzi.io/cluster: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092 + strimzi.io/cluster: my-cluster name: resources-streams-bootstrap-my-streams-app-error spec: config: From 8ea444c9123bdc066646b3b5976764291a590c73 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Fri, 6 Dec 2024 10:21:13 +0100 Subject: [PATCH 37/47] Update files --- kpops/manifests/strimzi/kafka_topic.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 81f1f3382..579744959 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -9,7 +9,6 @@ from kpops.components.common.topic import KafkaTopic from kpops.config import get_config from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta -from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel if TYPE_CHECKING: @@ -28,15 +27,9 @@ class TopicSpec(CamelCaseConfigModel): """ - partitions: int = Field( - default=1, ge=1, description=describe_attr("partitions", __doc__) - ) - replicas: int = Field( - default=1, ge=1, le=32767, description=describe_attr("replicas", __doc__) - ) - config: dict[str, str | int] = Field( - default_factory=dict, description=describe_attr("config", __doc__) - ) + partitions: int = Field(default=1, ge=1) + replicas: int = Field(default=1, ge=1, le=32767) + config: dict[str, str | int] = Field(default_factory=dict) model_config = ConfigDict(extra="allow") From 16d88c786bcb983bd17d1c71cc05779bfc2903bd Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 09:43:43 +0100 Subject: [PATCH 38/47] Update files --- kpops/manifests/strimzi/kafka_topic.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 579744959..9b16e9de1 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -1,14 +1,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from pydantic import ConfigDict, Field, model_validator -from typing_extensions import Any from kpops.api.exception import ValidationError from kpops.components.common.topic import KafkaTopic from kpops.config import get_config from kpops.manifests.kubernetes import KubernetesManifest, ObjectMeta +from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel if TYPE_CHECKING: @@ -27,9 +27,15 @@ class TopicSpec(CamelCaseConfigModel): """ - partitions: int = Field(default=1, ge=1) - replicas: int = Field(default=1, ge=1, le=32767) - config: dict[str, str | int] = Field(default_factory=dict) + partitions: int = Field( + default=1, ge=1, description=describe_attr("partitions", __doc__) + ) + replicas: int = Field( + default=1, ge=1, le=32767, description=describe_attr("replicas", __doc__) + ) + config: dict[str, str | int] = Field( + default_factory=dict, description=describe_attr("config", __doc__) + ) model_config = ConfigDict(extra="allow") From 3f712553833bcb6c4dc42574c8263268c9c7c094 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 09:50:14 +0100 Subject: [PATCH 39/47] Update files --- kpops/manifests/strimzi/kafka_topic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 9b16e9de1..ea6f66ce0 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -23,7 +23,7 @@ class TopicSpec(CamelCaseConfigModel): :param partitions: The number of partitions the topic should have. This cannot be decreased after topic creation. It can be increased after topic creation, but it is important to understand the consequences that has, especially for topics with semantic partitioning. When absent this will default to the broker configuration for `num.partitions`. :param replicas: The number of replicas the topic should have. When absent this will default to the broker configuration for `default.replication.factor`. - :param config: The topic configuration. + :param config: The topic configuration. Topic config reference: https://docs.confluent.io/platform/current/installation/configuration/topic-configs.html """ @@ -33,7 +33,7 @@ class TopicSpec(CamelCaseConfigModel): replicas: int = Field( default=1, ge=1, le=32767, description=describe_attr("replicas", __doc__) ) - config: dict[str, str | int] = Field( + config: dict[str, Any] = Field( default_factory=dict, description=describe_attr("config", __doc__) ) From 2f1b51035731279b65b10ffa0d5da9cb359e9ca3 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 14:20:26 +0100 Subject: [PATCH 40/47] Update files --- config.yaml | 3 +- .../resources/variables/config_env_vars.env | 10 ++-- .../resources/variables/config_env_vars.md | 42 +++++++-------- docs/docs/schema/config.json | 51 ++++++++++++++----- kpops/config/__init__.py | 39 ++++++++++++-- kpops/manifests/strimzi/kafka_topic.py | 11 ++-- 6 files changed, 104 insertions(+), 52 deletions(-) diff --git a/config.yaml b/config.yaml index 3c388de5c..13ef5e2c6 100644 --- a/config.yaml +++ b/config.yaml @@ -1,3 +1,4 @@ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" pipeline_base_dir: tests/pipeline -kafka_topic_resource_label: "my-cluster" +strimzi_topic: + kafka_topic_resource_label: "strimzi.io/cluster=my-cluster" diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index 554450f8b..baaf4fdbf 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -61,10 +61,6 @@ KPOPS_RETAIN_CLEAN_JOBS=False # operation_mode # The operation mode of KPOps (managed, manifest, argo). KPOPS_OPERATION_MODE=managed -# kafka_topic_resource_label -# The label to identify the KafkaTopic resources managed by the Topic -# Operator. This does not have to be the name of the Kafka cluster. It -# can be the label assigned to the KafkaTopic resource. If you deploy -# more than one Topic Operator, the labels must be unique for each. -# That is, the operators cannot manage the same resources. -KPOPS_KAFKA_TOPIC_RESOURCE_LABEL # No default value, not required +# strimzi_topic +# Configuration for Strimzi Kafka Topics. +KPOPS_STRIMZI_TOPIC # No default value, not required diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 86231771f..f7fcd5303 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -1,23 +1,23 @@ These variables take precedence over the settings in `config.yaml`. Variables marked as required can instead be set in the global config. -| Name | Default Value |Required| Description | Setting name | -|--------------------------------------------------|----------------------------------------|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -|KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | -|KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | -|KPOPS_SCHEMA_REGISTRY__ENABLED |False |False |Whether the Schema Registry handler should be initialized. |schema_registry.enabled | -|KPOPS_SCHEMA_REGISTRY__URL |http://localhost:8081/ |False |Address of the Schema Registry. |schema_registry.url | -|KPOPS_SCHEMA_REGISTRY__TIMEOUT |30 |False |Operation timeout in seconds. |schema_registry.timeout | -|KPOPS_KAFKA_REST__URL |http://localhost:8082/ |False |Address of the Kafka REST Proxy. |kafka_rest.url | -|KPOPS_KAFKA_REST__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_rest.timeout | -|KPOPS_KAFKA_CONNECT__URL |http://localhost:8083/ |False |Address of Kafka Connect. |kafka_connect.url | -|KPOPS_KAFKA_CONNECT__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_connect.timeout | -|KPOPS_CREATE_NAMESPACE |False |False |Flag for `helm upgrade --install`. Create the release namespace if not present. |create_namespace | -|KPOPS_HELM_CONFIG__CONTEXT | |False |Name of kubeconfig context (`--kube-context`) |helm_config.context | -|KPOPS_HELM_CONFIG__DEBUG |False |False |Run Helm in Debug mode |helm_config.debug | -|KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | -|KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | -|KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion. |retain_clean_jobs | -|KPOPS_OPERATION_MODE |managed |False |The operation mode of KPOps (managed, manifest, argo). |operation_mode | -|KPOPS_KAFKA_TOPIC_RESOURCE_LABEL | |False |The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.|kafka_topic_resource_label | +| Name | Default Value |Required| Description | Setting name | +|--------------------------------------------------|----------------------------------------|--------|----------------------------------------------------------------------------------|-------------------------------------------| +|KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | +|KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | +|KPOPS_SCHEMA_REGISTRY__ENABLED |False |False |Whether the Schema Registry handler should be initialized. |schema_registry.enabled | +|KPOPS_SCHEMA_REGISTRY__URL |http://localhost:8081/ |False |Address of the Schema Registry. |schema_registry.url | +|KPOPS_SCHEMA_REGISTRY__TIMEOUT |30 |False |Operation timeout in seconds. |schema_registry.timeout | +|KPOPS_KAFKA_REST__URL |http://localhost:8082/ |False |Address of the Kafka REST Proxy. |kafka_rest.url | +|KPOPS_KAFKA_REST__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_rest.timeout | +|KPOPS_KAFKA_CONNECT__URL |http://localhost:8083/ |False |Address of Kafka Connect. |kafka_connect.url | +|KPOPS_KAFKA_CONNECT__TIMEOUT |30 |False |Operation timeout in seconds. |kafka_connect.timeout | +|KPOPS_CREATE_NAMESPACE |False |False |Flag for `helm upgrade --install`. Create the release namespace if not present. |create_namespace | +|KPOPS_HELM_CONFIG__CONTEXT | |False |Name of kubeconfig context (`--kube-context`) |helm_config.context | +|KPOPS_HELM_CONFIG__DEBUG |False |False |Run Helm in Debug mode |helm_config.debug | +|KPOPS_HELM_CONFIG__API_VERSION | |False |Kubernetes API version used for `Capabilities.APIVersions` |helm_config.api_version | +|KPOPS_HELM_DIFF_CONFIG__IGNORE | |True |Set of keys that should not be checked. |helm_diff_config.ignore | +|KPOPS_RETAIN_CLEAN_JOBS |False |False |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs | +|KPOPS_OPERATION_MODE |managed |False |The operation mode of KPOps (managed, manifest, argo). |operation_mode | +|KPOPS_STRIMZI_TOPIC | |False |Configuration for Strimzi Kafka Topics. |strimzi_topic | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index 068c32921..a2c2c1cf2 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -162,6 +162,32 @@ "title": "SchemaRegistryConfig", "type": "object" }, + "StrimziTopicConfig": { + "additionalProperties": false, + "description": "Configuration for Strimzi Kafka Topics.\n:param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + "properties": { + "resource_label": { + "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "string" + }, + { + "type": "string" + } + ], + "title": "Resource Label", + "type": "array" + } + }, + "required": [ + "resource_label" + ], + "title": "StrimziTopicConfig", + "type": "object" + }, "TopicNameConfig": { "additionalProperties": false, "description": "Configure the topic name variables you can use in the pipeline definition.", @@ -248,19 +274,6 @@ }, "description": "Configuration for Kafka REST Proxy." }, - "kafka_topic_resource_label": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", - "title": "Kafka Topic Resource Label" - }, "operation_mode": { "allOf": [ { @@ -296,6 +309,18 @@ }, "description": "Configuration for Schema Registry." }, + "strimzi_topic": { + "anyOf": [ + { + "$ref": "#/$defs/StrimziTopicConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for Strimzi Kafka Topics." + }, "topic_name_config": { "allOf": [ { diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 89543df82..4e641c1a6 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -2,8 +2,9 @@ import logging from pathlib import Path -from typing import ClassVar +from typing import Any, ClassVar +import pydantic from pydantic import AnyHttpUrl, Field, PrivateAttr, TypeAdapter from pydantic_settings import ( BaseSettings, @@ -12,14 +13,40 @@ ) from typing_extensions import override +from kpops.api.exception import ValidationError from kpops.api.operation import OperationMode from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig -from kpops.utils.docstring import describe_object +from kpops.utils.docstring import describe_attr, describe_object from kpops.utils.pydantic import YamlConfigSettingsSource ENV_PREFIX = "KPOPS_" +class StrimziTopicConfig(BaseSettings): + """Configuration for Strimzi Kafka Topics.\ + + :param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources. + """ + + resource_label: tuple[str, str] = Field( + description=describe_attr("resource_label", __doc__) + ) + + @pydantic.field_validator("resource_label", mode="before") + @classmethod + def deserialize_topics(cls, resource_label: Any) -> tuple[str, str]: + match resource_label: + case str(value) if "=" in value: + key, value = value.split("=") + return key, value + case dict(value) if len(value) == 1: + key, value = next(iter(value.items())) + return key, value + case _: + msg = "'kafka_topic_resource_label' should be defined either like 'foo=bar' or as a valid dictionary." + raise ValidationError(msg) + + class TopicNameConfig(BaseSettings): """Configure the topic name variables you can use in the pipeline definition.""" @@ -125,9 +152,13 @@ class KpopsConfig(BaseSettings): default=OperationMode.MANAGED, description="The operation mode of KPOps (managed, manifest, argo).", ) - kafka_topic_resource_label: str | None = Field( + # kafka_topic_resource_label: str | None = Field( + # default=None, + # description="", + # ) + strimzi_topic: StrimziTopicConfig | None = Field( default=None, - description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + description=describe_object(StrimziTopicConfig.__doc__), ) model_config = SettingsConfigDict( diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index ea6f66ce0..c7245fc14 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -64,17 +64,16 @@ class StrimziKafkaTopic(KubernetesManifest): @classmethod def from_topic(cls, topic: KafkaTopic) -> Self: - topic_resource_label = get_config().kafka_topic_resource_label - if not topic_resource_label: - msg = "When manifesting KafkaTopic you must define 'kafka_topic_resource_label' in the config.yaml" + strimzi_topic = get_config().strimzi_topic + if not strimzi_topic: + msg = "When manifesting KafkaTopic you must define 'strimzi_topic.kafka_topic_resource_label' in the config.yaml" raise ValidationError(msg) + topic_resource_label = strimzi_topic.resource_label metadata = ObjectMeta.model_validate( { "name": topic.name, - "labels": { - "strimzi.io/cluster": topic_resource_label, - }, + "labels": {topic_resource_label[0]: topic_resource_label[1]}, } ) spec = TopicSpec.model_validate( From baae8e33851ccd7021e2366bcecdf0e8949de715 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 14:34:10 +0100 Subject: [PATCH 41/47] fix tests --- config.yaml | 2 +- docs/docs/schema/config.json | 2 +- kpops/config/__init__.py | 8 ++------ kpops/manifests/strimzi/kafka_topic.py | 2 +- .../test_init_project_include_optional/config.yaml | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/config.yaml b/config.yaml index 13ef5e2c6..c97450e85 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,4 @@ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" pipeline_base_dir: tests/pipeline strimzi_topic: - kafka_topic_resource_label: "strimzi.io/cluster=my-cluster" + resource_label: "strimzi.io/cluster=my-cluster" diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index a2c2c1cf2..b21df358f 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -164,7 +164,7 @@ }, "StrimziTopicConfig": { "additionalProperties": false, - "description": "Configuration for Strimzi Kafka Topics.\n:param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + "description": "Configuration for Strimzi Kafka Topics.\n\n:param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", "properties": { "resource_label": { "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 4e641c1a6..baff597f5 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -23,7 +23,7 @@ class StrimziTopicConfig(BaseSettings): - """Configuration for Strimzi Kafka Topics.\ + """Configuration for Strimzi Kafka Topics. :param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources. """ @@ -43,7 +43,7 @@ def deserialize_topics(cls, resource_label: Any) -> tuple[str, str]: key, value = next(iter(value.items())) return key, value case _: - msg = "'kafka_topic_resource_label' should be defined either like 'foo=bar' or as a valid dictionary." + msg = "'resource_label' should be defined either like 'foo=bar' or as a valid dictionary." raise ValidationError(msg) @@ -152,10 +152,6 @@ class KpopsConfig(BaseSettings): default=OperationMode.MANAGED, description="The operation mode of KPOps (managed, manifest, argo).", ) - # kafka_topic_resource_label: str | None = Field( - # default=None, - # description="", - # ) strimzi_topic: StrimziTopicConfig | None = Field( default=None, description=describe_object(StrimziTopicConfig.__doc__), diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index c7245fc14..215bf7c07 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -66,7 +66,7 @@ class StrimziKafkaTopic(KubernetesManifest): def from_topic(cls, topic: KafkaTopic) -> Self: strimzi_topic = get_config().strimzi_topic if not strimzi_topic: - msg = "When manifesting KafkaTopic you must define 'strimzi_topic.kafka_topic_resource_label' in the config.yaml" + msg = "When manifesting KafkaTopic you must define 'strimzi_topic.resource_label' in the config.yaml" raise ValidationError(msg) topic_resource_label = strimzi_topic.resource_label diff --git a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml index 8f04208f6..438094f12 100644 --- a/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml +++ b/tests/cli/snapshots/test_init/test_init_project_include_optional/config.yaml @@ -17,7 +17,6 @@ kafka_connect: kafka_rest: timeout: 30 url: http://localhost:8082/ -kafka_topic_resource_label: null operation_mode: managed pipeline_base_dir: . retain_clean_jobs: false @@ -25,6 +24,7 @@ schema_registry: enabled: false timeout: 30 url: http://localhost:8081/ +strimzi_topic: null topic_name_config: default_error_topic_name: ${pipeline.name}-${component.name}-error default_output_topic_name: ${pipeline.name}-${component.name} From ac780e4b222e6a169dc78bc8b991720295aad6c8 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 14:36:00 +0100 Subject: [PATCH 42/47] Update files --- docs/docs/schema/config.json | 2 +- kpops/config/__init__.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index b21df358f..54aa50763 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -164,7 +164,7 @@ }, "StrimziTopicConfig": { "additionalProperties": false, - "description": "Configuration for Strimzi Kafka Topics.\n\n:param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", + "description": "Configuration for Strimzi Kafka Topics.", "properties": { "resource_label": { "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index baff597f5..351cb711d 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -16,20 +16,17 @@ from kpops.api.exception import ValidationError from kpops.api.operation import OperationMode from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig -from kpops.utils.docstring import describe_attr, describe_object +from kpops.utils.docstring import describe_object from kpops.utils.pydantic import YamlConfigSettingsSource ENV_PREFIX = "KPOPS_" class StrimziTopicConfig(BaseSettings): - """Configuration for Strimzi Kafka Topics. - - :param resource_label: The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources. - """ + """Configuration for Strimzi Kafka Topics.""" resource_label: tuple[str, str] = Field( - description=describe_attr("resource_label", __doc__) + description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources." ) @pydantic.field_validator("resource_label", mode="before") From 253157f7a6929294c59ee404c71c8d1bf1336769 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 14:38:43 +0100 Subject: [PATCH 43/47] Update files --- kpops/manifests/strimzi/kafka_topic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 215bf7c07..55b74a1ee 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -33,8 +33,8 @@ class TopicSpec(CamelCaseConfigModel): replicas: int = Field( default=1, ge=1, le=32767, description=describe_attr("replicas", __doc__) ) - config: dict[str, Any] = Field( - default_factory=dict, description=describe_attr("config", __doc__) + config: dict[str, Any] | None = Field( + default=None, description=describe_attr("config", __doc__) ) model_config = ConfigDict(extra="allow") From b32b6a423c0f5570b4f0bb8a9918f25348aad350 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Mon, 9 Dec 2024 15:09:20 +0100 Subject: [PATCH 44/47] Update files --- tests/manifests/strimzi/__init__.py | 0 tests/manifests/strimzi/test_kafka_topic.py | 77 +++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tests/manifests/strimzi/__init__.py create mode 100644 tests/manifests/strimzi/test_kafka_topic.py diff --git a/tests/manifests/strimzi/__init__.py b/tests/manifests/strimzi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/manifests/strimzi/test_kafka_topic.py b/tests/manifests/strimzi/test_kafka_topic.py new file mode 100644 index 000000000..3aa1ef53d --- /dev/null +++ b/tests/manifests/strimzi/test_kafka_topic.py @@ -0,0 +1,77 @@ +from unittest.mock import MagicMock + +import pytest +from pydantic import ValidationError as PydanticValidationError +from pytest_mock import MockerFixture + +from kpops.api.exception import ValidationError +from kpops.components.common.topic import KafkaTopic, TopicConfig +from kpops.manifests.strimzi.kafka_topic import StrimziKafkaTopic, TopicSpec + + +@pytest.fixture +def kafka_topic() -> KafkaTopic: + return KafkaTopic( + name="test-topic", + config=TopicConfig.model_validate( + { + "partitions_count": 3, + "replication_factor": 2, + "configs": {"cleanup.policy": "compact"}, + }, + ), + ) + + +def test_topic_spec_defaults(): + spec = TopicSpec() + assert spec.partitions == 1 + assert spec.replicas == 1 + assert spec.config is None + + +def test_topic_spec_custom_values(): + spec = TopicSpec(partitions=3, replicas=2, config={"retention.ms": "60000"}) + assert spec.partitions == 3 + assert spec.replicas == 2 + assert spec.config == {"retention.ms": "60000"} + + +def test_topic_spec_validation(): + with pytest.raises(PydanticValidationError): + TopicSpec(partitions=0) # Less than 1, should raise validation error + + with pytest.raises(PydanticValidationError): + TopicSpec(replicas=40000) # Exceeds max value, should raise validation error + + +def test_strimzi_kafka_topic_from_topic(kafka_topic: KafkaTopic, mocker: MockerFixture): + mock_config = MagicMock() + mock_config.strimzi_topic.resource_label = ("bakdata.com/cluster", "my-cluster") + mocker.patch( + "kpops.manifests.strimzi.kafka_topic.get_config", return_value=mock_config + ) + + strimzi_topic = StrimziKafkaTopic.from_topic(kafka_topic) + + # Check metadata + assert strimzi_topic.metadata.name == kafka_topic.name + assert strimzi_topic.metadata.labels == {"bakdata.com/cluster": "my-cluster"} + + # Check spec + assert strimzi_topic.spec.partitions == kafka_topic.config.partitions_count + assert strimzi_topic.spec.replicas == kafka_topic.config.replication_factor + assert strimzi_topic.spec.config == kafka_topic.config.configs + + +def test_strimzi_kafka_topic_missing_config(kafka_topic, mocker): + mock_config = MagicMock() + mock_config.strimzi_topic = None + mocker.patch( + "kpops.manifests.strimzi.kafka_topic.get_config", return_value=mock_config + ) + + with pytest.raises( + ValidationError, match="must define 'strimzi_topic.resource_label'" + ): + StrimziKafkaTopic.from_topic(kafka_topic) From 794d1a6b903151c978da25102e82f5d2387ab76b Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 10 Dec 2024 16:04:08 +0100 Subject: [PATCH 45/47] change strimzi label --- config.yaml | 3 ++- docs/docs/schema/config.json | 21 +++++---------- kpops/config/__init__.py | 29 +++++++++------------ kpops/manifests/strimzi/kafka_topic.py | 4 +-- tests/manifests/strimzi/test_kafka_topic.py | 2 +- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/config.yaml b/config.yaml index c97450e85..aa32d7d3c 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,5 @@ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" pipeline_base_dir: tests/pipeline strimzi_topic: - resource_label: "strimzi.io/cluster=my-cluster" + label: + strimzi.io/cluster: my-cluster diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index 54aa50763..bbf12c5ac 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -166,24 +166,17 @@ "additionalProperties": false, "description": "Configuration for Strimzi Kafka Topics.", "properties": { - "resource_label": { + "label": { + "additionalProperties": { + "type": "string" + }, "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", - "maxItems": 2, - "minItems": 2, - "prefixItems": [ - { - "type": "string" - }, - { - "type": "string" - } - ], - "title": "Resource Label", - "type": "array" + "title": "Label", + "type": "object" } }, "required": [ - "resource_label" + "label" ], "title": "StrimziTopicConfig", "type": "object" diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 351cb711d..7760d7078 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -2,7 +2,7 @@ import logging from pathlib import Path -from typing import Any, ClassVar +from typing import ClassVar import pydantic from pydantic import AnyHttpUrl, Field, PrivateAttr, TypeAdapter @@ -13,7 +13,6 @@ ) from typing_extensions import override -from kpops.api.exception import ValidationError from kpops.api.operation import OperationMode from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig from kpops.utils.docstring import describe_object @@ -21,27 +20,25 @@ ENV_PREFIX = "KPOPS_" +log = logging.getLogger("KPOpsConfig") + class StrimziTopicConfig(BaseSettings): """Configuration for Strimzi Kafka Topics.""" - resource_label: tuple[str, str] = Field( - description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources." + label: dict[str, str] = Field( + description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", ) - @pydantic.field_validator("resource_label", mode="before") + @pydantic.field_validator("label", mode="after") @classmethod - def deserialize_topics(cls, resource_label: Any) -> tuple[str, str]: - match resource_label: - case str(value) if "=" in value: - key, value = value.split("=") - return key, value - case dict(value) if len(value) == 1: - key, value = next(iter(value.items())) - return key, value - case _: - msg = "'resource_label' should be defined either like 'foo=bar' or as a valid dictionary." - raise ValidationError(msg) + def label_validator(cls, label: dict[str, str]) -> dict[str, str]: + if len(label) > 1: + log.warning( + "'resource_label' only reads the first entry in the dictionary. Other defined labels will be ignored." + ) + + return label class TopicNameConfig(BaseSettings): diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index 55b74a1ee..d218e9361 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -68,12 +68,12 @@ def from_topic(cls, topic: KafkaTopic) -> Self: if not strimzi_topic: msg = "When manifesting KafkaTopic you must define 'strimzi_topic.resource_label' in the config.yaml" raise ValidationError(msg) - topic_resource_label = strimzi_topic.resource_label + cluster_key, cluster_name = next(iter(strimzi_topic.label.items())) metadata = ObjectMeta.model_validate( { "name": topic.name, - "labels": {topic_resource_label[0]: topic_resource_label[1]}, + "labels": {cluster_key: cluster_name}, } ) spec = TopicSpec.model_validate( diff --git a/tests/manifests/strimzi/test_kafka_topic.py b/tests/manifests/strimzi/test_kafka_topic.py index 3aa1ef53d..8f189a6e2 100644 --- a/tests/manifests/strimzi/test_kafka_topic.py +++ b/tests/manifests/strimzi/test_kafka_topic.py @@ -47,7 +47,7 @@ def test_topic_spec_validation(): def test_strimzi_kafka_topic_from_topic(kafka_topic: KafkaTopic, mocker: MockerFixture): mock_config = MagicMock() - mock_config.strimzi_topic.resource_label = ("bakdata.com/cluster", "my-cluster") + mock_config.strimzi_topic.label = {"bakdata.com/cluster": "my-cluster"} mocker.patch( "kpops.manifests.strimzi.kafka_topic.get_config", return_value=mock_config ) From c41af8d78bfd32854617ffefbd5d094425a0468c Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 10 Dec 2024 16:42:08 +0100 Subject: [PATCH 46/47] add property --- kpops/config/__init__.py | 10 ++++++++-- kpops/manifests/strimzi/kafka_topic.py | 4 ++-- tests/manifests/strimzi/test_kafka_topic.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 7760d7078..3bdbfb9c9 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -26,11 +26,17 @@ class StrimziTopicConfig(BaseSettings): """Configuration for Strimzi Kafka Topics.""" - label: dict[str, str] = Field( + label_: dict[str, str] = Field( + alias="label", description="The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.", ) - @pydantic.field_validator("label", mode="after") + @property + def cluster_labels(self) -> tuple[str, str]: + """Return the defined strimzi_topic.label as a tuple.""" + return next(iter(self.label_.items())) + + @pydantic.field_validator("label_", mode="after") @classmethod def label_validator(cls, label: dict[str, str]) -> dict[str, str]: if len(label) > 1: diff --git a/kpops/manifests/strimzi/kafka_topic.py b/kpops/manifests/strimzi/kafka_topic.py index d218e9361..4b9a4b667 100644 --- a/kpops/manifests/strimzi/kafka_topic.py +++ b/kpops/manifests/strimzi/kafka_topic.py @@ -68,12 +68,12 @@ def from_topic(cls, topic: KafkaTopic) -> Self: if not strimzi_topic: msg = "When manifesting KafkaTopic you must define 'strimzi_topic.resource_label' in the config.yaml" raise ValidationError(msg) - cluster_key, cluster_name = next(iter(strimzi_topic.label.items())) + cluster_domain, cluster_name = strimzi_topic.cluster_labels metadata = ObjectMeta.model_validate( { "name": topic.name, - "labels": {cluster_key: cluster_name}, + "labels": {cluster_domain: cluster_name}, } ) spec = TopicSpec.model_validate( diff --git a/tests/manifests/strimzi/test_kafka_topic.py b/tests/manifests/strimzi/test_kafka_topic.py index 8f189a6e2..a519c2e94 100644 --- a/tests/manifests/strimzi/test_kafka_topic.py +++ b/tests/manifests/strimzi/test_kafka_topic.py @@ -47,7 +47,7 @@ def test_topic_spec_validation(): def test_strimzi_kafka_topic_from_topic(kafka_topic: KafkaTopic, mocker: MockerFixture): mock_config = MagicMock() - mock_config.strimzi_topic.label = {"bakdata.com/cluster": "my-cluster"} + mock_config.strimzi_topic.cluster_labels = ("bakdata.com/cluster", "my-cluster") mocker.patch( "kpops.manifests.strimzi.kafka_topic.get_config", return_value=mock_config ) From 7f1adb72bc30af1572e09c9c3940499bb29f3318 Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 10 Dec 2024 17:39:44 +0100 Subject: [PATCH 47/47] add tests --- kpops/config/__init__.py | 4 ++++ tests/conftest.py | 2 +- tests/test_kpops_config.py | 25 +++++++++++++++++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/kpops/config/__init__.py b/kpops/config/__init__.py index 3bdbfb9c9..07ecce58c 100644 --- a/kpops/config/__init__.py +++ b/kpops/config/__init__.py @@ -13,6 +13,7 @@ ) from typing_extensions import override +from kpops.api.exception import ValidationError from kpops.api.operation import OperationMode from kpops.component_handlers.helm_wrapper.model import HelmConfig, HelmDiffConfig from kpops.utils.docstring import describe_object @@ -39,6 +40,9 @@ def cluster_labels(self) -> tuple[str, str]: @pydantic.field_validator("label_", mode="after") @classmethod def label_validator(cls, label: dict[str, str]) -> dict[str, str]: + if len(label) == 0: + msg = "'strimzi_topic.label' must contain a single key-value pair." + raise ValidationError(msg) if len(label) > 1: log.warning( "'resource_label' only reads the first entry in the dictionary. Other defined labels will be ignored." diff --git a/tests/conftest.py b/tests/conftest.py index 4983aa325..777c87416 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,8 +57,8 @@ def custom_components() -> Iterator[None]: def clear_kpops_config() -> Iterator[None]: from kpops.config import KpopsConfig - yield KpopsConfig._instance = None + yield KUBECONFIG = """ diff --git a/tests/test_kpops_config.py b/tests/test_kpops_config.py index 0f35ddf37..e86096b21 100644 --- a/tests/test_kpops_config.py +++ b/tests/test_kpops_config.py @@ -1,14 +1,17 @@ import re from pathlib import Path +import pydantic import pytest -from pydantic import AnyHttpUrl, AnyUrl, TypeAdapter, ValidationError +from pydantic import AnyHttpUrl, AnyUrl, TypeAdapter +from kpops.api.exception import ValidationError from kpops.config import ( KafkaConnectConfig, KafkaRestConfig, KpopsConfig, SchemaRegistryConfig, + StrimziTopicConfig, get_config, set_config, ) @@ -43,7 +46,7 @@ def test_kpops_config_with_default_values(): def test_kpops_config_with_different_invalid_urls(): - with pytest.raises(ValidationError): + with pytest.raises(pydantic.ValidationError): KpopsConfig( kafka_brokers="http://broker:9092", kafka_connect=KafkaConnectConfig( @@ -51,7 +54,7 @@ def test_kpops_config_with_different_invalid_urls(): ), ) - with pytest.raises(ValidationError): + with pytest.raises(pydantic.ValidationError): KpopsConfig( kafka_brokers="http://broker:9092", kafka_rest=KafkaRestConfig( @@ -59,7 +62,7 @@ def test_kpops_config_with_different_invalid_urls(): ), ) - with pytest.raises(ValidationError): + with pytest.raises(pydantic.ValidationError): KpopsConfig( kafka_brokers="http://broker:9092", schema_registry=SchemaRegistryConfig( @@ -69,6 +72,7 @@ def test_kpops_config_with_different_invalid_urls(): ) +@pytest.mark.usefixtures("clear_kpops_config") def test_global_kpops_config_not_initialized_error(): with pytest.raises( RuntimeError, @@ -90,3 +94,16 @@ def test_set_global_kpops_config(): ) set_config(config) assert get_config() == config + + +def test_strimzi_topic_config_valid(): + config = StrimziTopicConfig.model_validate({"label": {"key": "value"}}) + assert config.cluster_labels == ("key", "value") + + +def test_strimzi_topic_config_empty_label(): + with pytest.raises( + ValidationError, + match="'strimzi_topic.label' must contain a single key-value pair.", + ): + StrimziTopicConfig.model_validate({"label": {}})