diff --git a/.github/actions/update-docs/action.yml b/.github/actions/update-docs/action.yml index 48911af5f..bc7877497 100644 --- a/.github/actions/update-docs/action.yml +++ b/.github/actions/update-docs/action.yml @@ -1,8 +1,7 @@ name: "Update documentation in gh-pages" description: | - Compile markdown documents to html and deploy to docs branch. If a semver tag is given, this action strips the patch. + Compile markdown documents to html and deploy to docs branch. If a semver version is given, this action strips the patch. It then pushes the . as the latest alias to the documentation branch with mike. - In case no tag is given, it pushes to 'dev'. inputs: username: @@ -14,8 +13,12 @@ inputs: token: description: "GitHub Token (must be a PAT for repository dispatch)" required: true - tag: - description: "Version tag" + version: + description: "Version name to be deployed by mike" + required: true + release: + description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)" + default: 'false' required: false runs: @@ -27,23 +30,30 @@ runs: poetry-version: "1.5.1" python-version: "3.10" - - name: Update documentation branch with mike + - name: Install docs dependencies shell: bash - env: - USERNAME: ${{ inputs.username }} - EMAIL: ${{ inputs.email }} - TOKEN: ${{ inputs.token }} - NEW_TAG: ${{ inputs.tag }} run: | poetry install --with docs - git config --local user.name ${USERNAME} - git config --local user.email ${EMAIL} - git config --local user.password ${TOKEN} - git pull # make sure docs branch is up-to-date - if [ -z "$NEW_TAG" ]; then - poetry run mike deploy dev --push --rebase --config-file ./docs/mkdocs.yml - else - new_tag=${NEW_TAG%.*} - poetry run mike deploy "$new_tag" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml - fi + - name: Update ${{ github.head_ref }} branch + shell: bash + run: | + git config --local user.name ${{ inputs.username }} + git config --local user.email ${{ inputs.email }} + git config --local user.password ${{ inputs.token }} + + git pull + + - name: Deploy ${{ inputs.version }} version of the documentation with mike + shell: bash + if: ${{ inputs.release == 'false' }} + run: | + poetry run mike deploy ${{ inputs.version }} --push --rebase --config-file ./docs/mkdocs.yml + + - name: Deploy ${{ inputs.version }} version (latest) of the documentation with mike + shell: bash + if: ${{ inputs.release == 'true' }} + run: | + sem_version=${{ inputs.version }} + major_minor_version=${sem_version%.*} + poetry run mike deploy "$major_minor_version" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b0cda131..baa091133 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -69,6 +69,10 @@ jobs: - name: Install docs dependencies run: poetry install --with docs + + - name: Check markdown formatting + uses: dprint/check@v2.2 + if: runner.os == 'Linux' - name: Test docs build (mkdocs) run: poetry run mkdocs build -f docs/mkdocs.yml @@ -79,3 +83,46 @@ jobs: uses: bakdata/ci-templates/.github/workflows/python-poetry-publish-snapshot.yaml@v1.25.0 secrets: pypi-token: ${{ secrets.TEST_PYPI_TOKEN }} + + publish-docs-from-main: + runs-on: ubuntu-22.04 + if: ${{ github.ref == 'refs/heads/main' }} + needs: [test] + steps: + - uses: actions/checkout@v3 + + - name: Publish docs from main branch + uses: ./.github/actions/update-docs + with: + username: ${{ secrets.GH_USERNAME }} + email: ${{ secrets.GH_EMAIL }} + token: ${{ secrets.GH_TOKEN }} + version: main + + publish-dev-docs-from-pr: + runs-on: ubuntu-22.04 + if: ${{ github.event_name == 'pull_request' }} + needs: [test] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # Checks to see if any files in the PR match one of the listed file types. + # This will return true if there's a file in docs folder that was added, deleted, or modified in the PR. + - name: Check if files in docs folder have changed + uses: dorny/paths-filter@v2 + id: docs-changes + with: + filters: | + docs: + - added|deleted|modified: 'docs/**' + + - name: Publish dev docs from PR + if: steps.docs-changes.outputs.docs == 'true' + uses: ./.github/actions/update-docs + with: + username: ${{ secrets.GH_USERNAME }} + email: ${{ secrets.GH_EMAIL }} + token: ${{ secrets.GH_TOKEN }} + version: dev diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 1341075b0..0fe1f0573 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,6 +6,7 @@ on: - main paths: - .github/** + - actions/** jobs: actionlint: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b34323896..9171e6c70 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,7 +39,8 @@ jobs: - name: Update gh-pages uses: ./.github/actions/update-docs with: - tag: ${{ needs.create-github-release-push-tag.outputs.release-version }} username: ${{ secrets.GH_USERNAME }} email: ${{ secrets.GH_EMAIL }} token: ${{ secrets.GH_TOKEN }} + version: ${{ needs.create-github-release-push-tag.outputs.release-version }} + release: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f92a0c936..59639ac35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,3 +81,10 @@ repos: | kpops/components/.*\.py )$ require_serial: true + - id: dprint + name: dprint + entry: dprint fmt + language: system + types: [markdown] + require_serial: true + pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 5409cf299..66b262704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,80 @@ # Changelog +## [2.0.5](https://github.com/bakdata/kpops/releases/tag/2.0.5) - Release Date: [2023-08-30] + +### 🐛 Fixes + +- Fix versioning of docs when releasing - [#346](https://github.com/bakdata/kpops/pull/346) + + + + + + +## [2.0.4](https://github.com/bakdata/kpops/releases/tag/2.0.4) - Release Date: [2023-08-29] + +### 🐛 Fixes + +- Fix GitHub ref variable for pushing docs to main branch - [#343](https://github.com/bakdata/kpops/pull/343) + + +### 📝 Documentation + +- Add `dprint` as the markdown formatter - [#337](https://github.com/bakdata/kpops/pull/337) + +- Publish pre-release docs for PRs & main branch - [#339](https://github.com/bakdata/kpops/pull/339) + +- Align docs colours - [#345](https://github.com/bakdata/kpops/pull/345) + + +### 🌀 Miscellaneous + +- Exclude abstract components from pipeline schema - [#332](https://github.com/bakdata/kpops/pull/332) + + + + + + +## [2.0.3](https://github.com/bakdata/kpops/releases/tag/2.0.3) - Release Date: [2023-08-24] + +### 🐛 Fixes + +- Fix GitHub action error in non-Python projects - [#340](https://github.com/bakdata/kpops/pull/340) + + +### 🌀 Miscellaneous + +- Lint GitHub action - [#342](https://github.com/bakdata/kpops/pull/342) + + + + + + +## [2.0.2](https://github.com/bakdata/kpops/releases/tag/2.0.2) - Release Date: [2023-08-23] + +### 📝 Documentation + +- Add version dropdown to the documentation - [#336](https://github.com/bakdata/kpops/pull/336) + +- Break the documentation down into smaller subsection - [#329](https://github.com/bakdata/kpops/pull/329) + + + + + + +## [2.0.1](https://github.com/bakdata/kpops/releases/tag/2.0.1) - Release Date: [2023-08-22] + +### 🐛 Fixes + +- Fix optional flags in GitHub action - [#334](https://github.com/bakdata/kpops/pull/334) + + + + + + ## [2.0.0](https://github.com/bakdata/kpops/releases/tag/2.0.0) - Release Date: [2023-08-17] ### 🏗️ Breaking changes diff --git a/README.md b/README.md index 17707f0b3..9d2aaca2e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ the [documentation](https://bakdata.github.io/kpops/latest). ## Install KPOps -KPOps comes as a [PyPI package](https://pypi.org/project/kpops/). +KPOps comes as a [PyPI package](https://pypi.org/project/kpops/). You can install it with [pip](https://github.com/pypa/pip): ```shell diff --git a/actions/kpops-runner/README.md b/actions/kpops-runner/README.md index c8261d0b4..9ee35bfe1 100644 --- a/actions/kpops-runner/README.md +++ b/actions/kpops-runner/README.md @@ -1,5 +1,3 @@ # KPOps runner -Please refer to the -[GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) -for the documentation. +Please refer to the [GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) for the documentation. diff --git a/actions/kpops-runner/action.yaml b/actions/kpops-runner/action.yaml index 1d6eec417..ff2533251 100644 --- a/actions/kpops-runner/action.yaml +++ b/actions/kpops-runner/action.yaml @@ -13,20 +13,19 @@ inputs: default: "." pipeline-base-dir: description: "Directory where relative pipeline variables are initialized from" - default: "." + required: false defaults: description: "Defaults folder path" - default: "." + required: false config: description: "config.yaml file path" - default: "config.yaml" + required: false components: description: "Components package path" required: false filter-type: description: "Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS (default is include)" required: false - default: "include" python-version: description: "Python version to install (Defaults to the latest stable version of Python 3.11)" required: false @@ -46,26 +45,38 @@ inputs: runs: using: "composite" steps: - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - name: Set up Helm uses: azure/setup-helm@v3 with: version: ${{ inputs.helm-version }} token: ${{ inputs.token }} + - name: Create temporary requirements.txt for caching + shell: bash + id: requirements + run: | + TEMP_FILE="${{ github.action_path }}/kpops-runner-action-requirements.txt" + echo "kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }}" > "$TEMP_FILE" + cat "$TEMP_FILE" + echo "path=$TEMP_FILE">> $GITHUB_OUTPUT + + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + cache: pip + cache-dependency-path: '' + # FIXME: https://github.com/actions/setup-python/issues/361 + # "${{ steps.requirements.outputs.path }}" + - name: Install KPOps shell: bash run: | - echo "::group::pip install kpops package" - pip install kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }} + echo "::group::install kpops package" + pip install -r "${{ steps.requirements.outputs.path }}" echo "::endgroup::" - name: ${{ inputs.command }} ${{ inputs.pipeline }} pipeline shell: bash working-directory: ${{inputs.working-directory}} - run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} --defaults ${{ inputs.defaults }} --config ${{ inputs.config }} --pipeline-base-dir ${{ inputs.pipeline-base-dir }} --filter-type ${{ inputs.filter-type }} + run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.pipeline-base-dir != '' && format('--pipeline-base-dir {0}', inputs.pipeline-base-dir)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }} diff --git a/docs/docs/dev/development/autogeneration.md b/docs/docs/dev/development/autogeneration.md deleted file mode 100644 index b19e95c42..000000000 --- a/docs/docs/dev/development/autogeneration.md +++ /dev/null @@ -1,47 +0,0 @@ -# Autogeneration - -Autogeneration happens mostly in the `pre-commit` hooks in `/kpops/hooks`. - -## Generation scripts and their respective files - -### Documentation (`gen_docs/*`) - -#### Variables (`./kpops/docs/docs/resources/variables/*`) - -- `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file. -- `cli_env_vars.md` -- All CLI environment variables in a table. -- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in `PipelineConfig` from `./kpops/kpops/cli/pipeline_config.py` whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it. -- `config_env_vars.env` -- Almost all pipeline config environment variables in a table. -- `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substition. - -#### CLI commands (`./kpops/docs/docs/user/references/cli-commands.md`) - -Generated by `typer-cli` from the code in `./kpops/kpops/cli/main.py`. It is called with Python's `subprocess` module. - -#### Pipeline and defaults example definitions (`./kpops/docs/docs/resources/pipeline-components/` and `./kpops/docs/docs/resources/pipeline-defaults/`) - -Generates example `pipeline.yaml` and `defaults.yaml` for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition. - - -User input - -- `headers/*\.yaml` -- The top of each example. Includes a description comment, `type` and `name`. The headers for `pipeline.yaml` reside in the `pipeline-components` dir and the `defaults.yaml` headers reside in the `pipeline-defaults` dir. The names of the files must be equal to the respective component `type`. -- `sections/*\.yaml` -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component `type` and the attribute name. The sections are used for both `defaults.yaml` and `pipeline.yaml` generation and reside in the `pipeline-components` dir. - - -Generated - -- `pipeline-components/dependencies/*` -Cached information about KPOps components - - `pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `pipeline.yaml` generation. - - `defaults_pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `defaults.yaml` generation. - - `kpops_structure.yaml` -- Specifies the inheritance hierarchy of the components and what sections exist in each component. -- `pipeline-components/*\.yaml` -- All single-component pipeline definitions and one big (complete) `pipeline.yaml` that contains all of them. -- `pipeline-defaults/*\.yaml` -- All single-component defaults definitions and one big (complete) `defaults.yaml` that contains all of them. - -### Editor integration (`gen_schema.py`) - -#### Schemas (`./kpops/docs/docs/schema/*`) - -- `config.json` -- `pipeline.json` diff --git a/docs/docs/developer/auto-generation.md b/docs/docs/developer/auto-generation.md new file mode 100644 index 000000000..249f52b77 --- /dev/null +++ b/docs/docs/developer/auto-generation.md @@ -0,0 +1,45 @@ +# Auto generation + +Auto generation happens mostly with [`pre-commit`](https://pre-commit.com/) hooks. You can find the pre-commit configuration [here](https://github.com/bakdata/kpops/blob/main/.pre-commit-config.yaml). These pre-commit hooks call different [Python scripts](https://github.com/bakdata/kpops/tree/main/hooks) to auto generate code for the documentation. + +## Generation scripts and their respective files + +### [Documentation](https://github.com/bakdata/kpops/tree/main/hooks/gen_docs) + +#### [Variables](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/variables) + +- `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file. +- `cli_env_vars.md` -- All CLI environment variables in a table. +- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it. +- `config_env_vars.env` -- Almost all pipeline config environment variables in a table. +- `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substitution. + +#### [CLI commands](../user/references/cli-commands.md) + +Generated by `typer-cli` from the code in [`main.py`](https://github.com/bakdata/kpops/blob/main/kpops/cli/main.py). It is called with Python's `subprocess` module. + +#### [Pipeline](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/pipeline-components) and [defaults](https://github.com/bakdata/kpops/tree/main/docs/docs/resources/pipeline-defaults) example definitions + +Generates example `pipeline.yaml` and `defaults.yaml` for each individual component, stores them and also concatenates them into 1 big pipeline definition and 1 big pipeline defaults definition. + +User input + +- `headers/*\.yaml` -- The top of each example. Includes a description comment, `type` and `name`. The headers for `pipeline.yaml` reside in the `pipeline-components` dir and the `defaults.yaml` headers reside in the `pipeline-defaults` dir. The names of the files must be equal to the respective component `type`. +- `sections/*\.yaml` -- Each YAML file contains a single section (component attribute) definition. The intention is to keep the minimal set of definitions there from which any component definition can be built. The names of the files must be equal to the respective component `type` and the attribute name. The sections are used for both `defaults.yaml` and `pipeline.yaml` generation and reside in the `pipeline-components` dir. + +Generated + +- `pipeline-components/dependencies/*` + Cached information about KPOps components + - `pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `pipeline.yaml` generation. + - `defaults_pipeline_component_dependencies.yaml` -- Specifies per component which files in the `sections` dir should be used for the `defaults.yaml` generation. + - `kpops_structure.yaml` -- Specifies the inheritance hierarchy of the components and what sections exist in each component. +- `pipeline-components/*\.yaml` -- All single-component pipeline definitions and one big (complete) `pipeline.yaml` that contains all of them. +- `pipeline-defaults/*\.yaml` -- All single-component defaults definitions and one big (complete) `defaults.yaml` that contains all of them. + +### [Editor integration](https://github.com/bakdata/kpops/blob/main/hooks/gen_schema.py) + +#### [Schemas](https://github.com/bakdata/kpops/tree/main/docs/docs/schema) + +- [config.json](https://github.com/bakdata/kpops/blob/main/docs/docs/schema/config.json) +- [pipeline.json](https://github.com/bakdata/kpops/blob/main/docs/docs/schema/pipeline.json) diff --git a/docs/docs/developer/formatting.md b/docs/docs/developer/formatting.md new file mode 100644 index 000000000..54ea1b6d9 --- /dev/null +++ b/docs/docs/developer/formatting.md @@ -0,0 +1,12 @@ +# Formatting + +## Markdown + +To ensure a consistent markdown style, we use [dprint](https://dprint.dev) to check and reformat. + +```shell +dprint fmt +``` + +Use the [official documentation](https://dprint.dev/setup/) to set up dprint. +The configuration can be found [here](https://github.com/bakdata/kpops/blob/main/dprint.json). diff --git a/docs/docs/images/kpops-components_class-diagram.drawio b/docs/docs/images/kpops-components_class-diagram.drawio deleted file mode 100644 index ad7ea337a..000000000 --- a/docs/docs/images/kpops-components_class-diagram.drawio +++ /dev/null @@ -1 +0,0 @@ -7ZrJdpswFIafxsv0MJsu4ylje9I6qZulDNdGtUBUCA99+gojJuOmdpJSJ/Yqur8kJO7wCYhbetdfXjAUep+oC6SlKe6ypfdamqYauiH+JMpKKrbVTpUpw67UCmGIf4EUFanG2IWoMpBTSjgOq6JDgwAcXtEQY3RRHTahpLpqiKZQE4YOInV1hF3upaqttQv9EvDUy1ZWrY9pj4+ywfJOIg+5dFGS9H5L7zJKedryl10gifcyv4yuViNyO7Murr9EP9FD5+b+87ez9GKDfabkt8Ag4M+/dHT7dRbiiwd3dX9N7tvXoyvrTLXkvfFV5jBwhf+kSRn36JQGiPQLtSMcESa9MmSUCY3ROHAhWUoRVjHvltJQiKoQfwDnK5kgKOZUSB73ieyFJebfk+kfTGk9yosl7d6ybKykQdAYSAc5s+l6+S4lyV56LkxQTISnOhMa8AHyMUlmXAKZA8cOkh1yK6oq7fp0CNzzJAWFOCbUmaXSABMiNzARzWxaS9PHqutOEj3ijM6g1GMjZDnjvCfLRC3bivT3rpGW4yIaMweeCK/MXI7YFPgT47R0XBL6UtXIPLoA6gNnKzGAAUEcz6ulhWSFTvNxRRaKhkzEPZJS7nqOSCxXusMhEBwIj/ohDRLfbGZtNf8WHuYwDNHaOQtBtmquPT88tZD3zvv2oCv0/SI3B8Zh+aSvZa+e4ScDcGYvCprlmlcimaH8o/BkZN+LGf+FD3+t30OpU/W1609OvaM4qZU8l+yNXLI2ciQFhZy1kSb5Nl6QObXCbmnWmrQunovmNGnexGNgAXCIzsMw6xfLlYYcT/UbRjVieVWXq19psvq1Wgw7KIJeemZGrwToP7m5oXrdg86KWY+H2SiN1ZqrTzR+GY31HZ+aXp3aL0oEvVaYN2gyQ2uIHg0uTbOCS92q47LdaHUaz6jO0/vVoRHB2JEI+kERwagRYcgZID86LiYo1UcoY8sjlN0kEz6+nQNb3G5pUmI+ZtdLjGLa2npzB725Y1krB1XW5vaDvpufFkdT2lp2vB7M25Gqnc77d3DeWzuCwTwoMFjbwTDEwewI4aDqVThsexdo+MOpfoLDO4BD+03CoV2Dw7Zvr2terB1QIsZxf4LV2jtwxGiUI+aJI++AI/aOHDmsjwp2jSN3jLqxA+yoviqYzX1VEGbxI5P0X3HFb3X0/m8= diff --git a/docs/docs/images/kpops-components_class-diagram.png b/docs/docs/images/kpops-components_class-diagram.png deleted file mode 100644 index bec44874b..000000000 Binary files a/docs/docs/images/kpops-components_class-diagram.png and /dev/null differ diff --git a/docs/overrides/home.html b/docs/docs/overrides/home.html similarity index 84% rename from docs/overrides/home.html rename to docs/docs/overrides/home.html index 2986d9ec5..a69b1d4ca 100644 --- a/docs/overrides/home.html +++ b/docs/docs/overrides/home.html @@ -1,24 +1,22 @@ {% extends "base.html" %} +{% block outdated %} + You're not viewing the latest version. + + Click here to go to latest. + +{% endblock %} + {% block tabs %} diff --git a/docs/docs/resources/architecture/components-hierarchy.md b/docs/docs/resources/architecture/components-hierarchy.md index 644418cf3..fec927ddf 100644 --- a/docs/docs/resources/architecture/components-hierarchy.md +++ b/docs/docs/resources/architecture/components-hierarchy.md @@ -1,4 +1,20 @@ -
- ![kpops-component-hierarchy](../../images/kpops-components_class-diagram.png) -
KPOps component hierarchy
-
+```mermaid +flowchart BT + KubernetesApp --> PipelineComponent + KafkaConnector --> PipelineComponent + KafkaApp --> KubernetesApp + StreamsApp --> KafkaApp + ProducerApp --> KafkaApp + KafkaSourceConnector --> KafkaConnector + KafkaSinkConnector --> KafkaConnector + + click KubernetesApp "../kubernetes-app" + click KafkaApp "../kafka-app" + click StreamsApp "../streams-app" + click ProducerApp "../producer-app" + click KafkaConnector "../kafka-connector" + click KafkaSourceConnector "../kafka-source-connector" + click KafkaSinkConnector "../kafka-sink-connector" +``` + +

KPOps component hierarchy

diff --git a/docs/docs/resources/examples/defaults.md b/docs/docs/resources/examples/defaults.md index 2b4525493..a8740485e 100644 --- a/docs/docs/resources/examples/defaults.md +++ b/docs/docs/resources/examples/defaults.md @@ -2,18 +2,26 @@ ## [ATM Fraud Pipeline](https://github.com/bakdata/kpops/tree/main/examples/bakdata/atm-fraud-detection){target=_blank} + + ??? example "defaults.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/defaults.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/defaults.yaml + --8<-- ``` + + ## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops/defaults){target=_blank} + + ??? example "defaults.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults/defaults.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults/defaults.yaml + --8<-- ``` + + diff --git a/docs/docs/resources/examples/pipeline.md b/docs/docs/resources/examples/pipeline.md index 3bbec8039..4ef2632c8 100644 --- a/docs/docs/resources/examples/pipeline.md +++ b/docs/docs/resources/examples/pipeline.md @@ -2,18 +2,26 @@ ## [ATM Fraud Pipeline](https://github.com/bakdata/kpops/tree/main/examples/bakdata/atm-fraud-detection){target=_blank} + + ??? example "pipeline.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/pipeline.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops/main/examples/bakdata/atm-fraud-detection/pipeline.yaml + --8<-- ``` + + ## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank} + + ??? example "pipeline.yaml" ```yaml - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml - --8<-- + --8<-- + https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml + --8<-- ``` + + diff --git a/docs/docs/resources/pipeline-components/pipeline.md b/docs/docs/resources/pipeline-components/pipeline.md index b4b518ebb..e8d0b752b 100644 --- a/docs/docs/resources/pipeline-components/pipeline.md +++ b/docs/docs/resources/pipeline-components/pipeline.md @@ -3,7 +3,7 @@ [:material-download:](./pipeline.yaml) ```yaml - --8<-- - ./docs/resources/pipeline-components/pipeline.yaml - --8<-- +--8<-- +./docs/resources/pipeline-components/pipeline.yaml +--8<-- ``` diff --git a/docs/docs/resources/pipeline_config/config.yaml b/docs/docs/resources/pipeline-config/config.yaml similarity index 100% rename from docs/docs/resources/pipeline_config/config.yaml rename to docs/docs/resources/pipeline-config/config.yaml diff --git a/docs/docs/resources/pipeline-defaults/defaults.md b/docs/docs/resources/pipeline-defaults/defaults.md index 6f71a3cd6..914bca3e4 100644 --- a/docs/docs/resources/pipeline-defaults/defaults.md +++ b/docs/docs/resources/pipeline-defaults/defaults.md @@ -3,7 +3,7 @@ [:material-download:](./defaults.yaml) ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults.yaml - --8<-- +--8<-- +./docs/resources/pipeline-defaults/defaults.yaml +--8<-- ``` diff --git a/docs/docs/resources/setup/kafka.yaml b/docs/docs/resources/setup/kafka.yaml new file mode 100644 index 000000000..753b29152 --- /dev/null +++ b/docs/docs/resources/setup/kafka.yaml @@ -0,0 +1,109 @@ +cp-zookeeper: + enabled: true + servers: 1 + imageTag: 7.1.3 + heapOptions: "-Xms124M -Xmx124M" + overrideGroupId: k8kafka + fullnameOverride: "k8kafka-cp-zookeeper" + resources: + requests: + cpu: 50m + memory: 0.2G + limits: + cpu: 250m + memory: 0.2G + prometheus: + jmx: + enabled: false + +cp-kafka: + enabled: true + brokers: 1 + imageTag: 7.1.3 + podManagementPolicy: Parallel + configurationOverrides: + "auto.create.topics.enable": false + "offsets.topic.replication.factor": 1 + "transaction.state.log.replication.factor": 1 + "transaction.state.log.min.isr": 1 + "confluent.metrics.reporter.topic.replicas": 1 + resources: + requests: + cpu: 50m + memory: 0.5G + limits: + cpu: 250m + memory: 0.5G + prometheus: + jmx: + enabled: false + persistence: + enabled: false + +cp-schema-registry: + enabled: true + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-schema-registry" + overrideGroupId: k8kafka + kafka: + bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" + resources: + requests: + cpu: 50m + memory: 0.25G + limits: + cpu: 250m + memory: 0.25G + prometheus: + jmx: + enabled: false + +cp-kafka-connect: + enabled: true + replicaCount: 1 + image: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-kafka-connect" + overrideGroupId: k8kafka + kafka: + bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" + heapOptions: "-Xms256M -Xmx256M" + resources: + requests: + cpu: 500m + memory: 0.25G + limits: + cpu: 500m + memory: 0.25G + configurationOverrides: + "consumer.max.poll.records": "10" + "consumer.max.poll.interval.ms": "900000" + "config.storage.replication.factor": "1" + "offset.storage.replication.factor": "1" + "status.storage.replication.factor": "1" + cp-schema-registry: + url: http://k8kafka-cp-schema-registry:8081 + prometheus: + jmx: + enabled: false + +cp-kafka-rest: + enabled: true + imageTag: 7.1.3 + fullnameOverride: "k8kafka-cp-rest" + heapOptions: "-Xms256M -Xmx256M" + resources: + requests: + cpu: 50m + memory: 0.25G + limits: + cpu: 250m + memory: 0.5G + prometheus: + jmx: + enabled: false + +cp-ksql-server: + enabled: false +cp-control-center: + enabled: false diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index cc0000f72..9695ec9a2 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -93,215 +93,6 @@ "title": "InputTopicTypes", "type": "string" }, - "KafkaApp": { - "description": "Base component for Kafka-based components.\nProducer or streaming apps should inherit from this class.", - "properties": { - "app": { - "allOf": [ - { - "$ref": "#/definitions/KafkaAppConfig" - } - ], - "description": "Application-specific settings", - "title": "App" - }, - "from": { - "allOf": [ - { - "$ref": "#/definitions/FromSection" - } - ], - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-streams-bootstrap", - "url": "https://bakdata.github.io/streams-bootstrap/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component", - "title": "Repo Config" - }, - "to": { - "allOf": [ - { - "$ref": "#/definitions/ToSection" - } - ], - "description": "Topic(s) into which the component will write output", - "title": "To" - }, - "type": { - "default": "kafka-app", - "description": "Base component for Kafka-based components.\nProducer or streaming apps should inherit from this class.", - "enum": [ - "kafka-app" - ], - "title": "Component type", - "type": "string" - }, - "version": { - "default": "2.9.0", - "description": "Helm chart version", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "app" - ], - "title": "KafkaApp", - "type": "object" - }, - "KafkaAppConfig": { - "description": "Settings specific to Kafka Apps", - "properties": { - "nameOverride": { - "description": "Override name with this value", - "title": "Nameoverride", - "type": "string" - }, - "streams": { - "allOf": [ - { - "$ref": "#/definitions/KafkaStreamsConfig" - } - ], - "description": "Kafka streams config", - "title": "Streams" - } - }, - "required": [ - "streams" - ], - "title": "KafkaAppConfig", - "type": "object" - }, - "KafkaConnector": { - "description": "Base class for all Kafka connectors\nShould only be used to set defaults", - "properties": { - "app": { - "allOf": [ - { - "$ref": "#/definitions/KafkaConnectorConfig" - } - ], - "description": "Application-specific settings", - "title": "App" - }, - "from": { - "allOf": [ - { - "$ref": "#/definitions/FromSection" - } - ], - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "namespace": { - "description": "Namespace in which the component shall be deployed", - "title": "Namespace", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "repo_config": { - "allOf": [ - { - "$ref": "#/definitions/HelmRepoConfig" - } - ], - "default": { - "repo_auth_flags": { - "ca_file": null, - "cert_file": null, - "insecure_skip_tls_verify": false, - "password": null, - "username": null - }, - "repository_name": "bakdata-kafka-connect-resetter", - "url": "https://bakdata.github.io/kafka-connect-resetter/" - }, - "description": "Configuration of the Helm chart repo to be used for deploying the component", - "title": "Repo Config" - }, - "resetter_values": { - "description": "Overriding Kafka Connect Resetter Helm values. E.g. to override the Image Tag etc.", - "title": "Resetter Values", - "type": "object" - }, - "to": { - "allOf": [ - { - "$ref": "#/definitions/ToSection" - } - ], - "description": "Topic(s) into which the component will write output", - "title": "To" - }, - "type": { - "default": "kafka-connector", - "description": "Base class for all Kafka connectors\nShould only be used to set defaults", - "enum": [ - "kafka-connector" - ], - "title": "Component type", - "type": "string" - }, - "version": { - "default": "1.0.4", - "description": "Helm chart version", - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "namespace", - "app" - ], - "title": "KafkaConnector", - "type": "object" - }, "KafkaConnectorConfig": { "additionalProperties": { "type": "string" @@ -514,26 +305,6 @@ "title": "KafkaSourceConnector", "type": "object" }, - "KafkaStreamsConfig": { - "description": "Kafka Streams config", - "properties": { - "brokers": { - "description": "Brokers", - "title": "Brokers", - "type": "string" - }, - "schemaRegistryUrl": { - "description": "URL of the schema registry", - "title": "Schemaregistryurl", - "type": "string" - } - }, - "required": [ - "brokers" - ], - "title": "KafkaStreamsConfig", - "type": "object" - }, "KubernetesApp": { "description": "Base class for all Kubernetes apps.\nAll built-in components are Kubernetes apps, except for the Kafka connectors.", "properties": { @@ -607,8 +378,7 @@ "required": [ "name", "namespace", - "app", - "repo_config" + "app" ], "title": "KubernetesApp", "type": "object" @@ -628,54 +398,6 @@ "title": "OutputTopicTypes", "type": "string" }, - "PipelineComponent": { - "description": "Base class for all components", - "properties": { - "from": { - "allOf": [ - { - "$ref": "#/definitions/FromSection" - } - ], - "description": "Topic(s) and/or components from which the component will read input", - "title": "From" - }, - "name": { - "description": "Component name", - "title": "Name", - "type": "string" - }, - "prefix": { - "default": "${pipeline_name}-", - "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", - "title": "Prefix", - "type": "string" - }, - "to": { - "allOf": [ - { - "$ref": "#/definitions/ToSection" - } - ], - "description": "Topic(s) into which the component will write output", - "title": "To" - }, - "type": { - "default": "pipeline-component", - "description": "Base class for all components", - "enum": [ - "pipeline-component" - ], - "title": "Component type", - "type": "string" - } - }, - "required": [ - "name" - ], - "title": "PipelineComponent", - "type": "object" - }, "ProducerApp": { "description": "Producer component\nThis producer holds configuration to use as values for the streams bootstrap producer helm chart. Note that the producer does not support error topics.", "properties": { @@ -1218,24 +940,15 @@ "items": { "discriminator": { "mapping": { - "kafka-app": "#/definitions/KafkaApp", - "kafka-connector": "#/definitions/KafkaConnector", "kafka-sink-connector": "#/definitions/KafkaSinkConnector", "kafka-source-connector": "#/definitions/KafkaSourceConnector", "kubernetes-app": "#/definitions/KubernetesApp", - "pipeline-component": "#/definitions/PipelineComponent", "producer-app": "#/definitions/ProducerApp", "streams-app": "#/definitions/StreamsApp" }, "propertyName": "type" }, "oneOf": [ - { - "$ref": "#/definitions/KafkaApp" - }, - { - "$ref": "#/definitions/KafkaConnector" - }, { "$ref": "#/definitions/KafkaSinkConnector" }, @@ -1245,9 +958,6 @@ { "$ref": "#/definitions/KubernetesApp" }, - { - "$ref": "#/definitions/PipelineComponent" - }, { "$ref": "#/definitions/ProducerApp" }, diff --git a/docs/docs/stylesheets/extra.css b/docs/docs/stylesheets/extra.css new file mode 100644 index 000000000..ae2953c82 --- /dev/null +++ b/docs/docs/stylesheets/extra.css @@ -0,0 +1,24 @@ +:root > * { + /* The primary color of KPOps docs */ + --md-primary-fg-color: #a599ff; + + /* The second color of KPOps docs */ + --md-accent-fg-color: #B0A5FF; + + /* Create custom color variable to use for the buttons */ + --kpops-button-color: #7068AD; + + /* The footer color needs to be set sparately */ + --md-footer-bg-color: var(--md-primary-fg-color); + + /* Color the links with the primary color */ + --md-typeset-a-color: var(--md-primary-fg-color); +} + +.md-typeset .md-button, .md-button--primary { + background-color: var(--kpops-button-color); +} + +.md-typeset .md-button:hover, .md-button--primary:hover { + border-color: white; +} diff --git a/docs/docs/user/core-concepts/components/kafka-app.md b/docs/docs/user/core-concepts/components/kafka-app.md new file mode 100644 index 000000000..e69153c5e --- /dev/null +++ b/docs/docs/user/core-concepts/components/kafka-app.md @@ -0,0 +1,44 @@ +# KafkaApp + +Subclass of [_KubernetesApp_](kubernetes-app.md). + +### Usage + +- Defines a [streams-bootstrap](https://github.com/bakdata/streams-bootstrap#usage){target=_blank} component +- Should not be used in `pipeline.yaml` as the component can be defined as either a [StreamsApp](streams-app.md) or a [ProducerApp](producer-app.md) +- Often used in `defaults.yaml` + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/kafka-app.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +In addition to [KubernetesApp's `deploy`](kubernetes-app.md#deploy): + +- Create topics if provided (optional) +- Submit Avro schemas to the registry if provided (optional) + +#### destroy + +Uninstall Helm release. + +#### reset + +Do nothing. + +#### clean + +Do nothing. diff --git a/docs/docs/user/core-concepts/components/kafka-connector.md b/docs/docs/user/core-concepts/components/kafka-connector.md new file mode 100644 index 000000000..bf940bda1 --- /dev/null +++ b/docs/docs/user/core-concepts/components/kafka-connector.md @@ -0,0 +1,3 @@ +# KafkaConnector + +`KafkaConnector` is a component that deploys [Kafka Connectors](https://kafka.apache.org/documentation.html#connect_configuring){target=_blank}. Since a connector cannot be different from sink or source it is not recommended to use `KafkaConnector` for deployment in [`pipeline.yaml`](../../../resources/pipeline-components/pipeline.md). Instead, `KafkaConnector` should be used in [`defaults.yaml`](../defaults.md#kafkaconnector) to set defaults for all connectors in the pipeline as they can share some common settings. diff --git a/docs/docs/user/core-concepts/components/kafka-sink-connector.md b/docs/docs/user/core-concepts/components/kafka-sink-connector.md new file mode 100644 index 000000000..ab3ceece7 --- /dev/null +++ b/docs/docs/user/core-concepts/components/kafka-sink-connector.md @@ -0,0 +1,43 @@ +# KafkaSinkConnector + +Subclass of [_KafkaConnector_](./kafka-connector.md). + +### Usage + +Lets other systems pull data from Apache Kafka. + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/kafka-sink-connector.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +- Add the sink connector to the Kafka Connect cluster +- Create the output topics if provided (optional) +- Register schemas in the Schema Registry if provided (optional) + +#### destroy + +The associated sink connector is removed from the Kafka Connect cluster. + +#### reset + +Reset the consumer group offsets using +[bakdata's sink resetter](https://github.com/bakdata/kafka-connect-resetter/#sink-resetter){target=_blank}. + +#### clean + +- Delete associated consumer group +- Delete configured error topics diff --git a/docs/docs/user/core-concepts/components/kafka-source-connector.md b/docs/docs/user/core-concepts/components/kafka-source-connector.md new file mode 100644 index 000000000..d15a32bc1 --- /dev/null +++ b/docs/docs/user/core-concepts/components/kafka-source-connector.md @@ -0,0 +1,44 @@ +# KafkaSourceConnector + +Subclass of [_KafkaConnector_](./kafka-connector.md). + +### Usage + +Manages source connectors in your Kafka Connect cluster. + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/kafka-source-connector.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +- Add the source connector to the Kafka Connect cluster +- Create the output topics if provided (optional) +- Register schemas in the Schema registry if provided (optional) + +#### destroy + +Remove the source connector from the Kafka Connect cluster. + +#### reset + +Delete state associated with the connector using +[bakdata's sink resetter](https://github.com/bakdata/kafka-connect-resetter/#source-resetter){target=_blank}. + +#### clean + +- Delete all associated output topics +- Delete all associated schemas in the Schema Registry +- Delete state associated with the connector diff --git a/docs/docs/user/core-concepts/components/kubernetes-app.md b/docs/docs/user/core-concepts/components/kubernetes-app.md new file mode 100644 index 000000000..4a28dbe0e --- /dev/null +++ b/docs/docs/user/core-concepts/components/kubernetes-app.md @@ -0,0 +1,37 @@ +# KubernetesApp + +### Usage + +Can be used to deploy any app in Kubernetes using Helm, for example, a REST service that serves Kafka data. + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/kubernetes-app.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +Deploy using Helm. + +#### destroy + +Uninstall Helm release. + +#### reset + +Do nothing. + +#### clean + +Do nothing. diff --git a/docs/docs/user/core-concepts/components/overview.md b/docs/docs/user/core-concepts/components/overview.md new file mode 100644 index 000000000..39453dfcb --- /dev/null +++ b/docs/docs/user/core-concepts/components/overview.md @@ -0,0 +1,8 @@ +# Overview + +This section explains the different components of KPOps, their usage and configuration in the pipeline +definition [`pipeline.yaml`](../../../resources/pipeline-components/pipeline.md). + +--8<-- +./docs/resources/architecture/components-hierarchy.md +--8<-- diff --git a/docs/docs/user/core-concepts/components/producer-app.md b/docs/docs/user/core-concepts/components/producer-app.md new file mode 100644 index 000000000..1f55fa6d9 --- /dev/null +++ b/docs/docs/user/core-concepts/components/producer-app.md @@ -0,0 +1,43 @@ +# ProducerApp + +Subclass of [_KafkaApp_](kafka-app.md). + +### Usage + +Configures a [streams-bootstrap](https://github.com/bakdata/streams-bootstrap){target=_blank} [Kafka producer app](https://github.com/bakdata/streams-bootstrap#kafka-producer){target=_blank} + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/producer-app.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +In addition to [KubernetesApp's `deploy`](kubernetes-app.md#deploy): + +- Create topics if provided (optional) +- Submit Avro schemas to the registry if provided (optional) + +#### destroy + +Uninstall Helm release. + +#### reset + +Do nothing, producers are stateless. + +#### clean + +- Delete the output topics of the Kafka producer +- Delete all associated schemas in the Schema Registry diff --git a/docs/docs/user/core-concepts/components/streams-app.md b/docs/docs/user/core-concepts/components/streams-app.md new file mode 100644 index 000000000..ac881ade2 --- /dev/null +++ b/docs/docs/user/core-concepts/components/streams-app.md @@ -0,0 +1,48 @@ +# StreamsApp + +Subclass of [_KafkaApp_](kafka-app.md). + +### Usage + +Configures a +[streams-bootstrap](https://github.com/bakdata/streams-bootstrap){target=_blank} +[Kafka Streams app](https://github.com/bakdata/streams-bootstrap#kafka-streams){target=_blank} + +### Configuration + + + +??? example "`pipeline.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-components/streams-app.yaml + --8<-- + ``` + + + +### Operations + +#### deploy + +In addition to [KubernetesApp's `deploy`](kubernetes-app.md#deploy): + +- Create topics if provided (optional) +- Submit Avro schemas to the registry if provided (optional) + +#### destroy + +Uninstall Helm release. + +#### reset + +- Delete the consumer group offsets +- Delete Kafka Streams state + +#### clean + +Similar to [`reset`](#reset) with to additional steps: + +- Delete the app's output topics +- Delete all associated schemas in the Schema Registry diff --git a/docs/docs/user/core-concepts/config.md b/docs/docs/user/core-concepts/config.md new file mode 100644 index 000000000..dfc308fb4 --- /dev/null +++ b/docs/docs/user/core-concepts/config.md @@ -0,0 +1,22 @@ +# Configuration + +KPOps reads its global configuration that is unrelated to a pipeline's [components](./components/overview.md) from [`config.yaml`](#__codelineno-0-1). + +Consider enabling [KPOps' editor integration](../references/editor-integration.md) feature to enjoy the benefits of autocompletion and validation when configuring your pipeline. + +To learn about any of the available settings, take a look at the example below. + + + +??? example "`config.yaml`" + + ```yaml + --8<-- + ./docs/resources/pipeline-config/config.yaml + --8<-- + ``` + +!!! note "Environment-specific pipeline definitions" + Similarly to [defaults](defaults.md#configuration), it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form `_{environment}` to the filename. + + diff --git a/docs/docs/user/references/defaults.md b/docs/docs/user/core-concepts/defaults.md similarity index 53% rename from docs/docs/user/references/defaults.md rename to docs/docs/user/core-concepts/defaults.md index 888edd661..c07c2ce34 100644 --- a/docs/docs/user/references/defaults.md +++ b/docs/docs/user/core-concepts/defaults.md @@ -1,6 +1,6 @@ # Defaults -KPOps has a very efficient way of dealing with repeating settings which manifests as [`defaults.yaml`](../../../resources/pipeline-defaults/defaults). This file provides the user with the power to set defaults for any and all [components](../components), thus omitting the need to repeat the same settings in [`pipeline.yaml`](../../../resources/pipeline-components/pipeline). +KPOps has a very efficient way of dealing with repeating settings which manifests as [`defaults.yaml`](../../../resources/pipeline-defaults/defaults). This file provides the user with the power to set defaults for any and all [components](./components/overview.md), thus omitting the need to repeat the same settings in [`pipeline.yaml`](../../../resources/pipeline-components/pipeline). [See real-world examples for `defaults`](../../../resources/examples/defaults). @@ -18,86 +18,114 @@ It is possible to set specific `defaults` for each `environment` by adding files It is important to note that `defaults_{environment}.yaml` overrides only the settings that are explicitly set to be different from the ones in the base `defaults` file. + + !!! tip `defaults` is the default value of `defaults_filename_prefix`. Together with `defaults_path` and `environment` it can be changed in [`config.yaml`](../config/#__codelineno-0-16) + + ## Components The `defaults` codeblocks in this section contain the full set of settings that are specific to the component. If a setting already exists in a parent config, it will not be included in the child's. -### [KubernetesApp](../components/#kubernetesapp) +### [KubernetesApp](./components/kubernetes-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml + --8<-- ``` -### [KafkaApp](../components/#kafkaapp) + + +### [KafkaApp](./components/kafka-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-app.yaml + --8<-- ``` -### [StreamsApp](../components/#streamsapp) + + +### [StreamsApp](./components/streams-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-streams-app.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-streams-app.yaml + --8<-- ``` -### [ProducerApp](../components/#producerapp) + + +### [ProducerApp](./components/producer-app.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-producer.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-producer.yaml + --8<-- ``` -### KafkaConnector + -`KafkaConnector` is a component that deploys [Kafka Connectors](https://kafka.apache.org/documentation.html#connect_configuring){target=_blank}. Since a connector cannot be different from sink or source it is not recommended to use `KafkaConnector` for deployment in [`pipeline.yaml`](../../../resources/pipeline-components/pipeline). +### [KafkaConnector](./components/kafka-connector.md) -Instead, `KafkaConnector` should be used in [`defaults.yaml`](../../../resources/pipeline-defaults/defaults) to set defaults for all connectors in the pipeline as they can share some common settings. + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-connector.yaml + --8<-- ``` -### [KafkaSourceConnector](../components/#kafkasourceconnector) + + +### [KafkaSourceConnector](./components/kafka-source-connector.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-source-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-source-connector.yaml + --8<-- ``` -### [KafkaSinkConnector](../components/#kafkasinkconnector) + + +### [KafkaSinkConnector](./components/kafka-sink-connector.md) + + ??? example "`defaults.yaml`" ```yaml - --8<-- - ./docs/resources/pipeline-defaults/defaults-kafka-sink-connector.yaml - --8<-- + --8<-- + ./docs/resources/pipeline-defaults/defaults-kafka-sink-connector.yaml + --8<-- ``` + + diff --git a/docs/docs/user/references/variables/environment_variables.md b/docs/docs/user/core-concepts/variables/environment_variables.md similarity index 61% rename from docs/docs/user/references/variables/environment_variables.md rename to docs/docs/user/core-concepts/variables/environment_variables.md index 376b9a885..2a57aabea 100644 --- a/docs/docs/user/references/variables/environment_variables.md +++ b/docs/docs/user/core-concepts/variables/environment_variables.md @@ -2,9 +2,16 @@ Environment variables can be set by using the [export](https://www.unix.com/man-page/linux/1/export/){target=_blank} command in Linux or the [set](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/set_1){target=_blank} command in Windows. + + !!! tip "dotenv files" - Support for `.env` files is on the roadmap, but not implemented in KPOps yet. One of the possible ways to still use one and export the contents manually is with the following command: `#!sh export $(xargs < .env)`. This would work in `bash` suppose there are no spaces inside the values. + Support for `.env` files is on the [roadmap](https://github.com/bakdata/kpops/issues/20), + but not implemented in KPOps yet. One of the possible ways to still + use one and export the contents manually is with the following command: `#!sh export $(xargs < .env)`. + This would work in `bash` suppose there are no spaces inside the values. + + ## Config @@ -12,6 +19,8 @@ Environment variables can be set by using the [export](https://www.unix.com/man- ./docs/resources/variables/config_env_vars.md --8<-- + + ??? example "config_env_vars.env" ```py title="Exhaustive list of all config-related environment variables" @@ -20,12 +29,16 @@ Environment variables can be set by using the [export](https://www.unix.com/man- --8<-- ``` + + ## CLI --8<-- ./docs/resources/variables/cli_env_vars.md --8<-- + + ??? example "cli_env_vars.env" ```py title="Exhaustive list of all cli-related environment variables" @@ -33,3 +46,5 @@ Environment variables can be set by using the [export](https://www.unix.com/man- ./docs/resources/variables/cli_env_vars.env --8<-- ``` + + diff --git a/docs/docs/user/core-concepts/variables/substitution.md b/docs/docs/user/core-concepts/variables/substitution.md new file mode 100644 index 000000000..4ef9b1b25 --- /dev/null +++ b/docs/docs/user/core-concepts/variables/substitution.md @@ -0,0 +1,54 @@ +# Substitution + +KPOps supports the usage of placeholders and environment variables in [pipeline definition](../components/overview.md) and [defaults](../defaults.md). + +## Component-specific variables + +These variables can be used in a component's definition to refer to any of its attributes, including ones that the user has defined in the defaults. + +All of them are prefixed with `component_` and follow the following form: `component_{attribute_name}`. If the attribute itself contains attributes, they can be referred to like this: `component_{attribute_name}_{subattribute_name}`. + + + +??? Example + ```yaml + --8<-- + ./docs/resources/variables/variable_substitution.yaml + --8<-- + ``` + + + +## Pipeline-config-specific variables + +These variables include all fields in the [config](../config.md) and refer to the pipeline configuration that is independent of the components. + + + +!!! info Aliases + `error_topic_name` is an alias for `topic_name_config_default_error_topic_name` + `output_topic_name` is an alias for `topic_name_config_default_output_topic_name` + + + +## Environment variables + +Environment variables such as `$PATH` can be used in the pipeline definition and defaults without any transformation following the form `${ENV_VAR_NAME}`. This, of course, includes variables like the ones relevant to the [KPOps cli](../../references/cli-commands.md) that are exported by the user. + +[See all KPOps environment variables](environment_variables.md) + +## Pipeline name variables + +These are special variables that refer to the name and path of a pipeline. + +- `${pipeline_name}`: Concatenated path of the parent directory where pipeline.yaml is defined in. + For instance, `./data/pipelines/v1/pipeline.yaml`, here the value for the variable would be `data-pipelines-v1`. + +- `${pipeline_name_}`: Similar to the previous variable, each `` contains a part of the path to the `pipeline.yaml` file. + Consider the previous example, `${pipeline_name_0}` would be `data`, `${pipeline_name_1}` would be `pipelines`, and `${pipeline_name_2}` equals to `v1`. + +## Advanced use cases + +1. **Refer to default component field values**: As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the [pipeline schema](../../../schema/pipeline.json). +2. **Chaining variables**: It is possible to chain any number of variables, see the [example](#component-specific-variables) above. +3. **Cross-component substitution**: [YAML](https://yaml.org/){target=_blank} is quite an intricate language and with some of its [magic](https://yaml.org/spec/1.2.2/#692-node-anchors){target=_blank} one could write cross-component references. diff --git a/docs/docs/user/examples/atm-fraud-pipeline.md b/docs/docs/user/examples/atm-fraud-pipeline.md index da95effc7..06c249f3d 100644 --- a/docs/docs/user/examples/atm-fraud-pipeline.md +++ b/docs/docs/user/examples/atm-fraud-pipeline.md @@ -1,9 +1,9 @@ # ATM fraud detection pipeline -ATM fraud is a demo pipeline for ATM fraud detection. -The original by Confluent is written in KSQL -and outlined in this [blogpost](https://www.confluent.io/blog/atm-fraud-detection-apache-kafka-ksql/){target=_blank}. -The one used in this example is re-built from scratch using [bakdata](https://bakdata.com/){target=_blank}'s +ATM fraud is a demo pipeline for ATM fraud detection. +The original by Confluent is written in KSQL +and outlined in this [blogpost](https://www.confluent.io/blog/atm-fraud-detection-apache-kafka-ksql/){target=_blank}. +The one used in this example is re-built from scratch using [bakdata](https://bakdata.com/){target=_blank}'s [`streams-bootstrap`](https://github.com/bakdata/streams-bootstrap){target=_blank} library. ## What this will demonstrate @@ -14,7 +14,7 @@ The one used in this example is re-built from scratch using [bakdata](https://ba ## Prerequisites -Completed all steps in the [setup](../../getting-started/setup). +Completed all steps in the [setup](../getting-started/setup.md). ## Setup and deployment @@ -22,18 +22,22 @@ Completed all steps in the [setup](../../getting-started/setup). Deploy PostgreSQL using the [Bitnami Helm chart:](https://artifacthub.io/packages/helm/bitnami/postgresql){target=_blank} Add the helm repository: + ```shell helm repo add bitnami https://charts.bitnami.com/bitnami && \ helm repo update ``` Install the PostgreSQL with helm: + ```shell helm upgrade --install -f ./postgresql.yaml \ --namespace kpops \ postgresql bitnami/postgresql ``` + + ??? example "PostgreSQL Example Helm chart values (`postgresql.yaml`)" ```yaml auth: @@ -50,6 +54,8 @@ postgresql bitnami/postgresql enabled: true ``` + + ### ATM fraud detection example pipeline setup #### Port forwarding @@ -68,33 +74,38 @@ kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:808 1. Export environment variables in your terminal: - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 2. Deploy the pipeline - ```shell - poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/atm-fraud-detection/config.yaml \ - --execute - ``` - + ```shell + poetry run kpops deploy ./examples/bakdata/atm-fraud-detection/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/atm-fraud-detection/config.yaml \ + --execute + ``` + + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be deployed correctly. + + ### Check if the deployment is successful -You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to see the deployed pipeline. +You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to see the deployed pipeline. To do so, port-forward the service in a separate terminal session using the command below: ```shell kubectl port-forward -n kpops service/streams-explorer 8080:8080 ``` -After that open [http://localhost:8080](http://localhost:8080){target=_blank} in your browser. + +After that open [http://localhost:8080](http://localhost:8080){target=_blank} in your browser. You should be able to see pipeline shown in the image below:
@@ -102,6 +113,8 @@ You should be able to see pipeline shown in the image below:
An overview of ATM fraud pipeline shown in Streams Explorer
+ + !!! Attention Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka connect. @@ -135,23 +148,27 @@ helm --namespace kpops uninstall postgresql --verbose \ --execute ``` + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be destroyed correctly. !!! Attention - If you face any issues destroying this example see [Teardown](../../getting-started/teardown) for manual deletion. + If you face any issues destroying this example see [Teardown](../getting-started/teardown.md) for manual deletion. + + ## Common errors - `deploy` fails: - 1. Read the error message. - 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. Run `deploy --dry-run` to avoid havig to `clean` again. If an error is dropped, start over from step 1. - 5. If the dry-run is succesful, run `deploy`. + 1. Read the error message. + 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. Run `deploy --dry-run` to avoid havig to `clean` again. If an error is dropped, start over from step 1. + 5. If the dry-run is succesful, run `deploy`. - `clean` fails: - 1. Read the error message. - 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. If `clean` fails, follow the steps in [teardown](../../getting-started/teardown). + 1. Read the error message. + 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. If `clean` fails, follow the steps in [teardown](../getting-started/teardown.md). diff --git a/docs/docs/user/getting-started/quick-start.md b/docs/docs/user/getting-started/quick-start.md index 6127ecc30..3727f8ca8 100644 --- a/docs/docs/user/getting-started/quick-start.md +++ b/docs/docs/user/getting-started/quick-start.md @@ -20,18 +20,22 @@ Completed all steps in the [setup](../setup). Deploy Redis using the [Bitnami Helm chart:](https://artifacthub.io/packages/helm/bitnami/redis){target=_blank} Add the Helm repository: + ```shell helm repo add bitnami https://charts.bitnami.com/bitnami && \ helm repo update ``` Install Redis with Helm: + ```shell helm upgrade --install -f ./values-redis.yaml \ --namespace kpops \ redis bitnami/redis ``` + + ??? example "Redis example Helm chart values (`values-redis.yaml`)" ```yaml architecture: standalone @@ -44,6 +48,8 @@ redis bitnami/redis tag: 7.0.8 ``` + + ### Word-count example pipeline setup #### Port forwarding @@ -63,38 +69,42 @@ kubectl port-forward --namespace kpops service/k8kafka-cp-kafka-connect 8083:808 1. Copy the [configuration](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank} from the [kpops-examples repository](https://github.com/bakdata/kpops-examples/tree/main/word-count){target=_blank} into `kpops>examples>bakdata>word-count` like so: - ``` - kpops - ├── examples - | ├── bakdata - | | ├── word-count - | | | ├── config.yaml - | | | ├── defaults - | | | │   └── defaults.yaml - | | | └── pipeline.yaml - | | | - ``` + ``` + kpops + ├── examples + | ├── bakdata + | | ├── word-count + | | | ├── config.yaml + | | | ├── defaults + | | | │   └── defaults.yaml + | | | └── pipeline.yaml + | | | + ``` 2. Export environment variables in your terminal: - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 3. Deploy the pipeline - ```shell - kpops deploy ./examples/bakdata/word-count/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/word-count/config.yaml \ - --execute - ``` + ```shell + kpops deploy ./examples/bakdata/word-count/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/word-count/config.yaml \ + --execute + ``` + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be deployed correctly. + + ### Check if the deployment is successful You can use the [Streams Explorer](https://github.com/bakdata/streams-explorer){target=_blank} to inspect the deployed pipeline. @@ -113,11 +123,15 @@ You should be able to see pipeline shown in the image below:
An overview of Word-count pipeline shown in Streams Explorer
+ + !!! Attention Kafka Connect needs some time to set up the connector. Moreover, Streams Explorer needs a while to scrape the information from Kafka Connect. Therefore, it might take a bit until you see the whole graph. + + ## Teardown resources ### Redis @@ -132,20 +146,23 @@ helm --namespace kpops uninstall redis 1. Export environment variables in your terminal. - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` 2. Remove the pipeline - ```shell - kpops clean ./examples/bakdata/word-count/pipeline.yaml \ - --pipeline-base-dir ./examples \ - --config ./examples/bakdata/word-count/config.yaml \ - --verbose \ - --execute - ``` + ```shell + kpops clean ./examples/bakdata/word-count/pipeline.yaml \ + --pipeline-base-dir ./examples \ + --config ./examples/bakdata/word-count/config.yaml \ + --verbose \ + --execute + ``` + + + !!! Note You can use the `--dry-run` flag instead of the `--execute` flag and check the logs if your pipeline will be destroyed correctly. @@ -153,16 +170,18 @@ helm --namespace kpops uninstall redis !!! Attention If you face any issues destroying this example see [Teardown](../teardown) for manual deletion. + + ## Common errors - `deploy` fails: - 1. Read the error message. - 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. Run `deploy --dry-run` to avoid havig to `clean` again. If an error is dropped, start over from step 1. - 5. If the dry-run is succesful, run `deploy`. + 1. Read the error message. + 2. Try to correct the mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. Run `deploy --dry-run` to avoid having to `clean` again. If an error is dropped, start over from step 1. + 5. If the dry-run is successful, run `deploy`. - `clean` fails: - 1. Read the error message. - 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. - 3. Run `clean`. - 4. If `clean` fails, follow the steps in [teardown](../teardown). + 1. Read the error message. + 2. Try to correct the indicated mistakes if there were any. Likely the configuration is not correct or the port-forwarding is not working as intended. + 3. Run `clean`. + 4. If `clean` fails, follow the steps in [teardown](../teardown). diff --git a/docs/docs/user/getting-started/setup.md b/docs/docs/user/getting-started/setup.md index f30ca4688..1c608b77e 100644 --- a/docs/docs/user/getting-started/setup.md +++ b/docs/docs/user/getting-started/setup.md @@ -18,26 +18,30 @@ If you don't have access to an existing Kubernetes cluster, this section will gu 1. You can install k3d with its installation script: - ```shell - wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash - ``` + ```shell + wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/v5.4.6/install.sh | bash + ``` - For other ways of installing k3d, you can have a look at their [installation guide](https://k3d.io/v5.4.6/#installation){target=_blank}. + For other ways of installing k3d, you can have a look at their [installation guide](https://k3d.io/v5.4.6/#installation){target=_blank}. 2. The [Kafka deployment](#deploy-kafka) needs a modified Docker image. In that case the image is built and pushed to a Docker registry that holds it. If you do not have access to an existing Docker registry, you can use k3d's Docker registry: - ```shell - k3d registry create kpops-registry.localhost --port 12345 - ``` + ```shell + k3d registry create kpops-registry.localhost --port 12345 + ``` 3. Now you can create a new cluster called `kpops` that uses the previously created Docker registry: - ```shell - k3d cluster create kpops --k3s-arg "--no-deploy=traefik@server:*" --registry-use k3d-kpops-registry.localhost:12345 - ``` + ```shell + k3d cluster create kpops --k3s-arg "--no-deploy=traefik@server:*" --registry-use k3d-kpops-registry.localhost:12345 + ``` + + - !!! Note - Creating a new k3d cluster automatically configures `kubectl` to connect to the local cluster by modifying your `~/.kube/config`. In case you manually set the `KUBECONFIG` variable or don't want k3d to modify your config, k3d offers [many other options](https://k3d.io/v5.4.6/usage/kubeconfig/#handling-kubeconfigs){target=_blank}. +!!! Note + Creating a new k3d cluster automatically configures `kubectl` to connect to the local cluster by modifying your `~/.kube/config`. In case you manually set the `KUBECONFIG` variable or don't want k3d to modify your config, k3d offers [many other options](https://k3d.io/v5.4.6/usage/kubeconfig/#handling-kubeconfigs){target=_blank}. + + You can check the cluster status with `kubectl get pods -n kube-system`. If all returned elements have a `STATUS` of `Running` or `Completed`, then the cluster is up and running. @@ -47,157 +51,55 @@ You can check the cluster status with `kubectl get pods -n kube-system`. If all 1. To allow connectivity to other systems [Kafka Connect](https://docs.confluent.io/platform/current/connect/index.html#kafka-connect){target=_blank} needs to be extended with drivers. You can install a [JDBC driver](https://docs.confluent.io/kafka-connectors/jdbc/current/jdbc-drivers.html){target=_blank} for Kafka Connect by creating a new Docker image: - 1. Create a `Dockerfile` with the following content: + 1. Create a `Dockerfile` with the following content: - ```dockerfile - FROM confluentinc/cp-kafka-connect:7.1.3 + ```dockerfile + FROM confluentinc/cp-kafka-connect:7.1.3 - RUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0 - ``` + RUN confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0 + ``` - 2. Build and push the modified image to your private Docker registry: + 2. Build and push the modified image to your private Docker registry: - ```shell - docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 && \ - docker push localhost:12345/kafka-connect-jdbc:7.1.3 - ``` + ```shell + docker build . --tag localhost:12345/kafka-connect-jdbc:7.1.3 && \ + docker push localhost:12345/kafka-connect-jdbc:7.1.3 + ``` - Detailed instructions on building, tagging and pushing a docker image can be found in [Docker docs](https://docs.docker.com/){target=_blank}. + Detailed instructions on building, tagging and pushing a docker image can be found in [Docker docs](https://docs.docker.com/){target=_blank}. 2. Add Confluent's Helm chart repository and update the index: - ```shell - helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ && - helm repo update - ``` + ```shell + helm repo add confluentinc https://confluentinc.github.io/cp-helm-charts/ && + helm repo update + ``` 3. Install Kafka, Zookeeper, Confluent's Schema Registry, Kafka Rest Proxy, and Kafka Connect. A single Helm chart installs all five components. Below you can find an example for the `--values ./kafka.yaml` file configuring the deployment accordingly. Deploy the services: - ```shell - helm upgrade \ - --install \ - --version 0.6.1 \ - --values ./kafka.yaml \ - --namespace kpops \ - --create-namespace \ - --wait \ - k8kafka confluentinc/cp-helm-charts + ```shell + helm upgrade \ + --install \ + --version 0.6.1 \ + --values ./kafka.yaml \ + --namespace kpops \ + --create-namespace \ + --wait \ + k8kafka confluentinc/cp-helm-charts + ``` + + + +??? example "Kafka Helm chart values (`kafka.yaml`)" + An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources. + + ```yaml + --8<-- + ./docs/resources/setup/kafka.yaml + --8<-- ``` - ??? example "Kafka Helm chart values (`kafka.yaml`)" - An example value configuration for Confluent's Helm chart. This configuration deploys a single Kafka Broker, a Schema Registry, Zookeeper, Kafka Rest Proxy, and Kafka Connect with minimal resources. - - ```yaml - cp-zookeeper: - enabled: true - servers: 1 - imageTag: 7.1.3 - heapOptions: "-Xms124M -Xmx124M" - overrideGroupId: k8kafka - fullnameOverride: "k8kafka-cp-zookeeper" - resources: - requests: - cpu: 50m - memory: 0.2G - limits: - cpu: 250m - memory: 0.2G - prometheus: - jmx: - enabled: false - - cp-kafka: - enabled: true - brokers: 1 - imageTag: 7.1.3 - podManagementPolicy: Parallel - configurationOverrides: - "auto.create.topics.enable": false - "offsets.topic.replication.factor": 1 - "transaction.state.log.replication.factor": 1 - "transaction.state.log.min.isr": 1 - "confluent.metrics.reporter.topic.replicas": 1 - resources: - requests: - cpu: 50m - memory: 0.5G - limits: - cpu: 250m - memory: 0.5G - prometheus: - jmx: - enabled: false - persistence: - enabled: false - - cp-schema-registry: - enabled: true - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-schema-registry" - overrideGroupId: k8kafka - kafka: - bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" - resources: - requests: - cpu: 50m - memory: 0.25G - limits: - cpu: 250m - memory: 0.25G - prometheus: - jmx: - enabled: false - - cp-kafka-connect: - enabled: true - replicaCount: 1 - image: k3d-kpops-registry.localhost:12345/kafka-connect-jdbc - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-kafka-connect" - overrideGroupId: k8kafka - kafka: - bootstrapServers: "PLAINTEXT://k8kafka-cp-kafka-headless:9092" - heapOptions: "-Xms256M -Xmx256M" - resources: - requests: - cpu: 500m - memory: 0.25G - limits: - cpu: 500m - memory: 0.25G - configurationOverrides: - "consumer.max.poll.records": "10" - "consumer.max.poll.interval.ms": "900000" - "config.storage.replication.factor": "1" - "offset.storage.replication.factor": "1" - "status.storage.replication.factor": "1" - cp-schema-registry: - url: http://k8kafka-cp-schema-registry:8081 - prometheus: - jmx: - enabled: false - - cp-kafka-rest: - enabled: true - imageTag: 7.1.3 - fullnameOverride: "k8kafka-cp-rest" - heapOptions: "-Xms256M -Xmx256M" - resources: - requests: - cpu: 50m - memory: 0.25G - limits: - cpu: 250m - memory: 0.5G - prometheus: - jmx: - enabled: false - - cp-ksql-server: - enabled: false - cp-control-center: - enabled: false - ``` + ## Deploy Streams Explorer @@ -210,14 +112,16 @@ helm repo update Below you can find an example for the `--values ./streams-explorer.yaml` file configuring the deployment accordingly. Now, deploy the service: - ```shell - helm upgrade \ - --install \ - --version 0.2.3 \ - --values ./streams-explorer.yaml \ - --namespace kpops \ - streams-explorer streams-explorer/streams-explorer - ``` +```shell +helm upgrade \ + --install \ + --version 0.2.3 \ + --values ./streams-explorer.yaml \ + --namespace kpops \ + streams-explorer streams-explorer/streams-explorer +``` + + ??? example "Streams Explorer Helm chart values (`streams-explorer.yaml`)" An example value configuration for Steams Explorer Helm chart. @@ -237,6 +141,8 @@ Below you can find an example for the `--values ./streams-explorer.yaml` file co memory: 300Mi ``` + + ## Check the status of your deployments Now we will check if all the pods are running in our namespace. You can list all pods in the namespace with this command: diff --git a/docs/docs/user/getting-started/teardown.md b/docs/docs/user/getting-started/teardown.md index 00477cf16..47c839a18 100644 --- a/docs/docs/user/getting-started/teardown.md +++ b/docs/docs/user/getting-started/teardown.md @@ -8,31 +8,31 @@ ## KPOps-deployed pipeline -The [`kpops` CLI](../../references/cli-commands) can be used to destroy a pipeline that was previously deployed with KPOps. +The [`kpops` CLI](../references/cli-commands.md) can be used to destroy a pipeline that was previously deployed with KPOps. In case that doesn't work, the pipeline can always be taken down manually with `helm` (see section [Infrastructure](#infrastructure)). 1. Export environment variables. - ```shell - export DOCKER_REGISTRY=bakdata && \ - export NAMESPACE=kpops - ``` + ```shell + export DOCKER_REGISTRY=bakdata && \ + export NAMESPACE=kpops + ``` -2. Navigate to the `examples` folder. +2. Navigate to the `examples` folder. Replace the `` with the example you want to tear down. For example the `atm-fraud-detection`. 3. Remove the pipeline - ```shell - # Uncomment 1 line to either destroy, reset or clean. + ```shell + # Uncomment 1 line to either destroy, reset or clean. - # poetry run kpops destroy /pipeline.yaml \ - # poetry run kpops reset /pipeline.yaml \ - # poetry run kpops clean /pipeline.yaml \ - --config /config.yaml \ - --execute - ``` + # poetry run kpops destroy /pipeline.yaml \ + # poetry run kpops reset /pipeline.yaml \ + # poetry run kpops clean /pipeline.yaml \ + --config /config.yaml \ + --execute + ``` ## Infrastructure @@ -42,11 +42,15 @@ Delete namespace: kubectl delete namespace kpops ``` + + !!! Note In case `kpops destroy` is not working one can uninstall the pipeline services one by one. This is equivalent to running `kpops destroy`. In case a clean uninstall (like the one `kpops clean` does) is needed, one needs to also delete the topics and schemas created by deployment of the pipeline. + + ## Local cluster Delete local cluster: diff --git a/docs/docs/user/references/ci-integration/github-actions.md b/docs/docs/user/references/ci-integration/github-actions.md index 66cfe040d..7284254cc 100644 --- a/docs/docs/user/references/ci-integration/github-actions.md +++ b/docs/docs/user/references/ci-integration/github-actions.md @@ -1,4 +1,5 @@ # GitHub Actions integration + We provided a GitHub composite action called [kpops-runner](https://github.com/bakdata/kpops/tree/main/actions/kpops-runner/action.yaml) that installs all the necessary dependencies and runs KPOps commands with the given parameters. @@ -6,19 +7,19 @@ that installs all the necessary dependencies and runs KPOps commands with the gi ## Input Parameters | Name | Required | Default Value | Type | Description | -|-------------------|:--------:|:-------------:|:------:|-----------------------------------------------------------------------------------------------------------------------------------------------| -| command | ✅ | - | string | KPOps command to run. generate, deploy, destroy, reset, clean are possible values. Flags such as --dry-run and --execute need to be specified | -| pipeline | ✅ | - | string | Pipeline to run by KPOps | -| working-directory | ❌ | . | string | root directory used by KPOps to run pipelines | -| pipeline-base-dir | ❌ | . | string | directory where relative pipeline variables are initialized from | -| defaults | ❌ | . | string | defaults folder path | -| config | ❌ | config.yaml | string | config.yaml file path | -| components | ❌ | - | string | components package path | -| filter-type | ❌ | "include" | string | Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS | -| python-version | ❌ | "3.11.x" | string | Python version to install (Defaults to the latest stable version of Python 3.11) | -| kpops-version | ❌ | latest | string | KPOps version to install | -| helm-version | ❌ | latest | string | Helm version to install | -| token | ❌ | latest | string | secrets.GITHUB_TOKEN, needed for setup-helm action if helm-version is set to latest | +| ----------------- | :------: | :-----------: | :----: | --------------------------------------------------------------------------------------------------------------------------------------------- | +| command | ✅ | - | string | KPOps command to run. generate, deploy, destroy, reset, clean are possible values. Flags such as --dry-run and --execute need to be specified | +| pipeline | ✅ | - | string | Pipeline to run by KPOps | +| working-directory | ❌ | . | string | root directory used by KPOps to run pipelines | +| pipeline-base-dir | ❌ | - | string | directory where relative pipeline variables are initialized from | +| defaults | ❌ | - | string | defaults folder path | +| config | ❌ | - | string | config.yaml file path | +| components | ❌ | - | string | components package path | +| filter-type | ❌ | - | string | Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS | +| python-version | ❌ | "3.11.x" | string | Python version to install (Defaults to the latest stable version of Python 3.11) | +| kpops-version | ❌ | latest | string | KPOps version to install | +| helm-version | ❌ | latest | string | Helm version to install | +| token | ❌ | latest | string | secrets.GITHUB_TOKEN, needed for setup-helm action if helm-version is set to latest | ## Usage diff --git a/docs/docs/user/references/components.md b/docs/docs/user/references/components.md deleted file mode 100644 index 013fb1b5b..000000000 --- a/docs/docs/user/references/components.md +++ /dev/null @@ -1,252 +0,0 @@ -# Components - -This section explains the different components of KPOps, their usage and configuration in the pipeline definition [`pipeline.yaml`](../../../resources/pipeline-components/pipeline). - ---8<-- -./docs/resources/architecture/components-hierarchy.md ---8<-- - -[See real-world examples of `pipeline.yaml`](../../../resources/examples/pipeline) - - - - -!!! note "Environment-specific pipeline definitions" - Similarly to [defaults](../defaults/#configuration), it is possible to have an unlimited amount of additional environment-specific pipeline definitions. The naming convention is the same: add a suffix of the form `_{environment}` to the filename. Learn more about environments in the [Config](../config/#__codelineno-0-10) section. - -## KubernetesApp - -### Usage - -Can be used to deploy any app in Kubernetes using Helm, for example, a REST service that serves Kafka data. - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/kubernetes-app.yaml - --8<-- - ``` - -### Operations - -#### deploy - -Deploy using Helm. - -#### destroy - -Uninstall Helm release. - -#### reset - -Do nothing. - -#### clean - -Do nothing. - -## KafkaApp - -Sub class of [_KubernetesApp_](#kubernetesapp). - -### Usage - -- Defines a [streams-bootstrap](https://github.com/bakdata/streams-bootstrap#usage){target=_blank} component -- Should not be used in `pipeline.yaml` as the component can be defined as either a [StreamsApp](#streamsapp) or a [ProducerApp](#producerapp) -- Often used in `defaults.yaml` - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/kafka-app.yaml - --8<-- - ``` - -### Operations - -#### deploy - -In addition to [KubernetesApp's](#kubernetesapp) `deploy`: - -- Create topics if provided (optional) -- Submit Avro schemas to the registry if provided (optional) - -#### destroy - -Refer to [KubernetesApp](#kubernetesapp). - -#### reset - -Do nothing. - -#### clean - -Do nothing. - -## StreamsApp - -Sub class of [_KafkaApp_](#kafkaapp). - -### Usage - -Configures a -[streams-bootstrap](https://github.com/bakdata/streams-bootstrap){target=_blank} -[Kafka Streams app](https://github.com/bakdata/streams-bootstrap#kafka-streams){target=_blank} - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/streams-app.yaml - --8<-- - ``` - -### Operations - -#### deploy - -Refer to [KafkaApp](#kafkaapp). - -#### destroy - -Refer to [KafkaApp](#kafkaapp). - -#### reset - -- Reset the consumer group offsets -- Reset Kafka Streams state - -#### clean - - -- Reset Kafka Streams state -- Delete the app's output topics -- Delete all associated schemas in the Schema Registry -- Delete the consumer group - -## ProducerApp - -Sub class of [_KafkaApp_](#kafkaapp). - -### Usage - -Configures a -[streams-bootstrap](https://github.com/bakdata/streams-bootstrap){target=_blank} -[Kafka producer app](https://github.com/bakdata/streams-bootstrap#kafka-producer){target=_blank} - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/producer.yaml - --8<-- - ``` - -### Operations - -#### deploy - -Refer to [KafkaApp](#kafkaapp). - -#### destroy - -Refer to [KafkaApp](#kafkaapp). - -#### reset - -Do nothing, producers are stateless. - -#### clean - -- Delete the output topics of the Kafka producer -- Delete all associated schemas in the Schema Registry - -## KafkaSinkConnector - -Sub class of [_KafkaConnector_](../defaults/#kafkaconnector). - -### Usage - -Lets other systems pull data from Apache Kafka. - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/kafka-sink-connector.yaml - --8<-- - ``` - -### Operations - -#### deploy - -- Add the sink connector to the Kafka Connect cluster -- Create the output topics if provided (optional) -- Register schemas in the Schema Registry if provided (optional) - -#### destroy - -The associated sink connector is removed. - -#### reset - -Reset the consumer group offsets using [bakdata's sink resetter](https://github.com/bakdata/kafka-connect-resetter/#sink-resetter){target=_blank}. - -#### clean - -- Delete associated consumer group -- Delete configured error topics - -## KafkaSourceConnector - -Sub class of [_KafkaConnector_](../defaults/#kafkaconnector). - -### Usage - -Manages source connectors in your Kafka Connect cluster. - -### Configuration - -??? example "`pipeline.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline-components/kafka-source-connector.yaml - --8<-- - ``` - -### Operations - -#### deploy - -- Add the source connector to the Kafka Connect cluster -- Create the output topics if provided (optional) -- Register schemas in the Schema registry if provided (optional) - -#### destroy - -Remove the source connector from the Kafka Connect cluster. - -#### reset - -Delete state associated with the connector using [bakdata's sink resetter](https://github.com/bakdata/kafka-connect-resetter/#source-resetter){target=_blank}. - -#### clean - -- Delete all associated output topics -- Delete all associated schemas in the Schema Registry -- Delete state associated with the connector diff --git a/docs/docs/user/references/config.md b/docs/docs/user/references/config.md deleted file mode 100644 index 26c55d219..000000000 --- a/docs/docs/user/references/config.md +++ /dev/null @@ -1,15 +0,0 @@ -# Configuration - -KPOps reads its global configuration that is unrelated to a pipeline's [components](../components) from [`config.yaml`](#__codelineno-0-1). - -Consider enabling [KPOps' editor integration](../editor-integration) feature to enjoy the benefits of autocompletion and validation when configuring your pipeline. - -To learn about any of the available settings, take a look at the example below. - -??? example "`config.yaml`" - - ```yaml - --8<-- - ./docs/resources/pipeline_config/config.yaml - --8<-- - ``` diff --git a/docs/docs/user/references/editor-integration.md b/docs/docs/user/references/editor-integration.md index f58c73cd8..86b7e93d0 100644 --- a/docs/docs/user/references/editor-integration.md +++ b/docs/docs/user/references/editor-integration.md @@ -4,22 +4,25 @@ KPOps provides JSON schemas that enable autocompletion and validation for some o ## Supported files -- [`pipeline.yaml`](../components) -- [`config.yaml`](../config) +- [`pipeline.yaml`](../../resources/pipeline-components/pipeline.md) +- [`config.yaml`](../core-concepts/config.md) ## Usage -1. Install the -[yaml-language-server](https://github.com/redhat-developer/yaml-language-server#clients){target=_blank} in your editor of choice. (requires LSP support) +1. Install the [yaml-language-server](https://github.com/redhat-developer/yaml-language-server#clients){target=_blank} in your editor of choice. (requires LSP support) 2. Configure the extension with the settings below. + + ??? note "`settings.json`" ```json - --8<-- - ./docs/resources/editor_integration/settings.json - --8<-- + --8<-- + ./docs/resources/editor_integration/settings.json + --8<-- ``` !!! tip "Advanced usage" - It is possible to generate schemas with the [`kpops schema`](../cli-commands/#kpops-schema) command. Useful when using custom components or when using a pre-release version of KPOps. + It is possible to generate schemas with the [`kpops schema`](./cli-commands.md#kpops-schema) command. Useful when using custom components or when using a pre-release version of KPOps. + + diff --git a/docs/docs/user/references/variables/substitution.md b/docs/docs/user/references/variables/substitution.md deleted file mode 100644 index 79f34897f..000000000 --- a/docs/docs/user/references/variables/substitution.md +++ /dev/null @@ -1,51 +0,0 @@ -# Substitution - -KPOps supports the usage of placeholders and environment variables in [pipeline definition](../components.md) and [defaults](../defaults.md). - -## Component-specific variables - -These variables can be used in a component's definition to refer to any of its attributes, including ones that the user has defined in the defaults. - -All of them are prefixed with `component_` and follow the following form: `component_{attribute_name}`. If the attribute itself contains attributes, they can be referred to like this: `component_{attribute_name}_{subattribute_name}`. - -??? Example - ```yaml - --8<-- - ./docs/resources/variables/variable_substitution.yaml - --8<-- - ``` - -## Pipeline-config-specific variables - -These variables include all fields in the [config](../config.md) and refer to the pipeline configuration that is independent of the components. - -!!! info Aliases - `error_topic_name` is an alias for `topic_name_config_default_error_topic_name` - `output_topic_name` is an alias for `topic_name_config_default_output_topic_name` - -## Environment variables - -Environment variables such as `$PATH` can be used in the pipeline definition and defaults without any transformation following the form `${ENV_VAR_NAME}`. This, of course, includes variables like the ones relevant to the [KPOps cli](../cli-commands.md) that are exported by the user. - -[See all KPOps environment variables](environment_variables.md) - -## Pipeline name variables - -These are special variables that refer to the name and path of a pipeline. - -- `${pipeline_name}` - Concatenated path of the parent directory where pipeline.yaml is defined in. - For instance, `./data/pipelines/v1/pipeline.yaml`, here the value for the variable would be `data-pipelines-v1`. - -- `${pipeline_name_}` - Similar to the previous variable, each `` contains a part of the path to the `pipeline.yaml` file. - Consider the previous example, `${pipeline_name_0}` would be `data`, `${pipeline_name_1}` would be `pipelines`, and `${pipeline_name_2}` equals to `v1`. - -## Advanced use cases - -1. **Refer to default component field values** -As long as a value is assigned to a component attribute, it is possible to refer to it with a placeholder. To see all component fields, take a look at the [pipeline schema](../../../schema/pipeline.json). -2. **Chaining variables** -It is possible to chain any number of variables, see the [example](#component-specific-variables) above. -3. **Cross-component substitution** -[YAML](https://yaml.org/){target=_blank} is quite an intricate language and with some of its [magic](https://yaml.org/spec/1.2.2/#692-node-anchors){target=_blank} one could write cross-component references. \ No newline at end of file diff --git a/docs/docs/user/what-is-kpops.md b/docs/docs/user/what-is-kpops.md index 527e27a8d..ccac918c4 100644 --- a/docs/docs/user/what-is-kpops.md +++ b/docs/docs/user/what-is-kpops.md @@ -1,8 +1,10 @@ # What is KPOps? -With a couple of easy commands in the shell and a [`pipeline.yaml`](#example) of under 30 lines, [KPOps](/) can not only [`deploy`](../references/cli-commands/#deploy) a Kafka pipeline[^1] to a Kubernetes cluster, but also [`reset`](../references/cli-commands/#reset), [`clean`](../references/cli-commands/#clean) or [`destroy`](../references/cli-commands/#destroy) it! -[^1]: - A Kafka pipeline can consist of consecutive [streaming applications](../references/components/#streamsapp), [producers](../references/components/#producerapp), and [connectors](../references/defaults/#kafkaconnector). +With a couple of easy commands in the shell and a [`pipeline.yaml`](#example) of under 30 lines, KPOps can not only [`deploy`](./references/cli-commands.md#kpops-deploy) a Kafka pipeline[^1] to a Kubernetes cluster, but also [`reset`](./references/cli-commands.md#kpops-reset), [`clean`](./references/cli-commands.md#kpops-clean) or [`destroy`](./references/cli-commands.md#kpops-destroy) it! + +[^1]: A Kafka pipeline can consist of consecutive [streaming applications](./core-concepts/components/streams-app.md), +[producers](./core-concepts/components/producer-app.md), +and [connectors](./core-concepts/components/kafka-connector.md). ## Key features @@ -21,7 +23,7 @@ With a couple of easy commands in the shell and a [`pipeline.yaml`](#example) of ```yaml title="Word-count pipeline.yaml" - --8<-- - https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml - --8<-- +--8<-- +https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/pipeline.yaml +--8<-- ``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 77c03d39e..9a3c42720 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -4,15 +4,16 @@ site_url: https://bakdata.github.io/kpops/ remote_branch: gh-pages copyright: Copyright © 2023 bakdata +extra_css: + - stylesheets/extra.css + theme: name: "material" - custom_dir: overrides + custom_dir: docs/overrides language: "en" - palette: - primary: "custom" font: - text: "Roboto" - code: "Roboto Mono" + text: "SF Pro" + code: "DejaVu Sans Mono" features: - navigation.sections - navigation.tabs @@ -33,6 +34,10 @@ plugins: - search - glightbox +extra: + version: + provider: mike + markdown_extensions: - footnotes - toc: @@ -80,14 +85,27 @@ nav: - Teardown: user/getting-started/teardown.md - Examples: - ATM Fraud detection pipeline: user/examples/atm-fraud-pipeline.md + - Core Concepts: + - Components: + - Overview: user/core-concepts/components/overview.md + - KubernetesApp: user/core-concepts/components/kubernetes-app.md + - KafkaApp: user/core-concepts/components/kafka-app.md + - StreamsApp: user/core-concepts/components/streams-app.md + - ProducerApp: user/core-concepts/components/producer-app.md + - KafkaConnector: user/core-concepts/components/kafka-connector.md + - KafkaSinkConnector: user/core-concepts/components/kafka-sink-connector.md + - KafkaSourceConnector: user/core-concepts/components/kafka-source-connector.md + - Config: user/core-concepts/config.md + - Defaults: user/core-concepts/defaults.md + - Variables: + - Environment variables: user/core-concepts/variables/environment_variables.md + - Substitution: user/core-concepts/variables/substitution.md - References: - CLI usage: user/references/cli-commands.md - - Components: user/references/components.md - - Config: user/references/config.md - - Defaults: user/references/defaults.md - Editor integration: user/references/editor-integration.md - CI integration: - GitHub Actions: user/references/ci-integration/github-actions.md - - Variables: - - Environment variables: user/references/variables/environment_variables.md - - Substitution: user/references/variables/substitution.md + - Developer Guide: + - Auto generation: developer/auto-generation.md + - Formatting: developer/formatting.md + diff --git a/dprint.json b/dprint.json new file mode 100644 index 000000000..d702f9ea9 --- /dev/null +++ b/dprint.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://dprint.dev/schemas/v0.json", + "incremental": true, + "markdown": {}, + "includes": [ + "**/*.{md}" + ], + "excludes": [ + ".pytest_cache/**", + ".mypy_cache/**", + "**/.venv/**", + "CHANGELOG.md", + "**/cli-commands.md", + "**/cli_env_vars.md", + "**/config_env_vars.md" + ], + "plugins": [ + "https://plugins.dprint.dev/markdown-0.16.0.wasm" + ] +} diff --git a/justfile b/justfile index 9de7ea4e9..eb5116739 100644 --- a/justfile +++ b/justfile @@ -14,7 +14,3 @@ docs-config := base-directory / "docs/mkdocs.yml" # Serve current docs located in ./docs/docs serve-docs port="8000": poetry run mkdocs serve --config-file {{ docs-config }} --dev-addr localhost:{{ port }} - -# Generate CLI Usage page in documentation -generate-docs: - sh ./gen_docs.sh diff --git a/kpops/__init__.py b/kpops/__init__.py index 5d62ac728..590fd8bd2 100644 --- a/kpops/__init__.py +++ b/kpops/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.0" +__version__ = "2.0.5" # export public API functions from kpops.cli.main import clean, deploy, destroy, generate, reset diff --git a/kpops/component_handlers/helm_wrapper/helm.py b/kpops/component_handlers/helm_wrapper/helm.py index e73ba7209..2ad3f5f01 100644 --- a/kpops/component_handlers/helm_wrapper/helm.py +++ b/kpops/component_handlers/helm_wrapper/helm.py @@ -81,19 +81,17 @@ def upgrade_install( with tempfile.NamedTemporaryFile("w") as values_file: yaml.safe_dump(values, values_file) - command = ["helm"] - command.extend( - [ - "upgrade", - release_name, - chart, - "--install", - "--namespace", - namespace, - "--values", - values_file.name, - ] - ) + command = [ + "helm", + "upgrade", + release_name, + chart, + "--install", + "--namespace", + namespace, + "--values", + values_file.name, + ] command.extend(flags.to_command()) if dry_run: command.append("--dry-run") diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index c6eb8be38..c522e040e 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from abc import ABC from pydantic import BaseModel, Extra, Field from typing_extensions import override @@ -51,7 +52,7 @@ class KafkaAppConfig(KubernetesAppConfig): ) -class KafkaApp(KubernetesApp): +class KafkaApp(KubernetesApp, ABC): """Base component for Kafka-based components. Producer or streaming apps should inherit from this class. @@ -149,7 +150,7 @@ def __install_clean_up_job( """Install clean up job :param release_name: Name of the Helm release - :param suffix: Suffix to add to the realease name, e.g. "-clean" + :param suffix: Suffix to add to the release name, e.g. "-clean" :param values: The Helm values for the chart :param dry_run: Whether to do a dry run of the command :return: Install clean up job with helm, return the output of the installation diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index 1514231e2..88ef8380d 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -43,7 +43,8 @@ class KubernetesApp(PipelineComponent): :param app: Application-specific settings :param repo_config: Configuration of the Helm chart repo to be used for - deploying the component, defaults to None + deploying the component, defaults to None this means that the command "helm repo add" is not called and Helm + expects a path to local Helm chart. :param namespace: Namespace in which the component shall be deployed :param version: Helm chart version, defaults to None """ @@ -56,8 +57,8 @@ class KubernetesApp(PipelineComponent): default=..., description=describe_attr("app", __doc__), ) - repo_config: HelmRepoConfig = Field( - default=..., + repo_config: HelmRepoConfig | None = Field( + default=None, description=describe_attr("repo_config", __doc__), ) version: str | None = Field( @@ -102,8 +103,9 @@ def helm_chart(self) -> str: @property def helm_flags(self) -> HelmFlags: """Return shared flags for Helm commands""" + auth_flags = self.repo_config.repo_auth_flags.dict() if self.repo_config else {} return HelmFlags( - **self.repo_config.repo_auth_flags.dict(), + **auth_flags, version=self.version, create_namespace=self.config.create_namespace, ) diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index 468c5b54b..cf4bafa52 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -1,5 +1,7 @@ from __future__ import annotations +from abc import ABC + from pydantic import Extra, Field from kpops.components.base_components.base_defaults_component import ( @@ -19,7 +21,7 @@ from kpops.utils.pydantic import DescConfig -class PipelineComponent(BaseDefaultsComponent): +class PipelineComponent(BaseDefaultsComponent, ABC): """Base class for all components :param name: Component name diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index f66e575fd..470a1412d 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -1,4 +1,6 @@ +import inspect import logging +from abc import ABC from enum import Enum from typing import Annotated, Any, Literal, Sequence, Union @@ -43,12 +45,14 @@ def _is_valid_component( :param component: component type to be validated :return: Whether component is valid for schema generation """ + if inspect.isabstract(component) or ABC in component.__bases__: + log.warning(f"SKIPPED {component.__name__}, component is abstract.") + return False if component.type in defined_component_types: log.warning(f"SKIPPED {component.__name__}, component type must be unique.") return False - else: - defined_component_types.add(component.type) - return True + defined_component_types.add(component.type) + return True def _add_components( @@ -93,10 +97,12 @@ def gen_pipeline_schema( # Add stock components if enabled components: tuple[type[PipelineComponent]] = tuple() if include_stock_components: - components = tuple(_find_classes("kpops.components", PipelineComponent)) + components = _add_components("kpops.components") # Add custom components if provided if components_module: components = _add_components(components_module, components) + if not components: + raise RuntimeError("No valid components found.") # Create a type union that will hold the union of all component types PipelineComponents = Union[components] # type: ignore[valid-type] diff --git a/pyproject.toml b/pyproject.toml index cac422c7a..25c1160a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "kpops" -version = "2.0.0" +version = "2.0.5" description = "KPOps is a tool to deploy Kafka pipelines to Kubernetes" authors = ["bakdata "] license = "MIT" diff --git a/tests/cli/resources/empty_module.py b/tests/cli/resources/empty_module.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index 493db4c63..6c651dfa4 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from abc import ABC, abstractmethod from pathlib import Path import pytest @@ -8,6 +9,7 @@ from snapshottest.module import SnapshotTest from typer.testing import CliRunner +import tests.cli.resources.empty_module as empty_module from kpops.cli.main import app from kpops.components.base_components import PipelineComponent from kpops.utils.docstring import describe_attr @@ -24,6 +26,18 @@ class Config: anystr_strip_whitespace = True +# abstract component inheriting from ABC should be excluded +class AbstractBaseComponent(PipelineComponent, ABC): + ... + + +# abstract component with abstractmethods should be excluded +class AbstractPipelineComponent(AbstractBaseComponent): + @abstractmethod + def not_implemented(self) -> None: + ... + + class SubPipelineComponent(EmptyPipelineComponent): ... @@ -92,6 +106,20 @@ def test_gen_pipeline_schema_no_modules(self, caplog: pytest.LogCaptureFixture): ] assert result.exit_code == 0 + def test_gen_pipeline_schema_no_components(self): + with pytest.raises(RuntimeError, match="^No valid components found.$"): + result = runner.invoke( + app, + [ + "schema", + "pipeline", + "--no-include-stock-components", + empty_module.__name__, + ], + catch_exceptions=False, + ) + assert result.exit_code == 1 + def test_gen_pipeline_schema_only_stock_module(self): result = runner.invoke( app, @@ -121,7 +149,12 @@ def test_gen_pipeline_schema_only_stock_module(self): def test_gen_pipeline_schema_only_custom_module(self, snapshot: SnapshotTest): result = runner.invoke( app, - ["schema", "pipeline", MODULE, "--no-include-stock-components"], + [ + "schema", + "pipeline", + MODULE, + "--no-include-stock-components", + ], catch_exceptions=False, ) diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 93d530817..f1452d09b 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -3,6 +3,7 @@ import pytest from pytest_mock import MockerFixture +from typing_extensions import override from kpops.cli.pipeline_config import PipelineConfig from kpops.component_handlers import ComponentHandlers @@ -105,14 +106,15 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( self, - kubernetes_app: KubernetesApp, config: PipelineConfig, handlers: ComponentHandlers, helm_mock: MagicMock, mocker: MockerFixture, app_value: KubernetesTestValue, ): - repo_config = HelmRepoConfig(repository_name="test-repo", url="mock://test") + repo_config = HelmRepoConfig( + repository_name="test-repo", url="https://test.com/charts/" + ) kubernetes_app = KubernetesApp( name="test-kubernetes-app", config=config, @@ -135,7 +137,7 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( assert helm_mock.mock_calls == [ mocker.call.add_repo( "test-repo", - "mock://test", + "https://test.com/charts/", RepoAuthFlags(), ), mocker.call.upgrade_install( @@ -148,6 +150,42 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( ), ] + def test_should_deploy_app_with_local_helm_chart( + self, + config: PipelineConfig, + handlers: ComponentHandlers, + helm_mock: MagicMock, + app_value: KubernetesTestValue, + ): + class AppWithLocalChart(KubernetesApp): + repo_config: None = None + + @property + @override + def helm_chart(self) -> str: + return "path/to/helm/charts/" + + app_with_local_chart = AppWithLocalChart( + name="test-app-with-local-chart", + config=config, + handlers=handlers, + app=app_value, + namespace="test-namespace", + ) + + app_with_local_chart.deploy(dry_run=False) + + helm_mock.add_repo.assert_not_called() + + helm_mock.upgrade_install.assert_called_once_with( + "test-app-with-local-chart", + "path/to/helm/charts/", + False, + "test-namespace", + {"nameOverride": "test-value"}, + HelmUpgradeInstallFlags(), + ) + def test_should_raise_not_implemented_error_when_helm_chart_is_not_set( self, kubernetes_app: KubernetesApp,