Merge branch 'main' into refactor/component-name-prefix

.github/actions/update-docs/action.yml

@@ -1,8 +1,7 @@
 name: "Update documentation in gh-pages"
 description: |
-  Compile markdown documents to html and deploy to docs branch. If a semver tag is given, this action strips the patch.
+  Compile markdown documents to html and deploy to docs branch. If a semver version is given, this action strips the patch.
   It then pushes the <major>.<minor> as the latest alias to the documentation branch with mike.
-  In case no tag is given, it pushes to 'dev'.
 
 inputs:
   username:
@@ -14,8 +13,12 @@ inputs:
   token:
     description: "GitHub Token (must be a PAT for repository dispatch)"
     required: true
-  tag:
-    description: "Version tag"
+  version:
+    description: "Version name to be deployed by mike"
+    required: true 
+  release:
+    description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)"
+    default: 'false'
     required: false
 
 runs:
@@ -27,23 +30,30 @@ runs:
         poetry-version: "1.5.1"
         python-version: "3.10"
 
-    - name: Update documentation branch with mike
+    - name: Install docs dependencies
       shell: bash
-      env:
-        USERNAME: ${{ inputs.username }}
-        EMAIL: ${{ inputs.email }}
-        TOKEN: ${{ inputs.token }}
-        NEW_TAG: ${{ inputs.tag }}
       run: |
         poetry install --with docs
-        git config --local user.name ${USERNAME}
-        git config --local user.email ${EMAIL}
-        git config --local user.password ${TOKEN}
 
-        git pull # make sure docs branch is up-to-date
-        if [ -z "$NEW_TAG" ]; then
-          poetry run mike deploy dev --push --rebase --config-file ./docs/mkdocs.yml
-        else
-          new_tag=${NEW_TAG%.*}
-          poetry run mike deploy "$new_tag" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml
-        fi
+    - name: Update ${{ github.head_ref }} branch
+      shell: bash
+      run: |
+        git config --local user.name  ${{ inputs.username }}
+        git config --local user.email ${{ inputs.email }}
+        git config --local user.password ${{ inputs.token }}
+
+        git pull 
+
+    - name: Deploy ${{ inputs.version }} version of the documentation with mike
+      shell: bash
+      if: ${{ inputs.release == 'false' }}
+      run: |
+        poetry run mike deploy ${{ inputs.version }} --push --rebase --config-file ./docs/mkdocs.yml
+
+    - name: Deploy ${{ inputs.version }} version (latest) of the documentation with mike
+      shell: bash
+      if: ${{ inputs.release == 'true' }}
+      run: |
+        sem_version=${{ inputs.version }}
+        major_minor_version=${sem_version%.*}
+        poetry run mike deploy "$major_minor_version" latest --update-aliases --push --rebase --config-file ./docs/mkdocs.yml

.github/workflows/ci.yaml

@@ -69,6 +69,10 @@ jobs:
 
       - name: Install docs dependencies
         run: poetry install --with docs
+
+      - name: Check markdown formatting
+        uses: dprint/[email protected]
+        if: runner.os == 'Linux'
 
       - name: Test docs build (mkdocs)
         run: poetry run mkdocs build -f docs/mkdocs.yml
@@ -79,3 +83,46 @@ jobs:
     uses: bakdata/ci-templates/.github/workflows/[email protected]
     secrets:
       pypi-token: ${{ secrets.TEST_PYPI_TOKEN }}
+
+  publish-docs-from-main:
+    runs-on: ubuntu-22.04
+    if: ${{ github.ref == 'refs/heads/main' }}
+    needs: [test]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Publish docs from main branch
+        uses: ./.github/actions/update-docs
+        with:
+          username: ${{ secrets.GH_USERNAME }}
+          email: ${{ secrets.GH_EMAIL }}
+          token: ${{ secrets.GH_TOKEN }}
+          version: main 
+
+  publish-dev-docs-from-pr:
+    runs-on: ubuntu-22.04
+    if: ${{ github.event_name == 'pull_request' }}
+    needs: [test]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      # Checks to see if any files in the PR match one of the listed file types.
+      # This will return true if there's a file in docs folder that was added, deleted, or modified in the PR.
+      - name: Check if files in docs folder have changed
+        uses: dorny/paths-filter@v2
+        id: docs-changes
+        with:
+          filters: |
+            docs:
+              - added|deleted|modified: 'docs/**'
+
+      - name: Publish dev docs from PR 
+        if: steps.docs-changes.outputs.docs == 'true'
+        uses: ./.github/actions/update-docs
+        with:
+          username: ${{ secrets.GH_USERNAME }}
+          email: ${{ secrets.GH_EMAIL }}
+          token: ${{ secrets.GH_TOKEN }}
+          version: dev 

.github/workflows/lint.yaml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - .github/**
+      - actions/**
 
 jobs:
   actionlint:

.github/workflows/release.yaml

@@ -39,7 +39,8 @@ jobs:
       - name: Update gh-pages
         uses: ./.github/actions/update-docs
         with:
-          tag: ${{ needs.create-github-release-push-tag.outputs.release-version }}
           username: ${{ secrets.GH_USERNAME }}
           email: ${{ secrets.GH_EMAIL }}
           token: ${{ secrets.GH_TOKEN }}
+          version: ${{ needs.create-github-release-push-tag.outputs.release-version }}
+          release: true

.pre-commit-config.yaml

@@ -81,3 +81,10 @@ repos:
             | kpops/components/.*\.py
           )$
         require_serial: true
+      - id: dprint
+        name: dprint 
+        entry: dprint fmt
+        language: system
+        types: [markdown]
+        require_serial: true
+        pass_filenames: false

CHANGELOG.md

@@ -1,4 +1,80 @@
 # Changelog
+## [2.0.5](https://github.com/bakdata/kpops/releases/tag/2.0.5) - Release Date: [2023-08-30]
+
+### 🐛 Fixes
+
+- Fix versioning of docs when releasing - [#346](https://github.com/bakdata/kpops/pull/346)
+
+
+
+
+
+
+## [2.0.4](https://github.com/bakdata/kpops/releases/tag/2.0.4) - Release Date: [2023-08-29]
+
+### 🐛 Fixes
+
+- Fix GitHub ref variable for pushing docs to main branch - [#343](https://github.com/bakdata/kpops/pull/343)
+
+
+### 📝 Documentation
+
+- Add `dprint` as the markdown formatter - [#337](https://github.com/bakdata/kpops/pull/337)
+
+- Publish pre-release docs for PRs & main branch - [#339](https://github.com/bakdata/kpops/pull/339)
+
+- Align docs colours - [#345](https://github.com/bakdata/kpops/pull/345)
+
+
+### 🌀 Miscellaneous
+
+- Exclude abstract components from pipeline schema - [#332](https://github.com/bakdata/kpops/pull/332)
+
+
+
+
+
+
+## [2.0.3](https://github.com/bakdata/kpops/releases/tag/2.0.3) - Release Date: [2023-08-24]
+
+### 🐛 Fixes
+
+- Fix GitHub action error in non-Python projects - [#340](https://github.com/bakdata/kpops/pull/340)
+
+
+### 🌀 Miscellaneous
+
+- Lint GitHub action - [#342](https://github.com/bakdata/kpops/pull/342)
+
+
+
+
+
+
+## [2.0.2](https://github.com/bakdata/kpops/releases/tag/2.0.2) - Release Date: [2023-08-23]
+
+### 📝 Documentation
+
+- Add version dropdown to the documentation - [#336](https://github.com/bakdata/kpops/pull/336)
+
+- Break the documentation down into smaller subsection - [#329](https://github.com/bakdata/kpops/pull/329)
+
+
+
+
+
+
+## [2.0.1](https://github.com/bakdata/kpops/releases/tag/2.0.1) - Release Date: [2023-08-22]
+
+### 🐛 Fixes
+
+- Fix optional flags in GitHub action - [#334](https://github.com/bakdata/kpops/pull/334)
+
+
+
+
+
+
 ## [2.0.0](https://github.com/bakdata/kpops/releases/tag/2.0.0) - Release Date: [2023-08-17]
 
 ### 🏗️ Breaking changes

README.md

@@ -21,7 +21,7 @@ the [documentation](https://bakdata.github.io/kpops/latest).
 
 ## Install KPOps
 
-KPOps comes as a [PyPI package](https://pypi.org/project/kpops/). 
+KPOps comes as a [PyPI package](https://pypi.org/project/kpops/).
 You can install it with [pip](https://github.com/pypa/pip):
 
 ```shell

actions/kpops-runner/README.md

@@ -1,5 +1,3 @@
 # KPOps runner
 
-Please refer to the 
-[GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) 
-for the documentation. 
+Please refer to the [GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) for the documentation.

actions/kpops-runner/action.yaml

@@ -13,20 +13,19 @@ inputs:
     default: "."
   pipeline-base-dir:
     description: "Directory where relative pipeline variables are initialized from"
-    default: "."
+    required: false
   defaults:
     description: "Defaults folder path"
-    default: "."
+    required: false
   config:
     description: "config.yaml file path"
-    default: "config.yaml"
+    required: false
   components:
     description: "Components package path"
     required: false
   filter-type:
     description: "Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS (default is include)"
     required: false
-    default: "include"
   python-version:
     description: "Python version to install (Defaults to the latest stable version of Python 3.11)"
     required: false
@@ -46,26 +45,38 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Set up Python ${{ inputs.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ inputs.python-version }} 
-        cache: 'pip'
-
     - name: Set up Helm
       uses: azure/setup-helm@v3
       with:
         version: ${{ inputs.helm-version }}
         token: ${{ inputs.token }}
 
+    - name: Create temporary requirements.txt for caching
+      shell: bash
+      id: requirements
+      run: |
+        TEMP_FILE="${{ github.action_path }}/kpops-runner-action-requirements.txt"
+        echo "kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }}" > "$TEMP_FILE"
+        cat "$TEMP_FILE"
+        echo  "path=$TEMP_FILE">> $GITHUB_OUTPUT
+
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: pip
+        cache-dependency-path: ''
+        # FIXME: https://github.com/actions/setup-python/issues/361
+        # "${{ steps.requirements.outputs.path }}"
+
     - name: Install KPOps
       shell: bash
       run: |
-        echo "::group::pip install kpops package"
-        pip install kpops${{ inputs.kpops-version != 'latest' && format('=={0}', inputs.kpops-version) || '' }}
+        echo "::group::install kpops package"
+        pip install -r "${{ steps.requirements.outputs.path }}"
         echo "::endgroup::"
 
     - name: ${{ inputs.command }} ${{ inputs.pipeline }} pipeline
       shell: bash
       working-directory: ${{inputs.working-directory}}
-      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} --defaults ${{ inputs.defaults }} --config ${{ inputs.config }} --pipeline-base-dir ${{ inputs.pipeline-base-dir }} --filter-type ${{ inputs.filter-type }}
+      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.pipeline-base-dir != '' && format('--pipeline-base-dir {0}', inputs.pipeline-base-dir)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }}