Merge branch 'dev' into remove-consul

apache · Sep 10, 2023 · 7422268 · 7422268
2 parents 425eca6 + 8e4d866
commit 7422268
Show file tree

Hide file tree

Showing 174 changed files with 7,807 additions and 1,939 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -23,30 +23,31 @@ github:
     - analytics
     - stream-processing
     - self-service
+    - hacktoberfest
 
   features:
     wiki: false
     issues: true
     projects: false
 
   custom_subjects:
-    new_pr: "[PR] {title} ({repository})"
-    close_pr: "Re: [PR] {title} ({repository})"
-    comment_pr: "Re: [PR] {title} ({repository})"
-    diffcomment: "Re: [PR] {title} ({repository})"
-    merge_pr: "Re: [PR] {title} ({repository})"
-    new_issue: "[I] {title} ({repository})"
-    comment_issue: "Re: [I] {title} ({repository})"
-    close_issue: "Re: [I] {title} ({repository})"
-    catchall: "[GH] {title} ({repository})"
-    new_discussion: "[D] {title} ({repository})"
-    edit_discussion: "Re: [D] {title} ({repository})"
-    close_discussion: "Re: [D] {title} ({repository})"
-    close_discussion_with_comment: "Re: [D] {title} ({repository})"
-    reopen_discussion: "Re: [D] {title} ({repository})"
-    new_comment_discussion: "Re: [D] {title} ({repository})"
-    edit_comment_discussion: "Re: [D] {title} ({repository})"
-    delete_comment_discussion: "Re: [D] {title} ({repository})"
+    new_pr: "[PR] {title}"
+    close_pr: "Re: [PR] {title}"
+    comment_pr: "Re: [PR] {title}"
+    diffcomment: "Re: [PR] {title}"
+    merge_pr: "Re: [PR] {title}"
+    new_issue: "[I] {title}"
+    comment_issue: "Re: [I] {title}"
+    close_issue: "Re: [I] {title}"
+    catchall: "[GH] {title}"
+    new_discussion: "[D] {title}"
+    edit_discussion: "Re: [D] {title}"
+    close_discussion: "Re: [D] {title}"
+    close_discussion_with_comment: "Re: [D] {title}"
+    reopen_discussion: "Re: [D] {title}"
+    new_comment_discussion: "Re: [D] {title}"
+    edit_comment_discussion: "Re: [D] {title}"
+    delete_comment_discussion: "Re: [D] {title}"
 
   del_branch_on_merge: true
 

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -116,13 +116,13 @@ jobs:
           push: true
           tags: ${{ env.DOCKERHUB_APACHE_REPO }}/extensions-all-jvm:${{ env.MVN_VERSION }}
 
-      - name: Build and Push Docker Image pipeline-elements-experimental-flink
-        uses: docker/build-push-action@v4
-        with:
-          context: ./streampipes-extensions/streampipes-pipeline-elements-experimental-flink
-          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
-          push: true
-          tags: ${{ env.DOCKERHUB_APACHE_REPO }}/pipeline-elements-experimental-flink:${{ env.MVN_VERSION }}
+#      - name: Build and Push Docker Image pipeline-elements-experimental-flink
+#        uses: docker/build-push-action@v4
+#        with:
+#          context: ./streampipes-extensions/streampipes-pipeline-elements-experimental-flink
+#          platforms: linux/amd64,linux/arm64/v8,linux/arm/v7
+#          push: true
+#          tags: ${{ env.DOCKERHUB_APACHE_REPO }}/pipeline-elements-experimental-flink:${{ env.MVN_VERSION }}
 
       - name: Build and Push Docker Image extensions-all-iiot
         uses: docker/build-push-action@v4

diff --git a/.github/workflows/cypress-test.yml b/.github/workflows/cypress-test.yml
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -58,7 +58,7 @@ jobs:
         run: sleep 70
 
       - name: 'UI Tests'
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@v6
         with:
           install: false
           wait-on: 'http://localhost/#/login'

diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml
@@ -28,7 +28,7 @@ jobs:
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Pull OSV Scanner Docker image
         run: docker pull ghcr.io/google/osv-scanner:latest

diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Node
         uses: actions/setup-node@v3
@@ -64,7 +64,7 @@ jobs:
     needs: [python-quality-checks, python-unit-tests, run-format-and-lint-ui]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up JDK 17
         uses: actions/setup-java@v3
@@ -101,7 +101,7 @@ jobs:
         run: sleep 70
 
       - name: 'UI Tests'
-        uses: cypress-io/github-action@v5
+        uses: cypress-io/github-action@v6
         with:
           install: false
           wait-on: 'http://localhost/#/login'
@@ -112,81 +112,88 @@ jobs:
   python-quality-checks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
-          cache: 'pip'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
       - name: Install Python dependencies
         working-directory: ./streampipes-client-python
         run: |
-          python -m pip install --upgrade pip
-          pip install wheel
-          pip install -e ".[dev]"
+          poetry install --with dev,stubs --all-extras
 
       - name: Run style & code checks
         working-directory: ./streampipes-client-python
         run: |
           make pre-commit
-          make mypy
-          make lint
 
   python-unit-tests:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python: [3.8, 3.9, "3.10"]
+        python: [3.8, 3.9, "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python v${{ matrix.python }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-          cache: 'pip'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
       - name: Install Python dependencies
         working-directory: ./streampipes-client-python
         run: |
-          python -m pip install --upgrade pip
-          pip install wheel
-          pip install -e ".[dev]"
+          poetry install --with dev,stubs --all-extras
 
       - name: Run unit-tests
         working-directory: ./streampipes-client-python
         run: |
+          source $VENV  # see https://github.com/snok/install-poetry#running-on-windows
           make unit-tests
 
   python-doc-checks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
-      - name: Cache Python doc dependencies
-        uses: actions/cache@v3
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-doc-${{ hashFiles('**/setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-doc-
+          version: 1.5.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
       - name: Install Python dependencies
         working-directory: ./streampipes-client-python
         run: |
-          python -m pip install --upgrade pip
-          pip install wheel
-          pip install -e ".[docs]"
+          poetry install --with docs
 
       - name: Build MkDocs
         working-directory: ./streampipes-client-python

diff --git a/.github/workflows/pypi-deployment.yml b/.github/workflows/pypi-deployment.yml
@@ -25,24 +25,29 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.5.1
+          in-project: true
+          prefer-active-python: true
+
       - name: Install dependencies
         working-directory: ./streampipes-client-python
         run: |
-          python -m pip install --upgrade pip 
-          pip install build twine
-          pip install -e ".[docs]"
+          poetry install --with deployment,docs --all-extras
 
       - name: Build package
         working-directory: ./streampipes-client-python
         run: |
-          python -m build
+          poetry build
           twine check --strict dist/*
 
       - name: Publish package

diff --git a/.github/workflows/python-docs.yml b/.github/workflows/python-docs.yml
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: clone
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: create working branch & set GitHub config
         run: |
@@ -38,25 +38,29 @@ jobs:
         with:
           python-version: 3.8
 
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.6.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
       - name: Cache Python dependencies
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-doc-${{ hashFiles('**/setup.py') }}
+          path: .venv
+          key: ${{ runner.os }}-pip-doc-${{ hashFiles('**/pyproject.toml', '**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-pip-doc-
 
       - name: Install Python dependencies
         working-directory: ./streampipes-client-python
-        run: |
-          python -m pip install --upgrade pip
-          pip install wheel
-          pip install -e ".[docs]"
+        run: poetry install --with docs
 
       - name: Build Python docs
         working-directory: ./streampipes-client-python
         run: |
-          mike deploy dev -u --deploy-prefix ./streampipes-client-python/docs-tmp -b build-python-docs-${{ github.run_id }}
+          poetry run mike deploy dev -u --deploy-prefix ./streampipes-client-python/docs-tmp -b build-python-docs-${{ github.run_id }}
           git reset --hard  # undo local changes that delete generated doc files
 
       - name: Publish Python docs as artifact

diff --git a/README.md b/README.md
@@ -118,21 +118,20 @@ The quickest way to run StreamPipes including the latest extensions (adapters, p
 
 Please follow the instructions provided in the corresponding `README.md` to get started.
 
-For a more in-depth manual, read the [installation guide](https://streampipes.apache.org/docs/docs/try-installation.html).
+For a more in-depth manual, read the [installation guide](https://streampipes.apache.org/docs/try-installation.html).
 
 > TL;DR: Download the latest release, switch to the ``installer/compose`` directory and run ``docker-compose up -d``.
 
 ## Documentation
 
-The full documentation is available [here](https://streampipes.apache.org/docs/index.html).
+The full documentation is available [here](https://streampipes.apache.org/docs/user-guide-introduction).
 
 Quick Links:
 
-* [Installation](https://streampipes.apache.org/docs/docs/try-installation.html)
-* [Create adapters](https://streampipes.apache.org/docs/docs/use-connect.html)
-* [Create pipelines](https://streampipes.apache.org/docs/docs/use-pipeline-editor.html)
-* [Pipeline elements](https://streampipes.apache.org/pipeline-elements.html)
-* [Write you own pipeline elements](https://streampipes.apache.org/docs/docs/extend-archetypes.html)
+* [Installation](https://streampipes.apache.org/docs/try-installation.html)
+* [Create adapters](https://streampipes.apache.org/docs/use-connect.html)
+* [Create pipelines](https://streampipes.apache.org/docs/use-pipeline-editor.html)
+* [Write you own pipeline elements](https://streampipes.apache.org/docs/extend-archetypes.html)
 
 ## Building StreamPipes
 
@@ -174,10 +173,10 @@ The source code of all included pipeline elements and adapters can be found [her
 
 ## Extending StreamPipes
 
-You can easily add your own data streams, processors or sinks. A [Java-based SDK](https://streampipes.apache.org/docs/docs/extend-tutorial-data-processors.html) can be used to integrate your existing processing logic into StreamPipes. 
+You can easily add your own data streams, processors or sinks. A [Java-based SDK](https://streampipes.apache.org/docs/extend-tutorial-data-processors.html) can be used to integrate your existing processing logic into StreamPipes. 
 Pipeline elements are packaged as Docker images and can be installed at runtime, whenever your requirements change.
 
-👉 Check our [developer guide](https://streampipes.apache.org/docs/docs/extend-setup.html).
+👉 Check our [developer guide](https://streampipes.apache.org/docs/extend-setup.html).
 
 ## Bugs and Feature Requests
 

diff --git a/VULNERABILITY.md b/VULNERABILITY.md
@@ -2,9 +2,10 @@
 | OSV URL | CVSS | Ecosystem | Package | Version | Source |
 | --- | --- | --- | --- | --- | --- |
 | https://osv.dev/GHSA-6mjq-h674-j845 | 6.5 | Maven | io.netty:netty-handler | 4.1.72.Final | pom.xml |
-| https://osv.dev/GHSA-mjmj-j48q-9wg2 | 8.3 | Maven | org.yaml:snakeyaml | 1.33 | pom.xml |
+| https://osv.dev/GHSA-w596-4wvx-j9j6<br/>https://osv.dev/PYSEC-2022-42969 | 7.5 | PyPI | py | 1.11.0 | streampipes-client-python/poetry.lock |
+| https://osv.dev/GHSA-qppv-j76h-2rpx |  | PyPI | tornado | 6.3.2 | streampipes-client-python/poetry.lock |
 | https://osv.dev/GHSA-j8r2-6x86-q33q<br/>https://osv.dev/PYSEC-2023-74 | 6.1 | PyPI | requests | 2.24.0 | streampipes-wrapper-python/requirements.txt |
-| https://osv.dev/GHSA-p8p7-x288-28g6 | 6.1 | npm | @cypress/request | 2.88.11 | ui/package-lock.json |
+| https://osv.dev/GHSA-p8p7-x288-28g6 | 6.1 | npm | @cypress/request | 2.88.12 | ui/package-lock.json |
 | https://osv.dev/GHSA-ww39-953v-wcq6 | 7.5 | npm | glob-parent | 3.1.0 | ui/package-lock.json |
 | https://osv.dev/GHSA-4943-9vgg-gr5r | 6.1 | npm | quill | 1.3.7 | ui/package-lock.json |
-| https://osv.dev/GHSA-72xf-g2v4-qvf3 | 6.5 | npm | tough-cookie | 2.5.0 | ui/package-lock.json |
+| https://osv.dev/MAL-2022-6662 |  | npm | tsconfig-package | 7.0.0 | ui/package-lock.json |