diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 3c474a81d1..1182fe6e61 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -43,7 +43,7 @@ body: description: What Apache StreamPipes version are you using? multiple: false options: - - "0.92.0" + - "0.95.0" - "dev (current development state)" - "Other StreamPipes version (please specify below)" diff --git a/.github/ISSUE_TEMPLATE/doc_website_issue_report.yml b/.github/ISSUE_TEMPLATE/doc_website_issue_report.yml index c1c126fe5d..502df9f05e 100644 --- a/.github/ISSUE_TEMPLATE/doc_website_issue_report.yml +++ b/.github/ISSUE_TEMPLATE/doc_website_issue_report.yml @@ -22,12 +22,12 @@ body: attributes: value: "

- + StreamPipes Logo

- +
- + Thank you for taking the time to report the issue! We greatly appreciate the community's efforts to improve StreamPipes. @@ -40,11 +40,10 @@ body: - type: dropdown attributes: label: Where does your observation occur? - description: Please choose whether the issue you want to report occurs on the [website](https://streampipes.apache.org/), the [documentation](https://streampipes.apache.org/docs/user-guide-introduction/), or the [python client documentation](https://streampipes.apache.org/docs/docs/python/latest/). + description: Please choose whether the issue you want to report occurs on the [website](https://streampipes.apache.org/) or on the [documentation](https://streampipes.apache.org/docs/index.html). multiple: false options: - StreamPipes Documentation - - StreamPipes Python Documentation - StreamPipes Website - type: textarea @@ -90,4 +89,4 @@ body: - type: markdown attributes: - value: ":pray: Thanks a lot for completing our form!" + value: ":pray: Thanks a lot for completing our form!" \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 52ffa7f9a8..48fdf4a6c5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,16 +15,16 @@ ~ limitations under the License. ~ --> - + ' > VULNERABILITY.md + echo '' > VULNERABILITY.md docker run --rm -v $PWD:/repo -w /repo ghcr.io/google/osv-scanner --format markdown -r . 2>&1 | grep -vE '^((Scanning|Scanned|Failed).*)$' >> VULNERABILITY.md continue-on-error: true - name: Create Pull Request id: cpr - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> + committer: ${{ github.actor }} + author: ${{ github.actor }} <${{ github.actor }}@github.com> signoff: true title: Monthly update of vulnerability report commit-message: monthly update of vulnerability report body: | Update *Vulnerablity* report delete-branch: true - reviewers: "dominikriemer,tenthe,svenO3,smlabt,grainier,RobertIndie,bossenti" + reviewers: 'dominikriemer,tenthe,svenO3,smlabt,grainier,RobertIndie,bossenti' diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 3681c24514..350e9ef559 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -94,7 +94,7 @@ jobs: run: mvn clean package - name: Build and run streampipes - run: docker compose up --build -d + run: docker-compose up --build -d - name: Wait 70 seconds working-directory: ./installer/compose diff --git a/.github/workflows/pypi-deployment.yml b/.github/workflows/pypi-deployment.yml index f4b15e7237..1ce50f6deb 100644 --- a/.github/workflows/pypi-deployment.yml +++ b/.github/workflows/pypi-deployment.yml @@ -64,8 +64,8 @@ jobs: git reset --hard # undo local changes that delete generated doc files - name: Publish Python docs as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: streampipes-python-docs path: streampipes-client-python/docs-tmp/**/* - retention-days: 2 + retention-days: 2 \ No newline at end of file diff --git a/.github/workflows/python-docs.yml b/.github/workflows/python-docs.yml index 84db3979e8..224c3a1b19 100644 --- a/.github/workflows/python-docs.yml +++ b/.github/workflows/python-docs.yml @@ -18,7 +18,7 @@ name: "build-python-docs-and-share-as-artifact" on: workflow_dispatch: schedule: - - cron: "0 2 * * 6" # runs every saturday at 00:00:00 + - cron: "0 2 * * 6" # runs every saturday at 00:00:00 jobs: python-docs-artifact-building: @@ -26,8 +26,6 @@ jobs: steps: - name: clone uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: create working branch & set GitHub config run: | @@ -66,7 +64,7 @@ jobs: git reset --hard # undo local changes that delete generated doc files - name: Publish Python docs as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: streampipes-python-docs path: | diff --git a/.idea/runConfigurations/all_extensions_jvm.xml b/.idea/runConfigurations/all_extensions_jvm.xml new file mode 100644 index 0000000000..103f85d7e1 --- /dev/null +++ b/.idea/runConfigurations/all_extensions_jvm.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + diff --git a/.idea/runConfigurations/all_pipeline_elements_flink.xml b/.idea/runConfigurations/all_pipeline_elements_flink.xml new file mode 100644 index 0000000000..5675eac01d --- /dev/null +++ b/.idea/runConfigurations/all_pipeline_elements_flink.xml @@ -0,0 +1,14 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/all_pipeline_elements_jvm.xml b/.idea/runConfigurations/all_pipeline_elements_jvm.xml new file mode 100644 index 0000000000..60c64ead6a --- /dev/null +++ b/.idea/runConfigurations/all_pipeline_elements_jvm.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/connect_adapters.xml b/.idea/runConfigurations/connect_adapters.xml new file mode 100644 index 0000000000..7f920ddd31 --- /dev/null +++ b/.idea/runConfigurations/connect_adapters.xml @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/.idea/runConfigurations/connect_adapters_iiot.xml b/.idea/runConfigurations/connect_adapters_iiot.xml new file mode 100644 index 0000000000..ace168629a --- /dev/null +++ b/.idea/runConfigurations/connect_adapters_iiot.xml @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/.idea/runConfigurations/core.xml b/.idea/runConfigurations/core.xml new file mode 100644 index 0000000000..77258ea6bb --- /dev/null +++ b/.idea/runConfigurations/core.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/sinks_internal_jvm.xml b/.idea/runConfigurations/sinks_internal_jvm.xml new file mode 100644 index 0000000000..0a5d75aca8 --- /dev/null +++ b/.idea/runConfigurations/sinks_internal_jvm.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + diff --git a/.idea/runConfigurations/sources_vehicle_simulator_jvm.xml b/.idea/runConfigurations/sources_vehicle_simulator_jvm.xml new file mode 100644 index 0000000000..5ed43d80f1 --- /dev/null +++ b/.idea/runConfigurations/sources_vehicle_simulator_jvm.xml @@ -0,0 +1,15 @@ + + + + + + + + + + diff --git a/.idea/runConfigurations/ui.xml b/.idea/runConfigurations/ui.xml new file mode 100644 index 0000000000..ce35c11692 --- /dev/null +++ b/.idea/runConfigurations/ui.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe34d5a5cc..6d7d95f95a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ ## Contributing to StreamPipes -*Before opening a pull request*, review the [Get Involved](https://streampipes.apache.org/community/get-involved/) page. +*Before opening a pull request*, review the [Get Involved](https://streampipes.apache.org/getinvolved.html) page. It lists information that is required for contributing to StreamPipes. When you contribute code, you affirm that the contribution is your original work and that you diff --git a/README.md b/README.md index 6e86241c41..66fa337b79 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,14 @@


- StreamPipes Logo

Self-Service Data Analytics for the (Industrial) IoT

StreamPipes is a self-service (Industrial) IoT toolbox to enable non-technical users to connect , analyze and explore IoT data streams.

-

+

StreamPipes Overview

@@ -53,7 +53,7 @@ * [Installation](#installation) * [Documentation](#documentation) * [Building StreamPipes](#building-streampipes) - * [Pipeline Elements](#pipeline-elements) + * [Pipeline Elements](#pipeline-elements) * [Extending StreamPipes](#extending-streampipes) * [Bugs and Feature Requests](#bugs-and-feature-requests) * [Get help](#get-help) @@ -67,7 +67,7 @@ Apache StreamPipes makes industrial data analytics easy! -StreamPipes is an end-to-end toolbox for the industrial IoT. +StreamPipes is an end-to-end toolbox for the industrial IoT. It comes with a rich graphical user interface targeted at non-technical users and provides the following features: @@ -77,8 +77,8 @@ It comes with a rich graphical user interface targeted at non-technical users an * A live dashboard to display real-time data from data sources and pipelines, e.g., for shopfloor monitoring. -StreamPipes is highly extensible and includes a Java SDK to create new -pipeline elements and adapters. Python support is available in an early development stage - stay tuned! +StreamPipes is highly extensible and includes a Java SDK to create new +pipeline elements and adapters. Python support is available in an early development stage - stay tuned! Pipeline elements are standalone microservices that can run anywhere - centrally on your server or close at the edge. You want to employ your own machine learning model on live data? @@ -108,13 +108,13 @@ Besides that, StreamPipes includes features for production deployments: ## Installation -The quickest way to run StreamPipes including the latest extensions (adapters, pipeline elements) is by using our Docker-based [installation & operation options](installer), namely: +The quickest way to run StreamPipes including the latest extensions (adapters, pipeline elements) is by using our Docker-based [installation & operation options](installer), namely: * **[StreamPipes Compose](installer/compose)** - The User's Choice * **[StreamPipes CLI](installer/cli)** - The Developer's Favorite * **[StreamPipes k8s](installer/k8s)** - The Operator's Dream -> [!IMPORTANT] +> [!IMPORTANT] > StreamPipes CLI & k8s are highly recommended for developers or operators. Standard users should stick to StreamPipes Compose. Please follow the instructions provided in the corresponding `README.md` to get started. @@ -168,14 +168,14 @@ You can also use the installer or CLI as described in the ``Installation`` secti ## Pipeline Elements StreamPipes includes a repository of extensions for adapters and pipeline elements: -* **Connect adapters** for a variety of IoT data sources as well as -* **Data Processors** and **Data Sinks** as ready-to-use pipeline elements. +* **Connect adapters** for a variety of IoT data sources as well as +* **Data Processors** and **Data Sinks** as ready-to-use pipeline elements. The source code of all included pipeline elements and adapters can be found [here](https://github.com/apache/streampipes/tree/dev/streampipes-extensions). ## Extending StreamPipes -You can easily add your own data streams, processors or sinks. A [Java-based SDK](https://streampipes.apache.org/docs/extend-tutorial-data-processors.html) can be used to integrate your existing processing logic into StreamPipes. +You can easily add your own data streams, processors or sinks. A [Java-based SDK](https://streampipes.apache.org/docs/extend-tutorial-data-processors.html) can be used to integrate your existing processing logic into StreamPipes. Pipeline elements are packaged as Docker images and can be installed at runtime, whenever your requirements change. 👉 Check our [developer guide](https://streampipes.apache.org/docs/extend-setup.html). @@ -189,10 +189,10 @@ If you've found a bug or have a feature that you'd love to see in StreamPipes, f ## Get help -If you have any problems during the installation or questions around StreamPipes, you'll get help through one of our +If you have any problems during the installation or questions around StreamPipes, you'll get help through one of our community channels: -👉 [Mailing Lists](https://streampipes.apache.org/community/mailing-lists/) +👉 [Mailing Lists](https://streampipes.apache.org/mailinglists.html) Or directly subscribe to [users-subscribe@streampipes.apache.org](mailto:users-subscribe@streampipes.apache.org)! diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ccc2742754..88b3ee17cc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -206,322 +206,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). * [[#2145](https://github.com/apache/streampipes/pull/2145)]: chore: Maven housekeeping -# [0.93.0] - -## What's Changed - -### Enhancement 🌟 - -* [[#2092](https://github.com/apache/streampipes/issues/2092)]: Remove magic HTTP numbers in StreamPipes -* [[#2056](https://github.com/apache/streampipes/issues/2056)]: Make email templates configurable -* [[#2032](https://github.com/apache/streampipes/issues/2032)]: Provide endpoint to get measurement counts from core -* [[#2031](https://github.com/apache/streampipes/pull/2031)]: style: Add last message info to adapter overview -* [[#1992](https://github.com/apache/streampipes/issues/1992)]: Migration of Kafka source configuration. -* [[#1980](https://github.com/apache/streampipes/issues/1980)]: Arrays are not supported for S7 PLCs -* [[#1906](https://github.com/apache/streampipes/issues/1906)]: Revive streampipes-maven-plugin to auto-generate - pipeline element documentation -* [[#1875](https://github.com/apache/streampipes/issues/1875)]: Connect: Order measurement units by name -* [[#1814](https://github.com/apache/streampipes/issues/1814)]: Integrate extensions service discovery & configuration - management into core -* [[#1716](https://github.com/apache/streampipes/pull/1716)]: Enable creating CouchDB attachments for images -* [[#1688](https://github.com/apache/streampipes/issues/1688)]: New Processor: Round Numeric Values -* [[#1662](https://github.com/apache/streampipes/issues/1662)]: Support asynchronous browsing in OPC-UA adapter -* [[#1592](https://github.com/apache/streampipes/issues/1592)]: Connect IO-Link Sensor Data into StreamPipes -* [[#1374](https://github.com/apache/streampipes/issues/1374)]: Convenient `columns` query parameter for data lake - measure -* [[#1103](https://github.com/apache/streampipes/issues/1103)]: Support Python 3.11 in python client - -### Bug fixes 🧰 -* [[#2191](https://github.com/apache/streampipes/pull/2191)]: fix: tooltip in asset overview -* [[#2146](https://github.com/apache/streampipes/pull/2146)]: fix(#2002) Retry service registration in case services are - removed be… -* [[#2166](https://github.com/apache/streampipes/issues/2166)]: Protected names are not sanitized correctly in Data Lake - Sink / Influx sink -* [[#2165](https://github.com/apache/streampipes/issues/2165)]: Update `0.92.0` -> `0.93.0` of `Machine Data Simulator` - not working -* [[#2112](https://github.com/apache/streampipes/issues/2112)]: Changes on messaging layer configuration on UI not - persisted. -* [[#2044](https://github.com/apache/streampipes/issues/2044)]: Docker compose build error -* [[#2024](https://github.com/apache/streampipes/pull/2024)]: fix: make data retrieval of IOLink sensor more robust -* [[#1992](https://github.com/apache/streampipes/issues/1992)]: Migration of Kafka source configuration. -* [[#1983](https://github.com/apache/streampipes/issues/1983)]: Logo image broken in Footer -* [[#1956](https://github.com/apache/streampipes/issues/1956)]: NPE in ConsulConfigMigration -* [[#1938](https://github.com/apache/streampipes/issues/1938)]: Datetime selector in Data Explorer has issues with 12 am -* [[#1934](https://github.com/apache/streampipes/issues/1934)]: Improve adapter started dialog in StreamPipes connect -* [[#1876](https://github.com/apache/streampipes/issues/1876)]: Connect: Form validation in schema editor -* [[#1834](https://github.com/apache/streampipes/pull/1834)]: [hotfix] Fix MDC layout issue in permission dialog -* [[#1829](https://github.com/apache/streampipes/pull/1829)]: [hotfix] Fix layout issues and validation in data explorer -* [[#1794](https://github.com/apache/streampipes/issues/1794)]: Aggregation field in data explorer widget is broken -* [[#1770](https://github.com/apache/streampipes/issues/1770)]: Wrong base image in Maven archetypes -* [[#1769](https://github.com/apache/streampipes/issues/1769)]: The dashboard fails to load the element whose name - contiains '/' -* [[#1741](https://github.com/apache/streampipes/issues/1741)]: The status light widget in the live dashboard is broken -* [[#1713](https://github.com/apache/streampipes/issues/1713)]: OPC UA NullPointer Exception when Node Description is - Missing -* [[#1642](https://github.com/apache/streampipes/issues/1642)]: Data Lake default export period does not work -* [[#1637](https://github.com/apache/streampipes/issues/1637)]: Schema guessing from file is currently not implemented - in HTTP Server source -* [[#1629](https://github.com/apache/streampipes/pull/1629)]: fix: kafka consumer data loss promble -* [[#1597](https://github.com/apache/streampipes/issues/1597)]: apachestreampipes/sources-vehicle-simulator: - 0.92.0-SNAPSHOT not found -* [[#1546](https://github.com/apache/streampipes/issues/1546)]: Failed to upgrade the helm chart -* [[#1533](https://github.com/apache/streampipes/issues/1533)]: Notification counter is not reset -* [[#1481](https://github.com/apache/streampipes/issues/1481)]: URL Dereferencing Processor NotSerializableException - -### Breaking Change 💣 - -* [[#2143](https://github.com/apache/streampipes/pull/2143)]: refactor(#2128): deprecate legacy adapters -* [[#2088](https://github.com/apache/streampipes/issues/2088)]: Remove module `streampipes-logging` -* [[#2066](https://github.com/apache/streampipes/pull/2066)]: refactor: remove legacy demo resources -* [[#1912](https://github.com/apache/streampipes/pull/1912)]: Remove python wrapper -* [[#1583](https://github.com/apache/streampipes/issues/1583)]: Remove CumSum Pipeline Element -* [[#1289](https://github.com/apache/streampipes/issues/1289)]: Harmonize data set and data stream API - -### Deprecation ⚠️ - -* [[#2143](https://github.com/apache/streampipes/pull/2143)]: refactor(#2128): deprecate legacy adapters -* [[#1640](https://github.com/apache/streampipes/pull/1640)]: feature: retrieve credentials from SP environment - variables - -### Documentation & Website 📚 - -* [[#2143](https://github.com/apache/streampipes/pull/2143)]: refactor(#2128): deprecate legacy adapters -* [[#2138](https://github.com/apache/streampipes/pull/2138)]: Use os.environ dictionary to set environment variables. -* [[#2069](https://github.com/apache/streampipes/pull/2069)]: refactor: Add individual connector modules for adapters - and sinks -* [[#2067](https://github.com/apache/streampipes/issues/2067)]: Check references for watertank simulator and vehicle - simulator on the website -* [[#2066](https://github.com/apache/streampipes/pull/2066)]: refactor: remove legacy demo resources -* [[#1983](https://github.com/apache/streampipes/issues/1983)]: Logo image broken in Footer -* [[#1978](https://github.com/apache/streampipes/pull/1978)]: Add deployment of Prometheus and Grafana to K8s -* [[#1955](https://github.com/apache/streampipes/pull/1955)]: Introduce Quickstart deployment mode -* [[#1945](https://github.com/apache/streampipes/pull/1945)]: Add configuration hint for Kafka users. -* [[#1912](https://github.com/apache/streampipes/pull/1912)]: Remove python wrapper -* [[#1906](https://github.com/apache/streampipes/issues/1906)]: Revive streampipes-maven-plugin to auto-generate - pipeline element documentation -* [[#1844](https://github.com/apache/streampipes/pull/1844)]: chore: add Poetry badge to our README -* [[#1820](https://github.com/apache/streampipes/pull/1820)]: Add Pulsar's Messaging Layer to a Helm Deployment -* [[#1817](https://github.com/apache/streampipes/pull/1817)]: chore: introduce poetry as dependency management tool -* [[#1733](https://github.com/apache/streampipes/pull/1733)]: feature: introduce admonitions to warn about dependency - issue in docs -* [[#1694](https://github.com/apache/streampipes/pull/1694)]: chore: improve metadata for repository -* [[#1640](https://github.com/apache/streampipes/pull/1640)]: feature: retrieve credentials from SP environment - variables - -### Dependency Updates 📦 - -* [[#2177](https://github.com/apache/streampipes/pull/2177)]: deps: update Active MQ due to CVE -* [[#2140](https://github.com/apache/streampipes/pull/2140)]: chore(deps-dev): bump eslint from 8.37.0 to 8.53.0 in /ui -* [[#2127](https://github.com/apache/streampipes/pull/2127)]: chore(deps-dev): bump - @angular-eslint/eslint-plugin-template from 15.2.1 to 16.2.0 in /ui -* [[#2126](https://github.com/apache/streampipes/pull/2126)]: chore(deps): bump org.mockito:mockito-core from 5.6.0 to - 5.7.0 -* [[#2125](https://github.com/apache/streampipes/pull/2125)]: chore(deps): remove org.wildfly.common:wildfly-common -* [[#2119](https://github.com/apache/streampipes/pull/2119)]: chore(deps): bump com.google.protobuf:protobuf-java from - 3.24.0 to 3.25.0 -* [[#2113](https://github.com/apache/streampipes/pull/2113)]: chore(deps-dev): bump webpack from 5.88.2 to 5.89.0 in /ui -* [[#2091](https://github.com/apache/streampipes/pull/2091)]: chore(deps-dev): bump browserify-sign from 4.2.1 to 4.2.2 - in /ui -* [[#2087](https://github.com/apache/streampipes/pull/2087)]: chore(deps): bump com.rabbitmq:amqp-client from 5.19.0 to - 5.20.0 -* [[#2085](https://github.com/apache/streampipes/pull/2085)]: chore(deps-dev): bump jasmine-core from 4.6.0 to 5.1.1 in - /ui -* [[#2069](https://github.com/apache/streampipes/pull/2069)]: refactor: Add individual connector modules for adapters - and sinks -* [[#2066](https://github.com/apache/streampipes/pull/2066)]: refactor: remove legacy demo resources -* [[#2048](https://github.com/apache/streampipes/pull/2048)]: chore(deps-dev): bump assert from 2.0.0 to 2.1.0 in /ui -* [[#2038](https://github.com/apache/streampipes/pull/2038)]: chore(deps-dev): bump @babel/traverse from 7.22.5 to - 7.23.2 in /ui -* [[#2036](https://github.com/apache/streampipes/pull/2036)]: chore(deps): bump plotly.js from 2.22.0 to 2.26.2 in /ui -* [[#2035](https://github.com/apache/streampipes/pull/2035)]: chore(deps): bump org.apache.inlong:tubemq-client from - 1.7.0 to 1.9.0 -* [[#2027](https://github.com/apache/streampipes/pull/2027)]: chore(deps-dev): bump @types/jasmine from 4.3.1 to 5.1.0 - in /ui -* [[#2019](https://github.com/apache/streampipes/pull/2019)]: chore(deps): bump shepherd.js from 11.1.1 to 11.2.0 in /ui -* [[#2015](https://github.com/apache/streampipes/pull/2015)]: chore(deps): remove org.immutables -* [[#2011](https://github.com/apache/streampipes/pull/2011)]: chore(deps): bump org.simplejavamail:simple-java-mail from - 8.2.0 to 8.3.1 -* [[#2009](https://github.com/apache/streampipes/pull/2009)]: chore(deps-dev): bump webpack from 5.76.1 to 5.88.2 in /ui -* [[#1999](https://github.com/apache/streampipes/pull/1999)]: chore(deps): bump io.nats:jnats from 2.16.1 to 2.17.0 -* [[#1996](https://github.com/apache/streampipes/pull/1996)]: chore(deps): bump org.checkerframework:checker-qual from - 3.38.0 to 3.39.0 -* [[#1988](https://github.com/apache/streampipes/pull/1988)]: chore(deps): bump org.simplejavamail:simple-java-mail from - 8.1.3 to 8.2.0 -* [[#1984](https://github.com/apache/streampipes/pull/1984)]: chore(deps): bump org.yaml:snakeyaml from 2.1 to 2.2 -* [[#1977](https://github.com/apache/streampipes/pull/1977)]: chore(deps): bump com.rabbitmq:amqp-client from 5.18.0 to - 5.19.0 -* [[#1972](https://github.com/apache/streampipes/pull/1972)]: chore(deps-dev): bump org.testcontainers:testcontainers - from 1.18.3 to 1.19.0 -* [[#1970](https://github.com/apache/streampipes/pull/1970)]: Bump org.mockito:mockito-core from 5.4.0 to 5.5.0 -* [[#1964](https://github.com/apache/streampipes/pull/1964)]: Bump org.xerial.snappy:snappy-java from 1.1.10.1 to - 1.1.10.4 -* [[#1963](https://github.com/apache/streampipes/pull/1963)]: Bump tslib from 2.5.0 to 2.6.2 in /ui -* [[#1962](https://github.com/apache/streampipes/pull/1962)]: Bump com.google.guava:guava from 32.0.1-jre to 32.1.2-jre -* [[#1949](https://github.com/apache/streampipes/pull/1949)]: Bump com.nimbusds:nimbus-jose-jwt from 9.31 to 9.35 -* [[#1946](https://github.com/apache/streampipes/pull/1946)]: Bump typing-extensions from 4.5.0 to 4.8.0 in - /streampipes-client-python -* [[#1942](https://github.com/apache/streampipes/pull/1942)]: Bump org.boofcv:boofcv-core from 0.44 to 1.1.0 -* [[#1939](https://github.com/apache/streampipes/pull/1939)]: refactor: replace random password generation logic -* [[#1931](https://github.com/apache/streampipes/pull/1931)]: Bump org.eclipse.jetty:jetty-http from 10.0.14 to 10.0.16 -* [[#1930](https://github.com/apache/streampipes/pull/1930)]: Bump org.eclipse.jetty:jetty-servlets from 10.0.14 to - 10.0.16 -* [[#1919](https://github.com/apache/streampipes/pull/1919)]: Bump karma-jasmine-html-reporter from 2.0.0 to 2.1.0 in - /ui -* [[#1916](https://github.com/apache/streampipes/pull/1916)]: Bump net.minidev:json-smart from 2.4.9 to 2.5.0 -* [[#1912](https://github.com/apache/streampipes/pull/1912)]: Remove python wrapper -* [[#1881](https://github.com/apache/streampipes/pull/1881)]: Bump cz.habarta.typescript-generator: - typescript-generator-maven-plugin from 3.1.1185 to 3.2.1263 -* [[#1861](https://github.com/apache/streampipes/pull/1861)]: Bump roaster.version from 2.28.0.Final to 2.29.0.Final -* [[#1860](https://github.com/apache/streampipes/pull/1860)]: Bump @ctrl/ngx-codemirror from 5.1.1 to 6.1.0 in /ui -* [[#1850](https://github.com/apache/streampipes/pull/1850)]: Bump org.antlr:antlr4-runtime from 4.11.1 to 4.13.0 -* [[#1848](https://github.com/apache/streampipes/pull/1848)]: Bump @angular-eslint/builder from 15.2.1 to 16.1.1 in /ui -* [[#1837](https://github.com/apache/streampipes/pull/1837)]: Bump redis.clients:jedis from 4.3.1 to 4.4.3 -* [[#1836](https://github.com/apache/streampipes/pull/1836)]: Bump lint-staged from 13.2.0 to 14.0.0 in /ui -* [[#1831](https://github.com/apache/streampipes/pull/1831)]: Bump blacken-docs from 1.15.0 to 1.16.0 in - /streampipes-client-python -* [[#1830](https://github.com/apache/streampipes/pull/1830)]: Bump org.jetbrains.kotlin:kotlin-stdlib from 1.8.0 to - 1.9.0 -* [[#1825](https://github.com/apache/streampipes/pull/1825)]: Bump com.google.protobuf:protobuf-java from 3.21.12 to - 3.24.0 -* [[#1821](https://github.com/apache/streampipes/pull/1821)]: Bump org.boofcv:boofcv-core from 0.43.1 to 0.44 -* [[#1817](https://github.com/apache/streampipes/pull/1817)]: chore: introduce poetry as dependency management tool -* [[#1816](https://github.com/apache/streampipes/pull/1816)]: Bump eslint-config-prettier from 8.8.0 to 9.0.0 in /ui -* [[#1812](https://github.com/apache/streampipes/pull/1812)]: Bump konva from 8.4.0 to 9.2.0 in /ui -* [[#1810](https://github.com/apache/streampipes/pull/1810)]: Bump @types/node from 18.14.0 to 20.4.6 in /ui -* [[#1805](https://github.com/apache/streampipes/pull/1805)]: Bump pyupgrade from 3.9.0 to 3.10.1 in - /streampipes-client-python -* [[#1804](https://github.com/apache/streampipes/pull/1804)]: Bump flake8 from 6.0.0 to 6.1.0 in - /streampipes-client-python -* [[#1802](https://github.com/apache/streampipes/pull/1802)]: Bump org.boofcv:boofcv-core from 0.42 to 0.43.1 -* [[#1801](https://github.com/apache/streampipes/pull/1801)]: Bump mkdocs from 1.4.2 to 1.5.1 in - /streampipes-client-python -* [[#1790](https://github.com/apache/streampipes/pull/1790)]: Bump @jsplumb/browser-ui from 6.1.1 to 6.2.10 in /ui -* [[#1789](https://github.com/apache/streampipes/pull/1789)]: Bump com.opencsv:opencsv from 5.7.1 to 5.8 -* [[#1784](https://github.com/apache/streampipes/pull/1784)]: Bump @typescript-eslint/parser from 5.59.11 to 5.62.0 in - /ui -* [[#1780](https://github.com/apache/streampipes/pull/1780)]: Bump word-wrap from 1.2.3 to 1.2.4 in /ui -* [[#1767](https://github.com/apache/streampipes/pull/1767)]: remove dependency scala-xml_2.11 -* [[#1766](https://github.com/apache/streampipes/pull/1766)]: Bump semver from 5.7.1 to 5.7.2 in /ui -* [[#1765](https://github.com/apache/streampipes/pull/1765)]: Bump checkstyle from 10.6.0 to 10.12.1 -* [[#1764](https://github.com/apache/streampipes/pull/1764)]: Bump black from 23.3.0 to 23.7.0 in - /streampipes-client-python -* [[#1763](https://github.com/apache/streampipes/pull/1763)]: Bump pyupgrade from 3.8.0 to 3.9.0 in - /streampipes-client-python -* [[#1761](https://github.com/apache/streampipes/pull/1761)]: Bump cypress from 12.8.1 to 12.17.0 in /ui -* [[#1759](https://github.com/apache/streampipes/pull/1759)]: Bump amqp-client from 5.17.0 to 5.18.0 -* [[#1749](https://github.com/apache/streampipes/pull/1749)]: Bump blacken-docs from 1.14.0 to 1.15.0 in - /streampipes-client-python -* [[#1748](https://github.com/apache/streampipes/pull/1748)]: Bump extra-enforcer-rules from 1.6.1 to 1.7.0 -* [[#1746](https://github.com/apache/streampipes/pull/1746)]: Bump jquery from 3.6.3 to 3.7.0 in /ui -* [[#1739](https://github.com/apache/streampipes/pull/1739)]: Bump graalvm.js.version from 22.3.1 to 23.0.0 -* [[#1735](https://github.com/apache/streampipes/pull/1735)]: Bump jakarta.activation-api from 1.2.2 to 2.1.2 -* [[#1734](https://github.com/apache/streampipes/pull/1734)]: Bump shepherd.js from 11.0.1 to 11.1.1 in /ui -* [[#1733](https://github.com/apache/streampipes/pull/1733)]: feature: introduce admonitions to warn about dependency - issue in docs -* [[#1730](https://github.com/apache/streampipes/pull/1730)]: Bump javassist from 3.25.0-GA to 3.29.2-GA -* [[#1728](https://github.com/apache/streampipes/pull/1728)]: Bump pyupgrade from 3.7.0 to 3.8.0 in - /streampipes-client-python -* [[#1723](https://github.com/apache/streampipes/pull/1723)]: Bump jboss-logging from 3.4.0.Final to 3.5.2.Final -* [[#1721](https://github.com/apache/streampipes/pull/1721)]: Bump tubemq-client from 1.6.0 to 1.7.0 -* [[#1715](https://github.com/apache/streampipes/pull/1715)]: Bump okio from 1.16.0 to 3.3.0 -* [[#1712](https://github.com/apache/streampipes/pull/1712)]: Bump autoflake from 2.1.0 to 2.2.0 in - /streampipes-client-python -* [[#1711](https://github.com/apache/streampipes/pull/1711)]: Bump pytest from 7.3.0 to 7.4.0 in - /streampipes-client-python -* [[#1710](https://github.com/apache/streampipes/pull/1710)]: Bump formatter-maven-plugin from 2.21.0 to 2.23.0 -* [[#1707](https://github.com/apache/streampipes/pull/1707)]: Bump mypy from 1.3.0 to 1.4.0 in - /streampipes-client-python -* [[#1704](https://github.com/apache/streampipes/pull/1704)]: Bump spring-security-core from 6.0.3 to 6.1.1 & spring to - 6.0.10 -* [[#1702](https://github.com/apache/streampipes/pull/1702)]: Bump angular-plotly.js from 4.0.4 to 5.0.0 in /ui -* [[#1699](https://github.com/apache/streampipes/pull/1699)]: Bump @typescript-eslint/parser from 5.56.0 to 5.59.11 in - /ui -* [[#1698](https://github.com/apache/streampipes/pull/1698)]: Bump mockito-core from 5.3.1 to 5.4.0 -* [[#1697](https://github.com/apache/streampipes/pull/1697)]: Bump pyupgrade from 3.6.0 to 3.7.0 in - /streampipes-client-python -* [[#1692](https://github.com/apache/streampipes/pull/1692)]: Bump dependency-check-maven from 6.5.1 to 8.3.1 -* [[#1689](https://github.com/apache/streampipes/pull/1689)]: Bump snappy-java from 1.1.7.7 to 1.1.10.1 -* [[#1687](https://github.com/apache/streampipes/pull/1687)]: Bump guava from 31.1-jre to 32.0.1-jre -* [[#1686](https://github.com/apache/streampipes/pull/1686)]: Bump @swimlane/ngx-charts from 20.1.2 to 20.4.1 in /ui -* [[#1681](https://github.com/apache/streampipes/pull/1681)]: Bump testcontainers from 1.17.4 to 1.18.3 -* [[#1679](https://github.com/apache/streampipes/pull/1679)]: Bump nimbus-jose-jwt from 9.30.1 to 9.31 -* [[#1678](https://github.com/apache/streampipes/pull/1678)]: Bump blacken-docs from 1.13.0 to 1.14.0 in - /streampipes-client-python -* [[#1674](https://github.com/apache/streampipes/pull/1674)]: Bump pyupgrade from 3.4.0 to 3.6.0 in - /streampipes-client-python -* [[#1640](https://github.com/apache/streampipes/pull/1640)]: feature: retrieve credentials from SP environment - variables -* [[#1636](https://github.com/apache/streampipes/pull/1636)]: Bump mockito-core from 4.11.0 to 5.3.1 -* [[#1631](https://github.com/apache/streampipes/pull/1631)]: Bump mkdocstrings[python] from 0.21.1 to 0.22.0 in - /streampipes-client-python -* [[#1595](https://github.com/apache/streampipes/pull/1595)]: Bump spring-boot.version from 3.0.6 to 3.1.0 -* [[#1591](https://github.com/apache/streampipes/pull/1591)]: Bump pytest-cov from 4.0.0 to 4.1.0 in - /streampipes-client-python -* [[#1588](https://github.com/apache/streampipes/pull/1588)]: Bump types-requests from 2.30.0.0 to 2.31.0.0 in - /streampipes-client-python -* [[#1587](https://github.com/apache/streampipes/pull/1587)]: Bump socket.io-parser from 4.2.1 to 4.2.3 in /ui -* [[#1578](https://github.com/apache/streampipes/pull/1578)]: Bump postgresql from 42.4.3 to 42.6.0 -* [[#1576](https://github.com/apache/streampipes/pull/1576)]: Support pulsar messasging layer -* [[#1335](https://github.com/apache/streampipes/issues/1335)]: Replace `@angular/flex-layout` dependency - with `@ngbracket/ngx-layout` dependency - -### Uncategorized ❓ - -* [[#2209](https://github.com/apache/streampipes/pull/2209)]: build: change `outputHashing` in Angular to avoid caching - issue after new release -* [[#2190](https://github.com/apache/streampipes/pull/2190)]: ui: add link to LinkedIn in Support section -* [[#2135](https://github.com/apache/streampipes/issues/2135)]: Max health check intervals configurable -* [[#2130](https://github.com/apache/streampipes/issues/2130)]: Add E2E-Test for pipeline export and import -* [[#2129](https://github.com/apache/streampipes/pull/2129)]: feat: Use alpine-based Docker image for UI -* [[#2122](https://github.com/apache/streampipes/issues/2122)]: Cleanup extension bundles -* [[#2104](https://github.com/apache/streampipes/issues/2104)]: Improve lifecycle for managing core and extension - initialization actions -* [[#2098](https://github.com/apache/streampipes/issues/2098)]: Implement first migration for S7 adapter -* [[#2076](https://github.com/apache/streampipes/pull/2076)]: improve archetypes for adapter tutorial -* [[#2071](https://github.com/apache/streampipes/pull/2071)]: refactor: minor adaption & improvement -* [[#2068](https://github.com/apache/streampipes/pull/2068)]: refactor: Make interactive tutorial work again -* [[#2064](https://github.com/apache/streampipes/pull/2064)]: refactor: introduce convenience method for service url -* [[#2061](https://github.com/apache/streampipes/issues/2061)]: Create zip file during build phase with installer files - only -* [[#2041](https://github.com/apache/streampipes/pull/2041)]: refactor: remove references and artifacts for data sets -* [[#2018](https://github.com/apache/streampipes/pull/2018)]: test(#2017): Add cypress test for configuration -* [[#2017](https://github.com/apache/streampipes/issues/2017)]: Add more e2e tests to configuration view -* [[#2002](https://github.com/apache/streampipes/issues/2002)]: Harmonize registration of adapters and pipeline elements -* [[#1926](https://github.com/apache/streampipes/issues/1926)]: Improve handling of secrets in K8s -* [[#1852](https://github.com/apache/streampipes/pull/1852)]: Remove sources-vehicle-simulator from cli-installer full - env. -* [[#1843](https://github.com/apache/streampipes/pull/1843)]: chore: refine dependency constraints -* [[#1787](https://github.com/apache/streampipes/issues/1787)]: Improve logging of extensions services -* [[#1786](https://github.com/apache/streampipes/pull/1786)]: add probes to Streampipes' kubernetes deployment ( #1781 ) -* [[#1777](https://github.com/apache/streampipes/issues/1777)]: Add API endpoint to get available users -* [[#1771](https://github.com/apache/streampipes/issues/1771)]: Remove dependencies to specific protocols from the - StreamPipes core service -* [[#1726](https://github.com/apache/streampipes/issues/1726)]: Update Maven archetypes -* [[#1717](https://github.com/apache/streampipes/issues/1717)]: Support other protocols besides Kafka in Streampipes - Client for gathering live data -* [[#1683](https://github.com/apache/streampipes/pull/1683)]: Support migration of adapters in data import -* [[#1682](https://github.com/apache/streampipes/pull/1682)]: Harmonize OPC-UA adapter and sink, add timestamp to - metadata (#899) -* [[#1676](https://github.com/apache/streampipes/issues/1676)]: About Kafka consumer data loss problem -* [[#1673](https://github.com/apache/streampipes/pull/1673)]: Make ChangedValueDetectionProcessor dimension sensitive -* [[#1664](https://github.com/apache/streampipes/issues/1664)]: Unify the labels for OPC UA adapter & sink -* [[#1660](https://github.com/apache/streampipes/pull/1660)]: Improve CSS assets to ease configuration of custom layouts -* [[#1651](https://github.com/apache/streampipes/issues/1651)]: Integrate all experimental Flink pipeline elements into - a single module -* [[#1648](https://github.com/apache/streampipes/issues/1648)]: Move OPC-UA processor and sink into a single module -* [[#1632](https://github.com/apache/streampipes/issues/1632)]: Cleanup API to define data processors and sinks -* [[#1628](https://github.com/apache/streampipes/pull/1628)]: chore: add missing support of NATS as messaging protocol -* [[#1616](https://github.com/apache/streampipes/issues/1616)]: Modify .asf.yaml to better organize Github discussions - on mailing list -* [[#1590](https://github.com/apache/streampipes/issues/1590)]: Rename the interface `AdapterInterface` to `IAdapter` in - the `remove-set-adapter` branch -* [[#1589](https://github.com/apache/streampipes/pull/1589)]: add sample configuration of pulsar subscription-name -* [[#1581](https://github.com/apache/streampipes/issues/1581)]: HTTP Stream Adapter Stops Emitting Events When Running - Multiple Instances -* [[#1580](https://github.com/apache/streampipes/issues/1580)]: Include Set Adapters in CouchDB Backup During Migration - Script -* [[#1260](https://github.com/apache/streampipes/issues/1260)]: StreamPipes functions Python: `required_streams ` - vs `consumed_streams ` # [0.92.0] diff --git a/VULNERABILITY.md b/VULNERABILITY.md index 00255213a5..253128c4a0 100644 --- a/VULNERABILITY.md +++ b/VULNERABILITY.md @@ -5,7 +5,4 @@ | https://osv.dev/GHSA-w596-4wvx-j9j6
https://osv.dev/PYSEC-2022-42969 | 7.5 | PyPI | py | 1.11.0 | streampipes-client-python/poetry.lock | | https://osv.dev/GHSA-269g-pwp5-87pp | 4.4 | Maven | junit:junit (dev) | 4.8.2 | streampipes-maven-plugin/pom.xml | | https://osv.dev/GHSA-4943-9vgg-gr5r | 6.1 | npm | quill | 1.3.7 | ui/package-lock.json | -| https://osv.dev/GHSA-f5x3-32g6-xq36 | 6.5 | npm | tar (dev) | 6.2.0 | ui/package-lock.json | -| https://osv.dev/GHSA-9qxr-qj54-h672 | 2.6 | npm | undici (dev) | 6.7.1 | ui/package-lock.json | -| https://osv.dev/GHSA-m4v8-wqvr-p9f7 | 3.9 | npm | undici (dev) | 6.7.1 | ui/package-lock.json | -| https://osv.dev/GHSA-8jhw-289h-jh2g | 5.9 | npm | vite (dev) | 5.1.5 | ui/package-lock.json | +| https://osv.dev/GHSA-wr3j-pwj9-hqq6 | 7.4 | npm | webpack-dev-middleware (dev) | 6.1.1 | ui/package-lock.json | diff --git a/archetypes/streampipes-archetype-extensions-jvm/pom.xml b/archetypes/streampipes-archetype-extensions-jvm/pom.xml index 86df7a8d8f..c28a82e4a6 100644 --- a/archetypes/streampipes-archetype-extensions-jvm/pom.xml +++ b/archetypes/streampipes-archetype-extensions-jvm/pom.xml @@ -22,7 +22,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT ../../pom.xml streampipes-archetype-extensions-jvm diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/pom.xml index 84ecaae456..4e60493efd 100644 --- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/pom.xml @@ -24,7 +24,7 @@ ${version} - 0.97.0-SNAPSHOT + 0.95.0 5.9.1 diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/Init.java b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/Init.java index 47e910e837..d1b9cd51e7 100644 --- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/Init.java +++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/Init.java @@ -21,6 +21,10 @@ #set( $symbol_escape = '\' ) package ${package}; +import org.apache.streampipes.dataformat.cbor.CborDataFormatFactory; +import org.apache.streampipes.dataformat.fst.FstDataFormatFactory; +import org.apache.streampipes.dataformat.json.JsonDataFormatFactory; +import org.apache.streampipes.dataformat.smile.SmileDataFormatFactory; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder; import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory; @@ -28,7 +32,7 @@ import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory; import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory; import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory; -import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase; +import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter; import org.apache.streampipes.wrapper.standalone.runtime.StandaloneStreamPipesRuntimeProvider; import ${package}.pe.${packageName}.${classNamePrefix}DataProcessor; @@ -36,7 +40,7 @@ import ${package}.pe.${packageName}.${classNamePrefix}GenericAdapter; import ${package}.pe.${packageName}.${classNamePrefix}SpecificAdapter; -public class Init extends StreamPipesExtensionsServiceBase { +public class Init extends ExtensionsModelSubmitter { public static void main(String[] args) { new Init().init(); @@ -52,6 +56,11 @@ public SpServiceDefinition provideServiceDefinition() { .registerPipelineElement(new ${classNamePrefix}DataSink()) .registerAdapter(new ${classNamePrefix}GenericAdapter()) .registerAdapter(new ${classNamePrefix}SpecificAdapter()) + .registerMessagingFormats( + new JsonDataFormatFactory(), + new CborDataFormatFactory(), + new SmileDataFormatFactory(), + new FstDataFormatFactory()) .registerMessagingProtocols( new SpKafkaProtocolFactory(), new SpJmsProtocolFactory(), diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataProcessor.java b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataProcessor.java index 1bedf0d057..85b88fd115 100644 --- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataProcessor.java +++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataProcessor.java @@ -27,7 +27,6 @@ import org.apache.streampipes.extensions.api.pe.param.IDataProcessorParameters; import org.apache.streampipes.extensions.api.pe.routing.SpOutputCollector; import org.apache.streampipes.model.DataProcessorType; -import org.apache.streampipes.model.extensions.ExtensionAssetType; import org.apache.streampipes.model.runtime.Event; import org.apache.streampipes.sdk.builder.PrimitivePropertyBuilder; import org.apache.streampipes.sdk.builder.ProcessingElementBuilder; @@ -37,6 +36,7 @@ import org.apache.streampipes.sdk.helpers.Labels; import org.apache.streampipes.sdk.helpers.Locales; import org.apache.streampipes.sdk.helpers.OutputStrategies; +import org.apache.streampipes.sdk.utils.Assets; import org.apache.streampipes.sdk.utils.Datatypes; @@ -51,7 +51,7 @@ public IDataProcessorConfiguration declareConfig() { return DataProcessorConfiguration.create( ${classNamePrefix}DataProcessor::new, ProcessingElementBuilder.create("${package}.pe.${packageName}.processor") - .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON) + .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withLocales(Locales.EN) .category(DataProcessorType.AGGREGATE) .requiredStream(StreamRequirementsBuilder diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataSink.java b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataSink.java index 7abfb210c2..471677134a 100644 --- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataSink.java +++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/src/main/java/pe/__packageName__/__classNamePrefix__DataSink.java @@ -26,7 +26,6 @@ import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext; import org.apache.streampipes.extensions.api.pe.param.IDataSinkParameters; import org.apache.streampipes.model.DataSinkType; -import org.apache.streampipes.model.extensions.ExtensionAssetType; import org.apache.streampipes.model.runtime.Event; import org.apache.streampipes.sdk.builder.DataSinkBuilder; import org.apache.streampipes.sdk.builder.StreamRequirementsBuilder; @@ -34,8 +33,7 @@ import org.apache.streampipes.sdk.helpers.EpRequirements; import org.apache.streampipes.sdk.helpers.Labels; import org.apache.streampipes.sdk.helpers.Locales; -import org.apache.streampipes.model.AdapterType; - +import org.apache.streampipes.sdk.utils.Assets; public class ${classNamePrefix}DataSink implements IStreamPipesDataSink { @@ -50,7 +48,7 @@ public IDataSinkConfiguration declareConfig() { return DataSinkConfiguration.create( ${classNamePrefix}DataSink::new, DataSinkBuilder.create("${package}.pe.${packageName}.sink") - .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON) + .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withLocales(Locales.EN) .category(DataSinkType.UNCATEGORIZED) .requiredStream(StreamRequirementsBuilder diff --git a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml index e3b6590dd0..beacd3a8c3 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml @@ -22,7 +22,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT ../../pom.xml streampipes-archetype-pe-processors-flink diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml index 2d02a64406..d957b3585d 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml @@ -25,7 +25,7 @@ ${version} - 0.97.0-SNAPSHOT + 0.95.0 diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java index 1a53b264a8..58f30bf004 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/Init.java @@ -25,6 +25,10 @@ import ${package}.pe.processor.${packageName}.${classNamePrefix}Controller; +import org.apache.streampipes.dataformat.cbor.CborDataFormatFactory; +import org.apache.streampipes.dataformat.fst.FstDataFormatFactory; +import org.apache.streampipes.dataformat.json.JsonDataFormatFactory; +import org.apache.streampipes.dataformat.smile.SmileDataFormatFactory; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder; import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory; @@ -32,9 +36,9 @@ import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory; import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory; import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory; -import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase; +import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter; -public class Init extends StreamPipesExtensionsServiceBase { +public class Init extends ExtensionsModelSubmitter { public static void main(String[] args) throws Exception { new Init().init(); @@ -47,6 +51,11 @@ public SpServiceDefinition provideServiceDefinition() { "", 8090) .registerPipelineElement(new ${classNamePrefix}Controller()) + .registerMessagingFormats( + new JsonDataFormatFactory(), + new CborDataFormatFactory(), + new SmileDataFormatFactory(), + new FstDataFormatFactory()) .registerMessagingProtocols( new SpKafkaProtocolFactory(), new SpJmsProtocolFactory(), diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java index de85a9d338..dc810a0ce7 100644 --- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java @@ -35,9 +35,9 @@ import org.apache.streampipes.sdk.helpers.Labels; import org.apache.streampipes.sdk.helpers.OutputStrategies; import org.apache.streampipes.sdk.helpers.*; +import org.apache.streampipes.sdk.utils.Assets; import org.apache.streampipes.wrapper.flink.FlinkDataProcessorDeclarer; import org.apache.streampipes.wrapper.flink.FlinkDataProcessorRuntime; -import org.apache.streampipes.model.extensions.ExtensionAssetType; public class ${classNamePrefix}Controller extends FlinkDataProcessorDeclarer<${classNamePrefix}Parameters> { @@ -46,7 +46,7 @@ public class ${classNamePrefix}Controller extends FlinkDataProcessorDeclarer<${c @Override public DataProcessorDescription declareModel() { return ProcessingElementBuilder.create("${package}.pe.processor.${packageName}") - .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON) + .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withLocales(Locales.EN) .category(DataProcessorType.ENRICH) .requiredStream(StreamRequirementsBuilder diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml index 3605c02f38..5617412037 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml @@ -22,7 +22,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT ../../pom.xml streampipes-archetype-pe-sinks-flink diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml index 0acb6d4c15..8c7d55b3ed 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml @@ -25,7 +25,7 @@ ${version} - 0.97.0-SNAPSHOT + 0.95.0 diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java index 32f521f419..f0258d8ffc 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/Init.java @@ -21,6 +21,10 @@ #set( $symbol_escape = '\' ) package ${package}; +import org.apache.streampipes.dataformat.cbor.CborDataFormatFactory; +import org.apache.streampipes.dataformat.fst.FstDataFormatFactory; +import org.apache.streampipes.dataformat.json.JsonDataFormatFactory; +import org.apache.streampipes.dataformat.smile.SmileDataFormatFactory; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.extensions.management.model.SpServiceDefinitionBuilder; import org.apache.streampipes.messaging.jms.SpJmsProtocolFactory; @@ -28,12 +32,12 @@ import org.apache.streampipes.messaging.mqtt.SpMqttProtocolFactory; import org.apache.streampipes.messaging.pulsar.SpPulsarProtocolFactory; import org.apache.streampipes.messaging.nats.SpNatsProtocolFactory; -import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase; +import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter; import ${package}.config.ConfigKeys; import ${package}.pe.sink.${packageName}.${classNamePrefix}Controller; -public class Init extends StreamPipesExtensionsServiceBase { +public class Init extends ExtensionsModelSubmitter { public static void main(String[] args) throws Exception { new Init().init(); @@ -46,6 +50,11 @@ public SpServiceDefinition provideServiceDefinition() { "", 8090) .registerPipelineElement(new ${classNamePrefix}Controller()) + .registerMessagingFormats( + new JsonDataFormatFactory(), + new CborDataFormatFactory(), + new SmileDataFormatFactory(), + new FstDataFormatFactory()) .registerMessagingProtocols( new SpKafkaProtocolFactory(), new SpJmsProtocolFactory(), diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java index 3d15c73b7a..eaae81d257 100644 --- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java @@ -34,7 +34,7 @@ import org.apache.streampipes.wrapper.flink.FlinkDataSinkDeclarer; import org.apache.streampipes.wrapper.flink.FlinkDataSinkRuntime; import org.apache.streampipes.sdk.helpers.*; -import org.apache.streampipes.model.extensions.ExtensionAssetType; +import org.apache.streampipes.sdk.utils.Assets; public class ${classNamePrefix}Controller extends FlinkDataSinkDeclarer<${classNamePrefix}Parameters> { @@ -46,7 +46,7 @@ public class ${classNamePrefix}Controller extends FlinkDataSinkDeclarer<${classN public DataSinkDescription declareModel() { return DataSinkBuilder.create("${package}.pe.sink.${packageName}") .category(DataSinkType.NOTIFICATION) - .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON) + .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withLocales(Locales.EN) .requiredStream(StreamRequirementsBuilder .create() diff --git a/docker-compose.yml b/docker-compose.yml index e993154964..dfb554bc19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,16 +20,19 @@ # provided on https://streampipes.apache.org/download # + version: "3.8" # global logging -x-logging: &default-logging +x-logging: + &default-logging options: - max-size: "12m" - max-file: "5" + max-size: '12m' + max-file: '5' driver: json-file services: + #### apache/streampipes backend: build: @@ -132,19 +135,6 @@ services: networks: spnet: - mosquitto: - image: eclipse-mosquitto:1.5.4 - logging: *default-logging - networks: - spnet: - - opcua: - image: mcr.microsoft.com/iotedge/opc-plc - logging: *default-logging - command: --ut=true - networks: - spnet: - volumes: kafka: files: diff --git a/installer/cli/.env b/installer/cli/.env index 54817505b7..3e2c0f2c29 100644 --- a/installer/cli/.env +++ b/installer/cli/.env @@ -14,7 +14,7 @@ # limitations under the License. SP_DOCKER_REGISTRY=apachestreampipes -SP_VERSION=0.97.0-SNAPSHOT +SP_VERSION=0.95.0 SP_SUBNET=172.31.0.0/16 COMPOSE_PROJECT_NAME=streampipes diff --git a/installer/cli/README.md b/installer/cli/README.md index ebbd8edfac..cc3acd566f 100644 --- a/installer/cli/README.md +++ b/installer/cli/README.md @@ -23,7 +23,7 @@ The StreamPipes command-line interface (CLI) is focused on developers in order t * new core features for **backend** and **ui**. -**Current version:** 0.97.0-SNAPSHOT +**Current version:** 0.95.0 ## TL;DR @@ -107,7 +107,7 @@ streampipes up -d ``` Now you're good to go to write your new pipeline element :tada: :tada: :tada: -> **HINT for extensions**: Use our [Maven archetypes](https://streampipes.apache.org/docs/extend-archetypes/) to setup a project skeleton and use your IDE of choice for development. However, we do recommend using IntelliJ. +> **HINT for extensions**: Use our [Maven archetypes](https://streampipes.apache.org/docs/docs/dev-guide-archetype/) to setup a project skeleton and use your IDE of choice for development. However, we do recommend using IntelliJ. > **HINT for core**: To work on `backend` or `ui` features you need to set the template to `backend` and clone the core repository [streampipes](https://github.com/apache/streampipes) - check the prerequisites there for more information. @@ -163,7 +163,7 @@ streampipes start extensions-all-jvm **Restart** existing services ```bash # restart backend couchdb container -streampipes restart backend couchdb +streampipes restart backend couchdb # restart existing services by removing and recreating container instance streampipes restart --force-create extensions-all-jvm ``` @@ -263,7 +263,7 @@ If you've found a bug or have a feature that you'd love to see in StreamPipes, f If you have any problems during the installation or questions around StreamPipes, you'll get help through one of our community channels: - [Slack](https://slack.streampipes.org) -- [Mailing Lists](https://streampipes.apache.org/community/mailing-lists/) +- [Mailing Lists](https://streampipes.apache.org/mailinglists.html) And don't forget to follow us on [Twitter](https://twitter.com/streampipes)! diff --git a/installer/cli/bin/common b/installer/cli/bin/common index 9815433287..db4b216ac8 100644 --- a/installer/cli/bin/common +++ b/installer/cli/bin/common @@ -69,8 +69,8 @@ deployment_notice() { If your environment contains the StreamPipes UI, open your browser and follow the instructions to get started: - - Local Docker for Mac/Windows, Linux: http://localhost/ + + Local Docker for Mac/Windows, Linux: http://localhost/ Remote Docker host: http://HOST_IP/ EOF @@ -110,7 +110,7 @@ get_curr_environment() { concatenate_compose_files() { no_ports=$1 search_str="[environment:" - docker_compose_files="docker compose --env-file $STREAMPIPES_WORKDIR/.env" + docker_compose_files="docker-compose --env-file $STREAMPIPES_WORKDIR/.env" while IFS='' read -r line; do [[ -n "$line" && "$line" != *"$search_str"* && "$line" != [[:blank:]#]* ]] && diff --git a/installer/cli/deploy/standalone/iotdb/docker-compose.dev.yml b/installer/cli/deploy/standalone/iotdb/docker-compose.dev.yml index caac1831b0..8cfad035c3 100644 --- a/installer/cli/deploy/standalone/iotdb/docker-compose.dev.yml +++ b/installer/cli/deploy/standalone/iotdb/docker-compose.dev.yml @@ -16,7 +16,8 @@ version: "3.4" services: iotdb: - image: apache/iotdb:1.3.0-standalone + # 0.13.3 is the only recommended production ready version for edge side (standalone) deployment + image: apache/iotdb:0.13.3-node ports: # rpc port - "6667:6667" diff --git a/installer/cli/deploy/standalone/iotdb/docker-compose.yml b/installer/cli/deploy/standalone/iotdb/docker-compose.yml index 63c4012784..f45807b9d6 100644 --- a/installer/cli/deploy/standalone/iotdb/docker-compose.yml +++ b/installer/cli/deploy/standalone/iotdb/docker-compose.yml @@ -16,32 +16,20 @@ version: "3.4" services: iotdb: - image: apache/iotdb:1.3.0-standalone - ports: - - "6667:6667" - environment: - - cn_internal_address=iotdb - - cn_internal_port=10710 - - cn_consensus_port=10720 - - cn_seed_config_node=iotdb:10710 - - dn_rpc_address=iotdb - - dn_internal_address=iotdb - - dn_rpc_port=6667 - - dn_mpp_data_exchange_port=10740 - - dn_schema_region_consensus_port=10750 - - dn_data_region_consensus_port=10760 - - dn_seed_config_node=iotdb:10710 + # 0.13.3 is the only recommended production ready version for edge side (standalone) deployment + image: apache/iotdb:0.13.3-node volumes: - - data:/iotdb/data - - logs:/iotdb/logs + - iotdb:/iotdb + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + networks: + spnet: volumes: - data: - logs: - -# If you want to influence iotdb's logging behavior, e.g., to adapt the log retention -# then you need to adapt the log configurations and map them into the container -# see: https://stackoverflow.com/questions/78177657/for-apache-iotdb-service-deployed-by-docker-how-to-modify-to-only-print-specifi + iotdb: networks: spnet: diff --git a/installer/cli/deploy/standalone/opcua/docker-compose.yml b/installer/cli/deploy/standalone/opcua/docker-compose.yml index 116ac59a38..994d8b90b8 100644 --- a/installer/cli/deploy/standalone/opcua/docker-compose.yml +++ b/installer/cli/deploy/standalone/opcua/docker-compose.yml @@ -17,6 +17,8 @@ version: "3.4" services: opcua: image: mcr.microsoft.com/iotedge/opc-plc + volumes: + - ./config-generated/:/appdata ports: - 50000:50000 command: --ut=true diff --git a/installer/compose/.env b/installer/compose/.env index a0988a144e..f3c28a6020 100644 --- a/installer/compose/.env +++ b/installer/compose/.env @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SP_VERSION=0.97.0-SNAPSHOT +SP_VERSION=0.95.0 SP_DOCKER_REGISTRY=apachestreampipes SP_SUBNET=172.31.0.0/16 COMPOSE_PROJECT_NAME=streampipes diff --git a/installer/compose/README.md b/installer/compose/README.md index 7d04ebd46d..33fa9ce472 100644 --- a/installer/compose/README.md +++ b/installer/compose/README.md @@ -19,7 +19,7 @@ StreamPipes Compose is a simple collection of user-friendly `docker-compose` files that easily lets gain first-hand experience with Apache StreamPipes. -**Current version:** 0.97.0-SNAPSHOT +**Current version:** 0.95.0 > **NOTE**: We recommend StreamPipes Compose to only use for initial try-out and testing. If you are a developer and want to develop new pipeline elements or core feature, use the [StreamPipes CLI](../cli). @@ -29,7 +29,7 @@ StreamPipes Compose is a simple collection of user-friendly `docker-compose` fil ```bash docker-compose up -d ``` -Go to http://localhost to finish the installation in the browser. Once finished, switch to the pipeline editor and start the interactive tour or check the [online tour](https://streampipes.apache.org/docs/user-guide-tour/) to learn how to create your first pipeline! +Go to http://localhost to finish the installation in the browser. Once finished, switch to the pipeline editor and start the interactive tour or check the [online tour](https://streampipes.apache.org/docs/docs/user-guide-tour/) to learn how to create your first pipeline! ## Prerequisite * Docker >= 17.06.0 @@ -50,7 +50,7 @@ We provide three options to get you going: - **full**: contains experimental Flink wrappers - **quickstart**: contains pre-configured sample assets, including pipelines, dashboards, and data views. We recommend first-time StreamPipes users to use the Quickstart mode to experience the convenience of StreamPipes in IIoT! （Recommended for first-time users) -The ``nats`` version will become the default version in a later release. You can already try it for new installations, +The ``nats`` version will become the default version in a later release. You can already try it for new installations, but there's not yet an automatic migration from current Kafka-based installations to Nats. **Starting** the **default** option is as easy as simply running: @@ -60,7 +60,7 @@ but there's not yet an automatic migration from current Kafka-based installation docker-compose up -d # go to `http://localhost` after all services are started ``` -After all containers are successfully started just got to your browser and visit http://localhost to finish the installation. Once finished, switch to the pipeline editor and start the interactive tour or check the [documentation](https://streampipes.apache.org/docs/user-guide-introduction/) to learn more about StreamPipes! +After all containers are successfully started just got to your browser and visit http://localhost to finish the installation. Once finished, switch to the pipeline editor and start the interactive tour or check the [documentation](https://streampipes.apache.org/docs/docs/user-guide-introduction.html) to learn more about StreamPipes! **Stopping** the **default** option is similarly easy: ```bash @@ -123,7 +123,7 @@ Since we purely levarage Docker Compose, please see their [documentation](https: If you have any problems during the installation or questions around StreamPipes, you'll get help through one of our community channels: - [Slack](https://slack.streampipes.org) -- [Mailing Lists](https://streampipes.apache.org/community/mailing-lists/) +- [Mailing Lists](https://streampipes.apache.org/mailinglists.html) And don't forget to follow us on [Twitter](https://twitter.com/streampipes)! diff --git a/installer/k8s/Chart.yaml b/installer/k8s/Chart.yaml index 0b4fb79a82..ec6dc13df8 100644 --- a/installer/k8s/Chart.yaml +++ b/installer/k8s/Chart.yaml @@ -14,9 +14,9 @@ # limitations under the License. apiVersion: v1 -appVersion: "0.97.0-SNAPSHOT" +appVersion: "0.95.0" description: Self-Service Data Analytics for the Industrial IoT name: streampipes-helm-chart home: https://streampipes.apache.org -version: 0.97.0-SNAPSHOT +version: 0.95.0 icon: https://avatars1.githubusercontent.com/u/33908576 diff --git a/installer/k8s/README.md b/installer/k8s/README.md index 48afd5a59e..df0968d373 100644 --- a/installer/k8s/README.md +++ b/installer/k8s/README.md @@ -21,7 +21,7 @@ StreamPipes k8s is a helm chart to deploy StreamPipes on Kubernetes. -**Current version:** 0.97.0-SNAPSHOT +**Current version:** 0.95.0 We provide two helm chart templates to get you going: @@ -78,10 +78,11 @@ ui-b94bd9766-rm6zb 2/2 Running 0 3m27 For **minikube users**: > **NOTE**: If you're running Docker Desktop or Minikube with a local k8s cluster, the above step to use your host IP > might not work. Luckily, you can port-forward a service port to your localhost using the following command to be able to -> access the UI either via `http://localhost:8088` or `http://:8088` (If you want to use privileged ports such as 80, you need to run this command with sudo to bind to the privileged port). +> access the UI either via `http://localhost` or `http://` (you require sudo to run this command in order to bind +> to a privileged port). ```bash -kubectl port-forward svc/ui --address=0.0.0.0 8088:8088 +kubectl port-forward svc/ui --address=0.0.0.0 80:80 ``` **Deleting** the current helm chart deployment: @@ -116,41 +117,41 @@ rm -rf ${HOME}/streampipes-k8s ### StreamPipes common parameters -| Parameter Name | Description | Value | -|-----------------------------------------------|---------------------------------------------------------|------------------------------------------| -| streampipes.version | StreamPipes version | "0.97.0-SNAPSHOT" | -| streampipes.registry | StreamPipes registry URL | "apachestreampipes" | -| streampipes.auth.secretName | The secret name for storing secrets | "sp-secrets" | -| streampipes.auth.users.admin.user | The initial admin user | "admin@streampipes.apache.org" | -| streampipes.auth.users.admin.password | The initial admin password (leave empty for autogen) | "admin" | -| streampipes.auth.users.service.user | The initial service account user | "sp-service-client" | -| streampipes.auth.users.service.secret | The initial service account secret | empty (auto-generated) | -| streampipes.auth.encryption.passcode | Passcode for value encryption | empty (auto-generated) | -| streampipes.core.appName | StreamPipes backend application name | "backend" | -| streampipes.core.port | StreamPipes backend port | 8030 | -| streampipes.core.persistence.storageClassName | Storage class name for backend PVs | "hostpath" | -| streampipes.core.persistence.storageSize | Size of the backend PV | "1Gi" | -| streampipes.core.persistence.claimName | Name of the backend PersistentVolumeClaim | "backend-pvc" | -| streampipes.core.persistence.pvName | Name of the backend PersistentVolume | "backend-pv" | -| streampipes.core.service.name | Name of the backend service | "backend" | -| streampipes.core.service.port | TargetPort of the StreamPipes backend service | 8030 | -| streampipes.ui.appName | StreamPipes UI application name | "ui" | -| streampipes.ui.resolverActive | Flag for enabling DNS resolver for Nginx proxy | true | -| streampipes.ui.port | StreamPipes UI port | 8088 | +| Parameter Name | Description | Value | +|-----------------------------------------------|---------------------------------------------------------|-----------------------------------------| +| streampipes.version | StreamPipes version | "0.95.0" | +| streampipes.registry | StreamPipes registry URL | "apachestreampipes" | +| streampipes.auth.secretName | The secret name for storing secrets | "sp-secrets" | +| streampipes.auth.users.admin.user | The initial admin user | "admin@streampipes.apache.org" | +| streampipes.auth.users.admin.password | The initial admin password (leave empty for autogen) | "admin" | +| streampipes.auth.users.service.user | The initial service account user | "sp-service-client" | +| streampipes.auth.users.service.secret | The initial service account secret | empty (auto-generated) | +| streampipes.auth.encryption.passcode | Passcode for value encryption | empty (auto-generated) | +| streampipes.core.appName | StreamPipes backend application name | "backend" | +| streampipes.core.port | StreamPipes backend port | 8030 | +| streampipes.core.persistence.storageClassName | Storage class name for backend PVs | "hostpath" | +| streampipes.core.persistence.storageSize | Size of the backend PV | "1Gi" | +| streampipes.core.persistence.claimName | Name of the backend PersistentVolumeClaim | "backend-pvc" | +| streampipes.core.persistence.pvName | Name of the backend PersistentVolume | "backend-pv" | +| streampipes.core.service.name | Name of the backend service | "backend" | +| streampipes.core.service.port | TargetPort of the StreamPipes backend service | 8030 | +| streampipes.ui.appName | StreamPipes UI application name | "ui" | +| streampipes.ui.resolverActive | Flag for enabling DNS resolver for Nginx proxy | true | +| streampipes.ui.port | StreamPipes UI port | 8088 | | streampipes.ui.resolver | DNS resolver for Nginx proxy | "kube-dns.kube-system.svc.cluster.local" | -| streampipes.ui.service.name | Name of the UI service | "ui" | -| streampipes.ui.service.type | Type of the UI service | "ClusterIP" | -| streampipes.ui.service.nodePort | Node port for the UI service | 8088 | -| streampipes.ui.service.port | TargetPort of the StreamPipes UI service | 8088 | -| streampipes.ingress.active | Flag for enabling Ingress for StreamPipes | false | -| streampipes.ingress.annotations | Annotations for Ingress | {} | -| streampipes.ingress.host | Hostname for Ingress | "" | -| streampipes.ingressroute.active | Flag for enabling IngressRoute for StreamPipes | true | -| streampipes.ingressroute.annotations | Annotations for IngressRoute | {} | -| streampipes.ingressroute.entryPoints | Entry points for IngressRoute | ["web", "websecure"] | -| streampipes.ingressroute.host | Hostname for IngressRoute | "" | -| streampipes.ingressroute.certResolverActive | Flag for enabling certificate resolver for IngressRoute | true | -| streampipes.ingressroute.certResolver | Certificate resolver for IngressRoute | "" | +| streampipes.ui.service.name | Name of the UI service | "ui" | +| streampipes.ui.service.type | Type of the UI service | "ClusterIP" | +| streampipes.ui.service.nodePort | Node port for the UI service | 8088 | +| streampipes.ui.service.port | TargetPort of the StreamPipes UI service | 8088 | +| streampipes.ingress.active | Flag for enabling Ingress for StreamPipes | false | +| streampipes.ingress.annotations | Annotations for Ingress | {} | +| streampipes.ingress.host | Hostname for Ingress | "" | +| streampipes.ingressroute.active | Flag for enabling IngressRoute for StreamPipes | true | +| streampipes.ingressroute.annotations | Annotations for IngressRoute | {} | +| streampipes.ingressroute.entryPoints | Entry points for IngressRoute | ["web", "websecure"] | +| streampipes.ingressroute.host | Hostname for IngressRoute | "" | +| streampipes.ingressroute.certResolverActive | Flag for enabling certificate resolver for IngressRoute | true | +| streampipes.ingressroute.certResolver | Certificate resolver for IngressRoute | "" | ### Extensions common parameters @@ -313,7 +314,7 @@ If you have any problems during the installation or questions around StreamPipes community channels: - [Slack](https://slack.streampipes.org) -- [Mailing Lists](https://streampipes.apache.org/community/mailing-lists/) +- [Mailing Lists](https://streampipes.apache.org/mailinglists.html) And don't forget to follow us on [Twitter](https://twitter.com/streampipes)! diff --git a/installer/k8s/values.yaml b/installer/k8s/values.yaml index 509e2bef48..ba3d0d3f84 100644 --- a/installer/k8s/values.yaml +++ b/installer/k8s/values.yaml @@ -27,7 +27,7 @@ failureThreshold: 30 hostPath: "" streampipes: - version: "0.97.0-SNAPSHOT" + version: "0.95.0" registry: "apachestreampipes" auth: secretName: "sp-secrets" diff --git a/pom.xml b/pom.xml index 455675231a..b4f57f5b02 100644 --- a/pom.xml +++ b/pom.xml @@ -24,12 +24,12 @@ org.apache apache - 32 + 31 org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT pom Apache StreamPipes @@ -43,38 +43,37 @@ 5.21.0 1.2 1.1.0 - 1.78.1 3.9 - 3.43.0 - 1.17.0 + 3.42.0 + 1.16.0 3.2.2 - 1.27.1 - 2.17.0 + 1.26.0 + 2.15.1 3.14.0 - 1.3.2 2.12.0 - 1.12.0 + 1.11.0 1.0.0 - 0.6.14 + 0.6.9 3.1.0 1.13.5 1.0 + 2.57 1.14 2.2.0 23.0.0 2.10 - 33.2.0-jre + 33.1.0-jre 4.5.13 4.4.9 2.24 - 1.12.0 - 1.3.0 + 1.11.0 2.17.0 2.17.0 3.0.0 2.1.3 2.0.1 6.0.0 + 3.0.2 4.0.0 2.4.0-b180725.0427 1.5.6 @@ -83,20 +82,20 @@ 3.0.1 3.30.1-GA 3.5.2.Final - 5.1.4 + 4.4.3 3.1.3 1.5.1 0.2.0 0.11.2 1.19.0 - 3.7.1 + 3.4.0 0.2.0 - 3.13.0 + 3.12.0 1.4.3 1.12.0 1.12.0 1.12 - 2.19.1 + 2.17.0 4.1.72.Final 3.13.1 5.9 @@ -106,7 +105,7 @@ 1.1 4.0.0 42.7.2 - 4.28.2 + 4.26.1 3.2.2 2.3.2 @@ -114,39 +113,41 @@ 1.9.0 5.0.2 5.1.27 - 8.10.0 + 8.8.0 1.4.0 2.0.6 2.2 2.3.0 6.1.1 3.2.0 - 6.3.0 + 6.2.3 2.2.19 0.8.1 1.11 - 7.0.0 + 6.6.1 5.10.1 - 5.12.0 + 5.11.0 0.13 1.11 - 10.17.0 + 10.12.1 1.8.0 2.23.0 17 17 3.3.1 - 3.6.0 + 9.0.9 + 3.5.1 3.8.7 3.2.1263 UTF-8 false + true @@ -173,11 +174,6 @@ commons-io ${commons-io.version} - - commons-logging - commons-logging - ${commons-logging.version} - com.fasterxml.jackson.core jackson-annotations @@ -195,12 +191,17 @@ com.fasterxml.jackson.dataformat - jackson-dataformat-xml + jackson-dataformat-cbor ${jackson.version} com.fasterxml.jackson.dataformat - jackson-dataformat-yaml + jackson-dataformat-smile + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml ${jackson.version} @@ -228,6 +229,11 @@ opencsv ${opencsv.version} + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + com.squareup.okhttp3 okhttp @@ -238,6 +244,11 @@ geojson-jackson ${geojson-jackson.version} + + de.ruedigermoeller + fst + ${fst.version} + io.fogsy qudt @@ -318,6 +329,11 @@ jakarta.inject-api ${jakarta-inject-api.version} + + jakarta.validation + jakarta.validation-api + ${jakarta-validation.version} + jakarta.servlet jakarta.servlet-api @@ -373,11 +389,6 @@ httpcore ${httpcore.version} - - org.apache.iotdb - iotdb-session - ${iotdb.version} - org.apache.kafka kafka-clients @@ -399,16 +410,6 @@ - - org.bouncycastle - bcprov-jdk18on - ${bcprov-jdk18on.version} - - - org.bouncycastle - bcutil-jdk18on - ${bcprov-jdk18on.version} - org.eclipse.rdf4j rdf4j-rio-turtle @@ -821,13 +822,13 @@ streampipes-extensions-api streampipes-extensions-management streampipes-data-explorer - streampipes-data-explorer-api - streampipes-data-explorer-export - streampipes-data-explorer-influx - streampipes-data-explorer-iotdb - streampipes-data-explorer-management + streampipes-data-explorer-commons streampipes-data-export streampipes-dataformat + streampipes-dataformat-cbor + streampipes-dataformat-fst + streampipes-dataformat-json + streampipes-dataformat-smile streampipes-extensions streampipes-integration-tests streampipes-mail @@ -860,10 +861,9 @@ streampipes-service-discovery-api streampipes-service-extensions streampipes-storage-api - streampipes-storage-couchdb streampipes-storage-management + streampipes-storage-couchdb streampipes-test-utils - streampipes-test-utils-executors streampipes-user-management streampipes-vocabulary streampipes-wrapper @@ -1022,6 +1022,11 @@ true + + org.owasp + dependency-check-maven + ${maven.dependency.check.plugin.version} + org.springframework.boot spring-boot-maven-plugin @@ -1233,9 +1238,6 @@ ui/.husky/** - - **/go.sum - @@ -1274,6 +1276,28 @@ + + org.owasp + dependency-check-maven + + + ${owasp.check.skip} + HTML + false + false + false + tools/maven/owasp-dependency-check-suppression.xml + + + + + + check + aggregate + + + + maven-surefire-plugin @@ -1292,7 +1316,7 @@ scm:git:ssh://git@github.com/apache/streampipes.git scm:git:ssh://git@github.com/apache/streampipes.git https://github.com/apache/streampipes - HEAD + release/0.95.0 diff --git a/prometheus-grafana/README.md b/prometheus-grafana/README.md index 4451975565..a94c751394 100644 --- a/prometheus-grafana/README.md +++ b/prometheus-grafana/README.md @@ -18,7 +18,7 @@ ## Prometheus Configuration -In the [prometheus](./prometheus) directory, you can find sample configuration file for Prometheus. +In the [dashboards](./prometheus) directory, you can find sample configuration file for Prometheus. ## Grafana Dashboards @@ -26,4 +26,4 @@ In the [dashboards](./grafana/dashboards) directory, you can find sample grafana ## note -The metrics displayed in the [dashboards](./grafana/dashboards) directory dashboards show metrics that have a filter condition related to the job_name, if you change the job_name in the example you must also change the condition in the grafana dashboard. \ No newline at end of file +The metrics displayed in the [dashboards](. /grafana/dashboards) directory dashboards show metrics that have a filter condition related to the job_name, if you change the job_name in the example you must also change the condition in the grafana dashboard. \ No newline at end of file diff --git a/streampipes-client-api/pom.xml b/streampipes-client-api/pom.xml index 68cfa36073..ef8bbd5cd8 100644 --- a/streampipes-client-api/pom.xml +++ b/streampipes-client-api/pom.xml @@ -22,7 +22,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-client-api @@ -31,17 +31,17 @@ org.apache.streampipes streampipes-dataformat - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IStreamPipesClient.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IStreamPipesClient.java index 8e91c1ce45..b326b362b9 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IStreamPipesClient.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IStreamPipesClient.java @@ -28,8 +28,6 @@ import java.io.Serializable; public interface IStreamPipesClient extends Serializable { - - @Deprecated(since = "0.97.0", forRemoval = true) void registerDataFormat(SpDataFormatFactory spDataFormatFactory); void registerProtocol(SpProtocolDefinitionFactory spProtocolDefinitionFactory); diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/config/IStreamPipesClientConfig.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/config/IStreamPipesClientConfig.java index 5fef7a6ba4..4238eec3d7 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/config/IStreamPipesClientConfig.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/config/IStreamPipesClientConfig.java @@ -18,6 +18,7 @@ package org.apache.streampipes.client.api.config; +import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,6 +26,8 @@ public interface IStreamPipesClientConfig { ObjectMapper getSerializer(); + void addDataFormat(SpDataFormatFactory spDataFormatFactory); + void addTransportProtocol(SpProtocolDefinitionFactory protocolDefinitionFactory); ClientConnectionUrlResolver getConnectionConfig(); diff --git a/streampipes-client-go/examples/main.go b/streampipes-client-go/examples/main.go index fef7dbd9b4..cc053b8aff 100644 --- a/streampipes-client-go/examples/main.go +++ b/streampipes-client-go/examples/main.go @@ -18,10 +18,9 @@ package main import ( - "log" - "github.com/apache/streampipes/streampipes-client-go/streampipes" "github.com/apache/streampipes/streampipes-client-go/streampipes/config" + "log" ) /* diff --git a/streampipes-client-go/streampipes/data_lake_measure_api.go b/streampipes-client-go/streampipes/data_lake_measure_api.go index 012e7d1030..d524c72280 100644 --- a/streampipes-client-go/streampipes/data_lake_measure_api.go +++ b/streampipes-client-go/streampipes/data_lake_measure_api.go @@ -18,14 +18,13 @@ package streampipes import ( - "io" - "log" - "net/http" - "github.com/apache/streampipes/streampipes-client-go/streampipes/config" "github.com/apache/streampipes/streampipes-client-go/streampipes/internal/serializer" "github.com/apache/streampipes/streampipes-client-go/streampipes/internal/util" "github.com/apache/streampipes/streampipes-client-go/streampipes/model/data_lake" + "io" + "log" + "net/http" ) // DataLakeMeasure connects to the DataLakeMeasure endpoint of streamPipes. @@ -43,13 +42,13 @@ func NewDataLakeMeasures(clientConfig config.StreamPipesClientConfig) *DataLakeM } } -// GetAllDataLakeMeasure retrieves a list of all measurements series from the Data Lake. -func (d *DataLakeMeasure) GetAllDataLakeMeasure() ([]data_lake.DataLakeMeasure, error) { +// AllDataLakeMeasure retrieves a list of all measurements series from the Data Lake. +func (d *DataLakeMeasure) AllDataLakeMeasure() ([]data_lake.DataLakeMeasure, error) { endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measurements", nil) log.Printf("Get data from: %s", endPointUrl) - response, err := d.executeRequest("GET", endPointUrl, nil) + response, err := d.executeRequest("GET", endPointUrl) if err != nil { return nil, err } @@ -81,7 +80,7 @@ func (d *DataLakeMeasure) DeleteDataLakeMeasurements() error { endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measurements", nil) log.Printf("Delete data from: %s", endPointUrl) - response, err := d.executeRequest("DELETE", endPointUrl, nil) + response, err := d.executeRequest("DELETE", endPointUrl) if err != nil { return err } @@ -102,7 +101,7 @@ func (d *DataLakeMeasure) GetSingleDataLakeMeasure(elementId string) (data_lake. endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measure", []string{elementId}) log.Printf("Get data from: %s", endPointUrl) - response, err := d.executeRequest("GET", endPointUrl, nil) + response, err := d.executeRequest("GET", endPointUrl) if err != nil { return data_lake.DataLakeMeasure{}, err } @@ -134,7 +133,7 @@ func (d *DataLakeMeasure) DeleteSingleDataLakeMeasure(elementId string) error { endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measure", []string{elementId}) log.Printf("Delete data from: %s", endPointUrl) - response, err := d.executeRequest("DELETE", endPointUrl, nil) + response, err := d.executeRequest("DELETE", endPointUrl) if err != nil { return err } @@ -157,7 +156,7 @@ func (d *DataLakeMeasure) GetSingleDataSeries(measureId string) (*data_lake.Data endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measurements", []string{measureId}) log.Printf("Get data from: %s", endPointUrl) - response, err := d.executeRequest("GET", endPointUrl, nil) + response, err := d.executeRequest("GET", endPointUrl) if err != nil { return nil, err } @@ -190,7 +189,7 @@ func (d *DataLakeMeasure) ClearDataLakeMeasureData(measureId string) error { endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measurements", []string{measureId}) log.Printf("Clear data from: %s", endPointUrl) - response, err := d.executeRequest("DELETE", endPointUrl, nil) + response, err := d.executeRequest("DELETE", endPointUrl) if err != nil { return err } @@ -212,7 +211,7 @@ func (d *DataLakeMeasure) DeleteDataLakeMeasure(measureId string) error { endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v4/datalake/measurements", []string{measureId, "drop"}) log.Printf("Delete data from: %s", endPointUrl) - response, err := d.executeRequest("DELETE", endPointUrl, nil) + response, err := d.executeRequest("DELETE", endPointUrl) if err != nil { return err } diff --git a/streampipes-client-go/streampipes/endpoint.go b/streampipes-client-go/streampipes/endpoint.go index 5fccd5f3a5..a9247264a2 100644 --- a/streampipes-client-go/streampipes/endpoint.go +++ b/streampipes-client-go/streampipes/endpoint.go @@ -18,11 +18,9 @@ package streampipes import ( - "bytes" - "fmt" + "errors" "github.com/apache/streampipes/streampipes-client-go/streampipes/config" headers "github.com/apache/streampipes/streampipes-client-go/streampipes/internal/http_headers" - "io" "net/http" ) @@ -30,13 +28,9 @@ type endpoint struct { config config.StreamPipesClientConfig } -func (e *endpoint) executeRequest(method string, endPointUrl string, body []byte) (*http.Response, error) { +func (e *endpoint) executeRequest(method string, endPointUrl string) (*http.Response, error) { - var reader io.Reader - if body != nil { - reader = bytes.NewReader(body) - } - req, err := http.NewRequest(method, endPointUrl, reader) + req, err := http.NewRequest(method, endPointUrl, nil) if err != nil { return nil, err } @@ -58,23 +52,23 @@ func (e *endpoint) handleStatusCode(resp *http.Response) error { switch resp.StatusCode { case http.StatusUnauthorized: - return fmt.Errorf("response code %d:"+"The streamPipes Backend returned an unauthorized error.\nplease check your ApiUser and/or Apikey to be correct.", resp.StatusCode) + return errors.New("The streamPipes Backend returned an unauthorized error.\nplease check your ApiUser and/or Apikey to be correct.") case http.StatusForbidden: - return fmt.Errorf("response code %d:"+"There seems to be an issue with the access rights of the given user and the resource you queried.\n"+ - "Apparently, this user is not allowed to query the resource.\n"+ - "Please check the user's permissions or contact your StreamPipes admin.", resp.StatusCode) + return errors.New("There seems to be an issue with the access rights of the given user and the resource you queried.\n" + + "Apparently, this user is not allowed to query the resource.\n" + + "Please check the user's permissions or contact your StreamPipes admin.") case http.StatusNotFound: - return fmt.Errorf("response code %d:"+"There seems to be an issue with the Go Client calling the API inappropriately.\n"+ - "This should not happen, but unfortunately did.\n"+ - "If you don't mind, it would be awesome to let us know by creating an issue at https://github.com/apache/streampipes.\n", resp.StatusCode) + return errors.New("There seems to be an issue with the Go Client calling the API inappropriately.\n" + + "This should not happen, but unfortunately did.\n" + + "If you don't mind, it would be awesome to let us know by creating an issue at https://github.com/apache/streampipes.\n") case http.StatusMethodNotAllowed: - return fmt.Errorf("response code %d:"+"There seems to be an issue with the Go Client calling the API inappropriately.\n"+ - "This should not happen, but unfortunately did.\n"+ - "If you don't mind, it would be awesome to let us know by creating an issue at https://github.com/apache/streampipes.\n", resp.StatusCode) + return errors.New("There seems to be an issue with the Go Client calling the API inappropriately.\n" + + "This should not happen, but unfortunately did.\n" + + "If you don't mind, it would be awesome to let us know by creating an issue at https://github.com/apache/streampipes.\n") case http.StatusInternalServerError: - return fmt.Errorf("response code %d:"+"streamPipes internal error", resp.StatusCode) + return errors.New("streamPipes internal error") default: - return fmt.Errorf(resp.Status) + return errors.New(resp.Status) } } diff --git a/streampipes-client-go/streampipes/internal/serializer/deserializer.go b/streampipes-client-go/streampipes/internal/serializer/deserializer.go index 27283c85b4..d331bc01ef 100644 --- a/streampipes-client-go/streampipes/internal/serializer/deserializer.go +++ b/streampipes-client-go/streampipes/internal/serializer/deserializer.go @@ -19,19 +19,7 @@ package serializer import ( "encoding/json" - "github.com/apache/streampipes/streampipes-client-go/streampipes/model/adapter" - "log" - "strings" - - "github.com/apache/streampipes/streampipes-client-go/streampipes/model" "github.com/apache/streampipes/streampipes-client-go/streampipes/model/data_lake" - - "github.com/apache/streampipes/streampipes-client-go/streampipes/model/pipeline" - - "github.com/apache/streampipes/streampipes-client-go/streampipes/model/functions" - - "github.com/apache/streampipes/streampipes-client-go/streampipes/model/streampipes_user" - "github.com/apache/streampipes/streampipes-client-go/streampipes/model/streampipes_version" ) @@ -39,18 +27,20 @@ type Deserializer interface { Unmarshal(body []byte) (interface{}, error) } +var _ Deserializer = (*DataLakeMeasuresDeserializer)(nil) +var _ Deserializer = (*DataSeriesDeserializer)(nil) +var _ Deserializer = (*DataLakeMeasureDeserializer)(nil) + type DataLakeMeasuresDeserializer struct{} func NewDataLakeMeasuresDeserializer() *DataLakeMeasuresDeserializer { return &DataLakeMeasuresDeserializer{} } -func (d DataLakeMeasuresDeserializer) Unmarshal(data []byte) (interface{}, error) { +func (d *DataLakeMeasuresDeserializer) Unmarshal(data []byte) (interface{}, error) { var dataLakeMeasures []data_lake.DataLakeMeasure - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&dataLakeMeasures); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) + err := json.Unmarshal(data, &dataLakeMeasures) + if err != nil { return nil, err } return dataLakeMeasures, nil @@ -62,12 +52,10 @@ func NewDataLakeMeasureDeserializer() *DataLakeMeasureDeserializer { return &DataLakeMeasureDeserializer{} } -func (d DataLakeMeasureDeserializer) Unmarshal(data []byte) (interface{}, error) { +func (d *DataLakeMeasureDeserializer) Unmarshal(data []byte) (interface{}, error) { var dataLakeMeasure data_lake.DataLakeMeasure - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&dataLakeMeasure); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) + err := json.Unmarshal(data, &dataLakeMeasure) + if err != nil { return nil, err } return dataLakeMeasure, nil @@ -79,12 +67,10 @@ func NewDataSeriesDeserializer() *DataSeriesDeserializer { return &DataSeriesDeserializer{} } -func (d DataSeriesDeserializer) Unmarshal(data []byte) (interface{}, error) { +func (d *DataSeriesDeserializer) Unmarshal(data []byte) (interface{}, error) { var dataSeries data_lake.DataSeries - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&dataSeries); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) + err := json.Unmarshal(data, &dataSeries) + if err != nil { return nil, err } return dataSeries, nil @@ -96,317 +82,11 @@ func NewStreamPipesVersionDeserializer() *StreamPipesVersionDeserializer { return &StreamPipesVersionDeserializer{} } -func (d StreamPipesVersionDeserializer) Unmarshal(data []byte) (interface{}, error) { - var version streampipes_version.Versions - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&version); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return version, nil -} - -type ResponseMessageDeserializer struct{} - -func NewResponseMessageDeserializer() *ResponseMessageDeserializer { - return &ResponseMessageDeserializer{} -} - -func (r ResponseMessageDeserializer) Unmarshal(data []byte) (interface{}, error) { - var responseMessage model.ResponseMessage - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&responseMessage); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return responseMessage, nil -} - -type PipelineCategoriesDeserializer struct{} - -func NewPipelineCategoriesDeserializer() *PipelineCategoriesDeserializer { - return &PipelineCategoriesDeserializer{} -} - -func (p PipelineCategoriesDeserializer) Unmarshal(data []byte) (interface{}, error) { - var pipelineCategory []pipeline.PipelineCategory - - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - - if err := dec.Decode(&pipelineCategory); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Fatal(err) - return nil, err - } - - return pipelineCategory, nil -} - -type DataLakeDashboardDeserializer struct{} - -func NewDataLakeDashboardDeserializer() *DataLakeDashboardDeserializer { - return &DataLakeDashboardDeserializer{} -} - -func (d DataLakeDashboardDeserializer) Unmarshal(data []byte) (interface{}, error) { - var dashborad data_lake.Dashboard - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&dashborad); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) +func (d *StreamPipesVersionDeserializer) Unmarshal(data []byte) (interface{}, error) { + var dataSeries streampipes_version.Versions + err := json.Unmarshal(data, &dataSeries) + if err != nil { return nil, err } - return dashborad, nil -} - -type DataLakeDashboardsDeserializer struct{} - -func NewDataLakeDashboardsDeserializer() *DataLakeDashboardsDeserializer { - return &DataLakeDashboardsDeserializer{} -} - -func (d DataLakeDashboardsDeserializer) Unmarshal(data []byte) (interface{}, error) { - var dashborads []data_lake.Dashboard - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&dashborads); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - - return dashborads, nil -} - -type DataLakeWidgetDeserializer struct{} - -func NewDataLakeWidgetDeserializer() *DataLakeWidgetDeserializer { - return &DataLakeWidgetDeserializer{} -} - -func (d DataLakeWidgetDeserializer) Unmarshal(data []byte) (interface{}, error) { - var widget data_lake.DataExplorerWidgetModel - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&widget); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return widget, nil -} - -type DataLakeWidgetsDeserializer struct{} - -func NewDataLakeWidgetsDeserializer() *DataLakeWidgetsDeserializer { - return &DataLakeWidgetsDeserializer{} -} - -func (d DataLakeWidgetsDeserializer) Unmarshal(data []byte) (interface{}, error) { - var widgets []data_lake.DataExplorerWidgetModel - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&widgets); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return widgets, nil -} - -type SpLogEntriesDeserializer struct{} - -func NewSpLogEntriesDeserializer() *SpLogEntriesDeserializer { - return &SpLogEntriesDeserializer{} -} - -func (s SpLogEntriesDeserializer) Unmarshal(data []byte) (interface{}, error) { - var spLogEntry []functions.SpLogEntry - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&spLogEntry); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return spLogEntry, nil -} - -type SpMetricsEntryDeserializer struct{} - -func NewSpMetricsEntryDeserializer() *SpMetricsEntryDeserializer { - return &SpMetricsEntryDeserializer{} -} - -func (s SpMetricsEntryDeserializer) Unmarshal(data []byte) (interface{}, error) { - var spMetricsEntry functions.SpMetricsEntry - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&spMetricsEntry); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return spMetricsEntry, nil -} - -type FunctionDefinitionsDeserializer struct{} - -func NewFunctionDefinitionsDeserializer() *FunctionDefinitionsDeserializer { - return &FunctionDefinitionsDeserializer{} -} - -func (f FunctionDefinitionsDeserializer) Unmarshal(data []byte) (interface{}, error) { - var functionDefinitions []functions.FunctionDefinition - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&functionDefinitions); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return functionDefinitions, nil - -} - -type ShortUserInfosDeserializer struct{} - -func NewShortUserInfosDeserializer() *ShortUserInfosDeserializer { - return &ShortUserInfosDeserializer{} -} - -func (s ShortUserInfosDeserializer) Unmarshal(data []byte) (interface{}, error) { - var shortUserInfo []streampipes_user.ShortUserInfo - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&shortUserInfo); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return shortUserInfo, nil -} - -type UserAccountDeserializer struct{} - -func NewUserAccountDeserializer() *UserAccountDeserializer { - return &UserAccountDeserializer{} -} - -func (p UserAccountDeserializer) Unmarshal(data []byte) (interface{}, error) { - var userAccount streampipes_user.UserAccount - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&userAccount); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return userAccount, nil - -} - -type PipelineDeserializer struct{} - -func NewPipelineDeserializer() *PipelineDeserializer { - return &PipelineDeserializer{} -} - -func (p PipelineDeserializer) Unmarshal(data []byte) (interface{}, error) { - var pipeLine pipeline.Pipeline - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&pipeLine); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - - return pipeLine, nil - -} - -type PipelinesDeserializer struct{} - -func NewPipelinesDeserializer() *PipelinesDeserializer { - return &PipelinesDeserializer{} -} - -func (p PipelinesDeserializer) Unmarshal(data []byte) (interface{}, error) { - var pipelines []pipeline.Pipeline - - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&pipelines); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return pipelines, nil - -} - -type PipelineStatusMessagesDeserializer struct{} - -func NewPipelineStatusMessagesDeserializer() *PipelineStatusMessagesDeserializer { - return &PipelineStatusMessagesDeserializer{} -} - -func (p PipelineStatusMessagesDeserializer) Unmarshal(data []byte) (interface{}, error) { - var pipelineStatusMessage []pipeline.PipelineStatusMessage - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&pipelineStatusMessage); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return pipelineStatusMessage, nil -} - -type PipelineOperationStatusDeserializer struct{} - -func NewPipelineOperationStatusDeserializer() *PipelineOperationStatusDeserializer { - return &PipelineOperationStatusDeserializer{} -} - -func (p PipelineOperationStatusDeserializer) Unmarshal(data []byte) (interface{}, error) { - var pipelineOperationStatus pipeline.PipelineOperationStatus - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - if err := dec.Decode(&pipelineOperationStatus); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Println(err) - return nil, err - } - return pipelineOperationStatus, nil -} - -type AdapterDeserializer struct{} - -func NewAdapterDeserializer() *AdapterDeserializer { - return &AdapterDeserializer{} -} - -func (a AdapterDeserializer) Unmarshal(data []byte) (interface{}, error) { - var adapterDescription adapter.AdapterDescription - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - - if err := dec.Decode(&adapterDescription); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Fatal(err) - return nil, err - } - return adapterDescription, nil - -} - -type AdaptersDeserializer struct{} - -func NewAdaptersDeserializer() *AdaptersDeserializer { - return &AdaptersDeserializer{} -} - -func (a AdaptersDeserializer) Unmarshal(data []byte) (interface{}, error) { - var adapters []adapter.AdapterDescription - - dec := json.NewDecoder(strings.NewReader(string(data))) - dec.DisallowUnknownFields() - - if err := dec.Decode(&adapters); err != nil && !strings.Contains(err.Error(), "unknown field") { - log.Fatal(err) - return nil, err - } - return adapters, nil - + return dataSeries, nil } diff --git a/streampipes-client-go/streampipes/model/common.go b/streampipes-client-go/streampipes/model/common.go index b353308556..f0ba11637a 100644 --- a/streampipes-client-go/streampipes/model/common.go +++ b/streampipes-client-go/streampipes/model/common.go @@ -29,8 +29,10 @@ type EventProperty struct { Label string `json:"label,omitempty"` Description string `json:"description,omitempty"` RuntimeName string `json:"runtimeName,omitempty"` - SemanticType string `json:"semanticType,omitempty"` + Required bool `json:"required,omitempty"` + DomainProperties []string `json:"domainProperties,omitempty"` PropertyScope string `json:"propertyScope,omitempty"` + Index int `json:"index"` RuntimeID string `json:"runtimeId,omitempty"` RuntimeType string `json:"runtimeType"` MeasurementUnit string `json:"measurementUnit,omitempty"` @@ -42,8 +44,10 @@ type EventProperties struct { Label string `json:"label"` Description string `json:"description"` RuntimeName string `json:"runtimeName"` - SemanticType string `json:"semanticType"` + Required bool `json:"required"` + DomainProperties []string `json:"domainProperties"` PropertyScope string `json:"propertyScope"` + Index int `json:"index"` RuntimeID string `json:"runtimeId"` RuntimeType string `json:"runtimeType,omitempty"` MeasurementUnit string `json:"measurementUnit,omitempty"` @@ -62,128 +66,3 @@ type DataSeries struct { Headers []string `json:"http_headers"` Tags map[string]string `json:"tags"` } - -type ResponseMessage struct { - Success bool `json:"success"` - ElementName string `json:"elementName"` - Notifications []Notification `json:"notifications"` -} - -type Notification struct { - Title string `json:"title"` - Description interface{} `json:"description"` - AdditionalInformation string `json:"additionalInformation"` -} - -type StaticProperty struct { - Optional bool `json:"optional,omitempty"` - StaticPropertyType string `json:"staticPropertyType"` - Index int32 `json:"index"` - Label string `json:"label"` - Description string `json:"description"` - InternalName string `json:"internalName"` - Predefined bool `json:"predefined"` - Class string `json:"@class"` -} - -type MappingProperty struct { - StaticProperty - RequirementSelector string - MapsFromOptions []string - PropertyScope string -} - -type MappingPropertyUnary struct { - MappingProperty - SelectedProperty string -} - -type FreeTextStaticProperty struct { - StaticProperty - value string - MapsTo string - MultiLine bool - HtmlAllowed bool - HtmlFontFormat bool - PlaceholdersSupported bool -} - -type SelectionStaticProperty struct { - HorizontalRendering bool -} - -type OneOfStaticProperty struct { - SelectionStaticProperty -} - -type AnyStaticProperty struct { - SelectionStaticProperty -} - -type RuntimeResolvableAnyStaticProperty struct { - AnyStaticProperty - DependsOn []string -} - -type SlideToggleStaticProperty struct { - StaticProperty - Selected bool - DefaultValue bool -} - -type SpDataStream struct { - ElementId string `json:"elementId"` - Dom string `json:"dom"` - ConnectedTo []string `json:"connectedTo"` - Name string `json:"name"` - Description string `json:"description"` - IconUrl string `json:"iconUrl"` - AppId string `json:"appId"` - IncludesAssets bool `json:"includesAssets"` - IncludesLocales bool `json:"includesLocales"` - IncludedAssets []string `json:"includedAssets"` - IncludedLocales []string `json:"includedLocales"` - InternallyManaged bool `json:"internallyManaged"` - EventGrounding EventGrounding `json:"eventGrounding"` - EventSchema EventSchema `json:"eventSchema"` - Category []string `json:"category"` - Index int32 `json:"index"` - CorrespondingAdapterId string `json:"correspondingAdapterId"` - Rev string `json:"_rev"` -} - -type EventGrounding struct { - TransportProtocols []TransportProtocol `json:"transportProtocols"` - TransportFormats []TransportFormat `json:"transportFormats"` -} - -type TransportProtocol struct { - ElementId string `json:"elementId"` - BrokerHostname string `json:"brokerHostname"` - TopicDefinition TopicDefinition `json:"topicDefinition"` - Class string `json:"@class,omitempty"` - Port int `json:"port"` -} - -type TopicDefinition struct { - ActualTopicName string `json:"actualTopicName"` - Class string `json:"@class"` -} - -type TransportFormat struct { - RdfType []string `json:"rdfType"` -} - -type TransformationRuleDescription struct { - RulePriority int32 -} - -type SpServiceTag struct { - Prefix string `json:"prefix"` - Value string `json:"value"` -} - -type ExtensionDeploymentConfiguration struct { - DesiredServiceTags []SpServiceTag `json:"desiredServiceTags"` - SelectedEndpointUrl string `json:"selectedEndpointUrl"` -} diff --git a/streampipes-client-go/streampipes/model/data_lake/data_series.go b/streampipes-client-go/streampipes/model/data_lake/data_series.go index 0dffab756f..b273b42280 100644 --- a/streampipes-client-go/streampipes/model/data_lake/data_series.go +++ b/streampipes-client-go/streampipes/model/data_lake/data_series.go @@ -19,10 +19,9 @@ package data_lake import ( "fmt" + "github.com/apache/streampipes/streampipes-client-go/streampipes/model" "log" "strings" - - "github.com/apache/streampipes/streampipes-client-go/streampipes/model" ) type DataSeries struct { diff --git a/streampipes-client-go/streampipes/streampipes_client.go b/streampipes-client-go/streampipes/streampipes_client.go index e80e0a4158..557b2f95cf 100644 --- a/streampipes-client-go/streampipes/streampipes_client.go +++ b/streampipes-client-go/streampipes/streampipes_client.go @@ -19,11 +19,10 @@ package streampipes import ( "errors" - "net/url" - "strings" - "github.com/apache/streampipes/streampipes-client-go/streampipes/config" "github.com/apache/streampipes/streampipes-client-go/streampipes/utils" + "net/url" + "strings" ) // This is the central point of contact with StreamPipes and provides all the functionalities to interact with it. @@ -72,34 +71,3 @@ func (s *StreamPipesClient) StreamPipesVersion() *Versions { return NewVersions(s.config) } - -func (s *StreamPipesClient) Pipeline() *Pipeline { - - return NewPipeline(s.config) -} - -func (s *StreamPipesClient) Adapter() *Adapter { - return NewAdapter(s.config) -} - -func (s *StreamPipesClient) DataLakeDashboard() *DataLakeDashboard { - - return NewDataLakeDashborad(s.config) -} - -func (s *StreamPipesClient) DataLakeWidget() *DataLakeWidget { - - return NewDataLakeWidget(s.config) -} - -func (s *StreamPipesClient) Function() *Functions { - - return NewFunctions(s.config) - -} - -func (s *StreamPipesClient) UserInfo() *StreamPipesUserInfo { - - return NewStreamPipesUserInfo(s.config) - -} diff --git a/streampipes-client-go/streampipes/streampipes_version_api.go b/streampipes-client-go/streampipes/streampipes_version_api.go index 6c5b924e0e..54ef8ba108 100644 --- a/streampipes-client-go/streampipes/streampipes_version_api.go +++ b/streampipes-client-go/streampipes/streampipes_version_api.go @@ -18,14 +18,13 @@ package streampipes import ( - "io" - "log" - "net/http" - "github.com/apache/streampipes/streampipes-client-go/streampipes/config" "github.com/apache/streampipes/streampipes-client-go/streampipes/internal/serializer" "github.com/apache/streampipes/streampipes-client-go/streampipes/internal/util" "github.com/apache/streampipes/streampipes-client-go/streampipes/model/streampipes_version" + "io" + "log" + "net/http" ) type Versions struct { @@ -45,7 +44,7 @@ func (d *Versions) GetStreamPipesVersion() (streampipes_version.Versions, error) endPointUrl := util.NewStreamPipesApiPath(d.config.Url, "streampipes-backend/api/v2/info/versions", nil) log.Printf("Get data from: %s", endPointUrl) - response, err := d.executeRequest("GET", endPointUrl, nil) + response, err := d.executeRequest("GET", endPointUrl) if err != nil { return streampipes_version.Versions{}, err } diff --git a/streampipes-client-python/.pre-commit-config.yaml b/streampipes-client-python/.pre-commit-config.yaml index e9ac86e307..3c27eede8a 100644 --- a/streampipes-client-python/.pre-commit-config.yaml +++ b/streampipes-client-python/.pre-commit-config.yaml @@ -14,14 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -files: ^streampipes-client-python/(streampipes|tests)/ +files: streampipes/|tests/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v4.2.0 hooks: - id: check-json - id: check-yaml - - id: check-toml - id: end-of-file-fixer - id: trailing-whitespace - id: mixed-line-ending @@ -47,35 +46,35 @@ repos: name: pyupgrade language: python types: [ python ] - entry: pyupgrade --py38-plus --keep-runtime-typing + entry: pyupgrade --py38 --keep-runtime-typing verbose: true - id: autoflake name: autoflake language: python types: [ python ] - entry: autoflake + entry: autoflake --remove-all-unused-imports --expand-star-imports --in-place --remove-unused-variables --remove-duplicate-keys verbose: true - id: isort name: isort language: python types: [ python ] - entry: isort + entry: isort --profile black verbose: true - id: black name: black language: python types: [ python ] - entry: black + entry: black --line-length=120 verbose: true - id: blacken-docks name: blacken-docs language: python types: [ python ] - entry: black + entry: black --line-length=120 verbose: true - id: mypy @@ -90,4 +89,4 @@ repos: language: python types: [ python ] entry: flake8 --max-line-length 120 - verbose: true + verbose: true \ No newline at end of file diff --git a/streampipes-client-python/Makefile b/streampipes-client-python/Makefile index dc5510ad7b..2ac4ef1576 100644 --- a/streampipes-client-python/Makefile +++ b/streampipes-client-python/Makefile @@ -24,7 +24,7 @@ doc: .PHONY: lint lint: - poetry run flake8 streampipes tests --max-line-length 120 + poetry run flake8 streampipes --max-line-length 120 .PHONY: livedoc livedoc: doc @@ -32,7 +32,7 @@ livedoc: doc .PHONY: mypy mypy: - poetry run mypy streampipes tests --config-file pyproject.toml + poetry run mypy streampipes --config-file pyproject.toml .PHONY: rebase rebase: @@ -47,12 +47,7 @@ reformat-all: .PHONY: pre-commit pre-commit: - @CHANGED_FILES_PYTHON=$$(git diff --name-only HEAD~1 HEAD -- 'streampipes/**/*' 'tests/**/*'); \ - if [ -n "$$CHANGED_FILES_PYTHON" ]; then \ - poetry run pre-commit run --verbose --files $$CHANGED_FILES_PYTHON; \ - else \ - echo "No files changed in 'streampipes' or 'tests' directories."; \ - fi + git ls-files -- 'streampipes/**/*' | xargs poetry run pre-commit run --verbose --files .PHONY: unit-tests unit-tests: diff --git a/streampipes-client-python/docs/getting-started/developing.md b/streampipes-client-python/docs/getting-started/developing.md index 59e96cf568..10708dda96 100644 --- a/streampipes-client-python/docs/getting-started/developing.md +++ b/streampipes-client-python/docs/getting-started/developing.md @@ -24,7 +24,7 @@ This document describes how to easily set up your local dev environment to work 1) **Set up your Python environment** -Create a virtual Python environment using a tool of your choice. +Create a virtual Python environment using a tool of your choice. To manage dependencies, we use [Poetry](https://python-poetry.org/), so please install poetry in your local environment, e.g. via ```bash pip install poetry @@ -44,7 +44,7 @@ poetry install --with dev,stubs,docs # install all optional dependencies relate The pre-commit hook is run before every commit and takes care about code style, linting, type hints, import sorting, etc. It will stop your commit in case the changes do not apply the expected format. -Always check to have the recent version of the pre-commit hook installed otherwise the CI build might fail. +Always check to have the recent version of the pre-commit hook installed otherwise the CI build might fail. If you are interested, you can have a deeper look on the underlying library: [pre-commit](https://pre-commit.com/). ```bash @@ -67,7 +67,7 @@ Please stick to the `numpy` [style](https://numpydoc.readthedocs.io/en/latest/fo 2) **Provide tests** ✅
We are aiming for broad test coverage for the Python package and have therefore set a requirement of at least 90% unit test coverage. -Therefore, please remember to write (unit) tests already during development. +Therefore, please remember to write (unit) tests already during development. If you have problems with writing tests, don't hesitate to ask us for help directly in the PR or even before that via our mailing list (see above). @@ -127,7 +127,7 @@ Stay tuned! --- ## 👨‍💻 Contributing -*Before opening a pull request*, review the [Get Involved](https://streampipes.apache.org/community/get-involved/) page. +*Before opening a pull request*, review the [Get Involved](https://streampipes.apache.org/getinvolved.html) page. It lists information that is required for contributing to StreamPipes. When you contribute code, you affirm that the contribution is your original work and that you @@ -136,4 +136,4 @@ state this explicitly, by submitting any copyrighted material via pull request, other means you agree to license the material under the project's open source license and warrant that you have the legal authority to do so. ---- +--- \ No newline at end of file diff --git a/streampipes-client-python/docs/getting-started/first-steps.md b/streampipes-client-python/docs/getting-started/first-steps.md index 81df71eeac..a8f0288b42 100644 --- a/streampipes-client-python/docs/getting-started/first-steps.md +++ b/streampipes-client-python/docs/getting-started/first-steps.md @@ -48,11 +48,11 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES ... ... ... ... ... ... ... ``` Otherwise, you need to start docker first. -Please read the full guide on how to start StreamPipes with `docker compose` [here](https://streampipes.apache.org/docs/deploy-docker/). +Please read the full guide on how to start StreamPipes with `docker compose` [here](https://streampipes.apache.org/docs/docs/deploy-docker.html). #### Setup StreamPipes with NATS as message broker The following shows how you can set up a StreamPipes instance that uses [NATS](https://docs.nats.io/) as messaging layer. -So in this scenario, we will go with `docker-compose.nats.yml`. +So in this scenario, we will go with `docker-compose.nats.yml`. Thereby, when running locally, we need to add the following port mapping entry to `services.nats.ports`: ```yaml - 4222:4222 @@ -85,6 +85,6 @@ docker-compose -f docker-compose.yml up -d Once all services are started, you can access StreamPipes via `http://localhost`. In case you want to have more control over your StreamPipes setup, -you might take a look at our [deployment CLI](https://streampipes.apache.org/docs/extend-cli/). +you might take a look at our [deployment CLI](https://streampipes.apache.org/docs/docs/extend-cli.html). Have fun discovering StreamPipes and our Python library 🚀 diff --git a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb index 50eb8c3dfe..c7efd9a81f 100644 --- a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb +++ b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb @@ -7,9 +7,9 @@ "\n", "### Why there is an extra Python library for StreamPipes?\n", "[Apache StreamPipes](https://streampipes.apache.org/) aims to enable non-technical users to connect and analyze IoT data streams.\n", - "To achieve this, it provides an easy-to-use and convenient user interface that allows one to connect to an IoT data source and create some visual\n", + "To this end, it provides an easy-to-use and convenient user interface that allows one to connect to an IoT data source and create some visual\n", "graphs within a few minutes.
\n", - "While this is the primary use case for Apache StreamPipes, it also offers significant value to those interested in data analysis or data science with IoT data, without the need to handle the complexities of extracting data from devices in a suitable format.\n", + "Although this is the main use case of Apache StreamPipes, it can also provide great value for people who are eager to work on data analysis or data science with IoT data, but don't we do get in touch with all the hassle associated with extracting data from devices in a suitable format.\n", "In this scenario, StreamPipes helps you connect to your data source and extract the data for you.\n", "You then can make the data available outside StreamPipes by writing it into an external source, such as a database, Kafka, etc.\n", "While this requires another component, you can also extract your data directly from StreamPipes programmatically using the StreamPipes API.\n", @@ -87,7 +87,7 @@ "source": [ "### How to configure the Python client\n", "In order to access the resources available in StreamPipes, one must be able to authenticate against the backend.\n", - "For this purpose, the client so far only supports the authentication via an API token that can be generated via the StreamPipes UI, as you can see below.\n", + "For this purpose, the client sofar only supports the authentication via an API token that can be generated via the StreamPipes UI, as you see below.\n", "\n", "![how-to-get-api-key](https://raw.githubusercontent.com/apache/streampipes/dev/streampipes-client-python/docs/img/how-to-get-api-key.gif)\n", "\n", @@ -159,7 +159,7 @@ "cell_type": "markdown", "source": [ "To ensure that the above code works, you must set the environment variables as expected.\n", - "This can be done as follows:" + "This can be done like following:" ], "metadata": { "collapsed": false diff --git a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb index af3c0dc4b9..000f6fa96f 100644 --- a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb +++ b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb @@ -121,7 +121,7 @@ { "cell_type": "markdown", "source": [ - "So let's see how many measures are available:" + "So let's see how many measures are available" ], "metadata": { "collapsed": false @@ -179,7 +179,7 @@ { "cell_type": "markdown", "source": [ - "To get a more comprehensive overview, you can take a look at the [`pandas`](https://pandas.pydata.org/) representation:" + "To get a more comprehensive overview, you can take a look at the [`pandas`](https://pandas.pydata.org/) representation" ], "metadata": { "collapsed": false @@ -318,7 +318,7 @@ { "cell_type": "markdown", "source": [ - "As a final step, we want to create a plot of both attributes." + "As a final step, we want to create a plot of both attributes" ], "metadata": { "collapsed": false @@ -469,10 +469,10 @@ { "cell_type": "markdown", "source": [ - "... from this point on, we leave all future processing of the data up to your creativity.\n", + "... from this point on we leave all future processing of the data up to your creativity.\n", "Keep in mind: the general syntax used in this tutorial (`all()`, `to_pandas()`, `get()`) applies to all endpoints and associated resources of the StreamPipes Python client.\n", "\n", - "If you get further and create exciting stuff with data extracted from StreamPipes please [let us know](https://github.com/apache/streampipes/discussions/categories/show-and-tell).\n", + "If you get further and create exiting stuff with data extracted from StreamPipes please [let us know](https://github.com/apache/streampipes/discussions/categories/show-and-tell).\n", "We are thrilled to see what you as a community will build with the provided client.\n", "Furthermore, don't hesitate to discuss [feature requests](https://github.com/apache/streampipes/discussions/812) to extend the current functionality with us." ], @@ -483,7 +483,7 @@ { "cell_type": "markdown", "source": [ - "For now, that's all about the StreamPipes client. Read the next tutorial ([Getting live data from the StreamPipes data stream](../3-getting-live-data-from-the-streampipes-data-stream)) if you are interested in making use of the powerful [StreamPipes functions](https://streampipes.apache.org/docs/extend-sdk-functions.html) to interact with StreamPipes in an event-based manner." + "For now, that's all about the StreamPipes client. Read the next tutorial ([Getting live data from the StreamPipes data stream](../3-getting-live-data-from-the-streampipes-data-stream)) if you are interested in making use of the powerful [StreamPipes functions](https://streampipes.apache.org/docs/docs/extend-sdk-functions.html) to interact with StreamPipes event-based." ], "metadata": { "collapsed": false diff --git a/streampipes-client-python/docs/tutorials/5-applying-interoperable-machine-learning-in-streampipes.ipynb b/streampipes-client-python/docs/tutorials/5-applying-interoperable-machine-learning-in-streampipes.ipynb index 70cff543a5..f1b055aed4 100644 --- a/streampipes-client-python/docs/tutorials/5-applying-interoperable-machine-learning-in-streampipes.ipynb +++ b/streampipes-client-python/docs/tutorials/5-applying-interoperable-machine-learning-in-streampipes.ipynb @@ -381,7 +381,7 @@ "source": [ "Let's dive a little deeper into the different parts of the function\n", "\n", - "- **`__init__`**: First, we need to take care of the data stream that is required to send the predictions from our function to StreamPipes. Thus, we create a dedicated output data stream which we need to provide with the attributes our event will consist of (a timestamp attribute is always added automatically). This output data stream needs to be registered at the function definition which is to be passed to the parent class. Lastly, we need to define some instance variables that are mainly required for the ONNX runtime.\n", + "- **`__init__`**: First, we need to take care about the data stream that is required to send the predictions from our function to StreamPipes. Thus, we create a dedicated output data stream which we need to provide with the attributes our event will consist of (a timestamp attribute is always added automatically). This output data stream needs to be registered at the function definition which is to be passed to the parent class. Lastly, we need to define some instance variables that are mainly required for the ONNX runtime.\n", "\n", "- **`onServiceStarted`**: Here we prepare the ONNX runtime session by creating an `InferenceSession` and retrieving the corresponding configuration parameters.\n", "\n", diff --git a/streampipes-client-python/mkdocs.yml b/streampipes-client-python/mkdocs.yml index e8a73e6db2..81ece22708 100644 --- a/streampipes-client-python/mkdocs.yml +++ b/streampipes-client-python/mkdocs.yml @@ -32,16 +32,14 @@ theme: favicon: https://streampipes.apache.org/img/favicon.png custom_dir: docs/overrides features: - - content.action.edit - - content.code.copy - navigation.tabs - navigation.tabs.sticky - navigation.instant - navigation.indexes - navigation.tracking - - search.suggest font: false palette: + # Palette toggle for light mode - scheme: default toggle: @@ -64,9 +62,6 @@ extra: version: provider: mike -exclude_docs: | - /scripts/** - # Extensions markdown_extensions: - admonition @@ -99,17 +94,15 @@ plugins: docstring_style: numpy docstring_section_style: spacy show_source: False - merge_init_into_class: true - inherited_members: true + merge_init_into_class: True - mike: - canonical_version: "latest" + canonical_version: 'latest' version_selector: true css_dir: css javascript_dir: js - search: lang: - en - - git-revision-date-localized extra_css: - stylesheets/extra.css @@ -126,5 +119,4 @@ nav: - Exploring Live Data from a StreamPipes Data Stream: tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb - Using Online Machine Learning on a StreamPipes Data Stream: tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb - Applying Interoperable Machine Learning in StreamPipes: tutorials/5-applying-interoperable-machine-learning-in-streampipes.ipynb - - Creating Output Streams with StreamPipes Functions: tutorials/6-streampipes-function-output-stream.ipynb - 📚 Reference: reference/* diff --git a/streampipes-client-python/poetry.lock b/streampipes-client-python/poetry.lock index c79404ae27..4580bfd192 100644 --- a/streampipes-client-python/poetry.lock +++ b/streampipes-client-python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -154,33 +154,33 @@ lxml = ["lxml"] [[package]] name = "black" -version = "24.8.0" +version = "24.4.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, - {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, - {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, - {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, - {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, - {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, - {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, - {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, - {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, - {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, - {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, - {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, - {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, - {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, - {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, - {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, - {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, - {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, - {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, - {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, - {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, - {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, + {file = "black-24.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436"}, + {file = "black-24.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf"}, + {file = "black-24.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad"}, + {file = "black-24.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb"}, + {file = "black-24.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8"}, + {file = "black-24.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745"}, + {file = "black-24.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070"}, + {file = "black-24.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397"}, + {file = "black-24.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2"}, + {file = "black-24.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33"}, + {file = "black-24.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965"}, + {file = "black-24.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd"}, + {file = "black-24.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1"}, + {file = "black-24.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8"}, + {file = "black-24.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d"}, + {file = "black-24.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3"}, + {file = "black-24.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665"}, + {file = "black-24.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6"}, + {file = "black-24.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e"}, + {file = "black-24.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702"}, + {file = "black-24.4.0-py3-none-any.whl", hash = "sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e"}, + {file = "black-24.4.0.tar.gz", hash = "sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641"}, ] [package.dependencies] @@ -200,17 +200,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "blacken-docs" -version = "1.18.0" +version = "1.16.0" description = "Run Black on Python code blocks in documentation files." optional = false python-versions = ">=3.8" files = [ - {file = "blacken_docs-1.18.0-py3-none-any.whl", hash = "sha256:64f592246784131e9f84dad1db397f44eeddc77fdf01726bab920a3f00a3815c"}, - {file = "blacken_docs-1.18.0.tar.gz", hash = "sha256:47bed628679d008a8eb55d112df950582e68d0f57615223929e366348d935444"}, + {file = "blacken_docs-1.16.0-py3-none-any.whl", hash = "sha256:b0dcb84b28ebfb352a2539202d396f50e15a54211e204a8005798f1d1edb7df8"}, + {file = "blacken_docs-1.16.0.tar.gz", hash = "sha256:b4bdc3f3d73898dfbf0166f292c6ccfe343e65fc22ddef5319c95d1a8dcc6c1c"}, ] [package.dependencies] -black = ">=22.1" +black = ">=22.1.0" [[package]] name = "bleach" @@ -295,13 +295,13 @@ files = [ [[package]] name = "certifi" -version = "2024.7.4" +version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, - {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] [[package]] @@ -505,45 +505,45 @@ files = [ [[package]] name = "confluent-kafka" -version = "2.5.0" +version = "2.3.0" description = "Confluent's Python client for Apache Kafka" optional = false python-versions = "*" files = [ - {file = "confluent-kafka-2.5.0.tar.gz", hash = "sha256:551cabaade717bb56ec13eb860ce439bedbcf1c97f4a4aa26957572ed1bfa74f"}, - {file = "confluent_kafka-2.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5509a219128fb177fa4186a8669071cc52acd52eba436f339edb9063aabb486d"}, - {file = "confluent_kafka-2.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff98d8fbe7d3671cac3e1b692c13f160cf508b525c110a89906ffabd1cc140fe"}, - {file = "confluent_kafka-2.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:72ffae4387d283cb5657b6381a893c7231c26a9b4248557e7f030de76156290a"}, - {file = "confluent_kafka-2.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:75873087fd1bd753e082f74ab97f68cc3a0765d6b600c2ac3d3a0beffbdc569d"}, - {file = "confluent_kafka-2.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:2af917f93ac3a0aa88e6bee9b2056c1c176621e4a9c8f7051cc8646b81f91327"}, - {file = "confluent_kafka-2.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:efc8c48d5dbbcd1b56afe737df8156a74e62b50481ccffe581b9926eaa16c014"}, - {file = "confluent_kafka-2.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b9f867c7e955a48ed60dee0da9c157b0f84e67724f7e42591bbcf6867e3865f"}, - {file = "confluent_kafka-2.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b064baf6a93ab58199e63bddf73d9f2c855b89cc376d5313c2f89c633aa3254a"}, - {file = "confluent_kafka-2.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a55f3d761c8463c504012ad9b06be33ef07f301f246e61d656cc927d35763f82"}, - {file = "confluent_kafka-2.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c05a677b1dbdcf2a4532e2cf41e78d2e2ffb3a6829347caf2825f472cda59e69"}, - {file = "confluent_kafka-2.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:db987d8953d0d58a28a455e43a1da74a0e9dec7a12a74f5abd85a7cb308aefd4"}, - {file = "confluent_kafka-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d828ebb45db153cd462e72c575f8683c2c56ddba62b282aa36d9c365847e212"}, - {file = "confluent_kafka-2.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fb863f76605e9bbbb1d7f02abf05899cf1435421aa721a5be212c600bd054aa3"}, - {file = "confluent_kafka-2.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:92efb98908e29f597c77ab97faa064f670b681f4540c3eabc415b8a6e58df9bf"}, - {file = "confluent_kafka-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d668b5c426af595271bf6fce2917a6c3a15453656077a59db85f440958b5ccc2"}, - {file = "confluent_kafka-2.5.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:797250f1a66024dd8b1c94764cc75e1d7bd1b7224a0b982678eafbb39714874e"}, - {file = "confluent_kafka-2.5.0-cp36-cp36m-manylinux_2_28_aarch64.whl", hash = "sha256:e81dc0a2980e597848b73983ce6e0b4ae7d129c01370cd9f31599c15c5d02a5d"}, - {file = "confluent_kafka-2.5.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:ffda33c86f5fee6ae678cca039915a0c4c1863bbc592b6f2f82abfddc173b0d3"}, - {file = "confluent_kafka-2.5.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7410bd5b0d6f54df5fa3313c75801a6ebcfab7cbfb947c3f56149e38b0fe924c"}, - {file = "confluent_kafka-2.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9a29dc4b7d4a754037d7d8e3ad1873a27b16e7de8c0a06755456b20803a70b16"}, - {file = "confluent_kafka-2.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:089b68a43c3b911356a4ff08fa862245f1333387b79221ac7f60d99e5b4e24d6"}, - {file = "confluent_kafka-2.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c3a17ebdd97c803cf369c8615a474ca0bea39b5f5944e51f1c320aee8d6d5da9"}, - {file = "confluent_kafka-2.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:151656afaeb623b46c042a752091d8b17fd05ff7d309be6d8b4953b8dc0783bc"}, - {file = "confluent_kafka-2.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:570fc091cdcf9d1baf90c5f4965322cea8185ba8698d0f02cd1c8bd38bf6664a"}, - {file = "confluent_kafka-2.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bd57edf51434d6ec289339a0c9b627ca1f1e7c1130e348c0b411407183db53c6"}, - {file = "confluent_kafka-2.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:8975fea2ccd6927aad188e198e1688ef16589dc36b42f7a33ad07b1ca1341901"}, - {file = "confluent_kafka-2.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7c0b1a7774905c9e3c24d09d9b8463d771685e4105150c2503185537a6a590f9"}, - {file = "confluent_kafka-2.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:eaf01cd79b4d2cdbdf1e7b6ace9c846ae9ad9f4cf573617bbb5735a5c48cbd20"}, - {file = "confluent_kafka-2.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa789332fd40a9e99b9388f87f28db8fc7dd8ca54a1d24d0bcd0ad33f50f3528"}, - {file = "confluent_kafka-2.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e917db155dc3a64e1496b293a3ceb0a8edf23e0bd6f93d43576c40f0c59d3067"}, - {file = "confluent_kafka-2.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:a8bb3af6d1f109aaac5514c65a46cac933d78b3935f6fea52fe1f2ea6a9951bf"}, - {file = "confluent_kafka-2.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a1fb72461dcf7aa7e1834133eb733f824331aafda87ef48ec917d9b09c805a99"}, - {file = "confluent_kafka-2.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:4bda1b5fa87cb993bcd964d271a76cc11cafa2455de02ab5eff6efd9e688d55e"}, + {file = "confluent-kafka-2.3.0.tar.gz", hash = "sha256:4069e7b56e0baf9db18c053a605213f0ab2d8f23715dca7b3bd97108df446ced"}, + {file = "confluent_kafka-2.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5df845755cd3ebb9165ca00fd1d3a7d514c61e84d9fcbe7babb91193fe9b369c"}, + {file = "confluent_kafka-2.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ab2217875b731bd390582952e0f9cbe3e7b34774490f01afca70728f0d8b469"}, + {file = "confluent_kafka-2.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62046e8a75c7a6883a0f1f4a635573fd7e1665eeacace65e7f6d59cbaa94697d"}, + {file = "confluent_kafka-2.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1eba38061e9ed1c0a369c129bf01d07499286cc3cb295398b88a7037c14371fb"}, + {file = "confluent_kafka-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6abece28598fa2b59d2b9399fcec03440aaa73fd207fdad048a6030d7e897e1"}, + {file = "confluent_kafka-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d55fbdcd75586dd17fe3fe64f4b4efa1c93ce9dd09c275de46f75772826e8860"}, + {file = "confluent_kafka-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ec17b26d6155feeaded4a435ba949095aea9699afb65309d8f22e55722f53c48"}, + {file = "confluent_kafka-2.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9b42bf1b75fdd9aa20c77b27f166f6289440ac649f70622a0117a8e7aa6169d"}, + {file = "confluent_kafka-2.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f9f4099aaf2c5daae828d2f356e4277d0ef0485ec883dbe395f0c0e054450d0"}, + {file = "confluent_kafka-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1c6b29d57df99dabd45e67fd0aa46f17f195b057734ad84cf9cfdc2542855c10"}, + {file = "confluent_kafka-2.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6b46ce75bda0c092da103dbd55cb0ba429c73c232e70b476b19a0ab247ec9057"}, + {file = "confluent_kafka-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af60af786a7b8cbeafea51a9416664b96b0f5ef6243172b0bc59e5f75e8bd86a"}, + {file = "confluent_kafka-2.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08b601e09a584c6a4a8c323a71e92fca31a8826ed33b5b95b26783b7a996026"}, + {file = "confluent_kafka-2.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7fd1ab257d4fa0e2a98529e4eb2102cf8352ad6b3d22110d6cf0bb1f598893d9"}, + {file = "confluent_kafka-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1ccf6483d86535627cad7b94982ea95d9fa9ae04ddb552e097c1211ffcde5ea7"}, + {file = "confluent_kafka-2.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:030fb237927ec2296882a9bb96237ebf86e48388166b15ec0bbf3fdeb48df81a"}, + {file = "confluent_kafka-2.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc24c57a52c206648685e1c536afb8447d1cbbbf3871cacebccf2e5b67bdf535"}, + {file = "confluent_kafka-2.3.0-cp36-cp36m-manylinux_2_28_aarch64.whl", hash = "sha256:25292a9a8ef7765c85636851d6c4d5e5e98d6ead627b59637b24a5779e8a4b02"}, + {file = "confluent_kafka-2.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d634d4d9914b0a28ec3e37ab7b150173aa34c81fd5bd0b4dcac972b520ad56cc"}, + {file = "confluent_kafka-2.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ebf460d90478bcd1b4564023a5b081c6e5390b28dbabbb17ee664e223830465d"}, + {file = "confluent_kafka-2.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cec97f8c6564b16504d30fe42c22fd4a86c406dbcd45c337b93c21e876e20628"}, + {file = "confluent_kafka-2.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:128ddb28c19ab57c18c0e3d8209d089b6b90ff111b20108764f6798468432693"}, + {file = "confluent_kafka-2.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0470dc5e56e639693149961409bc6b663df94d68ceae296ae9c42e079fe65d00"}, + {file = "confluent_kafka-2.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b539064fef35386936a0d2dadf8a82b8b0ae325af95d9263a2431b82671c4702"}, + {file = "confluent_kafka-2.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4f9998f781a1da0c9dcb5506792a39799cb54e28c6f986ddc73e362887042f7c"}, + {file = "confluent_kafka-2.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e11facaf12130abd5d2d471db39d7cc89126c4d991527cf14e3da22c635c"}, + {file = "confluent_kafka-2.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f9842720ed0debcf4620710e01d356681a4812441f1ff49664fc205d1f9120e5"}, + {file = "confluent_kafka-2.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cf015e547b82a74a87d7363d0d42e4cd0ca23b01cdb479639a340f385581ea04"}, + {file = "confluent_kafka-2.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5c740ead14a2510e15f63e67b19d48ae48a7f30ef4823d5af125bad528033d1"}, + {file = "confluent_kafka-2.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6ae5e6a6dcd5ce85b9153c21c9f0b83e0cc88a5955b5334079db76c2267deb63"}, + {file = "confluent_kafka-2.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca36a8d1d49fd55cca1b7ec3090ca2684a933e63f196f0e3e506194b189fc31e"}, + {file = "confluent_kafka-2.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:210f2d346d1006e9b95c5204f7255735d4cb5ec962a3d1a68ac60c02e2763ae4"}, + {file = "confluent_kafka-2.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb279e369121e07ccb419220fc039127345a9e5f72f4abf7dda0e2e06a12b604"}, ] [package.extras] @@ -623,38 +623,43 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "43.0.1" +version = "42.0.5" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, - {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, - {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, - {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, - {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, - {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, - {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16"}, + {file = "cryptography-42.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278"}, + {file = "cryptography-42.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d"}, + {file = "cryptography-42.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da"}, + {file = "cryptography-42.0.5-cp37-abi3-win32.whl", hash = "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74"}, + {file = "cryptography-42.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940"}, + {file = "cryptography-42.0.5-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc"}, + {file = "cryptography-42.0.5-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc"}, + {file = "cryptography-42.0.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30"}, + {file = "cryptography-42.0.5-cp39-abi3-win32.whl", hash = "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413"}, + {file = "cryptography-42.0.5-cp39-abi3-win_amd64.whl", hash = "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c"}, + {file = "cryptography-42.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac"}, + {file = "cryptography-42.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd"}, + {file = "cryptography-42.0.5.tar.gz", hash = "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1"}, ] [package.dependencies] @@ -667,7 +672,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -814,19 +819,19 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "flake8" -version = "7.1.0" +version = "6.1.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" files = [ - {file = "flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a"}, - {file = "flake8-7.1.0.tar.gz", hash = "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"}, + {file = "flake8-6.1.0-py2.py3-none-any.whl", hash = "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"}, + {file = "flake8-6.1.0.tar.gz", hash = "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23"}, ] [package.dependencies] mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.12.0,<2.13.0" -pyflakes = ">=3.2.0,<3.3.0" +pycodestyle = ">=2.11.0,<2.12.0" +pyflakes = ">=3.1.0,<3.2.0" [[package]] name = "ghp-import" @@ -845,38 +850,6 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] -[[package]] -name = "gitdb" -version = "4.0.11" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - [[package]] name = "griffe" version = "0.42.0" @@ -1115,13 +1088,13 @@ trio = ["async_generator", "trio"] [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -1565,28 +1538,6 @@ files = [ [package.dependencies] mkdocs = ">=1.0.3" -[[package]] -name = "mkdocs-git-revision-date-localized-plugin" -version = "1.3.0" -description = "Mkdocs plugin that enables displaying the localized date of the last git modification of a markdown file." -optional = false -python-versions = ">=3.8" -files = [ - {file = "mkdocs_git_revision_date_localized_plugin-1.3.0-py3-none-any.whl", hash = "sha256:c99377ee119372d57a9e47cff4e68f04cce634a74831c06bc89b33e456e840a1"}, - {file = "mkdocs_git_revision_date_localized_plugin-1.3.0.tar.gz", hash = "sha256:439e2f14582204050a664c258861c325064d97cdc848c541e48bb034a6c4d0cb"}, -] - -[package.dependencies] -babel = ">=2.7.0" -GitPython = "*" -mkdocs = ">=1.0" -pytz = "*" - -[package.extras] -all = ["GitPython", "babel (>=2.7.0)", "click", "codecov", "mkdocs (>=1.0)", "mkdocs-gen-files", "mkdocs-git-authors-plugin", "mkdocs-material", "mkdocs-static-i18n", "pytest", "pytest-cov", "pytz"] -base = ["GitPython", "babel (>=2.7.0)", "mkdocs (>=1.0)", "pytz"] -dev = ["click", "codecov", "mkdocs-gen-files", "mkdocs-git-authors-plugin", "mkdocs-material", "mkdocs-static-i18n", "pytest", "pytest-cov"] - [[package]] name = "mkdocs-jupyter" version = "0.24.0" @@ -1664,13 +1615,13 @@ files = [ [[package]] name = "mkdocstrings" -version = "0.25.0" +version = "0.24.0" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings-0.25.0-py3-none-any.whl", hash = "sha256:df1b63f26675fcde8c1b77e7ea996cd2f93220b148e06455428f676f5dc838f1"}, - {file = "mkdocstrings-0.25.0.tar.gz", hash = "sha256:066986b3fb5b9ef2d37c4417255a808f7e63b40ff8f67f6cab8054d903fbc91d"}, + {file = "mkdocstrings-0.24.0-py3-none-any.whl", hash = "sha256:f4908560c10f587326d8f5165d1908817b2e280bbf707607f601c996366a2264"}, + {file = "mkdocstrings-0.24.0.tar.gz", hash = "sha256:222b1165be41257b494a9d29b14135d2b7ca43f38161d5b10caae03b87bd4f7e"}, ] [package.dependencies] @@ -1719,53 +1670,47 @@ files = [ [[package]] name = "mypy" -version = "1.13.0" +version = "1.9.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, - {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, - {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, - {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, - {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, - {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, - {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, - {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, - {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, - {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, - {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, - {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, - {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, - {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, - {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, - {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, - {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, - {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, - {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, - {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, - {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, - {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"}, + {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"}, + {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"}, + {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"}, + {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"}, + {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"}, + {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"}, + {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"}, + {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"}, + {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"}, + {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"}, + {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"}, + {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"}, + {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"}, + {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"}, + {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"}, + {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"}, + {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"}, + {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"}, + {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"}, + {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"}, + {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"}, + {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"}, ] [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.6.0" +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] -faster-cache = ["orjson"] install-types = ["pip"] mypyc = ["setuptools (>=50)"] reports = ["lxml"] @@ -1783,12 +1728,12 @@ files = [ [[package]] name = "nats-py" -version = "2.8.0" +version = "2.7.2" description = "NATS client for Python" optional = false python-versions = ">=3.7" files = [ - {file = "nats_py-2.8.0.tar.gz", hash = "sha256:e998dcacd711db3b90b469dcdd96c526cc56c421bad27be0f5f40e88005a7c44"}, + {file = "nats-py-2.7.2.tar.gz", hash = "sha256:0c97b4a57bed0ef1ff9ae6c19bc115ec7ca8ede5ab3e001fd00a377056a547cf"}, ] [package.extras] @@ -2357,13 +2302,13 @@ validation = ["cerberus"] [[package]] name = "pluggy" -version = "1.5.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, - {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] @@ -2440,13 +2385,13 @@ files = [ [[package]] name = "pycodestyle" -version = "2.12.0" +version = "2.11.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.12.0-py2.py3-none-any.whl", hash = "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"}, - {file = "pycodestyle-2.12.0.tar.gz", hash = "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c"}, + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, ] [[package]] @@ -2462,120 +2407,109 @@ files = [ [[package]] name = "pydantic" -version = "2.9.2" +version = "2.7.0" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.7.0-py3-none-any.whl", hash = "sha256:9dee74a271705f14f9a1567671d144a851c675b072736f0a7b2608fd9e495352"}, + {file = "pydantic-2.7.0.tar.gz", hash = "sha256:b5ecdd42262ca2462e2624793551e80911a1e989f462910bb81aef974b4bb383"}, ] [package.dependencies] -annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} +annotated-types = ">=0.4.0" +pydantic-core = "2.18.1" +typing-extensions = ">=4.6.1" [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.18.1" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ee9cf33e7fe14243f5ca6977658eb7d1042caaa66847daacbd2117adb258b226"}, + {file = "pydantic_core-2.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b7bbb97d82659ac8b37450c60ff2e9f97e4eb0f8a8a3645a5568b9334b08b50"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df4249b579e75094f7e9bb4bd28231acf55e308bf686b952f43100a5a0be394c"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0491006a6ad20507aec2be72e7831a42efc93193d2402018007ff827dc62926"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ae80f72bb7a3e397ab37b53a2b49c62cc5496412e71bc4f1277620a7ce3f52b"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58aca931bef83217fca7a390e0486ae327c4af9c3e941adb75f8772f8eeb03a1"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1be91ad664fc9245404a789d60cba1e91c26b1454ba136d2a1bf0c2ac0c0505a"}, + {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:667880321e916a8920ef49f5d50e7983792cf59f3b6079f3c9dac2b88a311d17"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f7054fdc556f5421f01e39cbb767d5ec5c1139ea98c3e5b350e02e62201740c7"}, + {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:030e4f9516f9947f38179249778709a460a3adb516bf39b5eb9066fcfe43d0e6"}, + {file = "pydantic_core-2.18.1-cp310-none-win32.whl", hash = "sha256:2e91711e36e229978d92642bfc3546333a9127ecebb3f2761372e096395fc649"}, + {file = "pydantic_core-2.18.1-cp310-none-win_amd64.whl", hash = "sha256:9a29726f91c6cb390b3c2338f0df5cd3e216ad7a938762d11c994bb37552edb0"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9ece8a49696669d483d206b4474c367852c44815fca23ac4e48b72b339807f80"}, + {file = "pydantic_core-2.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a5d83efc109ceddb99abd2c1316298ced2adb4570410defe766851a804fcd5b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7973c381283783cd1043a8c8f61ea5ce7a3a58b0369f0ee0ee975eaf2f2a1b"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54c7375c62190a7845091f521add19b0f026bcf6ae674bdb89f296972272e86d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd63cec4e26e790b70544ae5cc48d11b515b09e05fdd5eff12e3195f54b8a586"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:561cf62c8a3498406495cfc49eee086ed2bb186d08bcc65812b75fda42c38294"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68717c38a68e37af87c4da20e08f3e27d7e4212e99e96c3d875fbf3f4812abfc"}, + {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d5728e93d28a3c63ee513d9ffbac9c5989de8c76e049dbcb5bfe4b923a9739d"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0f17814c505f07806e22b28856c59ac80cee7dd0fbb152aed273e116378f519"}, + {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d816f44a51ba5175394bc6c7879ca0bd2be560b2c9e9f3411ef3a4cbe644c2e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win32.whl", hash = "sha256:09f03dfc0ef8c22622eaa8608caa4a1e189cfb83ce847045eca34f690895eccb"}, + {file = "pydantic_core-2.18.1-cp311-none-win_amd64.whl", hash = "sha256:27f1009dc292f3b7ca77feb3571c537276b9aad5dd4efb471ac88a8bd09024e9"}, + {file = "pydantic_core-2.18.1-cp311-none-win_arm64.whl", hash = "sha256:48dd883db92e92519201f2b01cafa881e5f7125666141a49ffba8b9facc072b0"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b6b0e4912030c6f28bcb72b9ebe4989d6dc2eebcd2a9cdc35fefc38052dd4fe8"}, + {file = "pydantic_core-2.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3202a429fe825b699c57892d4371c74cc3456d8d71b7f35d6028c96dfecad31"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3982b0a32d0a88b3907e4b0dc36809fda477f0757c59a505d4e9b455f384b8b"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25595ac311f20e5324d1941909b0d12933f1fd2171075fcff763e90f43e92a0d"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14fe73881cf8e4cbdaded8ca0aa671635b597e42447fec7060d0868b52d074e6"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca976884ce34070799e4dfc6fbd68cb1d181db1eefe4a3a94798ddfb34b8867f"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684d840d2c9ec5de9cb397fcb3f36d5ebb6fa0d94734f9886032dd796c1ead06"}, + {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:54764c083bbe0264f0f746cefcded6cb08fbbaaf1ad1d78fb8a4c30cff999a90"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:201713f2f462e5c015b343e86e68bd8a530a4f76609b33d8f0ec65d2b921712a"}, + {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd1a9edb9dd9d79fbeac1ea1f9a8dd527a6113b18d2e9bcc0d541d308dae639b"}, + {file = "pydantic_core-2.18.1-cp312-none-win32.whl", hash = "sha256:d5e6b7155b8197b329dc787356cfd2684c9d6a6b1a197f6bbf45f5555a98d411"}, + {file = "pydantic_core-2.18.1-cp312-none-win_amd64.whl", hash = "sha256:9376d83d686ec62e8b19c0ac3bf8d28d8a5981d0df290196fb6ef24d8a26f0d6"}, + {file = "pydantic_core-2.18.1-cp312-none-win_arm64.whl", hash = "sha256:c562b49c96906b4029b5685075fe1ebd3b5cc2601dfa0b9e16c2c09d6cbce048"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3e352f0191d99fe617371096845070dee295444979efb8f27ad941227de6ad09"}, + {file = "pydantic_core-2.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0295d52b012cbe0d3059b1dba99159c3be55e632aae1999ab74ae2bd86a33d7"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56823a92075780582d1ffd4489a2e61d56fd3ebb4b40b713d63f96dd92d28144"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd3f79e17b56741b5177bcc36307750d50ea0698df6aa82f69c7db32d968c1c2"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38a5024de321d672a132b1834a66eeb7931959c59964b777e8f32dbe9523f6b1"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ce426ee691319d4767748c8e0895cfc56593d725594e415f274059bcf3cb76"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2adaeea59849ec0939af5c5d476935f2bab4b7f0335b0110f0f069a41024278e"}, + {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9b6431559676a1079eac0f52d6d0721fb8e3c5ba43c37bc537c8c83724031feb"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:85233abb44bc18d16e72dc05bf13848a36f363f83757541f1a97db2f8d58cfd9"}, + {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:641a018af4fe48be57a2b3d7a1f0f5dbca07c1d00951d3d7463f0ac9dac66622"}, + {file = "pydantic_core-2.18.1-cp38-none-win32.whl", hash = "sha256:63d7523cd95d2fde0d28dc42968ac731b5bb1e516cc56b93a50ab293f4daeaad"}, + {file = "pydantic_core-2.18.1-cp38-none-win_amd64.whl", hash = "sha256:907a4d7720abfcb1c81619863efd47c8a85d26a257a2dbebdb87c3b847df0278"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:aad17e462f42ddbef5984d70c40bfc4146c322a2da79715932cd8976317054de"}, + {file = "pydantic_core-2.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:94b9769ba435b598b547c762184bcfc4783d0d4c7771b04a3b45775c3589ca44"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e0e57cc704a52fb1b48f16d5b2c8818da087dbee6f98d9bf19546930dc64b5"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76b86e24039c35280ceee6dce7e62945eb93a5175d43689ba98360ab31eebc4a"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12a05db5013ec0ca4a32cc6433f53faa2a014ec364031408540ba858c2172bb0"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:250ae39445cb5475e483a36b1061af1bc233de3e9ad0f4f76a71b66231b07f88"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a32204489259786a923e02990249c65b0f17235073149d0033efcebe80095570"}, + {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6395a4435fa26519fd96fdccb77e9d00ddae9dd6c742309bd0b5610609ad7fb2"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2533ad2883f001efa72f3d0e733fb846710c3af6dcdd544fe5bf14fa5fe2d7db"}, + {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b560b72ed4816aee52783c66854d96157fd8175631f01ef58e894cc57c84f0f6"}, + {file = "pydantic_core-2.18.1-cp39-none-win32.whl", hash = "sha256:582cf2cead97c9e382a7f4d3b744cf0ef1a6e815e44d3aa81af3ad98762f5a9b"}, + {file = "pydantic_core-2.18.1-cp39-none-win_amd64.whl", hash = "sha256:ca71d501629d1fa50ea7fa3b08ba884fe10cefc559f5c6c8dfe9036c16e8ae89"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e178e5b66a06ec5bf51668ec0d4ac8cfb2bdcb553b2c207d58148340efd00143"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:72722ce529a76a4637a60be18bd789d8fb871e84472490ed7ddff62d5fed620d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fe0c1ce5b129455e43f941f7a46f61f3d3861e571f2905d55cdbb8b5c6f5e2c"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4284c621f06a72ce2cb55f74ea3150113d926a6eb78ab38340c08f770eb9b4d"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0c3e718f4e064efde68092d9d974e39572c14e56726ecfaeebbe6544521f47"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2027493cc44c23b598cfaf200936110433d9caa84e2c6cf487a83999638a96ac"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76909849d1a6bffa5a07742294f3fa1d357dc917cb1fe7b470afbc3a7579d539"}, + {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ee7ccc7fb7e921d767f853b47814c3048c7de536663e82fbc37f5eb0d532224b"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ee2794111c188548a4547eccc73a6a8527fe2af6cf25e1a4ebda2fd01cdd2e60"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a139fe9f298dc097349fb4f28c8b81cc7a202dbfba66af0e14be5cfca4ef7ce5"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d074b07a10c391fc5bbdcb37b2f16f20fcd9e51e10d01652ab298c0d07908ee2"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c69567ddbac186e8c0aadc1f324a60a564cfe25e43ef2ce81bcc4b8c3abffbae"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf1c7b78cddb5af00971ad5294a4583188bda1495b13760d9f03c9483bb6203"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2684a94fdfd1b146ff10689c6e4e815f6a01141781c493b97342cdc5b06f4d5d"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:73c1bc8a86a5c9e8721a088df234265317692d0b5cd9e86e975ce3bc3db62a59"}, + {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e60defc3c15defb70bb38dd605ff7e0fae5f6c9c7cbfe0ad7868582cb7e844a6"}, + {file = "pydantic_core-2.18.1.tar.gz", hash = "sha256:de9d3e8717560eb05e28739d1b35e4eac2e458553a52a301e51352a7ffc86a35"}, ] [package.dependencies] @@ -2583,13 +2517,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyflakes" -version = "3.2.0" +version = "3.1.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.8" files = [ - {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, - {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, + {file = "pyflakes-3.1.0-py2.py3-none-any.whl", hash = "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774"}, + {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, ] [[package]] @@ -2627,13 +2561,13 @@ extra = ["pygments (>=2.12)"] [[package]] name = "pytest" -version = "8.3.1" +version = "8.1.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, - {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, + {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, + {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, ] [package.dependencies] @@ -2641,11 +2575,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.5,<2" +pluggy = ">=1.4,<2.0" tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-cov" @@ -2710,13 +2644,13 @@ files = [ [[package]] name = "pyupgrade" -version = "3.16.0" +version = "3.15.0" description = "A tool to automatically upgrade syntax for newer versions." optional = false python-versions = ">=3.8.1" files = [ - {file = "pyupgrade-3.16.0-py2.py3-none-any.whl", hash = "sha256:7a54ee28f3024d027048d49d101e5c702e88c85edc3a1d08b636c50ebef2a97d"}, - {file = "pyupgrade-3.16.0.tar.gz", hash = "sha256:237893a05d5b117259b31b423f23cbae4bce0b7eae57ba9a52c06098c2ddd76f"}, + {file = "pyupgrade-3.15.0-py2.py3-none-any.whl", hash = "sha256:8dc8ebfaed43566e2c65994162795017c7db11f531558a74bc8aa077907bc305"}, + {file = "pyupgrade-3.15.0.tar.gz", hash = "sha256:a7fde381060d7c224f55aef7a30fae5ac93bbc428367d27e70a603bc2acd4f00"}, ] [package.dependencies] @@ -3068,13 +3002,13 @@ files = [ [[package]] name = "requests" -version = "2.32.3" +version = "2.31.0" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] [package.dependencies] @@ -3271,29 +3205,28 @@ files = [ [[package]] name = "ruff" -version = "0.8.0" +version = "0.3.0" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.8.0-py3-none-linux_armv6l.whl", hash = "sha256:fcb1bf2cc6706adae9d79c8d86478677e3bbd4ced796ccad106fd4776d395fea"}, - {file = "ruff-0.8.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:295bb4c02d58ff2ef4378a1870c20af30723013f441c9d1637a008baaf928c8b"}, - {file = "ruff-0.8.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7b1f1c76b47c18fa92ee78b60d2d20d7e866c55ee603e7d19c1e991fad933a9a"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb0d4f250a7711b67ad513fde67e8870109e5ce590a801c3722580fe98c33a99"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e55cce9aa93c5d0d4e3937e47b169035c7e91c8655b0974e61bb79cf398d49c"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f4cd64916d8e732ce6b87f3f5296a8942d285bbbc161acee7fe561134af64f9"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c5c1466be2a2ebdf7c5450dd5d980cc87c8ba6976fb82582fea18823da6fa362"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2dabfd05b96b7b8f2da00d53c514eea842bff83e41e1cceb08ae1966254a51df"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:facebdfe5a5af6b1588a1d26d170635ead6892d0e314477e80256ef4a8470cf3"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a8e86bae0dbd749c815211ca11e3a7bd559b9710746c559ed63106d382bd9c"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:85e654f0ded7befe2d61eeaf3d3b1e4ef3894469cd664ffa85006c7720f1e4a2"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:83a55679c4cb449fa527b8497cadf54f076603cc36779b2170b24f704171ce70"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:812e2052121634cf13cd6fddf0c1871d0ead1aad40a1a258753c04c18bb71bbd"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:780d5d8523c04202184405e60c98d7595bdb498c3c6abba3b6d4cdf2ca2af426"}, - {file = "ruff-0.8.0-py3-none-win32.whl", hash = "sha256:5fdb6efecc3eb60bba5819679466471fd7d13c53487df7248d6e27146e985468"}, - {file = "ruff-0.8.0-py3-none-win_amd64.whl", hash = "sha256:582891c57b96228d146725975fbb942e1f30a0c4ba19722e692ca3eb25cc9b4f"}, - {file = "ruff-0.8.0-py3-none-win_arm64.whl", hash = "sha256:ba93e6294e9a737cd726b74b09a6972e36bb511f9a102f1d9a7e1ce94dd206a6"}, - {file = "ruff-0.8.0.tar.gz", hash = "sha256:a7ccfe6331bf8c8dad715753e157457faf7351c2b69f62f32c165c2dbcbacd44"}, + {file = "ruff-0.3.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944"}, + {file = "ruff-0.3.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a"}, + {file = "ruff-0.3.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49"}, + {file = "ruff-0.3.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e"}, + {file = "ruff-0.3.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933"}, + {file = "ruff-0.3.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2"}, + {file = "ruff-0.3.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f"}, + {file = "ruff-0.3.0-py3-none-win32.whl", hash = "sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b"}, + {file = "ruff-0.3.0-py3-none-win_amd64.whl", hash = "sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f"}, + {file = "ruff-0.3.0-py3-none-win_arm64.whl", hash = "sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83"}, + {file = "ruff-0.3.0.tar.gz", hash = "sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a"}, ] [[package]] @@ -3313,18 +3246,19 @@ jeepney = ">=0.6" [[package]] name = "setuptools" -version = "70.0.0" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, - {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -3337,17 +3271,6 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -[[package]] -name = "smmap" -version = "5.0.1" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.7" -files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, -] - [[package]] name = "snowballstemmer" version = "2.2.0" @@ -3591,22 +3514,22 @@ files = [ [[package]] name = "tornado" -version = "6.4.2" +version = "6.4" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false -python-versions = ">=3.8" +python-versions = ">= 3.8" files = [ - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, - {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, - {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, - {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, - {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, - {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, - {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, + {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, + {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, + {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, + {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, + {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, ] [[package]] @@ -3626,13 +3549,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "twine" -version = "5.1.0" +version = "5.0.0" description = "Collection of utilities for publishing packages on PyPI" optional = false python-versions = ">=3.8" files = [ - {file = "twine-5.1.0-py3-none-any.whl", hash = "sha256:fe1d814395bfe50cfbe27783cb74efe93abeac3f66deaeb6c8390e4e92bacb43"}, - {file = "twine-5.1.0.tar.gz", hash = "sha256:4d74770c88c4fcaf8134d2a6a9d863e40f08255ff7d8e2acb3cbbd57d25f6e9d"}, + {file = "twine-5.0.0-py3-none-any.whl", hash = "sha256:a262933de0b484c53408f9edae2e7821c1c45a3314ff2df9bdd343aa7ab8edc0"}, + {file = "twine-5.0.0.tar.gz", hash = "sha256:89b0cc7d370a4b66421cc6102f269aa910fe0f1861c124f573cf2ddedbc10cf4"}, ] [package.dependencies] @@ -3684,27 +3607,38 @@ files = [ [[package]] name = "types-requests" -version = "2.32.0.20240521" +version = "2.31.0.0" description = "Typing stubs for requests" optional = false -python-versions = ">=3.8" +python-versions = "*" files = [ - {file = "types-requests-2.32.0.20240521.tar.gz", hash = "sha256:c5c4a0ae95aad51f1bf6dae9eed04a78f7f2575d4b171da37b622e08b93eb5d3"}, - {file = "types_requests-2.32.0.20240521-py3-none-any.whl", hash = "sha256:ab728ba43ffb073db31f21202ecb97db8753ded4a9dc49cb480d8a5350c5c421"}, + {file = "types-requests-2.31.0.0.tar.gz", hash = "sha256:c1c29d20ab8d84dff468d7febfe8e0cb0b4664543221b386605e14672b44ea25"}, + {file = "types_requests-2.31.0.0-py3-none-any.whl", hash = "sha256:7c5cea7940f8e92ec560bbc468f65bf684aa3dcf0554a6f8c4710f5f708dc598"}, ] [package.dependencies] -urllib3 = ">=2" +types-urllib3 = "*" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] [[package]] name = "typing-extensions" -version = "4.12.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, - {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] @@ -3720,13 +3654,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.2" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -3876,20 +3810,20 @@ requests = "*" [[package]] name = "zipp" -version = "3.19.1" +version = "3.18.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, - {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, + {file = "zipp-3.18.0-py3-none-any.whl", hash = "sha256:c1bb803ed69d2cce2373152797064f7e79bc43f0a3748eb494096a867e0ebf79"}, + {file = "zipp-3.18.0.tar.gz", hash = "sha256:df8d042b02765029a09b157efd8e820451045890acc30f8e37dd2f94a060221f"}, ] [package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] lock-version = "2.0" python-versions = ">3.8.1,<3.12" -content-hash = "26e3ff7e9354c9a56e5d0a4391f98a376cba8da064ef0d10922089c5fb9668f0" +content-hash = "03e4bce64d5a592aa115a95cd76df71c31cf1fe20b7128a6400fa4bbf88006cd" diff --git a/streampipes-client-python/pyproject.toml b/streampipes-client-python/pyproject.toml index b1d807fca1..bc4dcd8089 100644 --- a/streampipes-client-python/pyproject.toml +++ b/streampipes-client-python/pyproject.toml @@ -66,17 +66,17 @@ optional = true [tool.poetry.group.dev.dependencies] autoflake = "2.3.0" -black = "24.8.0" -blacken-docs = "1.18.0" -flake8 = "7.1.0" +black = "24.4.0" +blacken-docs = "1.16.0" +flake8 = "6.1.0" interrogate = { version = "1.7.0", extras = ["png"] } isort = "5.13.0" -mypy = "1.13.0" -ruff = "0.8.0" +mypy = "1.9.0" +ruff = "0.3.0" pre-commit = "3.5.0" -pytest = "8.3.1" +pytest = "8.1.1" pytest-cov = "5.0.0" -pyupgrade = "3.16.0" +pyupgrade = "3.15.0" [tool.poetry.group.docs] optional = true @@ -86,11 +86,10 @@ mike = { git = "https://github.com/jimporter/mike.git", rev = "872f72def32f58890 mkdocs = "1.5.2" mkdocs-awesome-pages-plugin = "2.9.0" mkdocs-gen-files = "0.5.0" -mkdocs-git-revision-date-localized-plugin = "1.3.0" mkdocs-jupyter = "0.24.0" mkdocs-literate-nav = "0.6.0" mkdocs-material = "9.3.2" -mkdocstrings = { version = "0.25.0", extras = ["python"] } +mkdocstrings = { version = "0.24.0", extras = ["python"] } numpydoc = "1.7.0" pytkdocs = { version = "0.16.1", extras = ["numpy-style"] } @@ -100,28 +99,18 @@ optional = true [tool.poetry.group.stubs.dependencies] pandas-stubs = "2.0.2.230605" # do not upgrade to keep compability with Python 3.8 types-Jinja2 = "2.11.9" -types-requests = "2.32.0.20240521" +types-requests = "2.31.0.0" [tool.poetry.group.deployment] optional = true [tool.poetry.group.deployment.dependencies] -twine = "5.1.0" +twine = "5.0.0" [[tool.poetry.source]] name = "PyPI" priority = "primary" -[tool.autoflake] -remove-all-unused-imports = true -expand-star-imports = true -in-place = true -remove-unused-variables = true -remove-duplicate-keys = true - -[tool.black] -line-length = 120 - [tool.interrogate] ignore-init-method = true ignore-magic = true diff --git a/streampipes-client-python/streampipes/client/client.py b/streampipes-client-python/streampipes/client/client.py index 374f68569d..047e52cc89 100644 --- a/streampipes-client-python/streampipes/client/client.py +++ b/streampipes-client-python/streampipes/client/client.py @@ -137,7 +137,6 @@ def __init__( # this allows to centrally determine the behavior of all requests made self.request_session = Session() self.request_session.headers.update(self.http_headers) - self.request_session.headers.update(self.client_config.additional_headers) self.logging_level = logging_level self._set_up_logging(logging_level=self.logging_level) # type: ignore @@ -231,7 +230,7 @@ def http_headers(self) -> Dict[str, str]: Returns ------- http_headers: Dict[str, str] - Header information for HTTP requests as string key-value pairs. + header information for HTTP requests as string key-value pairs. """ # create HTTP headers from credential provider and add additional headers needed diff --git a/streampipes-client-python/streampipes/client/config.py b/streampipes-client-python/streampipes/client/config.py index 36b0618ad9..8149b3b0c1 100644 --- a/streampipes-client-python/streampipes/client/config.py +++ b/streampipes-client-python/streampipes/client/config.py @@ -20,8 +20,8 @@ """ -from dataclasses import dataclass, field -from typing import Dict, Optional +from dataclasses import dataclass +from typing import Optional __all__ = [ "StreamPipesClientConfig", @@ -49,8 +49,6 @@ class StreamPipesClientConfig: port: Optional[int] Specifies the port under which the StreamPipes API is available, e.g., `80` (with http) or `443` (with https) - additional_headers: Optional[Dict[str, str]] - Specifies additional HTTP headers that should be sent with each request, e.g., proxy headers Examples -------- @@ -63,4 +61,3 @@ class StreamPipesClientConfig: host_address: str https_disabled: Optional[bool] = False port: Optional[int] = 80 - additional_headers: Optional[Dict[str, str]] = field(default_factory=dict) diff --git a/streampipes-client-python/streampipes/client/credential_provider.py b/streampipes-client-python/streampipes/client/credential_provider.py index f2fd843b4d..d7493b40c2 100644 --- a/streampipes-client-python/streampipes/client/credential_provider.py +++ b/streampipes-client-python/streampipes/client/credential_provider.py @@ -112,7 +112,7 @@ class StreamPipesApiKeyCredentials(CredentialProvider): def from_env(cls, username_env: str, api_key_env: str) -> StreamPipesApiKeyCredentials: """DEPRECATED - use the class constructor instead - Returns an API key provider parameterized via environment variables. + Returns an api key provider parameterized via environment variables. Parameters ---------- diff --git a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py index 1ef107b1e7..0bdf256002 100644 --- a/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py +++ b/streampipes-client-python/streampipes/endpoint/api/data_lake_measure.py @@ -55,8 +55,7 @@ class MeasurementGetQueryConfig(BaseModel): If provided, the returned data only consists of the given columns.
Please be aware that the column `time` as an index is always included. end_date: Optional[datetime] - Limits the queried data to only include data that is older than the specified time. - In other words, any data that occurred after the end_date will not be included in the query results. + Restricts queried data to be younger than the specified time. limit: Optional[int] Amount of records returned at maximum (default: `1000`)
This needs to be at least `1` @@ -70,8 +69,7 @@ class MeasurementGetQueryConfig(BaseModel): Page number used for paging operation
This needs to be at least `1` start_date: Optional[datetime] - Limits the queried data to only include data that is newer than the specified time. - In other words, any data that occurred before the start_date will not be included in the query results. + Restricts queried data to be older than the specified time """ _regex_comma_separated_string = r"^[0-9a-zA-Z\_]+(,[0-9a-zA-Z\_]+)*$" @@ -318,21 +316,12 @@ def _resource_cls(self) -> Type[QueryResult]: that the resource class of the resource container is the return type of the get endpoint. Therefore, this is only a temporary implementation and will be removed soon. - - Returns - ------- - [QueryResult][streampipes.model.resource.QueryResult] """ return QueryResult @property def _container_cls(self) -> Type[ResourceContainer]: - """Defines the model container class the endpoint refers to. - - Returns - ------- - [DataLakeMeasures][streampipes.model.container.DataLakeMeasures] - """ + """Defines the model container class the endpoint refers to.""" return DataLakeMeasures @property @@ -355,14 +344,14 @@ def get(self, identifier: str, **kwargs: Optional[Dict[str, Any]]) -> QueryResul identifier: str The identifier of the data lake measure to be queried. **kwargs: Dict[str, Any] - Keyword arguments can be used to provide additional query parameters. + keyword arguments can be used to provide additional query parameters. The available query parameters are defined by the [MeasurementGetQueryConfig][streampipes.endpoint.api.data_lake_measure.MeasurementGetQueryConfig]. Returns ------- - measurement: QueryResult - The specified data lake measure + measurement: DataLakeMeasures + the specified data lake measure Examples -------- diff --git a/streampipes-client-python/streampipes/endpoint/api/version.py b/streampipes-client-python/streampipes/endpoint/api/version.py index 0b21544850..5ff4d1e60b 100644 --- a/streampipes-client-python/streampipes/endpoint/api/version.py +++ b/streampipes-client-python/streampipes/endpoint/api/version.py @@ -28,7 +28,6 @@ from streampipes.endpoint import APIEndpoint from streampipes.model.container import Versions from streampipes.model.container.resource_container import ResourceContainer -from streampipes.model.resource import Version from streampipes.model.resource.resource import Resource @@ -60,7 +59,7 @@ class VersionEndpoint(APIEndpoint): >>> client = StreamPipesClient.create(client_config=client_config) >>> client.versionApi.get(identifier="").to_dict(use_source_names=False) - {'backend_version': '0.92.0-SNAPSHOT'} + {'backend_version': '0.95.0'} """ @property @@ -74,16 +73,6 @@ def _container_cls(self) -> Type[ResourceContainer]: return Versions - @property - def _resource_cls(cls) -> Type[Version]: - """Returns the class of the resource that are bundled. - - Returns - ------- - [Version][streampipes.model.resource.Version] - """ - return Version - @property def _relative_api_path(self) -> Tuple[str, ...]: """Defines the relative api path to the DataStream endpoint. @@ -105,12 +94,12 @@ def all(self) -> ResourceContainer: Raises ------ NotImplementedError - This endpoint does not return multiple entries, therefore this method is not available. + this endpoint does not return multiple entries, therefore this method is not available """ raise NotImplementedError("The `all()` method is not supported by this endpoint.") - def get(self, identifier: str, **kwargs) -> Version: + def get(self, identifier: str, **kwargs) -> Resource: """Queries the resource from the API endpoint. For this endpoint only one resource is available. @@ -120,25 +109,13 @@ def get(self, identifier: str, **kwargs) -> Version: identifier: str Not supported by this endpoint, is set to an empty string. - Raises - ------ - ValueError - Non-empty `identifier` is not supported by this endpoint. Please set `identifier` to an empty string or `None`. - Returns ------- versions: Version The specified resource as an instance of the corresponding model class([Version][streampipes.model.resource.Version]). # noqa: 501 """ - if identifier: - raise ValueError( - "Non-empty 'identifier' is not supported by this endpoint. " - "Please set 'identifier' to an empty string or 'None'." - ) - - response = self._make_request(request_method=self._parent_client.request_session.get, url=self.build_url()) - return self._resource_cls(**response.json()) + return super().get(identifier="") def post(self, resource: Resource) -> None: """Usually, this method allows to create via this endpoint. @@ -147,7 +124,7 @@ def post(self, resource: Resource) -> None: Raises ------ NotImplementedError - This endpoint does not allow for POST requests, therefore this method is not available. + this endpoint does not allow for POST requests, therefore this method is not available """ raise NotImplementedError("The `post()` method is not supported by this endpoint.") diff --git a/streampipes-client-python/streampipes/endpoint/endpoint.py b/streampipes-client-python/streampipes/endpoint/endpoint.py index 2c491c6dd1..fb70fbf41a 100644 --- a/streampipes-client-python/streampipes/endpoint/endpoint.py +++ b/streampipes-client-python/streampipes/endpoint/endpoint.py @@ -18,7 +18,7 @@ """ General implementation for an endpoint. Provided classes and assets are aimed to be used for developing endpoints. -An endpoint provides all options to communicate with a dedicated part of StreamPipes in a handy way. +An endpoint provides all options to communicate with ad dedicated part of StreamPipes in a handy way. """ import json @@ -184,7 +184,7 @@ def all(self) -> ResourceContainer: Returns ------- container: ResourceContainer - Container element that bundles the returned resources + container element that bundles the returned resources """ response = self._make_request( @@ -284,7 +284,7 @@ def configure(self, broker: Broker) -> None: """Configures the message endpoint by setting the broker instance to be used. This configuration step is required before the endpoint can be actually used. - The based `broker` instance is passed to an internal property. + The based `broker` instance is passed to an internal property Parameters ---------- diff --git a/streampipes-client-python/streampipes/endpoint/exceptions.py b/streampipes-client-python/streampipes/endpoint/exceptions.py index 6803a836f4..6465146f92 100644 --- a/streampipes-client-python/streampipes/endpoint/exceptions.py +++ b/streampipes-client-python/streampipes/endpoint/exceptions.py @@ -16,7 +16,7 @@ # """ -Custom exceptions dedicated to the endpoints module. +Custom exceptions dedicated for the endpoints module """ __all__ = [ diff --git a/streampipes-client-python/streampipes/function_zoo/river_function.py b/streampipes-client-python/streampipes/function_zoo/river_function.py index 910097eef2..9bbcab3a74 100644 --- a/streampipes-client-python/streampipes/function_zoo/river_function.py +++ b/streampipes-client-python/streampipes/function_zoo/river_function.py @@ -178,7 +178,6 @@ def __init__( name="prediction", attributes=attributes, broker=get_broker_description(client.dataStreamApi.get(stream_ids[0])), # type: ignore - stream_id=stream_ids[0], ) function_definition = FunctionDefinition(consumed_streams=stream_ids).add_output_data_stream(output_stream) self.sp_function = RiverFunction( diff --git a/streampipes-client-python/streampipes/functions/streampipes_function.py b/streampipes-client-python/streampipes/functions/streampipes_function.py index 8d3c063768..90521f1934 100644 --- a/streampipes-client-python/streampipes/functions/streampipes_function.py +++ b/streampipes-client-python/streampipes/functions/streampipes_function.py @@ -34,7 +34,7 @@ class StreamPipesFunction(ABC): Parameters ---------- function_definition: FunctionDefinition - The definition of the function that contains metadata about the connected function + the definition of the function that contains metadata about the connected function Attributes ---------- @@ -50,7 +50,7 @@ def __init__(self, function_definition: Optional[FunctionDefinition] = None): } def add_output(self, stream_id: str, event: Dict[str, Any]): - """Send an event via an output data stream to StreamPipes. + """Send an event via an output data stream to StreamPipes Parameters ---------- @@ -78,7 +78,7 @@ def getFunctionId(self) -> FunctionId: return self.function_definition.function_id def stop(self) -> None: - """Stops the function and disconnects from the output streams.""" + """Stops the function and disconnects from the output streams""" for collector in self.output_collectors.values(): collector.disconnect() @@ -96,12 +96,12 @@ def requiredStreamIds(self) -> List[str]: @abstractmethod def onServiceStarted(self, context: FunctionContext) -> None: - """Is called when the function is started. + """Is called when the function gets started. Parameters ---------- context: FunctionContext - The context in which the function is started. + The context in which the function gets started. Returns ------- @@ -128,7 +128,7 @@ def onEvent(self, event: Dict[str, Any], streamId: str) -> None: @abstractmethod def onServiceStopped(self) -> None: - """Is called when the function is stopped. + """Is called when the function gets stopped. Returns ------- diff --git a/streampipes-client-python/streampipes/functions/utils/data_stream_generator.py b/streampipes-client-python/streampipes/functions/utils/data_stream_generator.py index 6587a2e53c..ee2ad36180 100644 --- a/streampipes-client-python/streampipes/functions/utils/data_stream_generator.py +++ b/streampipes-client-python/streampipes/functions/utils/data_stream_generator.py @@ -16,7 +16,7 @@ # from enum import Enum -from typing import Dict +from typing import Dict, Optional from streampipes.functions.broker import SupportedBroker from streampipes.model.common import ( @@ -53,7 +53,7 @@ class RuntimeType(Enum): def create_data_stream( name: str, attributes: Dict[str, str], - stream_id: str = None, + stream_id: Optional[str] = None, broker: SupportedBroker = SupportedBroker.NATS, ): """Creates a data stream @@ -78,7 +78,7 @@ def create_data_stream( EventProperty( # type: ignore label="timestamp", runtime_name="timestamp", - semantic_type="http://schema.org/DateTime", + domain_properties=["http://schema.org/DateTime"], property_scope="HEADER_PROPERTY", runtime_type="http://www.w3.org/2001/XMLSchema#long", ) @@ -87,6 +87,7 @@ def create_data_stream( EventProperty( # type: ignore label=attribute_name, runtime_name=attribute_name, + index=i, runtime_type=f"http://www.w3.org/2001/XMLSchema#{attribute_type}", ) for i, (attribute_name, attribute_type) in enumerate(attributes.items(), start=1) @@ -103,17 +104,9 @@ def create_data_stream( ) ] - sanitized_stream_id = stream_id.replace(" ", "") - - # Assign a default topic name incorporating the unique stream ID to each protocol. - # This ensures the topic name remains consistent across function restarts, - # avoiding reliance on client-side defaults. - for protocol in transport_protocols: - protocol.topic_definition.actual_topic_name = f"org.apache.streampipes.connect.{sanitized_stream_id}" - data_stream = DataStream( name=name, event_schema=event_schema, event_grounding=EventGrounding(transport_protocols=transport_protocols) ) - - data_stream.element_id = sanitized_stream_id + if stream_id: + data_stream.element_id = stream_id return data_stream diff --git a/streampipes-client-python/streampipes/model/common.py b/streampipes-client-python/streampipes/model/common.py index afcac4beac..49cd58caad 100644 --- a/streampipes-client-python/streampipes/model/common.py +++ b/streampipes-client-python/streampipes/model/common.py @@ -24,7 +24,7 @@ from typing import List, Optional from uuid import uuid4 -from pydantic.v1 import BaseModel, Field, StrictInt, StrictStr +from pydantic.v1 import BaseModel, Field, StrictBool, StrictInt, StrictStr __all__ = [ "BaseElement", @@ -82,14 +82,14 @@ class Config: class BaseElement(BasicModel): - """Structure of a basic element in the StreamPipes Backend.""" + """Structure of a basic element in the StreamPipes backend""" element_id: Optional[StrictStr] class ValueSpecification(BasicModel): """ - Data model of an `ValueSpecification` in compliance with the StreamPipes Backend. + Data model of an `ValueSpecification` in compliance to the StreamPipes Backend. """ class_name: Optional[StrictStr] = Field(alias="@class") @@ -101,7 +101,7 @@ class ValueSpecification(BasicModel): class EventProperty(BasicModel): """ - Data model of an `EventProperty` in compliance with the StreamPipes Backend. + Data model of an `EventProperty` in compliance to the StreamPipes Backend. """ class_name: StrictStr = Field(alias="@class", default="org.apache.streampipes.model.schema.EventPropertyPrimitive") @@ -109,8 +109,10 @@ class EventProperty(BasicModel): label: Optional[StrictStr] description: Optional[StrictStr] runtime_name: StrictStr - semantic_type: Optional[StrictStr] + required: StrictBool = Field(default=False) + domain_properties: Optional[List[StrictStr]] = Field(default_factory=list) property_scope: Optional[StrictStr] = Field(default="MEASUREMENT_PROPERTY") + index: StrictInt = Field(default=0) runtime_id: Optional[StrictStr] runtime_type: StrictStr = Field(default="http://www.w3.org/2001/XMLSchema#string") measurement_unit: Optional[StrictStr] @@ -119,7 +121,7 @@ class EventProperty(BasicModel): class EventSchema(BasicModel): """ - Data model of an `EventSchema` in compliance with the StreamPipes Backend. + Data model of an `EventSchema` in compliance to the StreamPipes Backend. """ event_properties: List[EventProperty] @@ -127,7 +129,7 @@ class EventSchema(BasicModel): class ApplicationLink(BasicModel): """ - Data model of an `ApplicationLink` in compliance with the StreamPipes Backend. + Data model of an `ApplicationLink` in compliance to the StreamPipes Backend. """ class_name: Optional[StrictStr] = Field(alias="@class") @@ -141,7 +143,7 @@ class ApplicationLink(BasicModel): class TopicDefinition(BasicModel): """ - Data model of a `TopicDefinition` in compliance with the StreamPipes Backend. + Data model of a `TopicDefinition` in compliance to the StreamPipes Backend. """ class_name: Optional[StrictStr] = Field( @@ -152,7 +154,7 @@ class TopicDefinition(BasicModel): class TransportProtocol(BasicModel): """ - Data model of a `TransportProtocol` in compliance with the StreamPipes Backend. + Data model of a `TransportProtocol` in compliance to the StreamPipes Backend. """ class_name: StrictStr = Field( @@ -166,7 +168,7 @@ class TransportProtocol(BasicModel): class TransportFormat(BasicModel): """ - Data model of a `TransportFormat` in compliance with the StreamPipes Backend. + Data model of a `TransportFormat` in compliance to the StreamPipes Backend. """ rdf_type: List[StrictStr] = Field(default=["http://sepa.event-processing.org/sepa#json"]) @@ -174,7 +176,7 @@ class TransportFormat(BasicModel): class EventGrounding(BasicModel): """ - Data model of an `EventGrounding` in compliance to with StreamPipes Backend. + Data model of an `EventGrounding` in compliance to the StreamPipes Backend. """ transport_protocols: List[TransportProtocol] = Field(default_factory=lambda: [TransportProtocol()]) @@ -183,7 +185,7 @@ class EventGrounding(BasicModel): class MeasurementCapability(BasicModel): """ - Data model of a `MeasurementCapability` in compliance with the StreamPipes Backend. + Data model of a `MeasurementCapability` in compliance to the StreamPipes Backend. """ capability: Optional[StrictStr] @@ -192,7 +194,7 @@ class MeasurementCapability(BasicModel): class MeasurementObject(BasicModel): """ - Data model of a `MeasurementObject` in compliance with the StreamPipes Backend. + Data model of a `MeasurementObject` in compliance to the StreamPipes Backend. """ element_id: Optional[StrictStr] diff --git a/streampipes-client-python/streampipes/model/container/resource_container.py b/streampipes-client-python/streampipes/model/container/resource_container.py index 03c017319e..b86a0c5ff8 100644 --- a/streampipes-client-python/streampipes/model/container/resource_container.py +++ b/streampipes-client-python/streampipes/model/container/resource_container.py @@ -166,7 +166,7 @@ def from_json(cls, json_string: str) -> ResourceContainer: Returns ------- container: ResourceContainer - Instance of the container derived from the JSON definition + instance of the container derived from the JSON definition Raises ------ diff --git a/streampipes-client-python/streampipes/model/resource/data_stream.py b/streampipes-client-python/streampipes/model/resource/data_stream.py index a44f9548f9..d8e424c4b0 100644 --- a/streampipes-client-python/streampipes/model/resource/data_stream.py +++ b/streampipes-client-python/streampipes/model/resource/data_stream.py @@ -16,7 +16,7 @@ # from typing import List, Optional -from pydantic.v1 import Field, StrictBool, StrictStr +from pydantic.v1 import Field, StrictBool, StrictInt, StrictStr from streampipes.model.common import ( ApplicationLink, @@ -102,6 +102,7 @@ def __init__(self, **kwargs): event_schema: Optional[EventSchema] measurement_capability: Optional[List[MeasurementCapability]] measurement_object: Optional[List[MeasurementObject]] + index: StrictInt = Field(default=0) corresponding_adapter_id: Optional[StrictStr] category: Optional[List[StrictStr]] uri: Optional[StrictStr] diff --git a/streampipes-client-python/streampipes/model/resource/query_result.py b/streampipes-client-python/streampipes/model/resource/query_result.py index fbd613e9ce..0f05a72747 100644 --- a/streampipes-client-python/streampipes/model/resource/query_result.py +++ b/streampipes-client-python/streampipes/model/resource/query_result.py @@ -16,7 +16,7 @@ # from itertools import chain -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Union import pandas as pd from pydantic.v1 import Field, StrictInt, StrictStr @@ -73,9 +73,6 @@ def convert_to_pandas_representation(self) -> Dict[str, Union[List[str], List[Li headers: List[StrictStr] all_data_series: List[DataSeries] query_status: Literal["OK", "TOO_MUCH_DATA"] = Field(alias="spQueryStatus") - source_index: StrictInt - for_id: Optional[str] - last_timestamp: StrictInt def to_pandas(self) -> pd.DataFrame: """Returns the data lake series in representation of a Pandas Dataframe. diff --git a/streampipes-client-python/streampipes/py.typed b/streampipes-client-python/streampipes/py.typed index 4d9d4b2ef6..17216e19b3 100644 --- a/streampipes-client-python/streampipes/py.typed +++ b/streampipes-client-python/streampipes/py.typed @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# +# \ No newline at end of file diff --git a/streampipes-client-python/tests/client/test_client.py b/streampipes-client-python/tests/client/test_client.py index b44898487d..6e92deb880 100644 --- a/streampipes-client-python/tests/client/test_client.py +++ b/streampipes-client-python/tests/client/test_client.py @@ -44,8 +44,10 @@ def test_client_init(self, server_version: MagicMock): "Application": "application/json", } result_headers = dict(result.request_session.headers) - for key, value in expected_headers.items(): - self.assertEqual(result_headers.get(key), value) + self.assertDictContainsSubset( + subset=expected_headers, + dictionary=result_headers, + ) self.assertTrue(isinstance(result.dataLakeMeasureApi, DataLakeMeasureEndpoint)) self.assertEqual(result.base_api_path, "http://localhost:80/streampipes-backend/") @@ -69,8 +71,10 @@ def test_client_create(self, server_version: MagicMock): "Application": "application/json", } result_headers = dict(result.request_session.headers) - for key, value in expected_headers.items(): - self.assertEqual(result_headers.get(key), value) + self.assertDictContainsSubset( + subset=expected_headers, + dictionary=result_headers, + ) self.assertTrue(isinstance(result.dataLakeMeasureApi, DataLakeMeasureEndpoint)) self.assertEqual(result.base_api_path, "https://localhost:443/streampipes-backend/") @@ -115,13 +119,7 @@ def simulate_response(*args, **kwargs): if "streams" in kwargs["url"]: return MockResponse( json.dumps( - [ - { - "elementId": "test-stream", - "name": "test", - "eventGrounding": {"transportProtocols": []}, - } - ] + [{"elementId": "test-stream", "name": "test", "eventGrounding": {"transportProtocols": []}}] ) ) if "versions" in kwargs["url"]: @@ -140,8 +138,7 @@ def simulate_response(*args, **kwargs): mocked_logger.info.assert_has_calls( calls=[ call( - "\nHi there!\nYou are connected to a StreamPipes instance running at " - "https://localhost:443 with version SP-dev.\n" + "\nHi there!\nYou are connected to a StreamPipes instance running at https://localhost:443 with version SP-dev.\n" "The following StreamPipes resources are available with this client:\n" "1x DataLakeMeasures\n1x DataStreams" ), diff --git a/streampipes-client-python/tests/client/test_data_lake_series.py b/streampipes-client-python/tests/client/test_data_lake_series.py index b83e71f13c..df0349a2f3 100644 --- a/streampipes-client-python/tests/client/test_data_lake_series.py +++ b/streampipes-client-python/tests/client/test_data_lake_series.py @@ -103,9 +103,6 @@ def test_to_pandas(self, server_version: MagicMock, http_session: MagicMock): "headers": self.headers, "spQueryStatus": "OK", "allDataSeries": [self.data_series], - "sourceIndex": 0, - "forId": None, - "lastTimestamp": 1717936808802, } result_pd = self.get_result_as_panda(http_session, query_result) @@ -128,9 +125,6 @@ def test_group_by_to_pandas(self, server_version: MagicMock, http_session: Magic "headers": self.headers, "spQueryStatus": "OK", "allDataSeries": [self.data_series, self.data_series], - "sourceIndex": 0, - "forId": None, - "lastTimestamp": 1717936808802, } result_pd = self.get_result_as_panda(http_session, query_result) @@ -153,9 +147,6 @@ def test_different_headers_exception(self, server_version: MagicMock, http_sessi "headers": ["one"], "spQueryStatus": "OK", "allDataSeries": [self.data_series], - "sourceIndex": 0, - "forId": None, - "lastTimestamp": 1717936808802, } with self.assertRaises(StreamPipesUnsupportedDataSeries): diff --git a/streampipes-client-python/tests/client/test_endpoint.py b/streampipes-client-python/tests/client/test_endpoint.py index 5b71d83531..69ab569b8d 100644 --- a/streampipes-client-python/tests/client/test_endpoint.py +++ b/streampipes-client-python/tests/client/test_endpoint.py @@ -53,8 +53,10 @@ def setUp(self) -> None: "label": "Density", "description": "Denotes the current density of the fluid", "runtimeName": "density", - "semanticType": "http://schema.org/Number", + "required": False, + "domainProperties": ["http://schema.org/Number"], "propertyScope": "MEASUREMENT_PROPERTY", + "index": 5, "runtimeId": None, "runtimeType": "http://www.w3.org/2001/XMLSchema#float", "measurementUnit": None, @@ -66,8 +68,10 @@ def setUp(self) -> None: "label": "Temperature", "description": "Denotes the current temperature in degrees celsius", "runtimeName": "temperature", - "semanticType": "http://schema.org/Number", + "required": False, + "domainProperties": ["http://schema.org/Number"], "propertyScope": "MEASUREMENT_PROPERTY", + "index": 4, "runtimeId": None, "runtimeType": "http://www.w3.org/2001/XMLSchema#float", "measurementUnit": "http://codes.wmo.int/common/unit/degC", @@ -131,8 +135,10 @@ def setUp(self) -> None: "label": "Density", "description": "Denotes the current density of the fluid", "runtimeName": "density", - "semanticType": "http://schema.org/Number", + "required": False, + "domainProperties": ["http://schema.org/Number"], "propertyScope": "MEASUREMENT_PROPERTY", + "index": 5, "runtimeId": None, "runtimeType": "http://www.w3.org/2001/XMLSchema#float", "measurementUnit": None, @@ -144,8 +150,10 @@ def setUp(self) -> None: "label": "Temperature", "description": "Denotes the current temperature in degrees celsius", "runtimeName": "temperature", - "semanticType": "http://schema.org/Number", + "required": False, + "domainProperties": ["http://schema.org/Number"], "propertyScope": "MEASUREMENT_PROPERTY", + "index": 4, "runtimeId": None, "runtimeType": "http://www.w3.org/2001/XMLSchema#float", "measurementUnit": "http://codes.wmo.int/common/unit/degC", @@ -161,6 +169,7 @@ def setUp(self) -> None: }, "measurementCapability": None, "measurementObject": None, + "index": 0, "correspondingAdapterId": "urn:streampipes.apache.org:spi:org.apache.streampipes.connect." "iiot.adapters.simulator.machine:11934d37-135b-4ef6-b5f1-4f520cc81a43", "category": None, @@ -285,6 +294,7 @@ def test_endpoint_data_stream_happy_path(self, server_version: MagicMock, http_s "includes_locales", "internally_managed", "measurement_object", + "index", "corresponding_adapter_id", "uri", "dom", diff --git a/streampipes-client-python/tests/client/test_versions.py b/streampipes-client-python/tests/client/test_versions.py index 14810e99c8..39c4535ef2 100644 --- a/streampipes-client-python/tests/client/test_versions.py +++ b/streampipes-client-python/tests/client/test_versions.py @@ -45,7 +45,3 @@ def test_get_development_version(self, http_session: MagicMock) -> None: ) self.assertEqual("development", result.backend_version) - - # Test get with non-empty identifier raises ValueError - with self.assertRaises(ValueError): - client.versionApi.get("any-incorrect-identifier") diff --git a/streampipes-client-python/tests/functions/test_function_handler.py b/streampipes-client-python/tests/functions/test_function_handler.py index d7fa3348aa..f1718f7aed 100644 --- a/streampipes-client-python/tests/functions/test_function_handler.py +++ b/streampipes-client-python/tests/functions/test_function_handler.py @@ -373,7 +373,7 @@ def save_event(self, event: Dict[str, Any]): ) ) - output_stream = create_data_stream("test", attributes={"number": RuntimeType.INTEGER.value}, stream_id="a1d") + output_stream = create_data_stream("test", attributes={"number": RuntimeType.INTEGER.value}) test_function = TestFunctionOutput( function_definition=FunctionDefinition( consumed_streams=["urn:streampipes.apache.org:eventstream:uPDKLI"] @@ -436,7 +436,7 @@ def save_event(self, event: Dict[str, Any]): ) output_stream = create_data_stream( - "test", stream_id="a1c", attributes={"number": RuntimeType.INTEGER.value}, broker=SupportedBroker.KAFKA + "test", attributes={"number": RuntimeType.INTEGER.value}, broker=SupportedBroker.KAFKA ) test_function = TestFunctionOutput( function_definition=FunctionDefinition( diff --git a/streampipes-client-python/tests/functions/test_river_function.py b/streampipes-client-python/tests/functions/test_river_function.py index 4d0e09e70b..597ee4963c 100644 --- a/streampipes-client-python/tests/functions/test_river_function.py +++ b/streampipes-client-python/tests/functions/test_river_function.py @@ -56,9 +56,7 @@ def predict_one(self, x): class TestRiverFunction(TestCase): def setUp(self) -> None: self.data_stream = create_data_stream( - "test", - attributes={"number": RuntimeType.FLOAT.value, "bool": RuntimeType.BOOLEAN.value}, - stream_id="sample-stream", + "test", attributes={"number": RuntimeType.FLOAT.value, "bool": RuntimeType.BOOLEAN.value} ).to_dict() self.test_stream_data = [ diff --git a/streampipes-client/pom.xml b/streampipes-client/pom.xml index e132a4fad7..82fba9d748 100644 --- a/streampipes-client/pom.xml +++ b/streampipes-client/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,32 +32,42 @@ org.apache.streampipes streampipes-client-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes - streampipes-dataformat - 0.97.0-SNAPSHOT + streampipes-dataformat-json + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-dataformat-cbor + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-dataformat-fst + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-security-jwt - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-serializers-json - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT @@ -72,11 +82,6 @@ org.apache.httpcomponents fluent-hc - - - commons-logging - commons-logging - diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/StreamPipesClient.java b/streampipes-client/src/main/java/org/apache/streampipes/client/StreamPipesClient.java index a8dfa12b14..1eee9f475d 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/StreamPipesClient.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/StreamPipesClient.java @@ -37,6 +37,9 @@ import org.apache.streampipes.client.model.StreamPipesClientConnectionConfig; import org.apache.streampipes.client.paths.ApiPath; import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.dataformat.cbor.CborDataFormatFactory; +import org.apache.streampipes.dataformat.fst.FstDataFormatFactory; +import org.apache.streampipes.dataformat.json.JsonDataFormatFactory; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; import org.apache.streampipes.model.mail.SpEmail; @@ -49,6 +52,9 @@ public class StreamPipesClient implements private StreamPipesClient(ClientConnectionUrlResolver connectionConfig) { this.config = new StreamPipesClientConfig(connectionConfig); + this.registerDataFormat(new JsonDataFormatFactory()); + this.registerDataFormat(new FstDataFormatFactory()); + this.registerDataFormat(new CborDataFormatFactory()); } private StreamPipesClient(String streamPipesHost, @@ -122,12 +128,11 @@ public static StreamPipesClient create(String streamPipesHost, /** * Register a new data format that is used by the live API * - * @deprecated * @param spDataFormatFactory The data format factory */ - @Deprecated(forRemoval = true, since = "0.97.0") @Override public void registerDataFormat(SpDataFormatFactory spDataFormatFactory) { + this.config.addDataFormat(spDataFormatFactory); } @Override diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index 2e87e1aacd..fa80926a46 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -29,6 +29,16 @@ public FileApi(StreamPipesClientConfig clientConfig) { super(clientConfig); } + + /** + * @deprecated As of release 0.95.0, replaced by {@link #getFileContent(String)}. + * The parameter isOriginalFileName is not used anymore. + */ + @Deprecated(since = "0.95.0", forRemoval = true) + public byte[] getFileContent(String filename, boolean isOriginalFileName) { + return this.getFileContent(filename); + } + @Override public byte[] getFileContent(String filename) { return new BinaryGetRequest( diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/live/ProducerManager.java b/streampipes-client/src/main/java/org/apache/streampipes/client/live/ProducerManager.java index 1452d6cfd1..6cf97026d1 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/live/ProducerManager.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/live/ProducerManager.java @@ -19,6 +19,7 @@ package org.apache.streampipes.client.live; import org.apache.streampipes.client.api.live.IConfiguredEventProducer; +import org.apache.streampipes.dataformat.SpDataFormatDefinition; import org.apache.streampipes.dataformat.SpDataFormatManager; import org.apache.streampipes.messaging.EventProducer; import org.apache.streampipes.messaging.SpProtocolManager; @@ -38,7 +39,7 @@ public IConfiguredEventProducer makeProducer() { return new ConfiguredEventProducer( producer, - SpDataFormatManager.getFormatDefinition() + findFormatDefinition() ); } @@ -50,4 +51,12 @@ private EventProducer findProducer() { .orElseThrow() .getProducer(protocol); } + + private SpDataFormatDefinition findFormatDefinition() { + var format = grounding.getTransportFormats().get(0); + return SpDataFormatManager + .INSTANCE + .findDefinition(format) + .orElseThrow(); + } } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/live/SubscriptionManager.java b/streampipes-client/src/main/java/org/apache/streampipes/client/live/SubscriptionManager.java index 947e190f60..961d9d3006 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/live/SubscriptionManager.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/live/SubscriptionManager.java @@ -57,31 +57,40 @@ public SubscriptionManager(IBrokerConfigOverride brokerConfigOverride, } public ISubscription subscribe() { + var formatDefinitionOpt = SpDataFormatManager + .INSTANCE + .findDefinition(this.grounding.getTransportFormats().get(0)); try { SpProtocolDefinition protocolDefinition = findProtocol(getTransportProtocol()); - final SpDataFormatDefinition converter = SpDataFormatManager.getFormatDefinition(); - var protocol = getTransportProtocol(); - if (overrideSettings) { - if (protocol instanceof KafkaTransportProtocol) { - brokerConfigOverride.overrideKafkaHostname((KafkaTransportProtocol) protocol); - } - brokerConfigOverride.overrideHostname(protocol); - brokerConfigOverride.overridePort(protocol); - } + if (formatDefinitionOpt.isPresent()) { + final SpDataFormatDefinition converter = formatDefinitionOpt.get(); - EventConsumer consumer = protocolDefinition.getConsumer(protocol); - consumer.connect(event -> { - try { - Event spEvent = EventFactory.fromMap(converter.toMap(event)); - callback.onEvent(spEvent); - } catch (SpRuntimeException e) { - e.printStackTrace(); + var protocol = getTransportProtocol(); + if (overrideSettings) { + if (protocol instanceof KafkaTransportProtocol) { + brokerConfigOverride.overrideKafkaHostname((KafkaTransportProtocol) protocol); + } + brokerConfigOverride.overrideHostname(protocol); + brokerConfigOverride.overridePort(protocol); } - }); - return new Subscription(consumer); + EventConsumer consumer = protocolDefinition.getConsumer(protocol); + consumer.connect(event -> { + try { + Event spEvent = EventFactory.fromMap(converter.toMap(event)); + callback.onEvent(spEvent); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + }); + + return new Subscription(consumer); + } else { + throw new SpRuntimeException( + "No converter found for data format - did you add a format factory (client.registerDataFormat)?"); + } } catch (NoSuchElementException e) { throw new SpRuntimeException( "Could not find an implementation for messaging protocol " diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/model/StreamPipesClientConfig.java b/streampipes-client/src/main/java/org/apache/streampipes/client/model/StreamPipesClientConfig.java index 88e22d56b5..716ed50b21 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/model/StreamPipesClientConfig.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/model/StreamPipesClientConfig.java @@ -19,6 +19,8 @@ import org.apache.streampipes.client.api.config.ClientConnectionUrlResolver; import org.apache.streampipes.client.api.config.IStreamPipesClientConfig; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.dataformat.SpDataFormatManager; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; import org.apache.streampipes.messaging.SpProtocolManager; import org.apache.streampipes.serializers.json.JacksonSerializer; @@ -40,6 +42,11 @@ public ObjectMapper getSerializer() { return serializer; } + @Override + public void addDataFormat(SpDataFormatFactory spDataFormatFactory) { + SpDataFormatManager.INSTANCE.register(spDataFormatFactory); + } + @Override public void addTransportProtocol(SpProtocolDefinitionFactory protocolDefinitionFactory) { SpProtocolManager.INSTANCE.register(protocolDefinitionFactory); diff --git a/streampipes-commons/pom.xml b/streampipes-commons/pom.xml index 14a40c5df5..4df5ace2fe 100644 --- a/streampipes-commons/pom.xml +++ b/streampipes-commons/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-commons diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java index 960c6d7406..983cc02951 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java @@ -24,6 +24,7 @@ public class DefaultEnvValues { public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client"; public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me"; + public static final int MAX_WAIT_TIME_AT_SHUTDOWN_DEFAULT = 10000; public static final String INSTALL_PIPELINE_ELEMENTS = "true"; public static final String DEFAULT_ENCRYPTION_PASSCODE = "eGgemyGBoILAu3xckoIp"; diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java index 2339475adb..4c58485e27 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java @@ -42,8 +42,6 @@ public enum Envs { SP_CLIENT_USER("SP_CLIENT_USER", DefaultEnvValues.INITIAL_CLIENT_USER_DEFAULT), SP_CLIENT_SECRET("SP_CLIENT_SECRET", DefaultEnvValues.INITIAL_CLIENT_SECRET_DEFAULT), SP_ENCRYPTION_PASSCODE("SP_ENCRYPTION_PASSCODE", DefaultEnvValues.DEFAULT_ENCRYPTION_PASSCODE), - SP_OAUTH_ENABLED("SP_OAUTH_ENABLED", "false"), - SP_OAUTH_REDIRECT_URI("SP_OAUTH_REDIRECT_URI"), SP_DEBUG("SP_DEBUG", "false"), SP_MAX_WAIT_TIME_AT_SHUTDOWN("SP_MAX_WAIT_TIME_AT_SHUTDOWN"), @@ -56,7 +54,6 @@ public enum Envs { // Time Series Storage - SP_TS_STORAGE("SP_TS_STORAGE", "influxdb"), SP_TS_STORAGE_PROTOCOL("SP_TS_STORAGE_PROTOCOL", "http"), SP_TS_STORAGE_HOST("SP_TS_STORAGE_HOST", "influxdb", DefaultEnvValues.LOCALHOST), SP_TS_STORAGE_PORT("SP_TS_STORAGE_PORT", "8086"), @@ -66,10 +63,6 @@ public enum Envs { SP_TS_STORAGE_ORG("SP_TS_STORAGE_ORG", "sp"), SP_TS_STORAGE_BUCKET("SP_TS_STORAGE_BUCKET", "sp"), - SP_TS_STORAGE_IOT_DB_SESSION_POOL_SIZE("SP_TS_STORAGE_IOT_DB_SESSION_POOL_SIZE", "10"), - SP_TS_STORAGE_IOT_DB_SESSION_POOL_ENABLE_COMPRESSION("SP_TS_STORAGE_IOT_DB_SESSION_POOL_ENABLE_COMPRESSION", "false"), - SP_TS_STORAGE_IOT_DB_USER("SP_TS_STORAGE_IOT_DB_USER", "root"), - SP_TS_STORAGE_IOT_DB_PASSWORD("SP_TS_STORAGE_IOT_DB_PASSWORD", "root"), SP_FLINK_JAR_FILE_LOC( "SP_FLINK_JAR_FILE_LOC", @@ -104,36 +97,7 @@ public enum Envs { SP_NATS_HOST("SP_NATS_HOST", "nats"), SP_NATS_PORT("SP_NATS_PORT", "4222"), - SP_PULSAR_URL("SP_PULSAR_URL", "pulsar://localhost:6650"), - - // expects a comma separated string of service names - SP_SERVICE_TAGS("SP_SERVICE_TAGS", ""), - SP_ALLOWED_UPLOAD_FILETYPES("SP_ALLOWED_UPLOAD_FILETYPES", "", ""), - - // OPC-UA security - SP_OPCUA_SECURITY_DIR("SP_OPCUA_SECURITY_DIR", "/streampipes-security/opcua"), - SP_OPCUA_KEYSTORE_FILE("SP_OPCUA_KEYSTORE_FILE", "keystore.pfx"), - SP_OPCUA_KEYSTORE_PASSWORD("SP_OPCUA_KEYSTORE_PASSWORD", "password"), - SP_OPCUA_KEYSTORE_TYPE("SP_OPCUA_KEYSTORE_TYPE", "PKCS12"), - SP_OPCUA_KEYSTORE_ALIAS("SP_OPCUA_KEYSTORE_ALIAS", "apache-streampipes"), - SP_OPCUA_APPLICATION_URI( - "SP_OPCUA_APPLICATION_URI", - "urn:org:apache:streampipes:opcua:client" - ), - - // Default keystore and truststore - SP_SECURITY_KEYSTORE_FILENAME( - "SP_SECURITY_KEYSTORE_FILENAME", - "/streampipes-security/keystore.pfx"), - SP_SECURITY_KEYSTORE_PASSWORD("SP_SECURITY_KEYSTORE_PASSWORD", ""), - SP_SECURITY_KEYSTORE_TYPE("SP_SECURITY_KEYSTORE_TYPE", "PKCS12"), - SP_SECURITY_KEY_PASSWORD("SP_SECURITY_KEY_PASSWORD", null), - SP_SECURITY_TRUSTSTORE_FILENAME( - "SP_SECURITY_TRUSTSTORE_FILENAME", - "/streampipes-security/truststore.pfx"), - SP_SECURITY_TRUSTSTORE_PASSWORD("SP_SECURITY_TRUSTSTORE_PASSWORD", ""), - SP_SECURITY_TRUSTSTORE_TYPE("SP_SECURITY_TRUSTSTORE_TYPE", "PKCS12"), - SP_SECURITY_ALLOW_SELFSIGNED("SP_SECURITY_ALLOW_SELFSIGNED", "false"); + SP_PULSAR_URL("SP_PULSAR_URL", "pulsar://localhost:6650"); private final String envVariableName; private String defaultValue; diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/GlobalStreamPipesConstants.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/GlobalStreamPipesConstants.java index 409a9a9090..c52eb6778b 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/GlobalStreamPipesConstants.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/GlobalStreamPipesConstants.java @@ -22,7 +22,8 @@ public class GlobalStreamPipesConstants { public static final String STD_ICON_NAME = "icon.png"; public static final String STD_DOCUMENTATION_NAME = "documentation.md"; + public static final String CONNECT_MASTER_SOURCES_ENDPOINT = "/streampipes-backend/api/v2/connect/master/sources"; + public static final String INTERNAL_TOPIC_PREFIX = "org-apache-streampipes-internal-"; - public static final String CONNECT_TOPIC_PREFIX = "org.apache.streampipes.connect."; } diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java index 3434924e3e..5e613e0909 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java @@ -19,14 +19,10 @@ package org.apache.streampipes.commons.environment; import org.apache.streampipes.commons.constants.Envs; -import org.apache.streampipes.commons.environment.model.OAuthConfiguration; -import org.apache.streampipes.commons.environment.parser.OAuthConfigurationParser; import org.apache.streampipes.commons.environment.variable.BooleanEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.IntEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.StringEnvironmentVariable; -import java.util.List; - public class DefaultEnvironment implements Environment { @Override @@ -59,11 +55,6 @@ public IntEnvironmentVariable getSpCorePort() { return new IntEnvironmentVariable(Envs.SP_CORE_PORT); } - @Override - public StringEnvironmentVariable getTsStorage() { - return new StringEnvironmentVariable(Envs.SP_TS_STORAGE); - } - @Override public StringEnvironmentVariable getTsStorageProtocol() { return new StringEnvironmentVariable(Envs.SP_TS_STORAGE_PROTOCOL); @@ -94,26 +85,6 @@ public StringEnvironmentVariable getTsStorageBucket() { return new StringEnvironmentVariable(Envs.SP_TS_STORAGE_BUCKET); } - @Override - public IntEnvironmentVariable getIotDbSessionPoolSize(){ - return new IntEnvironmentVariable(Envs.SP_TS_STORAGE_IOT_DB_SESSION_POOL_SIZE); - } - - @Override - public BooleanEnvironmentVariable getIotDbSessionEnableCompression(){ - return new BooleanEnvironmentVariable(Envs.SP_TS_STORAGE_IOT_DB_SESSION_POOL_ENABLE_COMPRESSION); - } - - @Override - public StringEnvironmentVariable getIotDbUser(){ - return new StringEnvironmentVariable(Envs.SP_TS_STORAGE_IOT_DB_USER); - } - - @Override - public StringEnvironmentVariable getIotDbPassword(){ - return new StringEnvironmentVariable(Envs.SP_TS_STORAGE_IOT_DB_PASSWORD); - } - @Override public StringEnvironmentVariable getCouchDbProtocol() { return new StringEnvironmentVariable(Envs.SP_COUCHDB_PROTOCOL); @@ -178,21 +149,6 @@ public StringEnvironmentVariable getEncryptionPasscode() { return new StringEnvironmentVariable(Envs.SP_ENCRYPTION_PASSCODE); } - @Override - public BooleanEnvironmentVariable getOAuthEnabled() { - return new BooleanEnvironmentVariable(Envs.SP_OAUTH_ENABLED); - } - - @Override - public StringEnvironmentVariable getOAuthRedirectUri() { - return new StringEnvironmentVariable(Envs.SP_OAUTH_REDIRECT_URI); - } - - @Override - public List getOAuthConfigurations() { - return new OAuthConfigurationParser().parse(System.getenv()); - } - @Override public StringEnvironmentVariable getKafkaRetentionTimeMs() { return new StringEnvironmentVariable(Envs.SP_KAFKA_RETENTION_MS); @@ -318,83 +274,4 @@ public StringEnvironmentVariable getPulsarUrl() { return new StringEnvironmentVariable(Envs.SP_PULSAR_URL); } - @Override - public StringEnvironmentVariable getCustomServiceTags() { - return new StringEnvironmentVariable(Envs.SP_SERVICE_TAGS); - } - - @Override - public StringEnvironmentVariable getAllowedUploadFiletypes() { - return new StringEnvironmentVariable(Envs.SP_ALLOWED_UPLOAD_FILETYPES); - } - - @Override - public StringEnvironmentVariable getOpcUaSecurityDir() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_SECURITY_DIR); - } - - @Override - public StringEnvironmentVariable getOpcUaKeystoreFile() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_KEYSTORE_FILE); - } - - @Override - public StringEnvironmentVariable getOpcUaKeystorePassword() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_KEYSTORE_PASSWORD); - } - - @Override - public StringEnvironmentVariable getOpcUaApplicationUri() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_APPLICATION_URI); - } - - @Override - public StringEnvironmentVariable getOpcUaKeystoreType() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_KEYSTORE_TYPE); - } - - @Override - public StringEnvironmentVariable getOpcUaKeystoreAlias() { - return new StringEnvironmentVariable(Envs.SP_OPCUA_KEYSTORE_ALIAS); - } - - @Override - public StringEnvironmentVariable getKeystoreFilename() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_KEYSTORE_FILENAME); - } - - @Override - public StringEnvironmentVariable getKeystorePassword() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_KEYSTORE_PASSWORD); - } - - @Override - public StringEnvironmentVariable getKeystoreType() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_KEYSTORE_TYPE); - } - - @Override - public StringEnvironmentVariable getKeyPassword() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_KEY_PASSWORD); - } - - @Override - public StringEnvironmentVariable getTruststoreFilename() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_TRUSTSTORE_FILENAME); - } - - @Override - public StringEnvironmentVariable getTruststorePassword() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_TRUSTSTORE_PASSWORD); - } - - @Override - public StringEnvironmentVariable getTruststoreType() { - return new StringEnvironmentVariable(Envs.SP_SECURITY_TRUSTSTORE_TYPE); - } - - @Override - public BooleanEnvironmentVariable getAllowSelfSignedCertificates() { - return new BooleanEnvironmentVariable(Envs.SP_SECURITY_ALLOW_SELFSIGNED); - } } diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java index d1c4adf6ae..092e6992d6 100644 --- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java @@ -18,13 +18,10 @@ package org.apache.streampipes.commons.environment; -import org.apache.streampipes.commons.environment.model.OAuthConfiguration; import org.apache.streampipes.commons.environment.variable.BooleanEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.IntEnvironmentVariable; import org.apache.streampipes.commons.environment.variable.StringEnvironmentVariable; -import java.util.List; - public interface Environment { BooleanEnvironmentVariable getSpDebug(); @@ -32,56 +29,61 @@ public interface Environment { // Service base configuration StringEnvironmentVariable getServiceHost(); + IntEnvironmentVariable getServicePort(); StringEnvironmentVariable getSpCoreScheme(); - StringEnvironmentVariable getSpCoreHost(); IntEnvironmentVariable getSpCorePort(); // Time series storage env variables - StringEnvironmentVariable getTsStorage(); StringEnvironmentVariable getTsStorageProtocol(); + StringEnvironmentVariable getTsStorageHost(); + IntEnvironmentVariable getTsStoragePort(); + StringEnvironmentVariable getTsStorageToken(); + StringEnvironmentVariable getTsStorageOrg(); - StringEnvironmentVariable getTsStorageBucket(); - IntEnvironmentVariable getIotDbSessionPoolSize(); - BooleanEnvironmentVariable getIotDbSessionEnableCompression(); - StringEnvironmentVariable getIotDbUser(); - StringEnvironmentVariable getIotDbPassword(); + StringEnvironmentVariable getTsStorageBucket(); // CouchDB env variables StringEnvironmentVariable getCouchDbProtocol(); + StringEnvironmentVariable getCouchDbHost(); + IntEnvironmentVariable getCouchDbPort(); + StringEnvironmentVariable getCouchDbUsername(); + StringEnvironmentVariable getCouchDbPassword(); // JWT & Authentication StringEnvironmentVariable getClientUser(); + StringEnvironmentVariable getClientSecret(); StringEnvironmentVariable getJwtSecret(); + StringEnvironmentVariable getJwtPublicKeyLoc(); + StringEnvironmentVariable getJwtPrivateKeyLoc(); + StringEnvironmentVariable getJwtSigningMode(); StringEnvironmentVariable getExtensionsAuthMode(); - StringEnvironmentVariable getEncryptionPasscode(); - BooleanEnvironmentVariable getOAuthEnabled(); - StringEnvironmentVariable getOAuthRedirectUri(); - List getOAuthConfigurations(); + StringEnvironmentVariable getEncryptionPasscode(); // Messaging StringEnvironmentVariable getKafkaRetentionTimeMs(); + StringEnvironmentVariable getPrioritizedProtocol(); @@ -123,36 +125,14 @@ public interface Environment { // Broker defaults StringEnvironmentVariable getKafkaHost(); - IntEnvironmentVariable getKafkaPort(); StringEnvironmentVariable getMqttHost(); - IntEnvironmentVariable getMqttPort(); StringEnvironmentVariable getNatsHost(); - IntEnvironmentVariable getNatsPort(); StringEnvironmentVariable getPulsarUrl(); - StringEnvironmentVariable getCustomServiceTags(); - - StringEnvironmentVariable getAllowedUploadFiletypes(); - - StringEnvironmentVariable getOpcUaSecurityDir(); - StringEnvironmentVariable getOpcUaKeystoreFile(); - StringEnvironmentVariable getOpcUaKeystorePassword(); - StringEnvironmentVariable getOpcUaApplicationUri(); - StringEnvironmentVariable getOpcUaKeystoreType(); - StringEnvironmentVariable getOpcUaKeystoreAlias(); - - StringEnvironmentVariable getKeystoreFilename(); - StringEnvironmentVariable getKeystorePassword(); - StringEnvironmentVariable getKeystoreType(); - StringEnvironmentVariable getKeyPassword(); - StringEnvironmentVariable getTruststoreFilename(); - StringEnvironmentVariable getTruststorePassword(); - StringEnvironmentVariable getTruststoreType(); - BooleanEnvironmentVariable getAllowSelfSignedCertificates(); } diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingFormatException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingFormatException.java new file mode 100644 index 0000000000..75e024b3d3 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingFormatException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoMatchingFormatException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -3381149054836186412L; + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingJsonSchemaException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingJsonSchemaException.java new file mode 100644 index 0000000000..b79ead019b --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingJsonSchemaException.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoMatchingJsonSchemaException extends Exception { +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingProtocolException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingProtocolException.java new file mode 100644 index 0000000000..b72747ebea --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingProtocolException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoMatchingProtocolException extends Exception { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingSchemaException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingSchemaException.java new file mode 100644 index 0000000000..05c1a961f3 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoMatchingSchemaException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoMatchingSchemaException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 1L; + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoSepaInPipelineException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoSepaInPipelineException.java new file mode 100644 index 0000000000..e844968d3e --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoSepaInPipelineException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoSepaInPipelineException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 1L; + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidConnectionException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidConnectionException.java new file mode 100644 index 0000000000..b2fb57c4fb --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidConnectionException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoValidConnectionException extends Exception { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSecTypeException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSecTypeException.java new file mode 100644 index 0000000000..8bffbbf8e0 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSecTypeException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoValidSecTypeException { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepTypeException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepTypeException.java new file mode 100644 index 0000000000..f20417ee10 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepTypeException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoValidSepTypeException { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaStructureException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaStructureException.java new file mode 100644 index 0000000000..7a9d7c5dd2 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaStructureException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoValidSepaStructureException { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaTypeException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaTypeException.java new file mode 100644 index 0000000000..d3712af38c --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/NoValidSepaTypeException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class NoValidSepaTypeException { + +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/RemoteServerNotAccessibleException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/RemoteServerNotAccessibleException.java new file mode 100644 index 0000000000..93c587f450 --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/RemoteServerNotAccessibleException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class RemoteServerNotAccessibleException extends Exception { + + /** + * + */ + private static final long serialVersionUID = 1L; + + private String serverUrl; + + public RemoteServerNotAccessibleException(String message, String serverUrl) { + super(message); + this.serverUrl = serverUrl; + } + + public RemoteServerNotAccessibleException(RemoteServerNotAccessibleException e) { + super(e.getMessage()); + this.serverUrl = e.getServerUrl(); + } + + public String getServerUrl() { + return serverUrl; + } + + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } +} diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/TooManyEdgesException.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/TooManyEdgesException.java new file mode 100644 index 0000000000..5b406f0b1a --- /dev/null +++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/exceptions/TooManyEdgesException.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.commons.exceptions; + +public class TooManyEdgesException extends Exception { + +} diff --git a/streampipes-commons/src/test/java/org/apache/streampipes/commons/file/FileHasherTest.java b/streampipes-commons/src/test/java/org/apache/streampipes/commons/file/FileHasherTest.java index 7b8d1b981d..17c976e675 100644 --- a/streampipes-commons/src/test/java/org/apache/streampipes/commons/file/FileHasherTest.java +++ b/streampipes-commons/src/test/java/org/apache/streampipes/commons/file/FileHasherTest.java @@ -19,6 +19,7 @@ package org.apache.streampipes.commons.file; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.File; @@ -27,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +@Disabled class FileHasherTest { private FileHasher fileHasher; @@ -52,4 +54,4 @@ void hash_throwsIOExceptionForNonExistingFile() { void hash_throwsIOExceptionForNullFile() { assertThrows(IOException.class, () -> fileHasher.hash(null)); } -} \ No newline at end of file +} diff --git a/streampipes-connect-management/pom.xml b/streampipes-connect-management/pom.xml index 62a449371a..38be5fcd83 100644 --- a/streampipes-connect-management/pom.xml +++ b/streampipes-connect-management/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,27 +32,27 @@ org.apache.streampipes streampipes-connect-shared - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-measurement-units - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-service-discovery-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-pipeline-management - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-storage-couchdb - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java index e21d29de0d..a8d07530c7 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/health/AdapterHealthCheck.java @@ -154,7 +154,7 @@ private void updateTotalEventsPublished(AdapterMetrics adapterMetrics, String ad */ public Map getAllAdaptersSupposedToRun() { Map result = new HashMap<>(); - List allRunningInstancesAdapterDescription = this.adapterStorage.findAll(); + List allRunningInstancesAdapterDescription = this.adapterStorage.getAllAdapters(); allRunningInstancesAdapterDescription .stream() .filter(AdapterDescription::isRunning) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java index 5c42478431..ec87909021 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMasterManagement.java @@ -23,7 +23,7 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.commons.prometheus.adapter.AdapterMetrics; import org.apache.streampipes.connect.management.util.GroundingUtils; -import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; +import org.apache.streampipes.connect.management.util.WorkerPaths; import org.apache.streampipes.manager.monitoring.pipeline.ExtensionsLogProvider; import org.apache.streampipes.manager.verification.DataStreamVerifier; import org.apache.streampipes.model.SpDataStream; @@ -32,11 +32,12 @@ import org.apache.streampipes.resource.management.AdapterResourceManager; import org.apache.streampipes.resource.management.DataStreamResourceManager; import org.apache.streampipes.storage.api.IAdapterStorage; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; +import org.apache.streampipes.storage.management.StorageDispatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.URISyntaxException; import java.util.List; import java.util.NoSuchElementException; @@ -53,56 +54,46 @@ public class AdapterMasterManagement { private final DataStreamResourceManager dataStreamResourceManager; - public AdapterMasterManagement( - IAdapterStorage adapterInstanceStorage, - AdapterResourceManager adapterResourceManager, - DataStreamResourceManager dataStreamResourceManager, - AdapterMetrics adapterMetrics + public AdapterMasterManagement(IAdapterStorage adapterStorage, + AdapterResourceManager adapterResourceManager, + DataStreamResourceManager dataStreamResourceManager, + AdapterMetrics adapterMetrics ) { - this.adapterInstanceStorage = adapterInstanceStorage; + this.adapterInstanceStorage = adapterStorage; this.adapterMetrics = adapterMetrics; this.adapterResourceManager = adapterResourceManager; this.dataStreamResourceManager = dataStreamResourceManager; } - public void addAdapter( - AdapterDescription adapterDescription, - String adapterId, - String principalSid - ) + public String addAdapter(AdapterDescription ad, + String principalSid) throws AdapterException { - // Create elementId for datastream + // Create elementId for adapter var dataStreamElementId = ElementIdGenerator.makeElementId(SpDataStream.class); - adapterDescription.setElementId(adapterId); - adapterDescription.setCreatedAt(System.currentTimeMillis()); - adapterDescription.setCorrespondingDataStreamElementId(dataStreamElementId); + ad.setElementId(ElementIdGenerator.makeElementId(ad)); + ad.setCreatedAt(System.currentTimeMillis()); + ad.setCorrespondingDataStreamElementId(dataStreamElementId); // Add EventGrounding to AdapterDescription var eventGrounding = GroundingUtils.createEventGrounding(); - adapterDescription.setEventGrounding(eventGrounding); + ad.setEventGrounding(eventGrounding); - this.adapterResourceManager.encryptAndCreate(adapterDescription); + var elementId = this.adapterResourceManager.encryptAndCreate(ad); - // Stream is only created if the adpater is successfully stored - createDataStreamForAdapter(adapterDescription, adapterId, dataStreamElementId, principalSid); - } - - private void createDataStreamForAdapter( - AdapterDescription adapterDescription, - String adapterId, - String streamId, - String principalSid - ) throws AdapterException { - var storedDescription = new SourcesManagement() - .createAdapterDataStream(adapterDescription, streamId); - storedDescription.setCorrespondingAdapterId(adapterId); + // Create stream + var storedDescription = new SourcesManagement().createAdapterDataStream(ad, dataStreamElementId); + storedDescription.setCorrespondingAdapterId(elementId); installDataSource(storedDescription, principalSid, true); - LOG.info("Install source (source URL: {} in backend", adapterDescription.getElementId()); + LOG.info("Install source (source URL: {} in backend", ad.getElementId()); + + return ad.getElementId(); } + + public AdapterDescription getAdapter(String elementId) throws AdapterException { - List allAdapters = adapterInstanceStorage.findAll(); + List allAdapters = adapterInstanceStorage.getAllAdapters(); if (allAdapters != null && elementId != null) { for (AdapterDescription ad : allAdapters) { @@ -130,7 +121,7 @@ public void deleteAdapter(String elementId) throws AdapterException { LOG.info("Could not stop adapter: " + elementId, e); } - AdapterDescription adapter = adapterInstanceStorage.getElementById(elementId); + AdapterDescription adapter = adapterInstanceStorage.getAdapter(elementId); // Delete adapter adapterResourceManager.delete(elementId); ExtensionsLogProvider.INSTANCE.remove(elementId); @@ -143,7 +134,7 @@ public void deleteAdapter(String elementId) throws AdapterException { public List getAllAdapterInstances() throws AdapterException { - List allAdapters = adapterInstanceStorage.findAll(); + List allAdapters = adapterInstanceStorage.getAllAdapters(); if (allAdapters == null) { throw new AdapterException("Could not get all adapters"); @@ -153,7 +144,7 @@ public List getAllAdapterInstances() throws AdapterException } public void stopStreamAdapter(String elementId) throws AdapterException { - AdapterDescription ad = adapterInstanceStorage.getElementById(elementId); + AdapterDescription ad = adapterInstanceStorage.getAdapter(elementId); WorkerRestClient.stopStreamAdapter(ad.getSelectedEndpointUrl(), ad); ExtensionsLogProvider.INSTANCE.reset(elementId); @@ -169,20 +160,15 @@ public void stopStreamAdapter(String elementId) throws AdapterException { public void startStreamAdapter(String elementId) throws AdapterException { - var ad = adapterInstanceStorage.getElementById(elementId); + var ad = adapterInstanceStorage.getAdapter(elementId); try { // Find endpoint to start adapter on - var baseUrl = new ExtensionsServiceEndpointGenerator().getEndpointBaseUrl( - ad.getAppId(), - SpServiceUrlProvider.ADAPTER, - ad.getDeploymentConfiguration() - .getDesiredServiceTags() - ); + var baseUrl = WorkerPaths.findEndpointUrl(ad.getAppId()); // Update selected endpoint URL of adapter ad.setSelectedEndpointUrl(baseUrl); - adapterInstanceStorage.updateElement(ad); + adapterInstanceStorage.updateAdapter(ad); // Invoke adapter instance WorkerRestClient.invokeStreamAdapter(baseUrl, elementId); @@ -191,16 +177,14 @@ public void startStreamAdapter(String elementId) throws AdapterException { adapterMetrics.register(ad.getElementId(), ad.getName()); LOG.info("Started adapter " + elementId + " on: " + baseUrl); - } catch (NoServiceEndpointsAvailableException e) { + } catch (NoServiceEndpointsAvailableException | URISyntaxException e) { throw new AdapterException("Could not start adapter due to unavailable service endpoint", e); } } - private void installDataSource( - SpDataStream stream, - String principalSid, - boolean publicElement - ) throws AdapterException { + private void installDataSource(SpDataStream stream, + String principalSid, + boolean publicElement) throws AdapterException { try { new DataStreamVerifier(stream).verifyAndAdd(principalSid, publicElement); } catch (SepaParseException e) { @@ -208,4 +192,8 @@ private void installDataSource( throw new AdapterException(); } } + + private IAdapterStorage getAdapterInstanceStorage() { + return StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage(); + } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java index b24e6494c7..26f158a5b7 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterMigrationManager.java @@ -22,7 +22,6 @@ import org.apache.streampipes.manager.migration.AbstractMigrationManager; import org.apache.streampipes.manager.migration.IMigrationHandler; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceRegistration; -import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.migration.ModelMigratorConfig; import org.apache.streampipes.storage.api.IAdapterStorage; @@ -37,12 +36,9 @@ public class AdapterMigrationManager extends AbstractMigrationManager implements private static final Logger LOG = LoggerFactory.getLogger(AdapterMigrationManager.class); private final IAdapterStorage adapterStorage; - private final IAdapterStorage adapterDescriptionStorage; - public AdapterMigrationManager(IAdapterStorage adapterStorage, - IAdapterStorage adapterDescriptionStorage) { + public AdapterMigrationManager(IAdapterStorage adapterStorage) { this.adapterStorage = adapterStorage; - this.adapterDescriptionStorage = adapterDescriptionStorage; } @Override @@ -87,7 +83,7 @@ public void handleMigrations(SpServiceRegistration extensionsServiceConfig, "Migration was performed by extensions service '{}'", extensionsServiceConfig.getServiceUrl()); - adapterStorage.updateElement(migrationResult.element()); + adapterStorage.updateAdapter(migrationResult.element()); LOG.info("Adapter description is updated - Migration successfully completed at Core."); } else { LOG.error("Migration failed with the following reason: {}", migrationResult.message()); @@ -114,9 +110,4 @@ public void handleMigrations(SpServiceRegistration extensionsServiceConfig, } } } - - @Override - protected boolean isInstalled(SpServiceTagPrefix modelType, String appId) { - return !adapterDescriptionStorage.getAdaptersByAppId(appId).isEmpty(); - } } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java index b64b661495..0b2b38a6ef 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/AdapterUpdateManagement.java @@ -19,8 +19,8 @@ package org.apache.streampipes.connect.management.management; import org.apache.streampipes.commons.exceptions.connect.AdapterException; -import org.apache.streampipes.manager.execution.PipelineExecutor; import org.apache.streampipes.manager.matching.PipelineVerificationHandlerV2; +import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.manager.pipeline.PipelineManager; import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.NamedStreamPipesEntity; @@ -77,23 +77,23 @@ public void updateAdapter(AdapterDescription ad) affectedPipelines.forEach(p -> { var shouldRestartPipeline = p.isRunning(); if (shouldRestartPipeline) { - new PipelineExecutor(p).stopPipeline(true); + Operations.stopPipeline(p, true); } var storedPipeline = PipelineManager.getPipeline(p.getPipelineId()); var pipeline = applyUpdatedDataStream(storedPipeline, ad); try { - var modificationMessage = new PipelineVerificationHandlerV2(pipeline).verifyPipeline(); + var modificationMessage = Operations.validatePipeline(pipeline); var updateInfo = makeUpdateInfo(modificationMessage, pipeline); - var modifiedPipeline = new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline().pipeline(); + var modifiedPipeline = new PipelineVerificationHandlerV2(pipeline).makeModifiedPipeline(); var canAutoMigrate = canAutoMigrate(modificationMessage); if (!canAutoMigrate) { modifiedPipeline.setHealthStatus(PipelineHealthStatus.REQUIRES_ATTENTION); modifiedPipeline.setPipelineNotifications(toNotification(updateInfo)); modifiedPipeline.setValid(false); } - StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineStorageAPI().updateElement(modifiedPipeline); + StorageDispatcher.INSTANCE.getNoSqlStore().getPipelineStorageAPI().updatePipeline(modifiedPipeline); if (shouldRestartPipeline && canAutoMigrate) { - new PipelineExecutor(PipelineManager.getPipeline(p.getPipelineId())).startPipeline(); + Operations.startPipeline(PipelineManager.getPipeline(p.getPipelineId())); } } catch (Exception e) { LOG.error("Could not update pipeline {}", pipeline.getName(), e); @@ -113,7 +113,7 @@ public List checkPipelineMigrations(AdapterDescription adapt affectedPipelines.forEach(pipeline -> { var updatedPipeline = applyUpdatedDataStream(pipeline, adapterDescription); try { - var modificationMessage = new PipelineVerificationHandlerV2(updatedPipeline).verifyPipeline(); + var modificationMessage = Operations.validatePipeline(updatedPipeline); var updateInfo = makeUpdateInfo(modificationMessage, updatedPipeline); updateInfos.add(updateInfo); } catch (Exception e) { diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/DescriptionManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/DescriptionManagement.java index 49306186b3..661abfaa16 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/DescriptionManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/DescriptionManagement.java @@ -32,7 +32,7 @@ public class DescriptionManagement { public List getAdapters() { IAdapterStorage adapterStorage = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage(); - return adapterStorage.findAll(); + return adapterStorage.getAllAdapters(); } public Optional getAdapter(String id) { @@ -43,9 +43,9 @@ public Optional getAdapter(String id) { public void deleteAdapterDescription(String id) throws SpRuntimeException { var adapterStorage = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage(); - var adapter = adapterStorage.getElementById(id); + var adapter = adapterStorage.getAdapter(id); if (!isAdapterUsed(adapter)) { - adapterStorage.deleteElementById(id); + adapterStorage.deleteAdapter(id); } else { throw new SpRuntimeException("This adapter is used by an existing instance and cannot be deleted"); } @@ -64,7 +64,7 @@ public String getDocumentationAsset(String baseUrl) throws AdapterException { } private boolean isAdapterUsed(AdapterDescription adapter) { - var allAdapters = StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage().findAll(); + var allAdapters = StorageDispatcher.INSTANCE.getNoSqlStore().getAdapterInstanceStorage().getAllAdapters(); return allAdapters .stream() diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java index b6dfbce037..3e1f546907 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/GuessManagement.java @@ -23,17 +23,13 @@ import org.apache.streampipes.connect.management.AdapterEventPreviewPipeline; import org.apache.streampipes.connect.management.util.WorkerPaths; import org.apache.streampipes.extensions.api.connect.exception.WorkerAdapterException; -import org.apache.streampipes.manager.api.extensions.IExtensionsServiceEndpointGenerator; import org.apache.streampipes.manager.execution.ExtensionServiceExecutions; -import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.guess.AdapterEventPreview; import org.apache.streampipes.model.connect.guess.GuessSchema; -import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTag; import org.apache.streampipes.model.monitoring.SpLogMessage; import org.apache.streampipes.resource.management.secret.SecretProvider; import org.apache.streampipes.serializers.json.JacksonSerializer; -import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -44,26 +40,22 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Set; public class GuessManagement { private static final Logger LOG = LoggerFactory.getLogger(GuessManagement.class); - private final IExtensionsServiceEndpointGenerator endpointGenerator; + private final WorkerUrlProvider workerUrlProvider; private final ObjectMapper objectMapper; public GuessManagement() { - this.endpointGenerator = new ExtensionsServiceEndpointGenerator(); + this.workerUrlProvider = new WorkerUrlProvider(); this.objectMapper = JacksonSerializer.getObjectMapper(); } public GuessSchema guessSchema(AdapterDescription adapterDescription) throws ParseException, WorkerAdapterException, NoServiceEndpointsAvailableException, IOException { + var workerUrl = getWorkerUrl(adapterDescription.getAppId()); SecretProvider.getDecryptionService().apply(adapterDescription); - var workerUrl = getWorkerUrl( - adapterDescription.getAppId(), - adapterDescription.getDeploymentConfiguration().getDesiredServiceTags() - ); var description = objectMapper.writeValueAsString(adapterDescription); LOG.info("Guess schema at: " + workerUrl); @@ -82,9 +74,8 @@ public GuessSchema guessSchema(AdapterDescription adapterDescription) } } - private String getWorkerUrl(String appId, - Set customServiceTags) throws NoServiceEndpointsAvailableException { - var baseUrl = endpointGenerator.getEndpointBaseUrl(appId, SpServiceUrlProvider.ADAPTER, customServiceTags); + private String getWorkerUrl(String appId) throws NoServiceEndpointsAvailableException { + var baseUrl = workerUrlProvider.getWorkerBaseUrl(appId); return baseUrl + WorkerPaths.getGuessSchemaPath(); } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/UnitMasterManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/UnitMasterManagement.java index b0f992031b..a9169f212b 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/UnitMasterManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/UnitMasterManagement.java @@ -68,22 +68,4 @@ public String getFittingUnits(UnitDescription unitDescription) throws AdapterExc return gson.toJson(unitDescriptionList); } - public List getAllUnitDescriptions(){ - List unitDescriptionList = new LinkedList<>(); - - List units = UnitProvider.INSTANCE.getAvailableUnits(); - - for (Unit unit : units) { - try { - UnitDescription unitDescriptionTmp = - new UnitDescription(unit.getResource().toString(), unit.getLabel()); - unitDescriptionList.add(unitDescriptionTmp); - } catch (NullPointerException e) { - logger.error("Unit has no resource and/or Label"); - } - } - - return unitDescriptionList; - } - } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java index 6cdad54c6b..e058c40cbf 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerAdministrationManagement.java @@ -91,7 +91,7 @@ public void checkAndRestore(int retryCount) { } public void performAdapterMigrations(List tags) { - var installedAdapters = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage().findAll(); + var installedAdapters = CouchDbStorageManager.INSTANCE.getAdapterDescriptionStorage().getAllAdapters(); var adminSid = new SpResourceManager().manageUsers().getAdminUser().getPrincipalId(); installedAdapters.stream() .filter(adapter -> tags.stream().anyMatch(tag -> tag.getValue().equals(adapter.getAppId()))) diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java index 854c806a93..c715459070 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerRestClient.java @@ -52,10 +52,10 @@ public class WorkerRestClient { private static final Logger LOG = LoggerFactory.getLogger(WorkerRestClient.class); - public static void invokeStreamAdapter(String baseUrl, + public static void invokeStreamAdapter(String endpointUrl, String elementId) throws AdapterException { var adapterStreamDescription = getAndDecryptAdapter(elementId); - var url = baseUrl + WorkerPaths.getStreamInvokePath(); + var url = endpointUrl + WorkerPaths.getStreamInvokePath(); startAdapter(url, adapterStreamDescription); updateStreamAdapterStatus(adapterStreamDescription.getElementId(), true); @@ -127,11 +127,11 @@ private static HttpResponse triggerPost(String url, return request.execute().returnResponse(); } - public static RuntimeOptionsResponse getConfiguration(String baseUrl, + public static RuntimeOptionsResponse getConfiguration(String workerEndpoint, String appId, RuntimeOptionsRequest runtimeOptionsRequest) throws AdapterException, SpConfigurationException { - String url = baseUrl + WorkerPaths.getRuntimeResolvablePath(appId); + String url = workerEndpoint + WorkerPaths.getRuntimeResolvablePath(appId); try { String payload = JacksonSerializer.getObjectMapper().writeValueAsString(runtimeOptionsRequest); @@ -199,7 +199,7 @@ public static String getDocumentationAsset(String baseUrl) throws AdapterExcepti private static AdapterDescription getAdapterDescriptionById(AdapterInstanceStorageImpl adapterStorage, String id) { AdapterDescription adapterDescription = null; - List allAdapters = adapterStorage.findAll(); + List allAdapters = adapterStorage.getAllAdapters(); for (AdapterDescription a : allAdapters) { if (a.getElementId().endsWith(id)) { adapterDescription = a; @@ -219,11 +219,11 @@ private static void updateStreamAdapterStatus(String adapterId, private static void encryptAndUpdateAdapter(AdapterDescription adapter) { AdapterDescription encryptedDescription = new Cloner().adapterDescription(adapter); SecretProvider.getEncryptionService().apply(encryptedDescription); - getAdapterStorage().updateElement(encryptedDescription); + getAdapterStorage().updateAdapter(encryptedDescription); } private static AdapterDescription getAndDecryptAdapter(String adapterId) { - AdapterDescription adapter = getAdapterStorage().getElementById(adapterId); + AdapterDescription adapter = getAdapterStorage().getAdapter(adapterId); SecretProvider.getDecryptionService().apply(adapter); return adapter; } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java new file mode 100644 index 0000000000..b9fc970ab4 --- /dev/null +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/management/WorkerUrlProvider.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.connect.management.management; + +import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; + +public class WorkerUrlProvider { + + + public WorkerUrlProvider() { + } + + public String getWorkerUrl(String appId) throws NoServiceEndpointsAvailableException { + return getEndpointGenerator(appId).getEndpointResourceUrl(); + } + + public String getWorkerBaseUrl(String appId) throws NoServiceEndpointsAvailableException { + return getEndpointGenerator(appId).getEndpointBaseUrl(); + } + + private ExtensionsServiceEndpointGenerator getEndpointGenerator(String appId) { + return new ExtensionsServiceEndpointGenerator(appId, SpServiceUrlProvider.ADAPTER); + } + +} diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java index 92870f490c..74c771358e 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/GroundingUtils.java @@ -29,6 +29,7 @@ import org.apache.streampipes.model.grounding.TopicDefinition; import org.apache.streampipes.model.grounding.TransportProtocol; +import java.util.Collections; import java.util.UUID; public class GroundingUtils { @@ -81,6 +82,9 @@ public static EventGrounding createEventGrounding() { ); } + eventGrounding.setTransportFormats(Collections + .singletonList(TransportFormatGenerator.getTransportFormat())); + return eventGrounding; } diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java new file mode 100644 index 0000000000..7038866af5 --- /dev/null +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/TransportFormatGenerator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.connect.management.util; + +import org.apache.streampipes.model.configuration.SpDataFormat; +import org.apache.streampipes.model.grounding.TransportFormat; +import org.apache.streampipes.sdk.helpers.SupportedFormats; +import org.apache.streampipes.vocabulary.MessageFormat; + +import java.util.Arrays; +import java.util.List; + +public class TransportFormatGenerator { + + public static TransportFormat getTransportFormat() { + var cfg = Utils.getCoreConfigStorage().get(); + List supportedFormats = + cfg.getMessagingSettings().getPrioritizedFormats(); + + if (supportedFormats.size() > 0) { + return new TransportFormat(supportedFormats.get(0).getMessageFormat()); + } else { + return new TransportFormat(MessageFormat.JSON); + } + } + + public static List getAllFormats() { + return Arrays.asList(SupportedFormats.cborFormat(), + SupportedFormats.jsonFormat(), + SupportedFormats.fstFormat(), + SupportedFormats.smileFormat()); + } +} diff --git a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java index 7bd2b0c1ed..29d89ebd1b 100644 --- a/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java +++ b/streampipes-connect-management/src/main/java/org/apache/streampipes/connect/management/util/WorkerPaths.java @@ -17,6 +17,13 @@ */ package org.apache.streampipes.connect.management.util; +import org.apache.streampipes.commons.exceptions.NoServiceEndpointsAvailableException; +import org.apache.streampipes.manager.execution.endpoint.ExtensionsServiceEndpointGenerator; +import org.apache.streampipes.svcdiscovery.api.model.SpServiceUrlProvider; + +import java.net.URI; +import java.net.URISyntaxException; + public class WorkerPaths { private static final String WorkerMainPath = "/api/v1/worker"; @@ -29,6 +36,14 @@ public static String getStreamStopPath() { return WorkerMainPath + "/stream/stop"; } + public static String getSetInvokePath() { + return WorkerMainPath + "/set/invoke"; + } + + public static String getSetStopPath() { + return WorkerMainPath + "/set/stop"; + } + public static String getRunningAdaptersPath() { return WorkerMainPath + "/running"; } @@ -41,4 +56,12 @@ public static String getGuessSchemaPath() { return WorkerMainPath + "/guess/schema"; } + public static String findEndpointUrl(String appId) throws NoServiceEndpointsAvailableException, URISyntaxException { + SpServiceUrlProvider serviceUrlProvider = SpServiceUrlProvider.ADAPTER; + String endpointUrl = new ExtensionsServiceEndpointGenerator(appId, serviceUrlProvider).getEndpointResourceUrl(); + URI uri = new URI(endpointUrl); + return uri.getScheme() + "://" + uri.getAuthority(); + } + + } diff --git a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java index 47612f1361..f4c3a3c2d5 100644 --- a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java +++ b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/health/AdapterHealthCheckTest.java @@ -44,7 +44,7 @@ public void setUp() { @Test public void getAllRunningInstancesAdapterDescriptionsEmpty() { - when(adapterInstanceStorageMock.findAll()).thenReturn(List.of()); + when(adapterInstanceStorageMock.getAllAdapters()).thenReturn(List.of()); var healthCheck = new AdapterHealthCheck( adapterInstanceStorageMock, @@ -74,7 +74,7 @@ public void getAllRunningInstancesAdapterDescriptionsMixed() { runningAdapter.setElementId(nameRunningAdapter); runningAdapter.setRunning(true); - when(adapterInstanceStorageMock.findAll()).thenReturn(List.of(stoppedAdapter, runningAdapter)); + when(adapterInstanceStorageMock.getAllAdapters()).thenReturn(List.of(stoppedAdapter, runningAdapter)); var healthCheck = new AdapterHealthCheck( adapterInstanceStorageMock, diff --git a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/management/AdapterMasterManagementTest.java b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/management/AdapterMasterManagementTest.java index e1e18324ce..ae18017699 100644 --- a/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/management/AdapterMasterManagementTest.java +++ b/streampipes-connect-management/src/test/java/org/apache/streampipes/connect/management/management/AdapterMasterManagementTest.java @@ -36,12 +36,12 @@ public class AdapterMasterManagementTest { @Test - public void getAdapter_FailNull() { - var adapterStorage = mock(AdapterInstanceStorageImpl.class); - var resourceManager = mock(AdapterResourceManager.class); - when(adapterStorage.findAll()).thenReturn(null); + public void getAdapterFailNull() { + AdapterInstanceStorageImpl adapterStorage = mock(AdapterInstanceStorageImpl.class); + AdapterResourceManager resourceManager = mock(AdapterResourceManager.class); + when(adapterStorage.getAllAdapters()).thenReturn(null); - var adapterMasterManagement = + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement( adapterStorage, resourceManager, @@ -53,13 +53,13 @@ public void getAdapter_FailNull() { } @Test - public void getAdapter_Fail() { - var adapterDescriptions = List.of(new AdapterDescription()); - var adapterStorage = mock(AdapterInstanceStorageImpl.class); - var resourceManager = mock(AdapterResourceManager.class); - when(adapterStorage.findAll()).thenReturn(adapterDescriptions); + public void getAdapterFail() { + List adapterDescriptions = List.of(new AdapterDescription()); + AdapterInstanceStorageImpl adapterStorage = mock(AdapterInstanceStorageImpl.class); + AdapterResourceManager resourceManager = mock(AdapterResourceManager.class); + when(adapterStorage.getAllAdapters()).thenReturn(adapterDescriptions); - var adapterMasterManagement = + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement( adapterStorage, resourceManager, @@ -71,11 +71,11 @@ public void getAdapter_Fail() { } @Test - public void getAllAdapters_Success() throws AdapterException { - var adapterDescriptions = List.of(new AdapterDescription()); - var adapterStorage = mock(AdapterInstanceStorageImpl.class); - var resourceManager = mock(AdapterResourceManager.class); - when(adapterStorage.findAll()).thenReturn(adapterDescriptions); + public void getAllAdaptersSuccess() throws AdapterException { + List adapterDescriptions = List.of(new AdapterDescription()); + AdapterInstanceStorageImpl adapterStorage = mock(AdapterInstanceStorageImpl.class); + AdapterResourceManager resourceManager = mock(AdapterResourceManager.class); + when(adapterStorage.getAllAdapters()).thenReturn(adapterDescriptions); AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement( @@ -91,12 +91,12 @@ public void getAllAdapters_Success() throws AdapterException { } @Test - public void getAllAdapters_Fail() { - var adapterStorage = mock(AdapterInstanceStorageImpl.class); - var resourceManager = mock(AdapterResourceManager.class); - when(adapterStorage.findAll()).thenReturn(null); + public void getAllAdaptersFail() { + AdapterInstanceStorageImpl adapterStorage = mock(AdapterInstanceStorageImpl.class); + AdapterResourceManager resourceManager = mock(AdapterResourceManager.class); + when(adapterStorage.getAllAdapters()).thenReturn(null); - var adapterMasterManagement = + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement( adapterStorage, resourceManager, diff --git a/streampipes-connect-shared/pom.xml b/streampipes-connect-shared/pom.xml index 6ee984881b..440349cf4d 100644 --- a/streampipes-connect-shared/pom.xml +++ b/streampipes-connect-shared/pom.xml @@ -22,7 +22,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-connect-shared @@ -31,23 +31,23 @@ org.apache.streampipes streampipes-extensions-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-measurement-units - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-sdk - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java index 8c95afb2f5..cdfaea25ce 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java @@ -28,19 +28,6 @@ public class DatatypeUtils { private static final Logger LOG = LoggerFactory.getLogger(DatatypeUtils.class); - /** - * Converts the given value to a specified XSD datatype. - * This method attempts to convert the input value to the target datatype specified by the XSD string. - * It supports conversion to string, double, float, boolean, integer, and long types. - * If the conversion is not possible due to a format mismatch, the original value is returned. - * A number format exception during conversion is logged as an error. - * - * @param value The value to be converted. It can be of any type. - * @param targetDatatypeXsd The target XSD datatype as a string. Supported types are XSD.STRING, - * XSD.DOUBLE, XSD.FLOAT, XSD.BOOLEAN, XSD.INTEGER, and XSD.LONG. - * @return The converted value as an Object. If conversion fails, the original value is returned. - * @throws NumberFormatException if the string does not contain a parsable number for numeric conversions. - */ public static Object convertValue(Object value, String targetDatatypeXsd) { var stringValue = String.valueOf(value); @@ -55,7 +42,7 @@ public static Object convertValue(Object value, } else if (XSD.BOOLEAN.toString().equals(targetDatatypeXsd)) { return Boolean.parseBoolean(stringValue); } else if (XSD.INTEGER.toString().equals(targetDatatypeXsd)) { - var floatingNumber = Double.parseDouble(stringValue); + var floatingNumber = Float.parseFloat(stringValue); return Integer.parseInt(String.valueOf(Math.round(floatingNumber))); } else if (XSD.LONG.toString().equals(targetDatatypeXsd)) { var floatingNumber = Double.parseDouble(stringValue); @@ -70,6 +57,11 @@ public static Object convertValue(Object value, return value; } + public static String getCanonicalTypeClassName(String value, + boolean preferFloat) { + return getTypeClass(value, preferFloat).getCanonicalName(); + } + public static String getXsdDatatype(String value, boolean preferFloat) { var clazz = getTypeClass(value, preferFloat); diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java index d4ee006249..86a36b7d6c 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/SupportsNestedTransformationRule.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.List; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java index 100029741a..c35ed1a791 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverter.java @@ -30,7 +30,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.model.schema.EventProperty; @@ -71,7 +70,7 @@ public void visit(DeleteRuleDescription rule) { @Override public void visit(MoveRuleDescription rule) { - var targetRuntimeKey = rule.getNewRuntimeKey() + Utils.DELIMITER + rule.getOldRuntimeKey(); + var targetRuntimeKey = rule.getNewRuntimeKey() + "." + rule.getOldRuntimeKey(); var existing = new Cloner().property(findProperty(properties, targetRuntimeKey)); var existingHierarchy = findPropertyHierarchy(this.properties, targetRuntimeKey); existingHierarchy.removeIf(property -> property.getRuntimeName().equals(existing.getRuntimeName())); @@ -91,11 +90,6 @@ public void visit(RenameRuleDescription rule) { property.setRuntimeName(rule.getOldRuntimeKey()); } - @Override - public void visit(RegexTransformationRuleDescription rule) { - // does not affect schema - } - @Override public void visit(EventRateTransformationRuleDescription rule) { // does not affect schema @@ -131,7 +125,7 @@ public void visit(CorrectionValueTransformationRuleDescription rule) { public void visit(TimestampTranfsformationRuleDescription rule) { var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); property.setRuntimeType(Datatypes.String.toString()); - property.setSemanticType(null); + property.setDomainProperties(List.of()); } @Override diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java index 00dcd7cc43..92b1674d0d 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverter.java @@ -31,7 +31,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.model.schema.EventProperty; @@ -96,7 +95,6 @@ public void visit(RenameRuleDescription rule) { property.setRuntimeName(rule.getNewRuntimeKey()); } - @Override public void visit(EventRateTransformationRuleDescription rule) { // does not affect schema @@ -129,7 +127,7 @@ public void visit(AddValueTransformationRuleDescription rule) { property.setPropertyScope(rule.getPropertyScope().name()); if (Objects.nonNull(rule.getSemanticType())) { - property.setSemanticType(rule.getSemanticType()); + property.setDomainProperties(List.of(URI.create(rule.getSemanticType()))); } if (Objects.nonNull(rule.getMeasurementUnit())) { property.setMeasurementUnit(URI.create(rule.getMeasurementUnit())); @@ -151,20 +149,10 @@ public void visit(CorrectionValueTransformationRuleDescription rule) { metadata.put("correctionValue", rule.getCorrectionValue()); } - @Override - public void visit(RegexTransformationRuleDescription rule) { - var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); - var metadata = property.getAdditionalMetadata(); - - metadata.put("regex", rule.getRegex()); - metadata.put("replaceWith", rule.getReplaceWith()); - metadata.put("replaceAll", rule.isReplaceAll()); - } - @Override public void visit(TimestampTranfsformationRuleDescription rule) { var property = findPrimitiveProperty(properties, rule.getRuntimeKey()); - property.setSemanticType("http://schema.org/DateTime"); + property.setDomainProperties(List.of(URI.create("http://schema.org/DateTime"))); property.setRuntimeType(Datatypes.Long.toString()); property.setPropertyScope(PropertyScope.HEADER_PROPERTY.toString()); var metadata = property.getAdditionalMetadata(); diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java index c3d6bb2445..2b6d5070b2 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/elements/AdapterTransformationPipelineElement.java @@ -19,9 +19,9 @@ package org.apache.streampipes.connect.shared.preprocessing.elements; import org.apache.streampipes.connect.shared.preprocessing.generator.TransformationRuleGeneratorVisitor; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import org.apache.streampipes.connect.shared.preprocessing.utils.Utils; import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; -import org.apache.streampipes.extensions.api.connect.TransformationRule; import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; import java.util.List; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java index b1d9c0956c..c8e3c4cb5d 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatefulTransformationRuleGeneratorVisitor.java @@ -30,7 +30,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; @@ -56,11 +55,6 @@ public void visit(RenameRuleDescription rule) { // skip (not a stateful transformation) } - @Override - public void visit(RegexTransformationRuleDescription rule) { - // skip (not a stateful transformation) - } - @Override public void visit(EventRateTransformationRuleDescription ruleDesc) { rules.add( diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java index 1ca7d68dbd..1728d8db27 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/StatelessTransformationRuleGeneratorVisitor.java @@ -26,7 +26,6 @@ import org.apache.streampipes.connect.shared.preprocessing.transform.value.AddTimestampTransformationRule; import org.apache.streampipes.connect.shared.preprocessing.transform.value.CorrectionValueTransformationRule; import org.apache.streampipes.connect.shared.preprocessing.transform.value.DatatypeTransformationRule; -import org.apache.streampipes.connect.shared.preprocessing.transform.value.RegexTransformationRule; import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTranformationRuleMode; import org.apache.streampipes.connect.shared.preprocessing.transform.value.TimestampTransformationRule; import org.apache.streampipes.connect.shared.preprocessing.transform.value.UnitTransformationRule; @@ -41,7 +40,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; @@ -73,15 +71,6 @@ public void visit(RenameRuleDescription ruleDesc) { Utils.getLastKey(ruleDesc.getNewRuntimeKey()))); } - @Override - public void visit(RegexTransformationRuleDescription rule) { - rules.add(new RegexTransformationRule( - Utils.toKeyArray(rule.getRuntimeKey()), - rule.getRegex(), - rule.getReplaceWith(), - rule.isReplaceAll())); - } - @Override public void visit(EventRateTransformationRuleDescription ruleDesc) { // Do nothing diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java index 87c9bb6881..2b6a4edaff 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/generator/TransformationRuleGeneratorVisitor.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.generator; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; import java.util.ArrayList; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java new file mode 100644 index 0000000000..f6443afc62 --- /dev/null +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/TransformationRule.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.shared.preprocessing.transform; + +import java.util.Map; + +public interface TransformationRule { + Map apply(Map event); +} diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java index 8a31382ebe..77b8b12f20 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/AddValueTransformationRule.java @@ -19,7 +19,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; import org.apache.streampipes.connect.shared.DatatypeUtils; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java index 358c32130e..5d2e158b63 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/CreateNestedTransformationRule.java @@ -24,11 +24,6 @@ import java.util.List; import java.util.Map; -/** - * @deprecated The functionlality to add nested rules was removed in version 0.97.0 form the UI - * For the next release we can also remove the functionality from the backend - */ -@Deprecated(since = "0.97.0", forRemoval = true) public class CreateNestedTransformationRule extends SupportsNestedTransformationRule { private final List key; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java index b584b9eff2..f6506f376c 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/MoveTransformationRule.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.HashMap; import java.util.List; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java index fedf106a24..bbca98fb1e 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/DuplicateFilterPipelineElement.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.stream; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.HashMap; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java index b5b586f65e..229952c8b3 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/stream/EventRateTransformationRule.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.stream; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java index f075933c33..c8267ef34a 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/AddTimestampTransformationRule.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java index df61f2e131..ce0f0509a5 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/DatatypeTransformationRule.java @@ -20,7 +20,7 @@ import org.apache.streampipes.connect.shared.DatatypeUtils; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import java.util.Map; diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java index 05810d6072..5f9f30de61 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/TimestampTransformationRule.java @@ -28,8 +28,6 @@ import java.util.List; import java.util.Map; -import static java.util.TimeZone.getTimeZone; - public class TimestampTransformationRule extends SupportsNestedTransformationRule { private final List eventKey; @@ -50,7 +48,6 @@ public TimestampTransformationRule(List eventKey, if (mode == TimestampTranformationRuleMode.FORMAT_STRING) { dateFormatter = new SimpleDateFormat(formatString); - dateFormatter.setTimeZone(getTimeZone("UTC")); } } diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java index 9a22a11124..50a5a5681c 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/utils/Utils.java @@ -26,10 +26,8 @@ public class Utils { - public static final String DELIMITER = "<-=>"; - public static String getLastKey(String s) { - String[] list = s.split(DELIMITER); + String[] list = s.split("\\."); if (list.length == 0) { return s; } else { @@ -38,7 +36,7 @@ public static String getLastKey(String s) { } public static List toKeyArray(String s) { - String[] split = s.split(DELIMITER); + String[] split = s.split("\\."); if (split.length == 0) { return List.of(s); } else { diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java index a8e823a520..ca8dd0c347 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/Helpers.java @@ -21,7 +21,6 @@ import org.apache.streampipes.model.connect.rules.schema.DeleteRuleDescription; import org.apache.streampipes.model.connect.rules.schema.MoveRuleDescription; import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventPropertyNested; @@ -35,8 +34,7 @@ public class Helpers { public static String getUnit(EventProperty eventProperty) { - return ((EventPropertyPrimitive) eventProperty).getMeasurementUnit() - .toString(); + return ((EventPropertyPrimitive) eventProperty).getMeasurementUnit().toString(); } public static List makeSimpleProperties(boolean addTimestamp) { @@ -69,35 +67,16 @@ public static UnitTransformRuleDescription makeUnitTransformationRule(String run return rule; } - public static RegexTransformationRuleDescription makeRegexTransformationRule( - String runtimeKey, - String regex, - String replaceWith, - boolean replaceAll - ) { - var rule = new RegexTransformationRuleDescription(); - rule.setRuntimeKey(runtimeKey); - rule.setRegex(regex); - rule.setReplaceWith(replaceWith); - rule.setReplaceAll(replaceAll); - - return rule; - } - - public static MoveRuleDescription makeMoveTransformationRule( - String runtimeKey, - String newRuntimeKey - ) { + public static MoveRuleDescription makeMoveTransformationRule(String runtimeKey, + String newRuntimeKey) { var rule = new MoveRuleDescription(); rule.setOldRuntimeKey(runtimeKey); rule.setNewRuntimeKey(newRuntimeKey); return rule; } - public static RenameRuleDescription makeRenameTransformationRule( - String runtimeKey, - String newRuntimeName - ) { + public static RenameRuleDescription makeRenameTransformationRule(String runtimeKey, + String newRuntimeName) { var rule = new RenameRuleDescription(); rule.setOldRuntimeKey(runtimeKey); rule.setNewRuntimeKey(newRuntimeName); diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java index e5f1e1ed3d..1b8d964dd6 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToOriginalSchemaConverterTest.java @@ -25,6 +25,7 @@ import org.apache.streampipes.sdk.helpers.EpProperties; import org.apache.streampipes.sdk.helpers.Labels; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -36,12 +37,9 @@ import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeNestedProperties; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeSimpleProperties; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeUnitTransformationRule; -import static org.junit.jupiter.api.Assertions.assertEquals; public class ToOriginalSchemaConverterTest { - private static final String NESTED_DELIMITER = "<-=>"; - @Test public void testSimpleUnitConversion() { List properties = makeSimpleProperties(true); @@ -52,8 +50,8 @@ public void testSimpleUnitConversion() { var resultProperties = executeAndReturnResult(properties, rules); - assertEquals(3, resultProperties.size()); - assertEquals("originalUnit", getUnit(resultProperties.get(0))); + Assertions.assertEquals(3, resultProperties.size()); + Assertions.assertEquals("originalUnit", getUnit(resultProperties.get(0))); } @Test @@ -62,39 +60,31 @@ public void testNestedUnitConversion() { var rules = new ArrayList(); - rules.add(makeUnitTransformationRule("nested" + NESTED_DELIMITER + "stringProp")); + rules.add(makeUnitTransformationRule("nested.stringProp")); var resultProperties = executeAndReturnResult(properties, rules); - var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties() - .get(0); + var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties().get(0); - assertEquals(2, resultProperties.size()); - assertEquals("originalUnit", getUnit(nestedResultProperty)); + Assertions.assertEquals(2, resultProperties.size()); + Assertions.assertEquals("originalUnit", getUnit(nestedResultProperty)); } @Test public void testSimpleMoveConversion() { List properties = makeNestedProperties(); var nestedProperty = ((EventPropertyNested) properties.get(1)); - nestedProperty.getEventProperties() - .add(EpProperties.stringEp(Labels.empty(), "epToBeMoved", "")); + nestedProperty.getEventProperties().add(EpProperties.stringEp(Labels.empty(), "epToBeMoved", "")); var rules = new ArrayList(); rules.add(makeMoveTransformationRule("epToBeMoved", "nested")); var result = executeAndReturnResult(properties, rules); - assertEquals(3, result.size()); - assertEquals( - "timestamp", - result.get(0) - .getRuntimeName() - ); - assertEquals( - 2, - ((EventPropertyNested) result.get(1)).getEventProperties() - .size() - ); + Assertions.assertEquals(3, result.size()); + Assertions.assertEquals("timestamp", + result.get(0).getRuntimeName()); + Assertions.assertEquals(2, + ((EventPropertyNested) result.get(1)).getEventProperties().size()); } @Test @@ -104,18 +94,13 @@ public void testDeleteRule() { rules.add(makeDeleteTransformationRule("epToBeRestored")); var result = executeAndReturnResult(properties, rules); - assertEquals(4, result.size()); - assertEquals( - "epToBeRestored", - result.get(3) - .getRuntimeName() - ); + Assertions.assertEquals(4, result.size()); + Assertions.assertEquals("epToBeRestored", + result.get(3).getRuntimeName()); } - private List executeAndReturnResult( - List properties, - List rules - ) { + private List executeAndReturnResult(List properties, + List rules) { var result = new SchemaConverter().toOriginalSchema(new EventSchema(properties), rules); return result.getEventProperties(); } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java index ab753dad24..d31615e3d7 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/convert/ToTransformedSchemaConverterTest.java @@ -25,6 +25,7 @@ import org.apache.streampipes.sdk.helpers.EpProperties; import org.apache.streampipes.sdk.helpers.Labels; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -34,12 +35,9 @@ import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeDeleteTransformationRule; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeMoveTransformationRule; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeNestedProperties; -import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeRegexTransformationRule; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeRenameTransformationRule; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeSimpleProperties; import static org.apache.streampipes.connect.shared.preprocessing.convert.Helpers.makeUnitTransformationRule; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; public class ToTransformedSchemaConverterTest { @@ -53,27 +51,8 @@ public void testSimpleUnitConversion() { var resultProperties = executeAndReturnResult(properties, rules); - assertEquals(3, resultProperties.size()); - assertEquals("targetUnit", getUnit(resultProperties.get(0))); - } - - - @Test - public void testRegexTransformationRule() { - List properties = makeSimpleProperties(true); - - var rules = new ArrayList(); - - rules.add(makeRegexTransformationRule("stringProp", ",", "", true)); - - var resultProperties = executeAndReturnResult(properties, rules); - - assertEquals(3, resultProperties.size()); - - assertEquals(3, resultProperties.get(0).getAdditionalMetadata().size()); - assertTrue(resultProperties.get(0).getAdditionalMetadata().containsKey("regex")); - assertTrue(resultProperties.get(0).getAdditionalMetadata().containsKey("replaceWith")); - assertTrue(resultProperties.get(0).getAdditionalMetadata().containsKey("replaceAll")); + Assertions.assertEquals(3, resultProperties.size()); + Assertions.assertEquals("targetUnit", getUnit(resultProperties.get(0))); } @@ -85,12 +64,9 @@ public void testSimpleRenameConversion() { rules.add(makeRenameTransformationRule("stringProp", "newStringProp")); var result = executeAndReturnResult(properties, rules); - assertEquals(3, result.size()); - assertEquals( - "newStringProp", - result.get(0) - .getRuntimeName() - ); + Assertions.assertEquals(3, result.size()); + Assertions.assertEquals("newStringProp", + result.get(0).getRuntimeName()); } @Test @@ -101,12 +77,9 @@ public void testSimpleDeleteConversion() { rules.add(makeDeleteTransformationRule("stringProp")); var result = executeAndReturnResult(properties, rules); - assertEquals(2, result.size()); - assertEquals( - "intProp", - result.get(0) - .getRuntimeName() - ); + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals("intProp", + result.get(0).getRuntimeName()); } @Test @@ -119,17 +92,11 @@ public void testSimpleMoveConversion() { rules.add(makeMoveTransformationRule("epToBeMoved", "nested")); var result = executeAndReturnResult(properties, rules); - assertEquals(2, result.size()); - assertEquals( - "timestamp", - result.get(0) - .getRuntimeName() - ); - assertEquals( - 3, - ((EventPropertyNested) result.get(1)).getEventProperties() - .size() - ); + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals("timestamp", + result.get(0).getRuntimeName()); + Assertions.assertEquals(3, + ((EventPropertyNested) result.get(1)).getEventProperties().size()); } @Test @@ -138,23 +105,21 @@ public void testNestedUnitConversion() { var rules = new ArrayList(); - rules.add(makeUnitTransformationRule("nested<-=>stringProp")); + rules.add(makeUnitTransformationRule("nested.stringProp")); var resultProperties = executeAndReturnResult(properties, rules); - var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties() - .get(0); + var nestedResultProperty = ((EventPropertyNested) resultProperties.get(1)).getEventProperties().get(0); - assertEquals(2, resultProperties.size()); - assertEquals("targetUnit", getUnit(nestedResultProperty)); + Assertions.assertEquals(2, resultProperties.size()); + Assertions.assertEquals("targetUnit", getUnit(nestedResultProperty)); } - private List executeAndReturnResult( - List properties, - List rules - ) { + private List executeAndReturnResult(List properties, + List rules) { var result = new SchemaConverter().toTransformedSchema(new EventSchema(properties), rules); return result.getEventProperties(); } + } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java index 7937d6875c..ef1235b4b8 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java @@ -21,192 +21,110 @@ import org.apache.streampipes.connect.shared.DatatypeUtils; import org.apache.streampipes.vocabulary.XSD; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Locale; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class DatatypeUtilsTest { + @Test /** - * The following tests ensure that timestamps represented as strings are correctly parsed. + * This test ensures that timestamps represented as strings are correctly parsed. * Often they are first parsed into floating point number before transformed back to long. * The data type for those values should be Double and not Float, because the transformation to Float might change * the value */ - @Test - public void convertValue_StringToStringValue() { - var inputValue = "testString"; - var actualValue = DatatypeUtils.convertValue(inputValue, XSD.STRING.toString()); - - assertEquals(inputValue, actualValue); - } - - @Test - public void convertValue_StringToDoubleValue() { - var actualValue = DatatypeUtils.convertValue("1667904471000", XSD.DOUBLE.toString()); - - assertEquals(1.667904471E12, actualValue); - } - - @Test - public void convertValue_StringToFloatValue() { - var actualValue = DatatypeUtils.convertValue("123.45", XSD.FLOAT.toString()); - - assertEquals(123.45f, actualValue); - } - - @Test - public void convertValue_StringToInteger() { - var actualValue = DatatypeUtils.convertValue("1623871500", XSD.INTEGER.toString()); - - assertEquals(1623871500, actualValue); - } - - @Test - public void convertValue_StringToIntegerValue() { - var actualValue = DatatypeUtils.convertValue("123", XSD.INTEGER.toString()); - - assertEquals(123, actualValue); - } - - @Test - public void convertValue_StringToLongValue() { - var actualValue = DatatypeUtils.convertValue("1623871500000", XSD.LONG.toString()); - - assertEquals(1623871500000L, actualValue); - } - - @Test - public void convertValue_StringToBooleanTrueValue() { - var actualValue = DatatypeUtils.convertValue("true", XSD.BOOLEAN.toString()); + public void convertTimestampValue() { + var inputValue = "1667904471000"; - assertEquals(true, actualValue); - } - - @Test - public void convertValue_StringToBooleanFalseValue() { - var actualValue = DatatypeUtils.convertValue("false", XSD.BOOLEAN.toString()); - - assertEquals(false, actualValue); - } - - @Test - public void convertValue_FloatToIntegerValue_Rounding() { - var actualValue = DatatypeUtils.convertValue(123.45f, XSD.INTEGER.toString()); + var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString()); + var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString()); - assertEquals(123, actualValue); + Assertions.assertEquals(Long.parseLong(inputValue), longValue); } - @Test - public void convertValue_DoubleToLongValue_Rounding1() { - var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString()); - - assertEquals(1234567890L, actualValue); - } - - @Test - public void convertValue_DoubleToLongValue() { - var actualValue = DatatypeUtils.convertValue(1.667904471E12, XSD.LONG.toString()); - - assertEquals(1667904471000L, actualValue); - } - - @Test - public void convertValue_DoubleToLongValue_Rounding() { - var actualValue = DatatypeUtils.convertValue(1234567890.12345, XSD.LONG.toString()); - - assertEquals(1234567890L, actualValue); - } - - String booleanInputValue = "true"; - @Test - public void getTypeClass_NoPrefereFloatingPointBoolean() { + public void getTypeClassNoPrefereFloatingPointBoolean() { var result = DatatypeUtils.getTypeClass(booleanInputValue, false); - assertEquals(Boolean.class, result); + Assertions.assertEquals(Boolean.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointBoolean() { + public void getTypeClassWithPrefereFloatingPointBoolean() { var result = DatatypeUtils.getTypeClass(booleanInputValue, true); - assertEquals(Boolean.class, result); + Assertions.assertEquals(Boolean.class, result); } String integerInputValue = "1"; - @Test - public void getTypeClass_NoPrefereFloatingPointInteger() { + public void getTypeClassNoPrefereFloatingPointInteger() { var result = DatatypeUtils.getTypeClass(integerInputValue, false); - assertEquals(Integer.class, result); + Assertions.assertEquals(Integer.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointInteger() { + public void getTypeClassWithPrefereFloatingPointInteger() { var result = DatatypeUtils.getTypeClass(integerInputValue, true); - assertEquals(Float.class, result); + Assertions.assertEquals(Float.class, result); } String floatInputValue = "1.0"; - @Test - public void getTypeClass_NoPrefereFloatingPointFloat() { + public void getTypeClassNoPrefereFloatingPointFloat() { var result = DatatypeUtils.getTypeClass(floatInputValue, false); - assertEquals(Float.class, result); + Assertions.assertEquals(Float.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointFloat() { + public void getTypeClassWithPrefereFloatingPointFloat() { var result = DatatypeUtils.getTypeClass(floatInputValue, true); - assertEquals(Float.class, result); + Assertions.assertEquals(Float.class, result); } String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE); - @Test - public void getTypeClass_NoPrefereFloatingPointDouble() { + public void getTypeClassNoPrefereFloatingPointDouble() { var result = DatatypeUtils.getTypeClass(doubleInputValue, false); - assertEquals(Double.class, result); + Assertions.assertEquals(Double.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointDouble() { + public void getTypeClassWithPrefereFloatingPointDouble() { var result = DatatypeUtils.getTypeClass(doubleInputValue, true); - assertEquals(Double.class, result); + Assertions.assertEquals(Double.class, result); } String longInputValue = "1667904471000"; - @Test - public void getTypeClass_NoPrefereFloatingPointLong() { + public void getTypeClassNoPrefereFloatingPointLong() { var result = DatatypeUtils.getTypeClass(longInputValue, false); - assertEquals(Long.class, result); + Assertions.assertEquals(Long.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointLong() { + public void getTypeClassWithPrefereFloatingPointLong() { var result = DatatypeUtils.getTypeClass(longInputValue, true); - assertEquals(Double.class, result); + Assertions.assertEquals(Double.class, result); } String stringInputValue = "one"; - @Test - public void getTypeClass_NoPrefereFloatingPointString() { + public void getTypeClassNoPrefereFloatingPointString() { var result = DatatypeUtils.getTypeClass(stringInputValue, false); - assertEquals(String.class, result); + Assertions.assertEquals(String.class, result); } @Test - public void getTypeClass_WithPrefereFloatingPointString() { + public void getTypeClassWithPrefereFloatingPointString() { var result = DatatypeUtils.getTypeClass(stringInputValue, true); - assertEquals(String.class, result); + Assertions.assertEquals(String.class, result); } + } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java index 56400a8066..ec87937de1 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/SchemaEventTransformerTest.java @@ -18,7 +18,7 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; -import org.apache.streampipes.extensions.api.connect.TransformationRule; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java index 5f3ba9717d..eaec0d6a85 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/CorrectionValueTest.java @@ -18,6 +18,11 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; +import org.apache.streampipes.model.schema.EventSchema; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,114 +30,126 @@ import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; public class CorrectionValueTest { private Map event; - private final String doubleProperty = "basicValue"; - private final String stringProperty = "otherValue"; + private final String propertyNameBasicValue = "basicValue"; + private final String propertyNameOtherValue = "otherValue"; @BeforeEach public void setUp() { + + EventSchema eventSchema = new EventSchema(); + EventProperty eventProperty = new EventPropertyPrimitive(); + eventProperty.setLabel(propertyNameBasicValue); + eventProperty.setRuntimeName(propertyNameBasicValue); + eventSchema.addEventProperty(eventProperty); + + EventProperty eventPropertyOther = new EventPropertyPrimitive(); + eventPropertyOther.setLabel(propertyNameBasicValue); + eventPropertyOther.setRuntimeName(propertyNameBasicValue); + eventSchema.addEventProperty(eventPropertyOther); + event = new HashMap<>(); - event.put(doubleProperty, 100.0); - event.put(stringProperty, "Hello"); + event.put(propertyNameBasicValue, 100.0); + event.put(propertyNameOtherValue, "Hello"); } @Test public void testAdd() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 10.0, "ADD" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(110.0, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(110.0, resultEvent.get(propertyNameBasicValue)); } @Test public void testSubtract() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 10.0, "SUBTRACT" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(90.0, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(90.0, resultEvent.get(propertyNameBasicValue)); } @Test public void testMultiply() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 1.5, "MULTIPLY" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(150.0, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(150.0, resultEvent.get(propertyNameBasicValue)); } @Test public void testDivide() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 5, "DIVIDE" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(20.0, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(20.0, resultEvent.get(propertyNameBasicValue)); } @Test public void testDivideByZero() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 0.0, "DIVIDE" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(Double.POSITIVE_INFINITY, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(Double.POSITIVE_INFINITY, resultEvent.get(propertyNameBasicValue)); } @Test public void testNonNumericValue() { var correctionRule = new CorrectionValueTransformationRule( - List.of(stringProperty), + List.of(propertyNameOtherValue), 10.0, "ADD" ); assertThrows( RuntimeException.class, - () -> correctionRule.apply(event).get(stringProperty) + () -> correctionRule.apply(event).get(propertyNameOtherValue) ); + + } @Test public void testUnsupportedOperation() { var correctionRule = new CorrectionValueTransformationRule( - List.of(doubleProperty), + List.of(propertyNameBasicValue), 10.0, "TEST" ); var resultEvent = correctionRule.apply(event); - assertNotNull(resultEvent); - assertEquals(100.0, resultEvent.get(doubleProperty)); + Assertions.assertNotNull(resultEvent); + Assertions.assertEquals(100.0, resultEvent.get(propertyNameBasicValue)); } } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java index 37416809ac..f6d9beaed0 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/UnitTransformRuleTest.java @@ -18,122 +18,134 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyList; +import org.apache.streampipes.model.schema.EventPropertyNested; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; +import org.apache.streampipes.model.schema.EventSchema; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; - +@SuppressWarnings("unchecked") public class UnitTransformRuleTest { - private static final String PARENT_PROPERTY = "parentProperty"; - private static final String CHILD_PROPERTY = "childProperty"; - private static final String PROPERTY_ONE = "property1"; - private static final String PROPERTY_TWO = "property2"; - - private static final String UNIT_DEGREE_CELSIUS = "http://qudt.org/vocab/unit#DegreeCelsius"; - private static final String UNIT_KELVIN = "http://qudt.org/vocab/unit#Kelvin"; - @Test public void transformList() { + EventSchema eventSchema = new EventSchema(); + EventPropertyList eventPropertyList = new EventPropertyList(); + eventPropertyList.setRuntimeName("list"); + EventProperty eventPropertyValue = new EventPropertyPrimitive(); + eventPropertyValue.setLabel("value"); + eventPropertyValue.setRuntimeName("value"); + eventPropertyList.setEventProperty(eventPropertyValue); + eventSchema.setEventProperties(Collections.singletonList(eventPropertyList)); + Map event = new HashMap<>(); Map subEvent = new HashMap<>(); - subEvent.put(CHILD_PROPERTY, 0.0); - event.put(PARENT_PROPERTY, subEvent); + subEvent.put("value", 0.0); + event.put("list", subEvent); - var unitTransformationRule = new UnitTransformationRule( - List.of(PARENT_PROPERTY, CHILD_PROPERTY), - UNIT_DEGREE_CELSIUS, - UNIT_KELVIN - ); + List keys = new ArrayList<>(); + keys.add("list"); + keys.add("value"); - var result = unitTransformationRule.apply(event); + UnitTransformationRule unitTransformationRule = new UnitTransformationRule(keys, + "http://qudt.org/vocab/unit#DegreeCelsius", "http://qudt.org/vocab/unit#Kelvin"); - assertEquals( - 1, - result.keySet() - .size() - ); + var result = unitTransformationRule.apply(event); - assertEquals(273.15, ((Map) result.get(PARENT_PROPERTY)) - .get(CHILD_PROPERTY)); + Assertions.assertEquals(1, + result.keySet().size()); + Assertions.assertEquals(273.15, ((Map) result.get(eventPropertyList.getRuntimeName())) + .get(eventPropertyValue.getRuntimeName())); } @Test public void transformNested() { + EventSchema eventSchema = new EventSchema(); + EventPropertyNested eventPropertyMainKey = new EventPropertyNested(); + eventPropertyMainKey.setLabel("mainKey"); + eventPropertyMainKey.setRuntimeName("mainKey"); + EventProperty eventPropertyValue = new EventPropertyPrimitive(); + eventPropertyValue.setLabel("value"); + eventPropertyValue.setRuntimeName("value"); + eventPropertyMainKey.setEventProperties(Collections.singletonList(eventPropertyValue)); + eventSchema.setEventProperties(Collections.singletonList(eventPropertyMainKey)); + Map event = new HashMap<>(); Map subEvent = new HashMap<>(); - subEvent.put(CHILD_PROPERTY, 10.0); - event.put(PARENT_PROPERTY, subEvent); + subEvent.put("value", 10.0); + event.put("mainKey", subEvent); - var unitTransformationRule = new UnitTransformationRule( - List.of(PARENT_PROPERTY, CHILD_PROPERTY), - UNIT_DEGREE_CELSIUS, - UNIT_KELVIN - ); + List keys = new ArrayList<>(); + keys.add("mainKey"); + keys.add("value"); + + UnitTransformationRule unitTransformationRule = new UnitTransformationRule(keys, + "http://qudt.org/vocab/unit#DegreeCelsius", "http://qudt.org/vocab/unit#Kelvin"); var result = unitTransformationRule.apply(event); - assertEquals( - 1, - result.keySet() - .size() - ); - assertEquals(283.15, ((Map) result.get(PARENT_PROPERTY)) - .get(CHILD_PROPERTY)); + Assertions.assertEquals(1, + result.keySet().size()); + Assertions.assertEquals(283.15, ((Map) result.get(eventPropertyMainKey.getRuntimeName())) + .get(eventPropertyValue.getRuntimeName())); } @Test public void transformMultiEvent() { - - var unitTransformationRule = new UnitTransformationRule( - List.of(PROPERTY_TWO), - UNIT_DEGREE_CELSIUS, - UNIT_KELVIN - ); - - Map event = getEventWithTwoProperties(0.0, 10.0); + EventSchema eventSchema = new EventSchema(); + EventProperty eventPropertyValue1 = new EventPropertyPrimitive(); + eventPropertyValue1.setLabel("value1"); + eventPropertyValue1.setRuntimeName("value1"); + EventProperty eventPropertyValue2 = new EventPropertyPrimitive(); + eventPropertyValue2.setLabel("value2"); + eventPropertyValue2.setRuntimeName("value2"); + eventSchema.addEventProperty(eventPropertyValue1); + eventSchema.addEventProperty(eventPropertyValue2); + + List keys = new ArrayList<>(); + keys.add("value2"); + + UnitTransformationRule unitTransformationRule = new UnitTransformationRule(keys, + "http://qudt.org/vocab/unit#DegreeCelsius", "http://qudt.org/vocab/unit#Kelvin"); + Map event = new HashMap<>(); + event.put("value1", 0.0); + event.put("value2", 10.0); var result = unitTransformationRule.apply(event); - assertEquals( - 2, - result.keySet() - .size() - ); - assertEquals(283.15, result.get(PROPERTY_TWO)); + Assertions.assertEquals(2, + result.keySet().size()); + Assertions.assertEquals(283.15, result.get(eventPropertyValue2.getLabel())); - event = getEventWithTwoProperties(20.0, 20.0); + event = new HashMap<>(); + event.put("value1", 20.0); + event.put("value2", 20.0); result = unitTransformationRule.apply(event); - assertEquals( - 2, - result.keySet() - .size() - ); - assertEquals(293.15, result.get(PROPERTY_TWO)); + Assertions.assertEquals(2, + result.keySet().size()); + Assertions.assertEquals(293.15, result.get(eventPropertyValue2.getRuntimeName())); - event = getEventWithTwoProperties(0.0, 0.0); + event = new HashMap<>(); + event.put("value1", 0.0); + event.put("value2", 0.0); result = unitTransformationRule.apply(event); - assertEquals( - 2, - result.keySet() - .size() - ); - assertEquals(273.15, result.get(PROPERTY_TWO)); - } - - private Map getEventWithTwoProperties(double value1, double value2) { - Map event = new HashMap<>(); - event.put(PROPERTY_ONE, value1); - event.put(PROPERTY_TWO, value2); - return event; + Assertions.assertEquals(2, + result.keySet().size()); + Assertions.assertEquals(273.15, result.get(eventPropertyValue2.getRuntimeName())); } } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java index bec73c2155..89cd25be8a 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/value/ValueEventTransformerTest.java @@ -18,45 +18,46 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.value; +import org.apache.streampipes.connect.shared.preprocessing.transform.TransformationRule; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; +import org.apache.streampipes.model.schema.EventSchema; + +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; public class ValueEventTransformerTest { - private static final String UNIT_DEGREE_CELSIUS = "http://qudt.org/vocab/unit#DegreeCelsius"; - private static final String UNIT_KELVIN = "http://qudt.org/vocab/unit#Kelvin"; - private static final String EVENT_PROPEPRTY = "property"; - @Test public void transform() { + EventSchema eventSchema = new EventSchema(); + EventProperty eventPropertyf = new EventPropertyPrimitive(); + eventPropertyf.setLabel("a"); + eventPropertyf.setRuntimeName("a"); + eventSchema.addEventProperty(eventPropertyf); Map event = new HashMap<>(); - event.put(EVENT_PROPEPRTY, 273.15); + event.put("a", 273.15); - var unitTransformationRule = new UnitTransformationRule( - List.of(EVENT_PROPEPRTY), - UNIT_KELVIN, - UNIT_DEGREE_CELSIUS - ); + List keys = new ArrayList<>(); + keys.add("a"); - var correctionRule = new CorrectionValueTransformationRule( - List.of(EVENT_PROPEPRTY), - 10.0, - "ADD" - ); + List rules = new ArrayList<>(); + rules.add(new UnitTransformationRule(keys, + "http://qudt.org/vocab/unit#Kelvin", "http://qudt.org/vocab/unit#DegreeCelsius")); - var rules = List.of(unitTransformationRule, correctionRule); - - for (var rule : rules) { + for (var rule: rules) { event = rule.apply(event); } - assertEquals(10.0, event.get(EVENT_PROPEPRTY)); + Assertions.assertEquals(0.0, event.get(eventPropertyf.getRuntimeName())); + } } diff --git a/streampipes-data-explorer-commons/pom.xml b/streampipes-data-explorer-commons/pom.xml new file mode 100644 index 0000000000..780d44b6c6 --- /dev/null +++ b/streampipes-data-explorer-commons/pom.xml @@ -0,0 +1,81 @@ + + + + + streampipes-parent + org.apache.streampipes + 0.95.1-SNAPSHOT + + 4.0.0 + + streampipes-data-explorer-commons + + + + org.apache.streampipes + streampipes-client-api + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-model + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-commons + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-test-utils + 0.95.1-SNAPSHOT + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.mockito + mockito-core + test + + + + + org.lightcouch + lightcouch + + + org.influxdb + influxdb-java + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java new file mode 100644 index 0000000000..6c28f9c145 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.commons; + +import org.apache.streampipes.client.api.IStreamPipesClient; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxNameSanitizer; +import org.apache.streampipes.dataexplorer.commons.sanitizer.MeasureNameSanitizer; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.schema.EventProperty; + +import java.util.List; +import java.util.stream.Collectors; + +public class DataExplorerUtils { + /** + * Sanitizes the event schema and stores the DataLakeMeasurement to the couchDB + * + * @param client StreamPipes client to store measure + * @param measure DataLakeMeasurement + */ + public static DataLakeMeasure sanitizeAndRegisterAtDataLake(IStreamPipesClient client, + DataLakeMeasure measure) throws SpRuntimeException { + sanitizeDataLakeMeasure(measure); + registerAtDataLake(client, measure); + + return measure; + } + + public static DataLakeMeasure sanitizeAndUpdateAtDataLake(IStreamPipesClient client, + DataLakeMeasure measure) throws SpRuntimeException { + sanitizeDataLakeMeasure(measure); + updateAtDataLake(client, measure); + return measure; + } + + private static void registerAtDataLake(IStreamPipesClient client, + DataLakeMeasure measure) throws SpRuntimeException { + client.dataLakeMeasureApi().create(measure); + } + + public static void updateAtDataLake(IStreamPipesClient client, + DataLakeMeasure measure) throws SpRuntimeException { + client.dataLakeMeasureApi().update(measure); + } + + + private static void sanitizeDataLakeMeasure(DataLakeMeasure measure) throws SpRuntimeException { + + // Removes selected timestamp from event schema + removeTimestampsFromEventSchema(measure); + + // Sanitize the data lake measure name + measure.setMeasureName(new MeasureNameSanitizer().sanitize(measure.getMeasureName())); + + // Removes all spaces with _ and validates that no special terms are used as runtime names + measure.getEventSchema() + .getEventProperties() + .forEach(ep -> ep.setRuntimeName(InfluxNameSanitizer.renameReservedKeywords(ep.getRuntimeName()))); + + } + + private static void removeTimestampsFromEventSchema(DataLakeMeasure measure) { + List eventPropertiesWithoutTimestamp = measure.getEventSchema().getEventProperties() + .stream() + .filter(eventProperty -> !measure.getTimestampField().endsWith(eventProperty.getRuntimeName())) + .collect(Collectors.toList()); + measure.getEventSchema().setEventProperties(eventPropertiesWithoutTimestamp); + } + +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java new file mode 100644 index 0000000000..4bc6628696 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons; + +import org.apache.streampipes.client.api.IStreamPipesClient; +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.commons.image.ImageStore; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxStore; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.runtime.Event; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class TimeSeriesStore { + + private static final Logger LOG = LoggerFactory.getLogger(TimeSeriesStore.class); + private final InfluxStore influxStore; + private ImageStore imageStore; + + + public TimeSeriesStore(Environment environment, + IStreamPipesClient client, + DataLakeMeasure measure, + boolean enableImageStore) { + + DataExplorerUtils.sanitizeAndRegisterAtDataLake(client, measure); + + if (enableImageStore) { + this.imageStore = new ImageStore(measure, environment); + } + + this.influxStore = new InfluxStore(measure, environment, new InfluxClientProvider()); + + } + + public boolean onEvent(Event event) throws SpRuntimeException { + // Store all images in image store and replace image with internal id + if (imageStore != null) { + this.imageStore.onEvent(event); + } + + // Store event in time series database + this.influxStore.onEvent(event); + + return true; + } + + public void close() throws SpRuntimeException { + if (imageStore != null) { + try { + this.imageStore.close(); + } catch (IOException e) { + LOG.error("Could not close couchDB connection"); + throw new SpRuntimeException(e); + } + } + + this.influxStore.close(); + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java new file mode 100644 index 0000000000..52c37c4b24 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.auth; + +import okhttp3.Interceptor; +import okhttp3.Response; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; + +public class AuthInterceptor implements Interceptor { + + private String token; + + public AuthInterceptor(String token) { + this.token = token; + } + + @NotNull + @Override + public Response intercept(@NotNull Chain chain) throws IOException { + var req = chain.request(); + var authHeaderValue = "Token " + token; + req = req.newBuilder().addHeader("Authorization", authHeaderValue).build(); + + return chain.proceed(req); + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java new file mode 100644 index 0000000000..e9323d111b --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.image; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.runtime.Event; +import org.apache.streampipes.model.schema.EventProperty; + +import org.apache.commons.codec.binary.Base64; +import org.lightcouch.CouchDbClient; +import org.lightcouch.CouchDbProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.List; +import java.util.UUID; + +public class ImageStore { + + private static final Logger LOG = LoggerFactory.getLogger(ImageStore.class); + private static final String DB_NAME = "images"; + + private List imageProperties; + private CouchDbClient couchDbClient; + + public ImageStore(DataLakeMeasure measure, + Environment environment) { + this.couchDbClient = new CouchDbClient(from(environment)); + this.imageProperties = ImageStoreUtils.getImageProperties(measure); + } + + private static CouchDbProperties from(Environment env) { + String couchDbProtocol = env.getCouchDbProtocol().getValueOrDefault(); + String couchDbHost = env.getCouchDbHost().getValueOrDefault(); + int couchDbPort = env.getCouchDbPort().getValueOrDefault(); + String username = env.getCouchDbUsername().getValueOrDefault(); + String password = env.getCouchDbPassword().getValueOrDefault(); + + return new CouchDbProperties(DB_NAME, true, couchDbProtocol, + couchDbHost, couchDbPort, username, password); + } + + public void onEvent(Event event) throws SpRuntimeException { + this.imageProperties.forEach(eventProperty -> { + String imageDocId = UUID.randomUUID().toString(); + String image = event.getFieldByRuntimeName(eventProperty.getRuntimeName()).getAsPrimitive().getAsString(); + + byte[] data = Base64.decodeBase64(image); + storeImage(data, imageDocId); + event.updateFieldBySelector("s0::" + eventProperty.getRuntimeName(), imageDocId); + }); + } + + public void storeImage(byte[] imageBytes, + String imageDocId) { + this.couchDbClient.saveAttachment( + new ByteArrayInputStream(imageBytes), + imageDocId, + "image/jpeg", + imageDocId, + null); + + } + + public void close() throws IOException { + this.couchDbClient.close(); + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java new file mode 100644 index 0000000000..42594827e4 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.image; + +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.vocabulary.SPSensor; + +import java.util.List; +import java.util.stream.Collectors; + +public class ImageStoreUtils { + + public static List getImageProperties(DataLakeMeasure measure) { + return measure.getEventSchema().getEventProperties().stream() + .filter(eventProperty -> eventProperty.getDomainProperties() != null + && eventProperty.getDomainProperties().size() > 0 + && eventProperty.getDomainProperties().get(0).toString().equals(SPSensor.IMAGE)) + .collect(Collectors.toList()); + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java new file mode 100644 index 0000000000..781829260c --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +public enum InfluxAuthMode { + + USERNAME_PASSWORD, + TOKEN; +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java new file mode 100644 index 0000000000..a95ecfac7e --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; + +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class InfluxClientProvider { + + private static final Logger LOG = LoggerFactory.getLogger(InfluxClientProvider.class); + + /** + * Create a new InfluxDB client from environment variables + * + * @return InfluxDB + */ + public static InfluxDB getInfluxDBClient() { + var env = getEnvironment(); + return getInfluxDBClient(InfluxConnectionSettings.from(env)); + } + + /** + * Create a new InfluxDB client from provided settings + * + * @param settings Connection settings + * @return InfluxDB + */ + public static InfluxDB getInfluxDBClient(InfluxConnectionSettings settings) { + if (settings.getAuthMode() == InfluxAuthMode.TOKEN) { + var okHttpClientBuilder = InfluxClientUtils.getHttpClientBuilder(settings.getToken()); + + return InfluxDBFactory.connect(settings.getConnectionUrl(), okHttpClientBuilder); + } else { + var okHttpClientBuilder = InfluxClientUtils.getHttpClientBuilder(); + return InfluxDBFactory.connect( + settings.getConnectionUrl(), + settings.getUsername(), + settings.getPassword(), + okHttpClientBuilder + ); + } + } + + + public InfluxDB getInitializedInfluxDBClient(Environment environment) { + + var settings = InfluxConnectionSettings.from(environment); + var influxDb = InfluxClientProvider.getInfluxDBClient(settings); + var databaseName = settings.getDatabaseName(); + + // Checking, if server is available + var response = influxDb.ping(); + if (response.getVersion() + .equalsIgnoreCase("unknown")) { + throw new SpRuntimeException("Could not connect to InfluxDb Server: " + settings.getConnectionUrl()); + } + + // Checking whether the database exists + if (!databaseExists(influxDb, databaseName)) { + LOG.info("Database '" + databaseName + "' not found. Gets created ..."); + createDatabase(influxDb, databaseName); + } + + // setting up the database + influxDb.setDatabase(databaseName); + var batchSize = 2000; + var flushDuration = 500; + influxDb.enableBatch(batchSize, flushDuration, TimeUnit.MILLISECONDS); + + return influxDb; + } + + /** + * Creates a new database with the given name + * + * @param influxDb The InfluxDB client + * @param dbName The name of the database which should be created + */ + public void createDatabase( + InfluxDB influxDb, + String dbName + ) throws SpRuntimeException { + if (!dbName.matches("^[a-zA-Z_]\\w*$")) { + throw new SpRuntimeException( + "Database name '" + dbName + "' not allowed. Allowed names: ^[a-zA-Z_][a-zA-Z0-9_]*$"); + } + influxDb.query(new Query("CREATE DATABASE \"" + dbName + "\"", "")); + } + + /** + * Checks whether the given database exists. + * + * @param influxDb The InfluxDB client instance + * @param dbName The name of the database, the method should look for + * @return True if the database exists, false otherwise + */ + public boolean databaseExists( + InfluxDB influxDb, + String dbName + ) { + var queryResult = influxDb.query(new Query("SHOW DATABASES", "")); + for (List a : queryResult.getResults() + .get(0) + .getSeries() + .get(0) + .getValues()) { + if (!a.isEmpty() && dbName.equals(a.get(0))) { + return true; + } + } + return false; + } + + + private static Environment getEnvironment() { + return Environments.getEnvironment(); + } + + +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java new file mode 100644 index 0000000000..1f7b04d998 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.dataexplorer.commons.auth.AuthInterceptor; + +import okhttp3.OkHttpClient; + +import java.util.concurrent.TimeUnit; + +public class InfluxClientUtils { + + public static OkHttpClient.Builder getHttpClientBuilder(String authToken) { + var builder = getHttpClientBuilder() + .addInterceptor(new AuthInterceptor(authToken)); + + return builder; + } + + public static OkHttpClient.Builder getHttpClientBuilder() { + return new OkHttpClient().newBuilder() + .connectTimeout(120, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .writeTimeout(120, TimeUnit.SECONDS); + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java new file mode 100644 index 0000000000..ac53b85730 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.commons.environment.Environment; + +public class InfluxConnectionSettings { + + private final Integer influxDbPort; + + private final String influxDbProtocol; + private final String influxDbHost; + private final String databaseName; + + private String username; + private String password; + private String token; + + private InfluxAuthMode authMode; + + + private InfluxConnectionSettings(String influxDbProtocol, + String influxDbHost, + Integer influxDbPort, + String databaseName) { + this.influxDbProtocol = influxDbProtocol; + this.influxDbHost = influxDbHost; + this.influxDbPort = influxDbPort; + this.databaseName = databaseName; + } + + private InfluxConnectionSettings(String influxDbProtocol, + String influxDbHost, + Integer influxDbPort, + String databaseName, + String token) { + this(influxDbProtocol, influxDbHost, influxDbPort, databaseName); + this.token = token; + this.authMode = InfluxAuthMode.TOKEN; + } + + private InfluxConnectionSettings(String influxDbProtocol, + String influxDbHost, + Integer influxDbPort, + String databaseName, + String username, + String password) { + this(influxDbProtocol, influxDbHost, influxDbPort, databaseName); + this.username = username; + this.password = password; + this.authMode = InfluxAuthMode.USERNAME_PASSWORD; + } + + public static InfluxConnectionSettings from(String influxDbProtocol, + String influxDbHost, + int influxDbPort, + String databaseName, + String username, + String password) { + return new InfluxConnectionSettings(influxDbProtocol, influxDbHost, influxDbPort, databaseName, username, password); + } + + public static InfluxConnectionSettings from(String influxDbProtocol, + String influxDbHost, + int influxDbPort, + String databaseName, + String token) { + return new InfluxConnectionSettings(influxDbProtocol, influxDbHost, influxDbPort, databaseName, token); + } + + public static InfluxConnectionSettings from(Environment environment) { + + return new InfluxConnectionSettings( + environment.getTsStorageProtocol().getValueOrDefault(), + environment.getTsStorageHost().getValueOrDefault(), + environment.getTsStoragePort().getValueOrDefault(), + environment.getTsStorageBucket().getValueOrDefault(), + environment.getTsStorageToken().getValueOrDefault()); + } + + public Integer getInfluxDbPort() { + return influxDbPort; + } + + public String getInfluxDbHost() { + return influxDbHost; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getToken() { + return token; + } + + public InfluxAuthMode getAuthMode() { + return authMode; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getConnectionUrl() { + return influxDbProtocol + "://" + influxDbHost + ":" + influxDbPort; + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java new file mode 100644 index 0000000000..48d8c9e8d1 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import java.util.Arrays; +import java.util.List; + +public class InfluxDbReservedKeywords { + + public static final List KEYWORD_LIST = Arrays.asList( + "ALL", + "ALTER", + "ANALYZE", + "ANY", + "AS", + "ASC", + "BEGIN", + "BY", + "CREATE", + "CONTINUOUS", + "DATABASE", + "DATABASES", + "DEFAULT", + "DELETE", + "DESC", + "DESTINATIONS", + "DIAGNOSTICS", + "DISTINCT", + "DROP", + "DURATION", + "END", + "EVERY", + "EXPLAIN", + "FIELD", + "FOR", + "FROM", + "GRANT", + "GRANTS", + "GROUP", + "GROUPS", + "IN", + "INF", + "INSERT", + "INTO", + "KEY", + "KEYS", + "KILL", + "LIMIT", + "SHOW", + "MEASUREMENT", + "MEASUREMENTS", + "NAME", + "OFFSET", + "ON", + "ORDER", + "PASSWORD", + "POLICY", + "POLICIES", + "PRIVILEGES", + "QUERIES", + "QUERY", + "READ", + "REPLICATION", + "RESAMPLE", + "RETENTION", + "REVOKE", + "SELECT", + "SERIES", + "SET", + "SHARD", + "SHARDS", + "SLIMIT", + "SOFFSET", + "STATS", + "SUBSCRIPTION", + "SUBSCRIPTIONS", + "TAG", + "TO", + "USER", + "USERS", + "VALUES", + "WHERE", + "WITH", + "WRITE" + ); +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java new file mode 100644 index 0000000000..353782ba8f --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +public class InfluxNameSanitizer { + + public static String renameReservedKeywords(String runtimeName) { + if (InfluxDbReservedKeywords.KEYWORD_LIST.stream().anyMatch(k -> k.equalsIgnoreCase(runtimeName))) { + return runtimeName + "_"; + } else { + return runtimeName; + } + } + +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java new file mode 100644 index 0000000000..1c16009ffa --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.runtime.Event; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class InfluxStore { + + private static final Logger LOG = LoggerFactory.getLogger(InfluxStore.class); + + private final DataLakeMeasure measure; + private final List allEventProperties; + private final Map sanitizedRuntimeNames = new HashMap<>(); + private final InfluxDB influxDb; + + private final PropertyHandler propertyHandler; + + + public InfluxStore( + DataLakeMeasure measure, + Environment environment, + InfluxClientProvider influxClientProvider + ) throws SpRuntimeException { + this.measure = measure; + storeSanitizedRuntimeNames(measure); + allEventProperties = getAllEventPropertiesExceptTimestamp(measure); + influxDb = influxClientProvider.getInitializedInfluxDBClient(environment); + propertyHandler = new PropertyHandler(); + } + + /** + * Takes an StreamPipes event, transforms it to an InfluxDB point and writes it to the InfluxDB + * + * @param event The event which should be saved + * @throws SpRuntimeException If the column name (key-value of the event map) is not allowed + */ + public void onEvent(Event event) throws SpRuntimeException { + + validateInputEventAndLogMissingFields(event); + + sanitizeRuntimeNamesInEvent(event); + + var point = initializePointWithTimestamp(event); + + iterateOverallEventProperties(event, point); + + influxDb.write(point.build()); + } + + private void validateInputEventAndLogMissingFields(Event event) { + checkEventIsNotNull(event); + + logMissingFields(event); + + logNullFields(event); + } + + /** + * Logs all fields which are present in the schema, but not in the provided event + */ + private void logMissingFields(Event event) { + var missingFields = getMissingProperties(allEventProperties, event); + if (!missingFields.isEmpty()) { + LOG.debug( + "Ignored {} fields which were present in the schema, but not in the provided event: {}", + missingFields.size(), + String.join(", ", missingFields) + ); + } + } + + + /** + * Logs all fields that contain null values + */ + private void logNullFields(Event event) { + List nullFields = allEventProperties + .stream() + .filter(EventPropertyPrimitive.class::isInstance) + .filter(ep -> { + var runtimeName = ep.getRuntimeName(); + var field = event.getOptionalFieldByRuntimeName(runtimeName); + + return field.isPresent() && field.get() + .getAsPrimitive() + .getRawValue() == null; + }) + .map(EventProperty::getRuntimeName) + .collect(Collectors.toList()); + + if (!nullFields.isEmpty()) { + LOG.warn("Ignored {} fields which had a value 'null': {}", nullFields.size(), String.join(", ", nullFields)); + } + } + + private void iterateOverallEventProperties( + Event event, + Point.Builder point + ) { + + allEventProperties.forEach(ep -> { + var runtimeName = ep.getRuntimeName(); + var sanitizedRuntimeName = sanitizedRuntimeNames.get(runtimeName); + var fieldOptional = event.getOptionalFieldByRuntimeName(runtimeName); + + fieldOptional.ifPresent(field -> { + if (ep instanceof EventPropertyPrimitive) { + propertyHandler.handlePrimitiveProperty( + point, + (EventPropertyPrimitive) ep, + field.getAsPrimitive(), + sanitizedRuntimeName + ); + } else { + propertyHandler.handleNonPrimitiveProperty( + point, + event, + sanitizedRuntimeName + ); + } + }); + }); + } + + + /** + * Returns a list of the runtime names that are missing within the event + */ + private List getMissingProperties( + List allEventProperties, + Event event + ) { + return allEventProperties.stream() + .map(EventProperty::getRuntimeName) + .filter(runtimeName -> event.getOptionalFieldByRuntimeName(runtimeName) + .isEmpty()) + .collect(Collectors.toList()); + } + + private void checkEventIsNotNull(Event event) { + if (event == null) { + throw new SpRuntimeException("event is null"); + } + } + + + /** + * Shuts down the connection to the InfluxDB server + */ + public void close() throws SpRuntimeException { + influxDb.flush(); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new SpRuntimeException(e); + } + influxDb.close(); + } + + + /** + * Creates a point object which is later written to the influxDB and adds the value of the timestamp field + */ + private Point.Builder initializePointWithTimestamp(Event event) { + var timestampValue = event.getFieldBySelector(measure.getTimestampField()) + .getAsPrimitive() + .getAsLong(); + return Point.measurement(measure.getMeasureName()) + .time((long) timestampValue, TimeUnit.MILLISECONDS); + } + + /** + * store sanitized target property runtime names in local variable + */ + private void storeSanitizedRuntimeNames(DataLakeMeasure measure) { + measure.getEventSchema() + .getEventProperties() + .forEach(ep -> sanitizedRuntimeNames.put( + ep.getRuntimeName(), + InfluxNameSanitizer.renameReservedKeywords(ep.getRuntimeName()) + )); + } + + /** + * Returns all measurements properties except the timestamp field + */ + private List getAllEventPropertiesExceptTimestamp(DataLakeMeasure dataLakeMeasure) { + return dataLakeMeasure.getEventSchema() + .getEventProperties() + .stream() + .filter(ep -> !dataLakeMeasure.getTimestampField() + .endsWith(ep.getRuntimeName())) + .collect(Collectors.toList()); + } + + /** + * Iterates over all properties of the event and renames the key if it is a reserved keywords in InfluxDB + */ + private void sanitizeRuntimeNamesInEvent(Event event) { + // sanitize event + for (var key : event.getRaw() + .keySet()) { + if (InfluxDbReservedKeywords.KEYWORD_LIST.stream() + .anyMatch(k -> k.equalsIgnoreCase(key))) { + event.renameFieldByRuntimeName(key, key + "_"); + } + } + } + +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java new file mode 100644 index 0000000000..9cc4a79d17 --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.commons.influx.serializer.RawFieldSerializer; +import org.apache.streampipes.model.runtime.Event; +import org.apache.streampipes.model.runtime.field.PrimitiveField; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; +import org.apache.streampipes.model.schema.PropertyScope; +import org.apache.streampipes.vocabulary.SO; +import org.apache.streampipes.vocabulary.XSD; + +import org.influxdb.dto.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PropertyHandler { + + private static final Logger LOG = LoggerFactory.getLogger(PropertyHandler.class); + + private RawFieldSerializer rawFieldSerializer; + + public PropertyHandler() { + rawFieldSerializer = new RawFieldSerializer(); + } + + /** + * Takes properties of type primitive and stores them as tags or fields to the given point + */ + public void handlePrimitiveProperty( + Point.Builder point, + EventPropertyPrimitive eventPropertyPrimitive, + PrimitiveField primitiveField, + String sanitizedRuntimeName + ) { + if (primitiveField.getRawValue() != null) { + // store property as tag when the field is a dimension property + if (PropertyScope.DIMENSION_PROPERTY.name() + .equals(eventPropertyPrimitive.getPropertyScope())) { + handleDimensionProperty( + point, + primitiveField, + sanitizedRuntimeName + ); + } else { + handleMeasurementProperty( + point, + eventPropertyPrimitive.getRuntimeType(), + sanitizedRuntimeName, + primitiveField + ); + } + } + } + + + /** + * Takes non-primitive properties of and stores them as strings to the given point + * Since InfluxDB can't store non-primitive types, store them as string + * and deserialize later in downstream processes + */ + public void handleNonPrimitiveProperty( + Point.Builder p, + Event event, + String preparedRuntimeName + ) { + try { + var json = rawFieldSerializer.serialize(event.getRaw() + .get(preparedRuntimeName)); + p.addField(preparedRuntimeName, json); + } catch (SpRuntimeException e) { + LOG.warn("Failed to serialize field {}, ignoring.", preparedRuntimeName); + } + } + + /** + * Add dimension properties as tags to point + */ + private void handleDimensionProperty( + Point.Builder point, + PrimitiveField primitiveField, + String sanitizedRuntimeName + ) { + point.tag(sanitizedRuntimeName, primitiveField.getAsString()); + } + + /** + * Transforms the given primitive property to the corresponding InfluxDB data type and adds it to the point + */ + private void handleMeasurementProperty( + Point.Builder p, + String runtimeType, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + // Store property according to property type + if (XSD.INTEGER.toString() + .equals(runtimeType)) { + handleIntegerProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else if (XSD.LONG.toString() + .equals(runtimeType)) { + handleLongProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else if (XSD.FLOAT.toString() + .equals(runtimeType)) { + handleFloatProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else if (XSD.DOUBLE.toString() + .equals(runtimeType)) { + handleDoubleProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else if (XSD.BOOLEAN.toString() + .equals(runtimeType)) { + handleBooleanProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else if (SO.NUMBER.equals(runtimeType)) { + handleDoubleProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } else { + handleStringProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } + } + + + private void handleStringProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsString()); + } + + private void handleBooleanProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsBoolean()); + } + + private void handleDoubleProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsDouble()); + } + + private void handleFloatProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsFloat()); + } + + private void handleLongProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + try { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsLong()); + } catch (NumberFormatException ef) { + handleFloatProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } + } + + private void handleIntegerProperty( + Point.Builder p, + String sanitizedRuntimeName, + PrimitiveField eventPropertyPrimitiveField + ) { + try { + p.addField(sanitizedRuntimeName, eventPropertyPrimitiveField.getAsInt()); + } catch (NumberFormatException ef) { + handleFloatProperty(p, sanitizedRuntimeName, eventPropertyPrimitiveField); + } + } + +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java new file mode 100644 index 0000000000..05380df10c --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx.serializer; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; + +public class RawFieldSerializer { + protected ObjectMapper objectMapper; + + public RawFieldSerializer() { + this.objectMapper = new ObjectMapper().activateDefaultTyping( + BasicPolymorphicTypeValidator.builder() + .allowIfBaseType(Object.class) + .build(), + ObjectMapper.DefaultTyping.EVERYTHING); + } + + public String serialize(Object object) { + try { + return objectMapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new SpRuntimeException(e.getCause()); + } + } + + public Object deserialize(String json) { + try { + return objectMapper.readValue(json, Object.class); + } catch (JsonProcessingException e) { + throw new SpRuntimeException(e.getCause()); + } + } +} diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java new file mode 100644 index 0000000000..1ffbd02fef --- /dev/null +++ b/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.sanitizer; + +import java.util.List; + +public class MeasureNameSanitizer { + + private final List forbiddenChars = List.of( + "/", + "?", + "=", + "\"" + ); + + private final String replacement = "_"; + + public String sanitize(String measureName) { + return forbiddenChars.stream() + .reduce(measureName, (str, forbiddenChar) -> str.replace(forbiddenChar, replacement)); + } +} diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java new file mode 100644 index 0000000000..79b0849812 --- /dev/null +++ b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +public class InfluxClientProviderTest { + + private InfluxDB influxDBMock; + private InfluxClientProvider influxClientProvider; + + private static final String DATABASE_NAME = "testDB"; + + @BeforeEach + public void setUp() { + influxDBMock = mock(InfluxDB.class); + influxClientProvider = new InfluxClientProvider(); + } + + @Test + public void createDatabaseSuccess() { + var expectedQuery = new Query("CREATE DATABASE test", ""); + + influxClientProvider.createDatabase(influxDBMock, "test"); + + var pointArgumentCaptor = ArgumentCaptor.forClass(Query.class); + verify(influxDBMock).query(pointArgumentCaptor.capture()); + var actualQuery = pointArgumentCaptor.getValue(); + + assertEquals(actualQuery.getDatabase(), expectedQuery.getDatabase()); + } + + @Test + public void createDatabaseIlligalName() { + assertThrows(SpRuntimeException.class, () -> influxClientProvider.createDatabase(influxDBMock, "test%^$")); + } + + @Test + public void dataBaseDoesExist() { + var actualResult = testDataBaseExists(List.of(DATABASE_NAME)); + assertTrue(actualResult); + } + + @Test + public void dataBaseDoesNotExist() { + var actualResult = testDataBaseExists(List.of()); + assertFalse(actualResult); + } + + private boolean testDataBaseExists(List databaseNames) { + var queryResult = getQueryResultWithDatabaseNames(databaseNames); + + when(influxDBMock.query(any())).thenReturn(queryResult); + + return influxClientProvider.databaseExists(influxDBMock, DATABASE_NAME); + } + + private QueryResult getQueryResultWithDatabaseNames(List databaseNames) { + var queryResult = mock(QueryResult.class); + var result = mock(QueryResult.Result.class); + when(queryResult.getResults()).thenReturn(List.of(result)); + var series = mock(QueryResult.Series.class); + when(result.getSeries()).thenReturn(List.of(series)); + + List> values = List.of(databaseNames); + when(series.getValues()).thenReturn(values); + return queryResult; + } +} \ No newline at end of file diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java new file mode 100644 index 0000000000..888e44e346 --- /dev/null +++ b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx; + + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.commons.influx.serializer.RawFieldSerializer; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.runtime.Event; +import org.apache.streampipes.model.runtime.EventFactory; +import org.apache.streampipes.model.runtime.SchemaInfo; +import org.apache.streampipes.model.runtime.SourceInfo; +import org.apache.streampipes.model.schema.EventSchema; +import org.apache.streampipes.model.schema.PropertyScope; +import org.apache.streampipes.test.generator.EventPropertyListTestBuilder; +import org.apache.streampipes.test.generator.EventPropertyNestedTestBuilder; +import org.apache.streampipes.test.generator.EventPropertyPrimitiveTestBuilder; +import org.apache.streampipes.test.generator.EventSchemaTestBuilder; +import org.apache.streampipes.vocabulary.SO; +import org.apache.streampipes.vocabulary.XSD; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class InfluxStoreTest { + + private InfluxDB influxDBMock; + + private RawFieldSerializer serializer; + + private static final String EXPECTED_MEASUREMENT = "testMeasure"; + private static final String FIELD_NAME = "testId"; + private static final String TIMESTAMP = "timestamp"; + private static final long SAMPLE_TIMESTAMP = 1701270890000L; + + @BeforeEach + public void setUp() { + influxDBMock = mock(InfluxDB.class); + serializer = new RawFieldSerializer(); + } + + @Test + public void onEventNull() { + var influxStore = getInfluxStore(new EventSchema()); + assertThrows(SpRuntimeException.class, () -> influxStore.onEvent(null)); + } + + + @Test + public void onEventWithInteger() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1) + .build(); + + var actualPoint = testEventWithOneField(XSD.INTEGER, 1); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithIntegerFloatValue() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1.0f) + .build(); + + var actualPoint = testEventWithOneField(XSD.INTEGER, 1.0f); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithLong() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1L) + .build(); + + var actualPoint = testEventWithOneField(XSD.LONG, 1L); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithLongFloatValue() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1.0f) + .build(); + + var actualPoint = testEventWithOneField(XSD.LONG, 1.0f); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithFloat() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1.0f) + .build(); + + var actualPoint = testEventWithOneField(XSD.FLOAT, 1.0f); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithDouble() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1.0) + .build(); + + var actualPoint = testEventWithOneField(XSD.DOUBLE, 1.0); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithBoolean() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, true) + .build(); + + var actualPoint = testEventWithOneField(XSD.BOOLEAN, true); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithNumber() throws URISyntaxException { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, 1.0) + .build(); + + var actualPoint = testEventWithOneField(new URI(SO.NUMBER), 1.0); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithString() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, "testValue") + .build(); + + var actualPoint = testEventWithOneField(XSD.STRING, "testValue"); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventReservedKeyWord() { + // name is a reserved keyword + var fieldName = "name"; + var sanitizedFieldName = "name_"; + var value = 1; + + var expected = getPointBuilderWithTimestamp() + .addField(sanitizedFieldName, value) + .build(); + + var eventSchema = getEventSchemaBuilderWithTimestamp() + .withEventProperty( + EventPropertyPrimitiveTestBuilder + .create() + .withRuntimeName(sanitizedFieldName) + .withRuntimeType(XSD.INTEGER) + .build()) + .build(); + + var event = getEvent(eventSchema, Map.of(fieldName, value)); + + var influxStore = getInfluxStore(eventSchema); + + var actualPoint = executeOnEvent(influxStore, event); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithTag() { + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, "value") + .tag("id", "id1") + .build(); + + var eventSchema = getEventSchemaBuilderWithTimestamp() + .withEventProperty( + EventPropertyPrimitiveTestBuilder + .create() + .withRuntimeName(FIELD_NAME) + .withRuntimeType(XSD.STRING) + .build()) + .withEventProperty( + EventPropertyPrimitiveTestBuilder + .create() + .withRuntimeName("id") + .withRuntimeType(XSD.STRING) + .withPropertyScope(PropertyScope.DIMENSION_PROPERTY) + .build()) + .build(); + + var event = getEvent(eventSchema, Map.of(FIELD_NAME, "value", "id", "id1")); + + var influxStore = getInfluxStore(eventSchema); + + var actualPoint = executeOnEvent(influxStore, event); + + assertEquals(expected, actualPoint); + } + + + @Test + public void onEventWithNestedProperty() { + Map value = new HashMap<>(); + value.put("one", "two"); + + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, serializer.serialize(value)) + .build(); + + var eventSchema = getEventSchemaBuilderWithTimestamp() + .withEventProperty( + EventPropertyNestedTestBuilder + .create() + .withRuntimeName(FIELD_NAME) + .build()) + .build(); + + var event = getEvent(eventSchema, Map.of(FIELD_NAME, value)); + + var influxStore = getInfluxStore(eventSchema); + + var actualPoint = executeOnEvent(influxStore, event); + + assertEquals(expected, actualPoint); + } + + @Test + public void onEventWithListProperty() { + String[] value = {"one", "two"}; + var expected = getPointBuilderWithTimestamp() + .addField(FIELD_NAME, serializer.serialize(value)) + .build(); + + var eventSchema = getEventSchemaBuilderWithTimestamp() + .withEventProperty( + EventPropertyListTestBuilder + .create() + .withRuntimeName(FIELD_NAME) + .build()) + .build(); + + var event = getEvent(eventSchema, Map.of(FIELD_NAME, value)); + + var influxStore = getInfluxStore(eventSchema); + + var actualPoint = executeOnEvent(influxStore, event); + + assertEquals(expected, actualPoint); + } + + /** + * Initialize a Point builder with a timestamp + */ + private Point.Builder getPointBuilderWithTimestamp() { + return Point.measurement(EXPECTED_MEASUREMENT) + .time(SAMPLE_TIMESTAMP, TimeUnit.MILLISECONDS); + } + + + /** + * Executes the onEvent method and returns the resuting data point using an argument captor + */ + private Point executeOnEvent(InfluxStore influxStore, Event event) { + influxStore.onEvent(event); + var pointArgumentCaptor = ArgumentCaptor.forClass(Point.class); + + verify(influxDBMock).write(pointArgumentCaptor.capture()); + + return pointArgumentCaptor.getValue(); + } + + /** + * Creates an event with the given data and a timestamp + */ + private Event getEvent(EventSchema eventSchema, Map inputData) { + Map data = new HashMap<>(); + data.put(TIMESTAMP, SAMPLE_TIMESTAMP); + data.putAll(inputData); + + return EventFactory.fromMap( + data, + new SourceInfo("test-topic", "s0"), + new SchemaInfo(eventSchema, new ArrayList<>()) + ); + } + + + /** + * This convenience method is used to test all cases for the different supported data types + */ + private Point testEventWithOneField( + URI type, + Object value + ) { + + var eventSchema = getEventSchemaBuilderWithTimestamp() + .withEventProperty( + EventPropertyPrimitiveTestBuilder + .create() + .withRuntimeName(FIELD_NAME) + .withRuntimeType(type) + .build()) + .build(); + + + var event = EventFactory.fromMap( + Map.of(TIMESTAMP, SAMPLE_TIMESTAMP, FIELD_NAME, value), + new SourceInfo("test-topic", "s0"), + new SchemaInfo(eventSchema, new ArrayList<>()) + ); + + var influxStore = getInfluxStore(eventSchema); + + influxStore.onEvent(event); + var pointArgumentCaptor = ArgumentCaptor.forClass(Point.class); + + verify(influxDBMock).write(pointArgumentCaptor.capture()); + return pointArgumentCaptor.getValue(); + } + + /** + * Returns the event schema builder with a default timestamp property + */ + private EventSchemaTestBuilder getEventSchemaBuilderWithTimestamp() { + return EventSchemaTestBuilder + .create() + .withEventProperty( + EventPropertyPrimitiveTestBuilder + .create() + .withRuntimeName(TIMESTAMP) + .build()); + } + + /** + * Initializes an influx store with the given event schema + */ + private InfluxStore getInfluxStore(EventSchema eventSchema) { + + DataLakeMeasure measure = new DataLakeMeasure( + EXPECTED_MEASUREMENT, + "s0::%s".formatted(TIMESTAMP), + eventSchema + ); + + var influxClientProviderMock = mock(InfluxClientProvider.class); + when(influxClientProviderMock.getInitializedInfluxDBClient(any())) + .thenReturn(influxDBMock); + + return new InfluxStore(measure, null, influxClientProviderMock); + } + +} \ No newline at end of file diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java new file mode 100644 index 0000000000..9708617418 --- /dev/null +++ b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.influx.serializer; + + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class TestRawFieldSerializer { + private RawFieldSerializer rawFieldSerializer = new RawFieldSerializer(); + private Map primitives = new HashMap<>(); + + public TestRawFieldSerializer() { + primitives.put("Integer", 1); + primitives.put("Long", 1L); + primitives.put("Float", 1.0f); + primitives.put("Double", 1.0d); + primitives.put("Boolean", true); + primitives.put("String", "1"); + } + + // Test able to deserialize back the original data + @Test + public void testRawFieldSerializerListInMap() { + var rawListField = new ArrayList<>(); + rawListField.addAll(primitives.values()); + + var rawNestedField = new HashMap(); + rawNestedField.putAll(primitives); + rawNestedField.put("List", rawListField); + + var json = rawFieldSerializer.serialize(rawNestedField); + + assertEquals(rawNestedField, rawFieldSerializer.deserialize(json)); + } + + @Test + public void testRawFieldSerializerMapInList() { + var rawNestedField = new HashMap(); + rawNestedField.putAll(primitives); + + var rawListField = new ArrayList<>(); + rawListField.addAll(primitives.values()); + rawListField.add(rawNestedField); + + var json = rawFieldSerializer.serialize(rawListField); + + assertEquals(rawListField, rawFieldSerializer.deserialize(json)); + } +} diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java new file mode 100644 index 0000000000..abbeb096f7 --- /dev/null +++ b/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.commons.sanitizer; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MeasureNameSanitizerTest { + + @Test + public void testSanitizedMeasure() { + String originalMeasureName = "test/001"; + String expected = "test_001"; + String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName); + + Assertions.assertEquals(expected, sanitizedMeasureName); + } + + @Test + public void testOriginalMeasure() { + String originalMeasureName = "test_001"; + String expected = "test_001"; + String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName); + + Assertions.assertEquals(expected, sanitizedMeasureName); + } + + @Test + public void testMultipleCharacters() { + String originalMeasureName = "t?\"est_001="; + String expected = "t__est_001_"; + String sanitizedMeasureName = new MeasureNameSanitizer().sanitize(originalMeasureName); + + Assertions.assertEquals(expected, sanitizedMeasureName); + } +} diff --git a/streampipes-data-explorer/pom.xml b/streampipes-data-explorer/pom.xml index 37b3a16552..ac4e4ed87d 100644 --- a/streampipes-data-explorer/pom.xml +++ b/streampipes-data-explorer/pom.xml @@ -21,43 +21,44 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 streampipes-data-explorer - - This Maven module encompasses all StreamPipes-specific implementation endeavors concerning data explorer functionalities. - - org.apache.streampipes - streampipes-data-explorer-api - 0.97.0-SNAPSHOT + streampipes-data-explorer-commons + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-storage-management - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT + + org.influxdb + influxdb-java + + - org.junit.jupiter - junit-jupiter-api + org.apache.streampipes + streampipes-test-utils + 0.95.1-SNAPSHOT test - org.apache.streampipes - streampipes-test-utils - 0.97.0-SNAPSHOT + org.junit.jupiter + junit-jupiter-api test diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java new file mode 100644 index 0000000000..07c5058b6c --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.dataexplorer.api.IDataExplorerQueryManagement; +import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; +import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.query.DeleteDataQuery; +import org.apache.streampipes.dataexplorer.query.QueryResultProvider; +import org.apache.streampipes.dataexplorer.query.StreamedQueryResultProvider; +import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.datalake.SpQueryResult; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class DataExplorerQueryManagement implements IDataExplorerQueryManagement { + + private final IDataExplorerSchemaManagement dataExplorerSchemaManagement; + + public DataExplorerQueryManagement(IDataExplorerSchemaManagement dataExplorerSchemaManagement) { + this.dataExplorerSchemaManagement = dataExplorerSchemaManagement; + } + + @Override + public SpQueryResult getData(ProvidedRestQueryParams queryParams, + boolean ignoreMissingData) throws IllegalArgumentException { + return new QueryResultProvider(queryParams, ignoreMissingData).getData(); + } + + @Override + public void getDataAsStream(ProvidedRestQueryParams params, + OutputFormat format, + boolean ignoreMissingValues, + OutputStream outputStream) throws IOException { + + new StreamedQueryResultProvider(params, format, ignoreMissingValues).getDataAsStream(outputStream); + } + + @Override + public boolean deleteAllData() { + List allMeasurements = getAllMeasurements(); + + for (DataLakeMeasure measure : allMeasurements) { + QueryResult queryResult = new DeleteDataQuery(measure).executeQuery(); + if (queryResult.hasError() || queryResult.getResults().get(0).getError() != null) { + return false; + } + } + return true; + } + + @Override + public boolean deleteData(String measurementID) { + List allMeasurements = getAllMeasurements(); + for (DataLakeMeasure measure : allMeasurements) { + if (measure.getMeasureName().equals(measurementID)) { + QueryResult queryResult = new DeleteDataQuery(new DataLakeMeasure(measurementID, null)).executeQuery(); + + return !queryResult.hasError(); + } + } + return false; + } + + @Override + public SpQueryResult deleteData(String measurementID, Long startDate, Long endDate) { + DeleteQueryParams params = + ProvidedRestQueryParamConverter.getDeleteQueryParams(measurementID, startDate, endDate); + return new DataExplorerInfluxQueryExecutor().executeQuery(params); + } + + @Override + public Map getTagValues(String measurementId, + String fields) { + InfluxDB influxDB = InfluxClientProvider.getInfluxDBClient(); + String databaseName = getEnvironment().getTsStorageBucket().getValueOrDefault(); + Map tags = new HashMap<>(); + if (fields != null && !(fields.isEmpty())) { + List fieldList = Arrays.asList(fields.split(",")); + fieldList.forEach(f -> { + String q = + "SHOW TAG VALUES ON \"" + databaseName + "\" FROM \"" + measurementId + + "\" WITH KEY = \"" + f + "\""; + Query query = new Query(q); + QueryResult queryResult = influxDB.query(query); + queryResult.getResults().forEach(res -> { + res.getSeries().forEach(series -> { + if (!series.getValues().isEmpty()) { + String field = series.getValues().get(0).get(0).toString(); + List values = + series.getValues().stream().map(v -> v.get(1).toString()).collect(Collectors.toList()); + tags.put(field, values); + } + }); + }); + }); + } + + return tags; + } + + private List getAllMeasurements() { + return this.dataExplorerSchemaManagement.getAllMeasurements(); + } + + private Environment getEnvironment() { + return Environments.getEnvironment(); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java index ef3db4c7d3..7db5bad83b 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagement.java @@ -23,8 +23,15 @@ import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy; import org.apache.streampipes.model.schema.EventProperty; -import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.api.IDataLakeStorage; +import org.apache.streampipes.storage.couchdb.utils.Utils; +import com.google.gson.JsonObject; +import org.lightcouch.CouchDbClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -34,9 +41,11 @@ public class DataExplorerSchemaManagement implements IDataExplorerSchemaManagement { - CRUDStorage dataLakeStorage; + private static final Logger LOG = LoggerFactory.getLogger(DataExplorerSchemaManagement.class); + + IDataLakeStorage dataLakeStorage; - public DataExplorerSchemaManagement(CRUDStorage dataLakeStorage) { + public DataExplorerSchemaManagement(IDataLakeStorage dataLakeStorage) { this.dataLakeStorage = dataLakeStorage; } @@ -47,7 +56,7 @@ public List getAllMeasurements() { @Override public DataLakeMeasure getById(String elementId) { - return dataLakeStorage.getElementById(elementId); + return dataLakeStorage.findOne(elementId); } /** @@ -71,7 +80,7 @@ public DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure) { } /** - * Distinguishes between the update strategy for existing measurements + * Destiguishes between the update straregy for existing measurments */ private void handleExistingMeasurement( DataLakeMeasure measure, @@ -92,7 +101,7 @@ private void handleExistingMeasurement( * Returns the existing measure that has the provided measure name */ private Optional getExistingMeasureByName(String measureName) { - return dataLakeStorage.findAll() + return dataLakeStorage.getAllDataLakeMeasures() .stream() .filter(m -> m.getMeasureName() .equals(measureName)) @@ -107,8 +116,8 @@ private static void setDefaultUpdateStrategyIfNoneProvided(DataLakeMeasure measu @Override public void deleteMeasurement(String elementId) { - if (dataLakeStorage.getElementById(elementId) != null) { - dataLakeStorage.deleteElementById(elementId); + if (dataLakeStorage.findOne(elementId) != null) { + dataLakeStorage.deleteDataLakeMeasure(elementId); } else { throw new IllegalArgumentException("Could not find measure with this ID"); } @@ -116,33 +125,52 @@ public void deleteMeasurement(String elementId) { @Override public boolean deleteMeasurementByName(String measureName) { - var measureToDeleteOpt = dataLakeStorage.findAll() - .stream() - .filter(measurement -> measurement.getMeasureName() - .equals(measureName)) - .findFirst(); - - return measureToDeleteOpt.map(measure -> { - dataLakeStorage.deleteElementById(measure.getElementId()); - return true; + boolean isSuccess = false; + CouchDbClient couchDbClient = Utils.getCouchDbDataLakeClient(); + List docs = couchDbClient.view("_all_docs") + .includeDocs(true) + .query(JsonObject.class); + + for (JsonObject document : docs) { + if (document.get("measureName") + .toString() + .replace("\"", "") + .equals(measureName)) { + couchDbClient.remove( + document.get("_id") + .toString() + .replace("\"", ""), + document.get("_rev") + .toString() + .replace("\"", "") + ); + isSuccess = true; + break; + } + } + + try { + couchDbClient.close(); + } catch (IOException e) { + LOG.error("Could not close CouchDB client", e); } - ).orElse(false); + return isSuccess; } @Override public void updateMeasurement(DataLakeMeasure measure) { - var existingMeasure = dataLakeStorage.getElementById(measure.getElementId()); + var existingMeasure = dataLakeStorage.findOne(measure.getElementId()); if (existingMeasure != null) { measure.setRev(existingMeasure.getRev()); - dataLakeStorage.updateElement(measure); + dataLakeStorage.updateDataLakeMeasure(measure); } else { - dataLakeStorage.persist(measure); + dataLakeStorage.storeDataLakeMeasure(measure); } } private void setSchemaVersionAndStoreMeasurement(DataLakeMeasure measure) { measure.setSchemaVersion(DataLakeMeasure.CURRENT_SCHEMA_VERSION); - dataLakeStorage.persist(measure); + dataLakeStorage.storeDataLakeMeasure(measure); } /** diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java new file mode 100644 index 0000000000..59620a21be --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.api; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.model.datalake.SpQueryResult; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +public interface IDataExplorerQueryManagement { + + SpQueryResult getData(ProvidedRestQueryParams queryParams, + boolean ignoreMissingData) throws IllegalArgumentException; + + void getDataAsStream(ProvidedRestQueryParams params, + OutputFormat format, + boolean ignoreMissingValues, + OutputStream outputStream) throws IOException; + + boolean deleteData(String measurementID); + + SpQueryResult deleteData(String measurementID, Long startDate, Long endDate); + + boolean deleteAllData(); + + Map getTagValues(String measurementId, + String fields); +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java new file mode 100644 index 0000000000..a210b515b3 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.api; + +import org.apache.streampipes.model.datalake.DataLakeMeasure; + +import java.util.List; + +public interface IDataExplorerSchemaManagement { + + List getAllMeasurements(); + + DataLakeMeasure getById(String elementId); + + DataLakeMeasure createOrUpdateMeasurement(DataLakeMeasure measure); + + void deleteMeasurement(String elementId); + + boolean deleteMeasurementByName(String measureName); + + void updateMeasurement(DataLakeMeasure measure); +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java new file mode 100644 index 0000000000..1e1a0f39a2 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.api; + +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; + +public interface IQueryStatement { + + void buildStatement(IDataLakeQueryBuilder builder); +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java new file mode 100644 index 0000000000..dbb8a9a0b2 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.influx; + +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; +import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.dataexplorer.query.DataExplorerQueryExecutor; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.DataSeries; +import org.apache.streampipes.model.datalake.SpQueryResult; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class DataExplorerInfluxQueryExecutor extends DataExplorerQueryExecutor { + + public DataExplorerInfluxQueryExecutor() { + super(); + } + + public DataExplorerInfluxQueryExecutor(String forId) { + super(forId); + } + + public DataExplorerInfluxQueryExecutor(int maximumAmountOfEvents) { + super(maximumAmountOfEvents); + } + + protected DataSeries convertResult(QueryResult.Series series, + boolean ignoreMissingValues) { + List columns = series.getColumns(); + List> values = series.getValues(); + + List> resultingValues = new ArrayList<>(); + + values.forEach(v -> { + if (ignoreMissingValues) { + if (!v.contains(null)) { + resultingValues.add(v); + } + } else { + resultingValues.add(v); + } + + }); + + return new DataSeries(values.size(), resultingValues, columns, series.getTags()); + } + + protected SpQueryResult postQuery(QueryResult queryResult, + boolean ignoreMissingValues) throws RuntimeException { + SpQueryResult result = new SpQueryResult(); + AtomicLong lastTimestamp = new AtomicLong(); + + if (hasResult(queryResult)) { + queryResult.getResults().get(0).getSeries().forEach(rs -> { + DataSeries series = convertResult(rs, ignoreMissingValues); + result.setHeaders(series.getHeaders()); + result.addDataResult(series); + List lastValue = rs.getValues().get(rs.getValues().size() - 1); + lastTimestamp.set(Math.max(lastTimestamp.get(), ((Double) lastValue.get(0)).longValue())); + }); + + result.setTotal(result.getAllDataSeries().stream().mapToInt(DataSeries::getTotal).sum()); + result.setLastTimestamp(lastTimestamp.get()); + } + + if (this.appendId) { + result.setForId(this.forId); + } + + return result; + } + + private IDataLakeQueryBuilder getQueryBuilder(String measurementId) { + return DataLakeInfluxQueryBuilder.create(measurementId); + } + + @Override + protected QueryResult executeQuery(Query query) { + try (final InfluxDB influxDB = InfluxClientProvider.getInfluxDBClient()) { + return influxDB.query(query, TimeUnit.MILLISECONDS); + } + } + + @Override + protected String asQueryString(Query query) { + return "(database:" + query.getDatabase() + "): " + query.getCommand(); + } + + @Override + protected Query makeDeleteQuery(DeleteQueryParams params) { + String query = "DELETE FROM \"" + params.getMeasurementId() + "\""; + if (params.isTimeRestricted()) { + query += "WHERE time > " + + params.getStartTime() * 1000000 + + " AND time < " + + params.getEndTime() * 1000000; + } + return new Query(query, getDatabaseName()); + } + + @Override + protected Query makeSelectQuery(SelectQueryParams params) { + var builder = getQueryBuilder(params.getIndex()); + return getQueryWithDatabaseName(params.toQuery(builder)); + } + + private boolean hasResult(QueryResult queryResult) { + return queryResult.getResults() != null + && !queryResult.getResults().isEmpty() + && queryResult.getResults().get(0).getSeries() != null; + } + + private Query getQueryWithDatabaseName(Query query) { + var databaseName = getDatabaseName(); + return new Query(query.getCommand(), databaseName); + } + + private String getDatabaseName() { + return Environments.getEnvironment().getTsStorageBucket().getValueOrDefault(); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java new file mode 100644 index 0000000000..d2f518246b --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.influx; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; +import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; +import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; + +import org.influxdb.dto.Query; +import org.influxdb.querybuilder.Ordering; +import org.influxdb.querybuilder.SelectionQueryImpl; +import org.influxdb.querybuilder.clauses.AndConjunction; +import org.influxdb.querybuilder.clauses.Clause; +import org.influxdb.querybuilder.clauses.ConjunctionClause; +import org.influxdb.querybuilder.clauses.NestedClause; +import org.influxdb.querybuilder.clauses.OrConjunction; +import org.influxdb.querybuilder.clauses.RawTextClause; +import org.influxdb.querybuilder.clauses.SimpleClause; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.asc; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.desc; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +public class DataLakeInfluxQueryBuilder implements IDataLakeQueryBuilder { + + private final String measurementId; + private final SelectionQueryImpl selectionQuery; + private final List whereClauses; + private final List groupByClauses; + private Ordering ordering; + private int limit = Integer.MIN_VALUE; + private int offset = Integer.MIN_VALUE; + + private Object fill; + + private final Environment env; + + private DataLakeInfluxQueryBuilder(String measurementId) { + this.measurementId = measurementId; + this.selectionQuery = select(); + this.whereClauses = new ArrayList<>(); + this.groupByClauses = new ArrayList<>(); + this.env = Environments.getEnvironment(); + } + + public static DataLakeInfluxQueryBuilder create(String measurementId) { + return new DataLakeInfluxQueryBuilder(measurementId); + } + + + @Override + public DataLakeInfluxQueryBuilder withAllColumns() { + this.selectionQuery.all(); + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withSimpleColumn(String columnName) { + this.selectionQuery.column(columnName); + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withSimpleColumns(List columnNames) { + columnNames.forEach(this.selectionQuery::column); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withAggregatedColumn(String columnName, + AggregationFunction aggregationFunction, + String aliasName) { + + this.selectionQuery.function(aggregationFunction.toDbName(), columnName).as(aliasName); + + return this; + } + + @Override + public IDataLakeQueryBuilder withAggregatedColumn(String columnName, AggregationFunction aggregationFunction) { + this.selectionQuery.function(aggregationFunction.toDbName(), columnName); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withStartTime(long startTime) { + this.whereClauses.add(new SimpleClause("time", ">=", startTime * 1000000)); + return this; + } + + + @Override + public DataLakeInfluxQueryBuilder withEndTime(long endTime) { + return withEndTime(endTime, true); + } + + @Override + public DataLakeInfluxQueryBuilder withEndTime(long endTime, + boolean includeEndTime) { + String operator = includeEndTime ? "<=" : "<"; + this.whereClauses.add(new SimpleClause("time", operator, endTime * 1000000)); + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withTimeBoundary(long startTime, + long endTime) { + this.withStartTime(startTime); + this.withEndTime(endTime); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withFilter(String field, + String operator, + Object value) { + this.whereClauses.add(new SimpleClause(field, operator, value)); + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withExclusiveFilter(String field, + String operator, + List values) { + List or = new ArrayList<>(); + values.forEach(value -> or.add(new OrConjunction(new SimpleClause(field, operator, value)))); + + addNestedWhereClause(or); + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withInclusiveFilter(String field, + String operator, + List values) { + List and = new ArrayList<>(); + values.forEach(value -> and.add(new AndConjunction(new SimpleClause(field, operator, value)))); + + addNestedWhereClause(and); + return this; + } + + @Override + public IDataLakeQueryBuilder withInclusiveFilter(List filterConditions) { + List and = new ArrayList<>(); + filterConditions.forEach(c -> and + .add(new AndConjunction(new SimpleClause(c.getField(), c.getOperator(), c.getCondition())))); + + addNestedWhereClause(and); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withFilter(NestedClause clause) { + this.whereClauses.add(clause); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withGroupByTime(String timeInterval) { + + this.groupByClauses.add(new RawTextClause("time(" + timeInterval + ")")); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withGroupByTime(String timeInterval, + String offsetInterval) { + + this.groupByClauses.add(new RawTextClause("time(" + + timeInterval + + "," + + offsetInterval + + ")")); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withGroupBy(String column) { + + this.groupByClauses.add(new RawTextClause(column)); + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withOrderBy(DataLakeQueryOrdering ordering) { + if (DataLakeQueryOrdering.ASC.equals(ordering)) { + this.ordering = asc(); + } else { + this.ordering = desc(); + } + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withLimit(int limit) { + this.limit = limit; + + return this; + } + + @Override + public DataLakeInfluxQueryBuilder withOffset(int offset) { + this.offset = offset; + + return this; + } + + @Override + public IDataLakeQueryBuilder withFill(Object fill) { + this.fill = fill; + + return this; + } + + @Override + public Query build() { + var selectQuery = + this.selectionQuery.from(env.getTsStorageBucket().getValueOrDefault(), escapeIndex(measurementId)); + this.whereClauses.forEach(selectQuery::where); + + if (this.groupByClauses.size() > 0) { + selectQuery.groupBy(this.groupByClauses.toArray()); + } + + if (this.ordering != null) { + selectQuery.orderBy(this.ordering); + } + + if (this.limit != Integer.MIN_VALUE) { + selectQuery.limit(this.limit); + } + + if (this.offset > 0) { + selectQuery.limit(this.limit, this.offset); + } + + if (Objects.nonNull(fill)) { + if (fill instanceof String) { + selectQuery.fill((String) fill); + } else { + selectQuery.fill((Number) fill); + } + } + + return selectQuery; + } + + private void addNestedWhereClause(List clauses) { + NestedClause nestedClause = new NestedClause(clauses); + this.whereClauses.add(nestedClause); + } + + private String escapeIndex(String index) { + return "\"" + index + "\""; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java new file mode 100644 index 0000000000..af8619fef5 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.influx; + +import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.datalake.SpQueryResult; +import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.PropertyScope; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +public class DataLakeMeasurementCount { + + private final List allMeasurements; + private final List measurementNames; + + private static final String COUNT_FIELD = "count"; + + public DataLakeMeasurementCount(List allMeasurements, + List measurementNames) { + this.allMeasurements = allMeasurements; + this.measurementNames = measurementNames; + } + + public Map countMeasurementSizes() { + Map> futures = measurementNames.stream() + .distinct() + .map(this::getMeasure) + .collect(Collectors.toMap(DataLakeMeasure::getMeasureName, m -> CompletableFuture.supplyAsync(() -> { + var firstColumn = getFirstColumn(m); + var builder = DataLakeInfluxQueryBuilder + .create(m.getMeasureName()).withEndTime(System.currentTimeMillis()) + .withAggregatedColumn(firstColumn, AggregationFunction.COUNT); + var queryResult = new DataExplorerInfluxQueryExecutor().executeQuery(builder.build(), true); + if (queryResult.getTotal() > 0) { + var headers = queryResult.getHeaders(); + return extractResult(queryResult, headers); + } else { + return 0; + } + }))); + + Map result = new HashMap<>(); + futures.entrySet().forEach((entry -> { + try { + result.put(entry.getKey(), entry.getValue().get()); + } catch (InterruptedException | ExecutionException e) { + result.put(entry.getKey(), 0); + } + })); + + return result; + } + + private Integer extractResult(SpQueryResult queryResult, + List headers) { + return ((Double) ( + queryResult.getAllDataSeries().get(0).getRows().get(0).get(headers.indexOf(COUNT_FIELD))) + ).intValue(); + } + + private DataLakeMeasure getMeasure(String measureName) { + return allMeasurements + .stream() + .filter(m -> m.getMeasureName().equals(measureName)) + .findFirst() + .orElse(null); + } + + private String getFirstColumn(DataLakeMeasure measure) { + return measure.getEventSchema().getEventProperties() + .stream() + .filter(ep -> ep.getPropertyScope() != null + && ep.getPropertyScope().equals(PropertyScope.MEASUREMENT_PROPERTY.name())) + .map(EventProperty::getRuntimeName) + .findFirst() + .orElse(null); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/DeleteQueryParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/DeleteQueryParams.java index 1fa6948c38..8bdda43de4 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/DeleteQueryParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/DeleteQueryParams.java @@ -18,11 +18,43 @@ package org.apache.streampipes.dataexplorer.param; +public class DeleteQueryParams { -public record DeleteQueryParams(String measurementName, Long startTime, Long endTime, boolean timeRestricted) { + private final String measurementId; - public DeleteQueryParams(String measurementId, Long startTime, Long endTime) { - this(measurementId, startTime != null ? startTime : 0, endTime != null ? endTime : 99999999999999L, true); + private long startTime; + + private long endTime; + + private boolean timeRestricted; + + public DeleteQueryParams(String measurementId) { + this.measurementId = measurementId; + this.timeRestricted = false; + } + + public DeleteQueryParams(String measurementId, + Long startTime, + Long endTime) { + this(measurementId); + this.startTime = startTime; + this.endTime = endTime; + this.timeRestricted = true; + } + + public String getMeasurementId() { + return measurementId; + } + + public long getStartTime() { + return startTime; } -} + public long getEndTime() { + return endTime; + } + + public boolean isTimeRestricted() { + return timeRestricted; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java index 5628b5f992..9b6f341dd7 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java @@ -26,13 +26,24 @@ import org.apache.streampipes.dataexplorer.param.model.OrderByClauseParams; import org.apache.streampipes.dataexplorer.param.model.SelectClauseParams; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; -import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; -import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COLUMNS; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COUNT_ONLY; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_END_DATE; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_FILTER; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_GROUP_BY; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_OFFSET; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_ORDER; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_PAGE; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_START_DATE; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_TIME_INTERVAL; + public class ProvidedRestQueryParamConverter { @@ -44,76 +55,74 @@ public class ProvidedRestQueryParamConverter { public static SelectQueryParams getSelectQueryParams(ProvidedRestQueryParams params) { SelectQueryParams queryParameters = new SelectQueryParams(params.getMeasurementId()); - if (params.has(SupportedRestQueryParams.QP_COUNT_ONLY) - && params.getAsBoolean(SupportedRestQueryParams.QP_COUNT_ONLY)) { - queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(SupportedRestQueryParams.QP_COLUMNS), - true) - ); + if (params.has(QP_COUNT_ONLY) && params.getAsBoolean(QP_COUNT_ONLY)) { + queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(QP_COLUMNS), true)); } else { - queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(SupportedRestQueryParams.QP_COLUMNS), - params.getAsString(SupportedRestQueryParams.QP_AGGREGATION_FUNCTION))); + queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(QP_COLUMNS), + params.getAsString(QP_AGGREGATION_FUNCTION))); } - String filterConditions = params.getAsString(SupportedRestQueryParams.QP_FILTER); + String filterConditions = params.getAsString(QP_FILTER); if (hasTimeParams(params)) { queryParameters.withWhereParams(WhereClauseParams.from( - params.getAsLong(SupportedRestQueryParams.QP_START_DATE), - params.getAsLong(SupportedRestQueryParams.QP_END_DATE), + params.getAsLong(QP_START_DATE), + params.getAsLong(QP_END_DATE), filterConditions)); } else if (filterConditions != null) { queryParameters.withWhereParams(WhereClauseParams.from(filterConditions)); } - if (params.has(SupportedRestQueryParams.QP_TIME_INTERVAL)) { - String timeInterval = params.getAsString(SupportedRestQueryParams.QP_TIME_INTERVAL); - if (!params.has(SupportedRestQueryParams.QP_GROUP_BY)) { + if (params.has(QP_TIME_INTERVAL)) { + String timeInterval = params.getAsString(QP_TIME_INTERVAL); + if (!params.has(QP_GROUP_BY)) { queryParameters.withGroupByTimeParams(GroupByTimeClauseParams.from(timeInterval)); } else { - params.update(SupportedRestQueryParams.QP_GROUP_BY, - params.getAsString(SupportedRestQueryParams.QP_GROUP_BY) + ",time(" + timeInterval + ")"); + params.update(QP_GROUP_BY, params.getAsString(QP_GROUP_BY) + ",time(" + timeInterval + ")"); } queryParameters.withFillParams(FillClauseParams.from()); } - if (params.has(SupportedRestQueryParams.QP_GROUP_BY)) { - queryParameters.withGroupByTagsParams(GroupByTagsClauseParams.from( - params.getAsString(SupportedRestQueryParams.QP_GROUP_BY)) - ); + if (params.has(QP_GROUP_BY)) { + queryParameters.withGroupByTagsParams(GroupByTagsClauseParams.from(params.getAsString(QP_GROUP_BY))); } - if (params.has(SupportedRestQueryParams.QP_ORDER)) { - String order = params.getAsString(SupportedRestQueryParams.QP_ORDER); + if (params.has(QP_ORDER)) { + String order = params.getAsString(QP_ORDER); if (order.equals(ORDER_DESCENDING)) { queryParameters.withOrderByParams(OrderByClauseParams.from(order)); } } - if (params.has(SupportedRestQueryParams.QP_LIMIT)) { - queryParameters.withLimitParams(LimitClauseParams.from(params.getAsInt(SupportedRestQueryParams.QP_LIMIT))); + if (params.has(QP_LIMIT)) { + queryParameters.withLimitParams(LimitClauseParams.from(params.getAsInt(QP_LIMIT))); } - if (params.has(SupportedRestQueryParams.QP_OFFSET)) { - queryParameters.withOffsetParams(OffsetClauseParams.from(params.getAsInt(SupportedRestQueryParams.QP_OFFSET))); - } else if (params.has(SupportedRestQueryParams.QP_LIMIT) && params.has(SupportedRestQueryParams.QP_PAGE)) { + if (params.has(QP_OFFSET)) { + queryParameters.withOffsetParams(OffsetClauseParams.from(params.getAsInt(QP_OFFSET))); + } else if (params.has(QP_LIMIT) && params.has(QP_PAGE)) { queryParameters.withOffsetParams(OffsetClauseParams.from( - params.getAsInt(SupportedRestQueryParams.QP_PAGE) * params.getAsInt(SupportedRestQueryParams.QP_LIMIT))); + params.getAsInt(QP_PAGE) * params.getAsInt(QP_LIMIT))); } return queryParameters; } - public static DeleteQueryParams getDeleteQueryParams(String measurementName, - Long startTime, - Long endTime) { - return new DeleteQueryParams(measurementName, startTime, endTime); + public static DeleteQueryParams getDeleteQueryParams(String measurementId, + Long startTime, + Long endTime) { + if (startTime != null || endTime != null) { + return new DeleteQueryParams(measurementId, startTime, endTime); + } else { + return new DeleteQueryParams(measurementId); + } } private static boolean hasTimeParams(ProvidedRestQueryParams params) { - return params.has(SupportedRestQueryParams.QP_START_DATE) - || params.has(SupportedRestQueryParams.QP_END_DATE); + return params.has(QP_START_DATE) + || params.has(QP_END_DATE); } public static List buildConditions(String queryPart) { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java new file mode 100644 index 0000000000..d703eb934b --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.param; + +import java.util.HashMap; +import java.util.Map; + +public class ProvidedRestQueryParams { + + private final String measurementId; + private final Map providedParams; + + public ProvidedRestQueryParams(String measurementId, + Map providedParams) { + this.measurementId = measurementId; + this.providedParams = providedParams; + } + + public ProvidedRestQueryParams(ProvidedRestQueryParams params) { + this.measurementId = params.getMeasurementId(); + this.providedParams = new HashMap<>(); + providedParams.putAll(params.getProvidedParams()); + } + + public boolean has(String key) { + return providedParams.containsKey(key); + } + + public Long getAsLong(String key) { + return has(key) ? Long.parseLong(providedParams.get(key)) : null; + } + + public Integer getAsInt(String key) { + return has(key) ? Integer.parseInt(String.valueOf(providedParams.get(key))) : null; + } + + public String getAsString(String key) { + return has(key) ? providedParams.get(key) : null; + } + + public boolean getAsBoolean(String key) { + return has(key) && Boolean.parseBoolean(providedParams.get(key)); + } + + public String getMeasurementId() { + return measurementId; + } + + public void update(String key, String value) { + this.providedParams.put(key, value); + } + + public void update(String key, long value) { + update(key, String.valueOf(value)); + } + + public void update(String key, Integer value) { + update(key, String.valueOf(value)); + } + + public void update(String key, boolean value) { + update(key, String.valueOf(value)); + } + + public void remove(String key) { + this.providedParams.remove(key); + } + + public Map getProvidedParams() { + return providedParams; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java index cca5c1b024..be5392d8c3 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java @@ -17,7 +17,6 @@ */ package org.apache.streampipes.dataexplorer.param; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.param.model.FillClauseParams; import org.apache.streampipes.dataexplorer.param.model.GroupByTagsClauseParams; import org.apache.streampipes.dataexplorer.param.model.GroupByTimeClauseParams; @@ -26,6 +25,7 @@ import org.apache.streampipes.dataexplorer.param.model.OrderByClauseParams; import org.apache.streampipes.dataexplorer.param.model.SelectClauseParams; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import java.util.Objects; @@ -95,7 +95,7 @@ public T toQuery(IDataLakeQueryBuilder builder) { public int getLimit() { if (Objects.nonNull(limitParams)) { - return limitParams.limit(); + return limitParams.getLimit(); } else { return Integer.MIN_VALUE; } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java new file mode 100644 index 0000000000..7c3e3f829c --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.param; + +import java.util.Arrays; +import java.util.List; + +public class SupportedRestQueryParams { + + public static final String QP_COLUMNS = "columns"; + public static final String QP_START_DATE = "startDate"; + public static final String QP_END_DATE = "endDate"; + public static final String QP_PAGE = "page"; + public static final String QP_LIMIT = "limit"; + public static final String QP_OFFSET = "offset"; + public static final String QP_GROUP_BY = "groupBy"; + public static final String QP_ORDER = "order"; + public static final String QP_AGGREGATION_FUNCTION = "aggregationFunction"; + public static final String QP_TIME_INTERVAL = "timeInterval"; + public static final String QP_FORMAT = "format"; + public static final String QP_CSV_DELIMITER = "delimiter"; + + public static final String QP_MISSING_VALUE_BEHAVIOUR = "missingValueBehaviour"; + public static final String QP_COUNT_ONLY = "countOnly"; + public static final String QP_AUTO_AGGREGATE = "autoAggregate"; + public static final String QP_FILTER = "filter"; + public static final String QP_MAXIMUM_AMOUNT_OF_EVENTS = "maximumAmountOfEvents"; + + public static final List SUPPORTED_PARAMS = Arrays.asList( + QP_COLUMNS, + QP_START_DATE, + QP_END_DATE, + QP_PAGE, + QP_LIMIT, + QP_OFFSET, + QP_GROUP_BY, + QP_ORDER, + QP_AGGREGATION_FUNCTION, + QP_TIME_INTERVAL, + QP_FORMAT, + QP_CSV_DELIMITER, + QP_COUNT_ONLY, + QP_AUTO_AGGREGATE, + QP_MISSING_VALUE_BEHAVIOUR, + QP_FILTER, + QP_MAXIMUM_AMOUNT_OF_EVENTS + ); + +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java new file mode 100644 index 0000000000..0d899c2b94 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.param.model; + +public enum AggregationFunction { + + MEAN("MEAN"), + MIN("MIN"), + MAX("MAX"), + COUNT("COUNT"), + FIRST("FIRST"), + LAST("LAST"), + MODE("MODE"), + SUM("SUM"); + + private final String dbName; + + AggregationFunction(String dbName) { + this.dbName = dbName; + } + + public String toDbName() { + return this.dbName; + } + +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java index ccece054ba..5204ce5a0b 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class FillClauseParams implements IQueryStatement { String fill = "none"; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java index 3a4c8fd84d..833fb3363a 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import java.util.ArrayList; import java.util.Arrays; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java index aa3f3f9422..0d83953ceb 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class GroupByTimeClauseParams implements IQueryStatement { private final String timeInterval; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java index b22292d3bc..bdff3d070d 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java @@ -18,15 +18,25 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; -public record LimitClauseParams(Integer limit) implements IQueryStatement { +public class LimitClauseParams implements IQueryStatement { + + private final Integer limit; + + public LimitClauseParams(Integer limit) { + this.limit = limit; + } public static LimitClauseParams from(Integer limit) { return new LimitClauseParams(limit); } + public Integer getLimit() { + return limit; + } + @Override public void buildStatement(IDataLakeQueryBuilder builder) { builder.withLimit(limit); diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java index 5eb445d90e..b8dbc94aea 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java @@ -18,15 +18,25 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; -public record OffsetClauseParams(Integer offset) implements IQueryStatement { +public class OffsetClauseParams implements IQueryStatement { + + private final Integer offset; + + public OffsetClauseParams(Integer offset) { + this.offset = offset; + } public static OffsetClauseParams from(Integer offset) { return new OffsetClauseParams(offset); } + public Integer getOffset() { + return offset; + } + @Override public void buildStatement(IDataLakeQueryBuilder builder) { builder.withOffset(offset); diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java index 6c184084a1..612a91674a 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java @@ -18,9 +18,9 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.model.datalake.DataLakeQueryOrdering; +import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class OrderByClauseParams implements IQueryStatement { private final String ordering; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java index af87b0778d..fe7f8d0f65 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java @@ -19,9 +19,8 @@ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.model.datalake.AggregationFunction; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import java.util.Arrays; import java.util.List; @@ -33,6 +32,10 @@ public class SelectClauseParams implements IQueryStatement { private List selectedColumnsCountOnly; private boolean selectWildcard = false; + public SelectClauseParams() { + this.selectWildcard = true; + } + public SelectClauseParams(String columns, boolean countOnly) { this.selectWildcard = false; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java index 3e4fd1048c..1fb7661e90 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java @@ -17,10 +17,9 @@ */ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.model.datalake.AggregationFunction; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class SelectColumn implements IQueryStatement { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java index 3381983268..ab5a6712b8 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java @@ -17,10 +17,10 @@ */ package org.apache.streampipes.dataexplorer.param.model; -import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.model.datalake.FilterCondition; +import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; +import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import org.apache.commons.lang3.math.NumberUtils; @@ -98,7 +98,7 @@ private void buildConditions(String whereConditions) { } private Object returnCondition(String inputCondition) { - if (NumberUtils.isParsable(inputCondition)) { + if (NumberUtils.isCreatable(inputCondition)) { return Double.parseDouble(inputCondition); } else if (isBoolean(inputCondition)) { return Boolean.parseBoolean(inputCondition); diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java new file mode 100644 index 0000000000..68bf23171a --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.query; + +import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement; +import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement; +import org.apache.streampipes.dataexplorer.api.IDataExplorerQueryManagement; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.param.model.SelectColumn; +import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; +import org.apache.streampipes.model.datalake.SpQueryResult; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COLUMNS; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COUNT_ONLY; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_ORDER; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_TIME_INTERVAL; + +public class AutoAggregationHandler { + + private static final Logger LOG = LoggerFactory.getLogger(AutoAggregationHandler.class); + private static final double MAX_RETURN_LIMIT = 2000; + private static final String TIMESTAMP_FIELD = "time"; + private static final String COMMA = ","; + + private final SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + private final SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + private final IDataExplorerQueryManagement dataLakeQueryManagement; + private final ProvidedRestQueryParams queryParams; + + public AutoAggregationHandler(ProvidedRestQueryParams params) { + this.queryParams = params; + this.dataLakeQueryManagement = getDataLakeQueryManagement(); + } + + private IDataExplorerQueryManagement getDataLakeQueryManagement() { + var dataLakeStorage = StorageDispatcher.INSTANCE + .getNoSqlStore() + .getDataLakeStorage(); + return new DataExplorerQueryManagement(new DataExplorerSchemaManagement(dataLakeStorage)); + } + + public ProvidedRestQueryParams makeAutoAggregationQueryParams() throws IllegalArgumentException { + try { + SpQueryResult newest = getSingleRecord(DataLakeQueryOrdering.DESC); + SpQueryResult oldest = getSingleRecord(DataLakeQueryOrdering.ASC); + if (newest.getTotal() > 0) { + String sampleField = getSampleField(newest); + Integer count = getCount(sampleField); + if (count <= MAX_RETURN_LIMIT) { + LOG.debug("Auto-Aggregation disabled as {} results <= max return limit {}", count, MAX_RETURN_LIMIT); + return disableAutoAgg(this.queryParams); + } else { + LOG.debug("Performing auto-aggregation"); + + int aggValue = getAggregationValue(newest, oldest); + LOG.debug("Setting auto-aggregation value to {} ms", aggValue); + queryParams.update(QP_TIME_INTERVAL, aggValue + "ms"); + return disableAutoAgg(queryParams); + } + } else { + return disableAutoAgg(this.queryParams); + } + } catch (ParseException e) { + e.printStackTrace(); + } + return null; + } + + private ProvidedRestQueryParams disableAutoAgg(ProvidedRestQueryParams params) { + params.remove(QP_AUTO_AGGREGATE); + return params; + } + + public Integer getCount(String fieldName) { + ProvidedRestQueryParams countParams = disableAutoAgg(new ProvidedRestQueryParams(queryParams)); + countParams.remove(QP_TIME_INTERVAL); + countParams.remove(QP_AGGREGATION_FUNCTION); + countParams.update(QP_COUNT_ONLY, true); + countParams.update(QP_COLUMNS, fieldName); + + SpQueryResult result = dataLakeQueryManagement.getData(countParams, true); + + return result.getTotal() > 0 ? ( + (Double) result.getAllDataSeries() + .get(0) + .getRows() + .get(0) + .get(1) + ).intValue() : 0; + } + + private SpQueryResult fireQuery(ProvidedRestQueryParams params) { + return dataLakeQueryManagement.getData(params, true); + } + + private int getAggregationValue(SpQueryResult newest, SpQueryResult oldest) throws ParseException { + long timerange = extractTimestamp(newest) - extractTimestamp(oldest); + double v = timerange / MAX_RETURN_LIMIT; + return Double.valueOf(v) + .intValue(); + } + + private SpQueryResult getSingleRecord(DataLakeQueryOrdering order) throws ParseException { + ProvidedRestQueryParams singleEvent = disableAutoAgg(new ProvidedRestQueryParams(queryParams)); + singleEvent.remove(QP_AGGREGATION_FUNCTION); + singleEvent.update(QP_LIMIT, 1); + singleEvent.update(QP_ORDER, order.name()); + singleEvent.update(QP_COLUMNS, transformColumns(singleEvent.getAsString(QP_COLUMNS))); + + return fireQuery(singleEvent); + } + + private String transformColumns(String rawQuery) { + List columns = + Arrays.stream(rawQuery.split(COMMA)) + .map(SelectColumn::fromApiQueryString) + .collect(Collectors.toList()); + return columns.stream() + .map(SelectColumn::getOriginalField) + .collect(Collectors.joining(COMMA)); + } + + private String getSampleField(SpQueryResult result) { + for (String column : result.getHeaders()) { + if (!column.equals(TIMESTAMP_FIELD)) { + return column; + } + } + throw new IllegalArgumentException("No columns present"); + } + + private long extractTimestamp(SpQueryResult result) throws ParseException { + int timestampIndex = result.getHeaders() + .indexOf(TIMESTAMP_FIELD); + return tryParseDate(result.getAllDataSeries() + .get(0) + .getRows() + .get(0) + .get(timestampIndex) + .toString()).getTime(); + } + + private Date tryParseDate(String v) throws ParseException { + try { + return dateFormat1.parse(v); + } catch (ParseException e) { + return dateFormat2.parse(v); + } + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java index 347070a804..c28c949780 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java @@ -18,9 +18,9 @@ package org.apache.streampipes.dataexplorer.query; +import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; import org.apache.streampipes.dataexplorer.param.SelectQueryParams; -import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataSeries; import org.apache.streampipes.model.datalake.SpQueryResult; import org.apache.streampipes.model.datalake.SpQueryStatus; @@ -28,40 +28,53 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; -import java.util.Optional; - public abstract class DataExplorerQueryExecutor { - private static final Logger LOG = LoggerFactory.getLogger(DataExplorerQueryExecutor.class); + private static final Logger LOG = LoggerFactory.getLogger(DataExplorerInfluxQueryExecutor.class); + protected int maximumAmountOfEvents; + + protected boolean appendId = false; + protected String forId; + + public DataExplorerQueryExecutor() { + this.maximumAmountOfEvents = -1; + } + + public DataExplorerQueryExecutor(String forId) { + this(); + this.appendId = true; + this.forId = forId; + } + + public DataExplorerQueryExecutor(int maximumAmountOfEvents) { + this(); + this.maximumAmountOfEvents = maximumAmountOfEvents; + } /** * Execute the data explorer query and return the result or a warning message * in case the maximum amount of events to return is defined */ public SpQueryResult executeQuery(SelectQueryParams params, - int maximumAmountOfEvents, - Optional forIdOpt, boolean ignoreMissingValues) throws RuntimeException { X query = makeSelectQuery(params); - var result = executeQuery(query, forIdOpt, ignoreMissingValues); - if (maximumAmountOfEvents != -1) { - return validateAndReturnQueryResult(result, params.getLimit(), maximumAmountOfEvents); + var result = executeQuery(query, ignoreMissingValues); + if (this.maximumAmountOfEvents != -1) { + return validateAndReturnQueryResult(result, params.getLimit()); } else { return result; } } private SpQueryResult validateAndReturnQueryResult(SpQueryResult queryResult, - int limit, - int maximumAmountOfEvents) { + int limit) { var amountOfResults = queryResult.getAllDataSeries() .stream() .mapToInt(DataSeries::getTotal) .sum(); var amountOfQueryResults = limit == Integer.MIN_VALUE ? amountOfResults : Math.min(amountOfResults, limit); - if (amountOfQueryResults > maximumAmountOfEvents) { + if (amountOfQueryResults > this.maximumAmountOfEvents) { return makeTooMuchDataResult(amountOfQueryResults); } else { return queryResult; @@ -76,36 +89,31 @@ private SpQueryResult makeTooMuchDataResult(int amountOfQueryResults) { } public SpQueryResult executeQuery(DeleteQueryParams params) { - return executeQuery(makeDeleteQuery(params), Optional.empty(), true); + return executeQuery(makeDeleteQuery(params), true); } public SpQueryResult executeQuery(X query, - Optional forIdOpt, boolean ignoreMissingValues) { if (LOG.isDebugEnabled()) { - LOG.debug("Data Lake Query {}", asQueryString(query)); + LOG.debug("Data Lake Query " + asQueryString(query)); } W result = executeQuery(query); if (LOG.isDebugEnabled()) { - LOG.debug("Data Lake Query Result: {}", result.toString()); + LOG.debug("Data Lake Query Result: " + result.toString()); } - return postQuery(result, forIdOpt, ignoreMissingValues); + return postQuery(result, ignoreMissingValues); } protected abstract SpQueryResult postQuery(W queryResult, - Optional forIdOpt, boolean ignoreMissingValues); - public abstract W executeQuery(X query); + protected abstract W executeQuery(X query); protected abstract String asQueryString(X query); protected abstract X makeDeleteQuery(DeleteQueryParams params); protected abstract X makeSelectQuery(SelectQueryParams params); - - public abstract Map getTagValues(String measurementId, String fields); - public abstract boolean deleteData(DataLakeMeasure measure); } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java new file mode 100644 index 0000000000..6ef1d68564 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.query; + +import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.commons.environment.Environments; +import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.model.datalake.DataLakeMeasure; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; + +public class DeleteDataQuery { + + private final DataLakeMeasure measure; + + public DeleteDataQuery(DataLakeMeasure measure) { + this.measure = measure; + } + + private String getQuery() { + return "DROP MEASUREMENT \"" + measure.getMeasureName() + "\""; + } + + public QueryResult executeQuery() throws RuntimeException { + try (final InfluxDB influxDB = InfluxClientProvider.getInfluxDBClient()) { + var databaseName = getEnvironment().getTsStorageBucket().getValueOrDefault(); + + var query = new Query(getQuery(), databaseName); + return influxDB.query(query); + } + } + + private Environment getEnvironment() { + return Environments.getEnvironment(); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java new file mode 100644 index 0000000000..ce88fd1b5d --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query; + +import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.model.datalake.SpQueryResult; + +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS; + +public class QueryResultProvider { + + public static final String FOR_ID_KEY = "forId"; + protected final boolean ignoreMissingData; + protected ProvidedRestQueryParams queryParams; + + public QueryResultProvider(ProvidedRestQueryParams queryParams, + boolean ignoreMissingData) { + this.queryParams = queryParams; + this.ignoreMissingData = ignoreMissingData; + } + + public SpQueryResult getData() { + if (queryParams.has(QP_AUTO_AGGREGATE)) { + queryParams = new AutoAggregationHandler(queryParams).makeAutoAggregationQueryParams(); + } + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(queryParams); + + if (queryParams.getProvidedParams().containsKey(QP_MAXIMUM_AMOUNT_OF_EVENTS)) { + int maximumAmountOfEvents = Integer.parseInt(queryParams.getProvidedParams().get(QP_MAXIMUM_AMOUNT_OF_EVENTS)); + return new DataExplorerInfluxQueryExecutor(maximumAmountOfEvents).executeQuery(qp, ignoreMissingData); + } + + if (queryParams.getProvidedParams().containsKey(FOR_ID_KEY)) { + String forWidgetId = queryParams.getProvidedParams().get(FOR_ID_KEY); + return new DataExplorerInfluxQueryExecutor(forWidgetId).executeQuery(qp, ignoreMissingData); + } else { + return new DataExplorerInfluxQueryExecutor().executeQuery(qp, ignoreMissingData); + } + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java new file mode 100644 index 0000000000..e1a8f756e6 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams; +import org.apache.streampipes.dataexplorer.query.writer.ConfiguredOutputWriter; +import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.dataexplorer.utils.DataExplorerUtils; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.datalake.SpQueryResult; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Optional; + +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; + +public class StreamedQueryResultProvider extends QueryResultProvider { + + private static final int MAX_RESULTS_PER_QUERY = 200000; + private static final String TIME_FIELD = "time"; + + private final OutputFormat format; + + public StreamedQueryResultProvider(ProvidedRestQueryParams params, + OutputFormat format, + boolean ignoreMissingValues) { + super(params, ignoreMissingValues); + this.format = format; + } + + public void getDataAsStream(OutputStream outputStream) throws IOException { + var usesLimit = queryParams.has(QP_LIMIT); + var configuredWriter = ConfiguredOutputWriter + .getConfiguredWriter(format, queryParams, ignoreMissingData); + + if (!queryParams.has(QP_LIMIT)) { + queryParams.update(QP_LIMIT, MAX_RESULTS_PER_QUERY); + } + + var limit = queryParams.getAsInt(QP_LIMIT); + var measurement = findByMeasurementName(queryParams.getMeasurementId()).get(); + + SpQueryResult dataResult; + + boolean isFirstDataItem = true; + var resultsCount = 0; + configuredWriter.beforeFirstItem(outputStream); + do { + dataResult = getData(); + long lastTimestamp = 0; + resultsCount = dataResult.getTotal() > 0 ? dataResult.getAllDataSeries().get(0).getTotal() : 0; + if (resultsCount > 0) { + + changeTimestampHeader(measurement, dataResult); + var columns = dataResult.getHeaders(); + for (List row : dataResult.getAllDataSeries().get(0).getRows()) { + configuredWriter.writeItem(outputStream, row, columns, isFirstDataItem); + isFirstDataItem = false; + lastTimestamp = dataResult.getLastTimestamp(); + } + } + queryParams.update(SupportedRestQueryParams.QP_START_DATE, lastTimestamp + 1); + } while (queryNextPage(resultsCount, usesLimit, limit)); + configuredWriter.afterLastItem(outputStream); + } + + private boolean queryNextPage(int lastResultsCount, + boolean usesLimit, + int limit) { + if (usesLimit) { + return lastResultsCount >= limit; + } else { + return lastResultsCount > 0; + } + } + + private Optional findByMeasurementName(String measurementName) { + return DataExplorerUtils.getInfos() + .stream() + .filter(measurement -> measurement.getMeasureName().equals(measurementName)) + .findFirst(); + } + + /** + * Replaces the field 'time' of the data result with the actual timestamp field name of the measurement + * + * @param measurement contains the actual timestamp name value + * @param dataResult the query result of the database with 'time' as timestamp field name + */ + private void changeTimestampHeader(DataLakeMeasure measurement, + SpQueryResult dataResult) { + var timeFieldIndex = dataResult.getHeaders().indexOf(TIME_FIELD); + if (timeFieldIndex > -1) { + dataResult.getHeaders().set(timeFieldIndex, measurement.getTimestampFieldName()); + } + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java new file mode 100644 index 0000000000..03ca75530c --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.query.writer.item.CsvItemWriter; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.StringJoiner; + +import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_CSV_DELIMITER; + +public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter { + + private static final String LINE_SEPARATOR = "\n"; + private static final String COMMA = ","; + private static final String SEMICOLON = ";"; + + private CsvItemWriter itemWriter; + private String delimiter = COMMA; + + @Override + public void configure(ProvidedRestQueryParams params, + boolean ignoreMissingValues) { + if (params.has(QP_CSV_DELIMITER)) { + delimiter = params.getAsString(QP_CSV_DELIMITER).equals("comma") ? COMMA : SEMICOLON; + } + this.itemWriter = new CsvItemWriter(delimiter); + } + + @Override + public void beforeFirstItem(OutputStream outputStream) { + // do nothing + } + + @Override + public void afterLastItem(OutputStream outputStream) { + // do nothing + } + + @Override + public void writeItem(OutputStream outputStream, + List row, + List columnNames, + boolean firstObject) throws IOException { + if (firstObject) { + outputStream.write(toBytes(makeHeaderLine(columnNames))); + } + + outputStream.write(toBytes(itemWriter.createItem(row, columnNames) + LINE_SEPARATOR)); + } + + private String makeHeaderLine(List columns) { + StringJoiner joiner = new StringJoiner(this.delimiter); + columns.forEach(joiner::add); + return joiner + LINE_SEPARATOR; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java new file mode 100644 index 0000000000..0781c7334a --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.query.writer.item.ItemGenerator; +import org.apache.streampipes.dataexplorer.query.writer.item.JsonItemWriter; + +import com.google.gson.Gson; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +public class ConfiguredJsonOutputWriter extends ConfiguredOutputWriter { + + private static final String BEGIN_ARRAY = "["; + private static final String END_ARRAY = "]"; + + private final ItemGenerator jsonObjectWriter; + + public ConfiguredJsonOutputWriter() { + Gson gson = new Gson(); + this.jsonObjectWriter = new JsonItemWriter(gson); + } + + @Override + public void configure(ProvidedRestQueryParams params, + boolean ignoreMissingValues) { + // do nothing + } + + @Override + public void beforeFirstItem(OutputStream outputStream) throws IOException { + outputStream.write(toBytes(BEGIN_ARRAY)); + } + + @Override + public void afterLastItem(OutputStream outputStream) throws IOException { + outputStream.write(toBytes(END_ARRAY)); + } + + @Override + public void writeItem(OutputStream outputStream, + List row, + List columnNames, + boolean firstObject) throws IOException { + if (!firstObject) { + outputStream.write(toBytes(",")); + } + + var item = jsonObjectWriter.createItem(row, columnNames); + outputStream.write(toBytes(item)); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java new file mode 100644 index 0000000000..ed60687ae5 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +public abstract class ConfiguredOutputWriter { + + public static ConfiguredOutputWriter getConfiguredWriter(OutputFormat format, + ProvidedRestQueryParams params, + boolean ignoreMissingValues) { + var writer = format.getWriter(); + writer.configure(params, ignoreMissingValues); + + return writer; + } + + public abstract void configure(ProvidedRestQueryParams params, + boolean ignoreMissingValues); + + public abstract void beforeFirstItem(OutputStream outputStream) throws IOException; + + public abstract void afterLastItem(OutputStream outputStream) throws IOException; + + public abstract void writeItem(OutputStream outputStream, + List row, + List columnNames, + boolean firstObject) throws IOException; + + protected byte[] toBytes(String value) { + return value.getBytes(); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java new file mode 100644 index 0000000000..c855bf1c8f --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import java.util.function.Supplier; + +public enum OutputFormat { + JSON(ConfiguredJsonOutputWriter::new), + CSV(ConfiguredCsvOutputWriter::new); + + private final Supplier writerSupplier; + + OutputFormat(Supplier writerSupplier) { + this.writerSupplier = writerSupplier; + } + + public ConfiguredOutputWriter getWriter() { + return writerSupplier.get(); + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemWriter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemWriter.java new file mode 100644 index 0000000000..bd80f2d9ab --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemWriter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package org.apache.streampipes.dataexplorer.query.writer.item; + +public class CsvItemWriter extends ItemGenerator { + + public CsvItemWriter(String delimiter) { + super(delimiter); + } + + @Override + protected String makeItemString(String key, Object value) { + return value != null ? value.toString() : ""; + } + + @Override + protected String finalizeItem(String item) { + return item; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java new file mode 100644 index 0000000000..31e9eb61d5 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package org.apache.streampipes.dataexplorer.query.writer.item; + +import java.util.List; +import java.util.StringJoiner; + +public abstract class ItemGenerator { + + protected static final String COMMA_SEPARATOR = ","; + + private final String separator; + + public ItemGenerator(String separator) { + this.separator = separator; + } + + public String createItem(List row, + List columns) { + StringJoiner joiner = new StringJoiner(separator); + + for (int i = 0; i < row.size(); i++) { + Object value = row.get(i); + if (i == 0) { + value = getTimestampValue((Double) row.get(i)); + } + joiner.add(makeItemString(columns.get(i), value)); + } + + return finalizeItem(joiner.toString()); + } + + protected abstract String makeItemString(String key, + Object value); + + protected abstract String finalizeItem(String item); + + private long getTimestampValue(Double value) { + return value.longValue(); + } + +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemWriter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemWriter.java new file mode 100644 index 0000000000..fde43721d4 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemWriter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package org.apache.streampipes.dataexplorer.query.writer.item; + +import com.google.gson.Gson; + +public class JsonItemWriter extends ItemGenerator { + + private static final String BEGIN_OBJECT = "{"; + private static final String END_OBJECT = "}"; + + private final Gson gson; + + public JsonItemWriter(Gson gson) { + super(COMMA_SEPARATOR); + this.gson = gson; + } + + @Override + protected String makeItemString(String key, + Object value) { + var stringValue = value != null ? gson.toJson(value) : null; + return "\"" + + key + + "\": " + + stringValue; + } + + @Override + protected String finalizeItem(String item) { + return BEGIN_OBJECT + item + END_OBJECT; + } +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java new file mode 100644 index 0000000000..f8825c157f --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.querybuilder; + +public enum DataLakeQueryOrdering { + ASC, DESC +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java new file mode 100644 index 0000000000..396bed21d2 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataexplorer.querybuilder; + +public class FilterCondition { + + private String field; + private String operator; + private Object condition; + + public FilterCondition(String field, String operator, Object condition) { + this.field = field; + this.operator = operator; + this.condition = condition; + } + + public String getField() { + return field; + } + + public String getOperator() { + return operator; + } + + public Object getCondition() { + return condition; + } + +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java new file mode 100644 index 0000000000..065c185621 --- /dev/null +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.querybuilder; + +import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; + +import org.influxdb.querybuilder.clauses.NestedClause; + +import java.util.List; + +public interface IDataLakeQueryBuilder { + + IDataLakeQueryBuilder withAllColumns(); + + IDataLakeQueryBuilder withSimpleColumn(String columnName); + + IDataLakeQueryBuilder withSimpleColumns(List columnNames); + + IDataLakeQueryBuilder withAggregatedColumn(String columnName, + AggregationFunction aggregationFunction, + String targetName); + + IDataLakeQueryBuilder withAggregatedColumn(String columnName, + AggregationFunction aggregationFunction); + + IDataLakeQueryBuilder withStartTime(long startTime); + + IDataLakeQueryBuilder withEndTime(long endTime); + + IDataLakeQueryBuilder withEndTime(long endTime, + boolean includeEndTime); + + IDataLakeQueryBuilder withTimeBoundary(long startTime, + long endTime); + + IDataLakeQueryBuilder withFilter(String field, + String operator, + Object value); + + IDataLakeQueryBuilder withExclusiveFilter(String field, + String operator, + List values); + + IDataLakeQueryBuilder withInclusiveFilter(String field, + String operator, + List values); + + IDataLakeQueryBuilder withInclusiveFilter(List filterConditions); + + IDataLakeQueryBuilder withFilter(NestedClause clause); + + IDataLakeQueryBuilder withGroupByTime(String timeInterval); + + IDataLakeQueryBuilder withGroupByTime(String timeInterval, + String offsetInterval); + + IDataLakeQueryBuilder withGroupBy(String column); + + IDataLakeQueryBuilder withOrderBy(DataLakeQueryOrdering ordering); + + IDataLakeQueryBuilder withLimit(int limit); + + IDataLakeQueryBuilder withOffset(int offset); + + IDataLakeQueryBuilder withFill(Object fill); + + T build(); +} diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/utils/DataExplorerUtils.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/utils/DataExplorerUtils.java index b9c5942fd9..a0b5125653 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/utils/DataExplorerUtils.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/utils/DataExplorerUtils.java @@ -28,6 +28,6 @@ public static List getInfos() { return StorageDispatcher.INSTANCE .getNoSqlStore() .getDataLakeStorage() - .findAll(); + .getAllDataLakeMeasures(); } } diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java index 4dc34531ce..1c98f3422c 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java @@ -21,7 +21,7 @@ import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataLakeMeasureSchemaUpdateStrategy; import org.apache.streampipes.model.schema.EventProperty; -import org.apache.streampipes.storage.api.CRUDStorage; +import org.apache.streampipes.storage.api.IDataLakeStorage; import org.apache.streampipes.test.generator.EventPropertyPrimitiveTestBuilder; import org.apache.streampipes.test.generator.EventSchemaTestBuilder; import org.apache.streampipes.vocabulary.XSD; @@ -41,21 +41,21 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; - public class DataExplorerSchemaManagementTest { public static final String NEW_PROPERTY = "newProperty"; public static final String OLD_PROPERTY = "oldProperty"; - private CRUDStorage dataLakeStorageMock; + private IDataLakeStorage dataLakeStorageMock; @BeforeEach public void setUp() { - dataLakeStorageMock = mock(CRUDStorage.class); + dataLakeStorageMock = mock(IDataLakeStorage.class); } @Test public void createMeasurementThatNotExisted() { - when(dataLakeStorageMock.findAll()).thenReturn(List.of()); + when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of()); + when(dataLakeStorageMock.storeDataLakeMeasure(any())).thenReturn(true); var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock); var oldMeasure = getSampleMeasure( @@ -66,7 +66,7 @@ public void createMeasurementThatNotExisted() { assertEquals(oldMeasure.getMeasureName(), resultingMeasure.getMeasureName()); verify(dataLakeStorageMock, Mockito.times(1)) - .persist(any()); + .storeDataLakeMeasure(any()); } @@ -80,8 +80,8 @@ public void createMeasurementWithUpdateStrategy() { ) ); - when(dataLakeStorageMock.findAll()).thenReturn(List.of(oldMeasure)); - when(dataLakeStorageMock.getElementById(any())).thenReturn(oldMeasure); + when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure)); + when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure); var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock); var newMeasure = getNewMeasure(DataLakeMeasureSchemaUpdateStrategy.UPDATE_SCHEMA); @@ -90,7 +90,7 @@ public void createMeasurementWithUpdateStrategy() { assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName()); verify(dataLakeStorageMock, Mockito.times(1)) - .updateElement(any()); + .updateDataLakeMeasure(any()); assertFalse(containsPropertyWithName(resultMeasure, OLD_PROPERTY)); assertTrue(containsPropertyWithName(resultMeasure, NEW_PROPERTY)); @@ -106,15 +106,15 @@ public void createMeasurementWithExtendSchemaStrategy() { getEventProperty(OLD_PROPERTY, XSD.STRING) ) ); - when(dataLakeStorageMock.findAll()).thenReturn(List.of(oldMeasure)); - when(dataLakeStorageMock.getElementById(any())).thenReturn(oldMeasure); + when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure)); + when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure); var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock); var newMeasure = getNewMeasure(DataLakeMeasureSchemaUpdateStrategy.EXTEND_EXISTING_SCHEMA); var resultMeasure = schemaManagement.createOrUpdateMeasurement(newMeasure); assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName()); - verify(dataLakeStorageMock, Mockito.times(1)).updateElement(any()); + verify(dataLakeStorageMock, Mockito.times(1)).updateDataLakeMeasure(any()); assertTrue(containsPropertyWithName(resultMeasure, OLD_PROPERTY)); assertTrue(containsPropertyWithName(resultMeasure, NEW_PROPERTY)); } @@ -130,8 +130,8 @@ public void createMeasurementWithExtendSchemaStrategyAndDifferentPropertyTypes() ) ); - when(dataLakeStorageMock.findAll()).thenReturn(List.of(oldMeasure)); - when(dataLakeStorageMock.getElementById(any())).thenReturn(oldMeasure); + when(dataLakeStorageMock.getAllDataLakeMeasures()).thenReturn(List.of(oldMeasure)); + when(dataLakeStorageMock.findOne(any())).thenReturn(oldMeasure); var schemaManagement = new DataExplorerSchemaManagement(dataLakeStorageMock); @@ -139,7 +139,7 @@ public void createMeasurementWithExtendSchemaStrategyAndDifferentPropertyTypes() var resultMeasure = schemaManagement.createOrUpdateMeasurement(newMeasure); assertEquals(newMeasure.getMeasureName(), resultMeasure.getMeasureName()); - verify(dataLakeStorageMock, Mockito.times(1)).updateElement(any()); + verify(dataLakeStorageMock, Mockito.times(1)).updateDataLakeMeasure(any()); assertEquals( 2, resultMeasure.getEventSchema() @@ -201,4 +201,4 @@ private boolean containsPropertyWithName( .equals(runtimeName) ); } -} +} \ No newline at end of file diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java new file mode 100644 index 0000000000..5c254ee87b --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.param; + +import org.apache.streampipes.dataexplorer.influx.DataLakeInfluxQueryBuilder; +import org.apache.streampipes.dataexplorer.utils.ProvidedQueryParameterBuilder; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SelectQueryParamsTest { + + @Test + public void testWildcardTimeBoundQuery() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT * FROM \"abc\" WHERE (time < 2000000 AND time > 1000000);", query); + } + + @Test + public void testSimpleColumnQuery() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000);", query); + } + + @Test + public void testSimpleColumnQueryWithBooleanFilter() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .withFilter("[p1;=;true]") + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000 AND p1 = true);", query); + } + + @Test + public void testSimpleColumnQueryWithStringFilter() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .withFilter("[p1;=;def]") + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000 AND p1 = 'def');", query); + } + + @Test + public void testSimpleColumnQueryWithIntFilter() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .withFilter("[p1;=;1]") + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000 AND p1 = 1.0);", query); + } + + @Test + public void testSimpleColumnQueryWithFloatFilter() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .withFilter("[p1;>;1.0]") + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000 AND p1 > 1.0);", query); + } + + @Test + public void testSimpleColumnQueryWithTwoFilters() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withSimpleColumns(Arrays.asList("p1", "p2")) + .withFilter("[p1;>;1.0],[p2;<;2]") + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT p1,p2 FROM \"abc\" WHERE (time < 2000000 AND time > 1000000 AND p1 > 1.0 AND" + + " p2 < 2.0);", query); + } + + @Test + public void testAggregatedColumn() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withQueryColumns(List.of("[p1;MEAN;p1_mean]")) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT MEAN(p1) AS p1_mean FROM \"abc\" WHERE (time < 2000000 AND time > 1000000);", query); + } + + @Test + public void testAggregatedColumns() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withQueryColumns(Arrays.asList("[p1;MEAN;p1_mean]", "[p2;COUNT;p2_count]")) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT MEAN(p1) AS p1_mean,COUNT(p2) AS p2_count FROM \"abc\" WHERE (time < 2000000 AND" + + " time > 1000000);", query); + } + + @Test + public void testGroupByTag() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withQueryColumns(Arrays.asList("[p1;MEAN;p1_mean]", "[p2;COUNT;p2_count]")) + .withGroupBy(List.of("sensorId")) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT MEAN(p1) AS p1_mean,COUNT(p2) AS p2_count FROM \"abc\" WHERE (time < 2000000 AND" + + " time > 1000000) GROUP BY sensorId;", query); + } + + @Test + public void testGroupByTags() { + var params = ProvidedQueryParameterBuilder.create("abc") + .withStartDate(1) + .withEndDate(2) + .withQueryColumns(Arrays.asList("[p1;MEAN;p1_mean]", "[p2;COUNT;p2_count]")) + .withGroupBy(Arrays.asList("sensorId", "sensorId2")) + .build(); + + SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(params); + + String query = qp.toQuery(DataLakeInfluxQueryBuilder.create("abc")).getCommand(); + + assertEquals("SELECT MEAN(p1) AS p1_mean,COUNT(p2) AS p2_count FROM \"abc\" WHERE (time < 2000000 AND" + + " time > 1000000) GROUP BY sensorId,sensorId2;", query); + } + +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java index 4a9a69dc55..6859a41531 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java @@ -19,7 +19,7 @@ package org.apache.streampipes.dataexplorer.param; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; -import org.apache.streampipes.model.datalake.FilterCondition; +import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; import org.junit.jupiter.api.Test; @@ -50,20 +50,12 @@ public void filterString() { assertWhereCondition(result, expected); } - @Test - public void filterString2() { - WhereClauseParams result = WhereClauseParams.from("[fieldName;=;3312476503F]"); - FilterCondition expected = new FilterCondition("fieldName", "=", "3312476503F"); - - assertWhereCondition(result, expected); - } - private void assertWhereCondition(WhereClauseParams result, FilterCondition expected) { assertEquals(1, result.getWhereConditions().size()); FilterCondition resultingFilterCondition = result.getWhereConditions().get(0); - assertEquals(expected.field(), resultingFilterCondition.field()); - assertEquals(expected.operator(), resultingFilterCondition.operator()); - assertEquals(expected.condition(), resultingFilterCondition.condition()); + assertEquals(expected.getField(), resultingFilterCondition.getField()); + assertEquals(expected.getOperator(), resultingFilterCondition.getOperator()); + assertEquals(expected.getCondition(), resultingFilterCondition.getCondition()); } diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java new file mode 100644 index 0000000000..2aef0dbf37 --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestConfiguredCsvOutputWriter extends TestConfiguredOutputWriter { + + private static final String Expected = "time,string,number\n1668578077051,test,1\n1668578127050,test2,2\n"; + + @Test + public void testCsvOutputWriter() throws IOException { + var writer = new ConfiguredCsvOutputWriter(); + writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true); + + try (var outputStream = new ByteArrayOutputStream()) { + writer.beforeFirstItem(outputStream); + + for (int i = 0; i < rows.size(); i++) { + writer.writeItem(outputStream, rows.get(i), columns, i == 0); + } + + writer.afterLastItem(outputStream); + assertEquals(Expected, outputStream.toString(StandardCharsets.UTF_8)); + } + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java new file mode 100644 index 0000000000..e008f8847a --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestConfiguredJsonOutputWriter extends TestConfiguredOutputWriter { + + private static final String Expected = "[{\"time\": 1668578077051,\"string\": \"test\",\"number\": 1}" + + ",{\"time\": 1668578127050,\"string\": \"test2\",\"number\": 2}]"; + + @Test + public void testJsonOutputWriter() throws IOException { + var writer = new ConfiguredJsonOutputWriter(); + writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true); + + try (var outputStream = new ByteArrayOutputStream()) { + writer.beforeFirstItem(outputStream); + + for (int i = 0; i < rows.size(); i++) { + writer.writeItem(outputStream, rows.get(i), columns, i == 0); + } + + writer.afterLastItem(outputStream); + assertEquals(Expected, outputStream.toString(StandardCharsets.UTF_8)); + } + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java new file mode 100644 index 0000000000..14f453fcee --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer; + +import org.junit.jupiter.api.BeforeEach; + +import java.util.Arrays; +import java.util.List; + +public abstract class TestConfiguredOutputWriter { + + protected List> rows; + protected List columns; + + @BeforeEach + public void before() { + this.rows = Arrays.asList( + Arrays.asList(1668578077051.0, "test", 1), + Arrays.asList(1668578127050.0, "test2", 2) + ); + + this.columns = Arrays.asList("time", "string", "number"); + } + + +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemWriter.java new file mode 100644 index 0000000000..bb45358a7c --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemWriter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer.item; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestCsvItemWriter extends TestItemWriter { + + private static final String ExpectedComma = "1668578077051,test,1"; + private static final String ExpectedSemicolon = "1668578077051;test;1"; + + @Test + public void testCsvItemWriterCommaSeparated() { + var writer = new CsvItemWriter(","); + + String result = writer.createItem(row, columns); + + assertEquals(ExpectedComma, result); + } + + @Test + public void testCsvItemWriterSemicolonSeparated() { + var writer = new CsvItemWriter(";"); + + String result = writer.createItem(row, columns); + + assertEquals(ExpectedSemicolon, result); + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemWriter.java new file mode 100644 index 0000000000..0c113ca78d --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemWriter.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer.item; + +import org.junit.jupiter.api.BeforeEach; + +import java.util.Arrays; +import java.util.List; + +public abstract class TestItemWriter { + + protected List row; + protected List columns; + + @BeforeEach + public void before() { + this.row = Arrays.asList(1668578077051.0, "test", 1); + this.columns = Arrays.asList("time", "string", "number"); + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemWriter.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemWriter.java new file mode 100644 index 0000000000..518194b4d4 --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemWriter.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.query.writer.item; + +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestJsonItemWriter extends TestItemWriter { + + private static final String Expected = "{\"time\": 1668578077051,\"string\": \"test\",\"number\": 1}"; + + @Test + public void testJsonWriter() { + var writer = new JsonItemWriter(new Gson()); + + String result = writer.createItem(row, columns); + + assertEquals(Expected, result); + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java new file mode 100644 index 0000000000..3c288e1ac0 --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.sdk; + +import org.apache.streampipes.dataexplorer.influx.DataLakeInfluxQueryBuilder; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DataLakeQueryBuilderTest { + + private static final String MEASUREMENT = "measurement"; + @Test + public void withSimpleColumnsTest() { + var result = DataLakeInfluxQueryBuilder + .create(MEASUREMENT) + .withSimpleColumns(List.of("one", "two")) + .build(); + + var expected = String.format("SELECT one,two FROM \"%s\";", MEASUREMENT); + assertEquals(expected , result.getCommand()); + } +} diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java new file mode 100644 index 0000000000..c4cd73cebc --- /dev/null +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.utils; + +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ProvidedQueryParameterBuilder { + + private final String measurementId; + private final Map queryParams = new HashMap<>(); + + public static ProvidedQueryParameterBuilder create(String measurementId) { + return new ProvidedQueryParameterBuilder(measurementId); + } + + public ProvidedQueryParameterBuilder(String measurementId) { + this.measurementId = measurementId; + } + + public ProvidedQueryParameterBuilder withStartDate(long startDate) { + this.queryParams.put(SupportedRestQueryParams.QP_START_DATE, String.valueOf(startDate)); + + return this; + } + + public ProvidedQueryParameterBuilder withEndDate(long endDate) { + this.queryParams.put(SupportedRestQueryParams.QP_END_DATE, String.valueOf(endDate)); + + return this; + } + + public ProvidedQueryParameterBuilder withSimpleColumns(List simpleColumns) { + this.queryParams.put(SupportedRestQueryParams.QP_COLUMNS, String.join(",", simpleColumns)); + + return this; + } + + public ProvidedQueryParameterBuilder withQueryColumns(List rawQueryColumns) { + this.queryParams.put(SupportedRestQueryParams.QP_COLUMNS, String.join(",", rawQueryColumns)); + + return this; + } + + public ProvidedQueryParameterBuilder withGroupBy(List groupBy) { + this.queryParams.put(SupportedRestQueryParams.QP_GROUP_BY, String.join(",", groupBy)); + + return this; + } + + public ProvidedQueryParameterBuilder withFilter(String filter) { + this.queryParams.put(SupportedRestQueryParams.QP_FILTER, filter); + + return this; + } + + public ProvidedQueryParameterBuilder withPage(int page) { + this.queryParams.put(SupportedRestQueryParams.QP_PAGE, String.valueOf(page)); + + return this; + } + + public ProvidedQueryParameterBuilder withLimit(int limit) { + this.queryParams.put(SupportedRestQueryParams.QP_LIMIT, String.valueOf(limit)); + + return this; + } + + public ProvidedRestQueryParams build() { + return new ProvidedRestQueryParams(measurementId, queryParams); + } +} diff --git a/streampipes-data-export/pom.xml b/streampipes-data-export/pom.xml index 0d2d9ab959..d87f219d2b 100644 --- a/streampipes-data-export/pom.xml +++ b/streampipes-data-export/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -30,22 +30,22 @@ org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-pipeline-management - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-resource-management - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-storage-management - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java index d22ae5ba72..8a5a445835 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java @@ -29,7 +29,7 @@ public class AdapterResolver extends AbstractResolver { @Override public AdapterDescription findDocument(String resourceId) { - return getNoSqlStore().getAdapterInstanceStorage().getElementById(resourceId); + return getNoSqlStore().getAdapterInstanceStorage().getAdapter(resourceId); } @Override @@ -53,7 +53,7 @@ public ExportItem convert(AdapterDescription document) { @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getAdapterInstanceStorage().persist(deserializeDocument(document)); + getNoSqlStore().getAdapterInstanceStorage().storeAdapter(deserializeDocument(document)); } public void writeDocument(String document, @@ -62,7 +62,7 @@ public void writeDocument(String document, if (overrideDocument) { overrideProtocol(adapterDescription.getEventGrounding()); } - getNoSqlStore().getAdapterInstanceStorage().persist(adapterDescription); + getNoSqlStore().getAdapterInstanceStorage().storeAdapter(adapterDescription); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java index 35354044f8..9ce7713c3b 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java @@ -33,18 +33,18 @@ public class DashboardResolver extends AbstractResolver { @Override public DashboardModel findDocument(String resourceId) { - return getNoSqlStore().getDashboardStorage().getElementById(resourceId); + return getNoSqlStore().getDashboardStorage().getDashboard(resourceId); } @Override public DashboardModel modifyDocumentForExport(DashboardModel doc) { - doc.setRev(null); + doc.setCouchDbRev(null); return doc; } @Override protected boolean existsDoc(DashboardModel doc) { - return Objects.nonNull(doc) && doc.getElementId() != null; + return Objects.nonNull(doc) && doc.getCouchDbId() != null; } @Override @@ -54,12 +54,12 @@ public DashboardModel readDocument(String serializedDoc) throws JsonProcessingEx @Override public ExportItem convert(DashboardModel document) { - return new ExportItem(document.getElementId(), document.getName(), true); + return new ExportItem(document.getCouchDbId(), document.getName(), true); } @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getDashboardStorage().persist(deserializeDocument(document)); + getNoSqlStore().getDashboardStorage().storeDashboard(deserializeDocument(document)); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java index 73632e97e3..2c50b6ca7f 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardWidgetResolver.java @@ -28,7 +28,7 @@ public class DashboardWidgetResolver extends AbstractResolver { @Override public DashboardModel findDocument(String resourceId) { - return getNoSqlStore().getDataExplorerDashboardStorage().getElementById(resourceId); + return getNoSqlStore().getDataExplorerDashboardStorage().getDashboard(resourceId); } @Override public DashboardModel modifyDocumentForExport(DashboardModel doc) { - doc.setRev(null); + doc.setCouchDbRev(null); return doc; } @Override protected boolean existsDoc(DashboardModel doc) { - return Objects.nonNull(doc) && doc.getElementId() != null; + return Objects.nonNull(doc) && doc.getCouchDbId() != null; } @Override @@ -55,12 +55,12 @@ public DashboardModel readDocument(String serializedDoc) throws JsonProcessingEx @Override public ExportItem convert(DashboardModel document) { - return new ExportItem(document.getElementId(), document.getName(), true); + return new ExportItem(document.getCouchDbId(), document.getName(), true); } @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getDataExplorerDashboardStorage().persist(deserializeDocument(document)); + getNoSqlStore().getDataExplorerDashboardStorage().storeDashboard(deserializeDocument(document)); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java index a1b004b19e..a39926a0ac 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataViewWidgetResolver.java @@ -28,7 +28,7 @@ public class DataViewWidgetResolver extends AbstractResolver { @Override public FileMetadata findDocument(String resourceId) { - return getNoSqlStore().getFileMetadataStorage().getElementById(resourceId); + return getNoSqlStore().getFileMetadataStorage().getMetadataById(resourceId); } @Override @@ -49,7 +49,7 @@ public ExportItem convert(FileMetadata document) { @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getFileMetadataStorage().persist(deserializeDocument(document)); + getNoSqlStore().getFileMetadataStorage().addFileMetadata(deserializeDocument(document)); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java index e94012c91d..91c55c5a1b 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java @@ -28,7 +28,7 @@ public class MeasurementResolver extends AbstractResolver { @Override public DataLakeMeasure findDocument(String resourceId) { - return getNoSqlStore().getDataLakeStorage().getElementById(resourceId); + return getNoSqlStore().getDataLakeStorage().findOne(resourceId); } @Override @@ -49,7 +49,7 @@ public ExportItem convert(DataLakeMeasure document) { @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getDataLakeStorage().persist(deserializeDocument(document)); + getNoSqlStore().getDataLakeStorage().storeDataLakeMeasure(deserializeDocument(document)); } @Override diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java index 53830c166f..bd71f676de 100644 --- a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java +++ b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java @@ -30,7 +30,7 @@ public class PipelineResolver extends AbstractResolver { @Override public Pipeline findDocument(String resourceId) { - return getNoSqlStore().getPipelineStorageAPI().getElementById(resourceId); + return getNoSqlStore().getPipelineStorageAPI().getPipeline(resourceId); } @Override @@ -58,7 +58,7 @@ public ExportItem convert(Pipeline document) { @Override public void writeDocument(String document) throws JsonProcessingException { - getNoSqlStore().getPipelineStorageAPI().persist(deserializeDocument(document)); + getNoSqlStore().getPipelineStorageAPI().storePipeline(deserializeDocument(document)); } public void writeDocument(String document, @@ -81,7 +81,7 @@ public void writeDocument(String document, }).collect(Collectors.toList())); } - getNoSqlStore().getPipelineStorageAPI().persist(pipeline); + getNoSqlStore().getPipelineStorageAPI().storePipeline(pipeline); } @Override diff --git a/streampipes-dataformat-cbor/pom.xml b/streampipes-dataformat-cbor/pom.xml new file mode 100644 index 0000000000..2406b539dc --- /dev/null +++ b/streampipes-dataformat-cbor/pom.xml @@ -0,0 +1,59 @@ + + + + + + streampipes-parent + org.apache.streampipes + 0.95.1-SNAPSHOT + + 4.0.0 + + streampipes-dataformat-cbor + + + + + org.apache.streampipes + streampipes-dataformat + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-vocabulary + 0.95.1-SNAPSHOT + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + diff --git a/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatDefinition.java b/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatDefinition.java new file mode 100644 index 0000000000..95464b59e8 --- /dev/null +++ b/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatDefinition.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.cbor; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataformat.SpDataFormatDefinition; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.cbor.CBORFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class CborDataFormatDefinition implements SpDataFormatDefinition { + + private ObjectMapper objectMapper; + + public CborDataFormatDefinition() { + this.objectMapper = new ObjectMapper(new CBORFactory()); + } + + @Override + public Map toMap(byte[] event) throws SpRuntimeException { + try { + return objectMapper.readValue(event, HashMap.class); + } catch (IOException e) { + throw new SpRuntimeException("Could not convert event to map data structure"); + } + } + + @Override + public byte[] fromMap(Map event) throws SpRuntimeException { + try { + return objectMapper.writeValueAsBytes(event); + } catch (JsonProcessingException e) { + throw new SpRuntimeException("Could not convert map data structure to JSON string"); + } + } +} diff --git a/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatFactory.java b/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatFactory.java new file mode 100644 index 0000000000..5814a4fb5c --- /dev/null +++ b/streampipes-dataformat-cbor/src/main/java/org/apache/streampipes/dataformat/cbor/CborDataFormatFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.cbor; + +import org.apache.streampipes.dataformat.SpDataFormatDefinition; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.vocabulary.MessageFormat; + +public class CborDataFormatFactory extends SpDataFormatFactory { + + @Override + public String getTransportFormatRdfUri() { + return MessageFormat.CBOR; + } + + @Override + public SpDataFormatDefinition createInstance() { + return new CborDataFormatDefinition(); + } +} diff --git a/streampipes-dataformat-fst/pom.xml b/streampipes-dataformat-fst/pom.xml new file mode 100644 index 0000000000..ac915ef8ef --- /dev/null +++ b/streampipes-dataformat-fst/pom.xml @@ -0,0 +1,59 @@ + + + + + + streampipes-parent + org.apache.streampipes + 0.95.1-SNAPSHOT + + 4.0.0 + + streampipes-dataformat-fst + + + + + org.apache.streampipes + streampipes-dataformat + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-vocabulary + 0.95.1-SNAPSHOT + + + + + de.ruedigermoeller + fst + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + diff --git a/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatDefinition.java b/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatDefinition.java new file mode 100644 index 0000000000..39095143fc --- /dev/null +++ b/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatDefinition.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.fst; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataformat.SpDataFormatDefinition; + +import org.nustaq.serialization.FSTConfiguration; + +import java.util.Map; + +public class FstDataFormatDefinition implements SpDataFormatDefinition { + + private static FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration(); + + public FstDataFormatDefinition() { + + } + + @Override + public Map toMap(byte[] event) throws SpRuntimeException { + return (Map) conf.asObject(event); + } + + @Override + public byte[] fromMap(Map event) throws SpRuntimeException { + return conf.asByteArray(event); + } +} diff --git a/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatFactory.java b/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatFactory.java new file mode 100644 index 0000000000..285ea9cc25 --- /dev/null +++ b/streampipes-dataformat-fst/src/main/java/org/apache/streampipes/dataformat/fst/FstDataFormatFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.fst; + +import org.apache.streampipes.dataformat.SpDataFormatDefinition; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.vocabulary.MessageFormat; + +public class FstDataFormatFactory extends SpDataFormatFactory { + + @Override + public String getTransportFormatRdfUri() { + return MessageFormat.FST; + } + + @Override + public SpDataFormatDefinition createInstance() { + return new FstDataFormatDefinition(); + } +} diff --git a/streampipes-dataformat-json/pom.xml b/streampipes-dataformat-json/pom.xml new file mode 100644 index 0000000000..53900c5844 --- /dev/null +++ b/streampipes-dataformat-json/pom.xml @@ -0,0 +1,63 @@ + + + + + + streampipes-parent + org.apache.streampipes + 0.95.1-SNAPSHOT + + 4.0.0 + + streampipes-dataformat-json + + + + + org.apache.streampipes + streampipes-dataformat + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-vocabulary + 0.95.1-SNAPSHOT + + + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + diff --git a/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatDefinition.java b/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatDefinition.java new file mode 100644 index 0000000000..f3bbacaf06 --- /dev/null +++ b/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatDefinition.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataformat.json; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataformat.SpDataFormatDefinition; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class JsonDataFormatDefinition implements SpDataFormatDefinition { + + private ObjectMapper objectMapper; + + public JsonDataFormatDefinition() { + this.objectMapper = new ObjectMapper(); + } + + @Override + public Map toMap(byte[] event) throws SpRuntimeException { + try { + return objectMapper.readValue(event, HashMap.class); + } catch (IOException e) { + throw new SpRuntimeException("Could not convert event to map data structure"); + } + } + + @Override + public byte[] fromMap(Map event) throws SpRuntimeException { + try { + return objectMapper.writeValueAsBytes(event); + } catch (JsonProcessingException e) { + throw new SpRuntimeException("Could not convert map data structure to JSON string"); + } + } +} diff --git a/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatFactory.java b/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatFactory.java new file mode 100644 index 0000000000..87821cd37c --- /dev/null +++ b/streampipes-dataformat-json/src/main/java/org/apache/streampipes/dataformat/json/JsonDataFormatFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataformat.json; + +import org.apache.streampipes.dataformat.SpDataFormatDefinition; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.vocabulary.MessageFormat; + +public class JsonDataFormatFactory extends SpDataFormatFactory { + + @Override + public String getTransportFormatRdfUri() { + return MessageFormat.JSON; + } + + @Override + public SpDataFormatDefinition createInstance() { + return new JsonDataFormatDefinition(); + } +} diff --git a/streampipes-dataformat-smile/pom.xml b/streampipes-dataformat-smile/pom.xml new file mode 100644 index 0000000000..ff4103eb99 --- /dev/null +++ b/streampipes-dataformat-smile/pom.xml @@ -0,0 +1,59 @@ + + + + + + streampipes-parent + org.apache.streampipes + 0.95.1-SNAPSHOT + + 4.0.0 + + streampipes-dataformat-smile + + + + + org.apache.streampipes + streampipes-dataformat + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-vocabulary + 0.95.1-SNAPSHOT + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + diff --git a/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatDefinition.java b/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatDefinition.java new file mode 100644 index 0000000000..6c6bbdf26f --- /dev/null +++ b/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatDefinition.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.smile; + +import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataformat.SpDataFormatDefinition; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.smile.SmileFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class SmileDataFormatDefinition implements SpDataFormatDefinition { + + private ObjectMapper objectMapper; + + public SmileDataFormatDefinition() { + this.objectMapper = new ObjectMapper(new SmileFactory()); + } + + @Override + public Map toMap(byte[] event) throws SpRuntimeException { + try { + return objectMapper.readValue(event, HashMap.class); + } catch (IOException e) { + throw new SpRuntimeException("Could not convert event to map data structure"); + } + } + + @Override + public byte[] fromMap(Map event) throws SpRuntimeException { + try { + return objectMapper.writeValueAsBytes(event); + } catch (JsonProcessingException e) { + throw new SpRuntimeException("Could not convert map data structure to JSON string"); + } + } +} diff --git a/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatFactory.java b/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatFactory.java new file mode 100644 index 0000000000..d04b15197f --- /dev/null +++ b/streampipes-dataformat-smile/src/main/java/org/apache/streampipes/dataformat/smile/SmileDataFormatFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.dataformat.smile; + +import org.apache.streampipes.dataformat.SpDataFormatDefinition; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.vocabulary.MessageFormat; + +public class SmileDataFormatFactory extends SpDataFormatFactory { + @Override + public String getTransportFormatRdfUri() { + return MessageFormat.SMILE; + } + + @Override + public SpDataFormatDefinition createInstance() { + return new SmileDataFormatDefinition(); + } +} diff --git a/streampipes-dataformat/pom.xml b/streampipes-dataformat/pom.xml index daefc05026..554445e4ca 100644 --- a/streampipes-dataformat/pom.xml +++ b/streampipes-dataformat/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,15 +32,7 @@ org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core + 0.95.1-SNAPSHOT diff --git a/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatFactory.java b/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatFactory.java index ca7d9afc66..b805a23d0d 100644 --- a/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatFactory.java +++ b/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatFactory.java @@ -18,6 +18,15 @@ package org.apache.streampipes.dataformat; +import org.apache.streampipes.model.grounding.TransportFormat; + public abstract class SpDataFormatFactory { + public TransportFormat getTransportFormat() { + return new TransportFormat(getTransportFormatRdfUri()); + } + + public abstract String getTransportFormatRdfUri(); + + public abstract SpDataFormatDefinition createInstance(); } diff --git a/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatManager.java b/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatManager.java index 6b8ff4d7ef..342a9b40d8 100644 --- a/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatManager.java +++ b/streampipes-dataformat/src/main/java/org/apache/streampipes/dataformat/SpDataFormatManager.java @@ -18,9 +18,43 @@ package org.apache.streampipes.dataformat; -public class SpDataFormatManager { +import org.apache.streampipes.model.grounding.TransportFormat; - public static SpDataFormatDefinition getFormatDefinition() { - return new JsonDataFormatDefinition(); +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public enum SpDataFormatManager { + + INSTANCE; + + private List availableDataFormats; + + SpDataFormatManager() { + this.availableDataFormats = new ArrayList<>(); + } + + public void register(SpDataFormatFactory dataFormatDefinition) { + availableDataFormats.add(dataFormatDefinition); } + + public List getAvailableDataFormats() { + return availableDataFormats; + } + + public Optional findDefinition(TransportFormat transportFormat) { + // TODO why is transportFormat.getRdfType a list? + return this.availableDataFormats + .stream() + .filter + (adf -> transportFormat + .getRdfType() + .stream() + .anyMatch(tf -> tf.toString().equals(adf + .getTransportFormatRdfUri()))) + .map(SpDataFormatFactory::createInstance) + .findFirst(); + + } + } diff --git a/streampipes-extensions-api/pom.xml b/streampipes-extensions-api/pom.xml index 4b51b9a4aa..7fb3349f49 100644 --- a/streampipes-extensions-api/pom.xml +++ b/streampipes-extensions-api/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -31,17 +31,17 @@ org.apache.streampipes streampipes-client-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/AssetResolver.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/AssetResolver.java index c032168647..16006e631b 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/AssetResolver.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/AssetResolver.java @@ -23,18 +23,8 @@ public interface AssetResolver { - default byte[] getAsset(String assetName) throws IOException { - return getAsset(this.getClass().getClassLoader(), assetName); - } + byte[] getAsset(String assetName) throws IOException; - default Properties getLocale(String localeName) throws IOException { - return getLocale(this.getClass().getClassLoader(), localeName); - } - - byte[] getAsset(ClassLoader classLoader, - String assetName) throws IOException; - - Properties getLocale(ClassLoader classLoader, - String localeName) throws IOException; + Properties getLocale(String localeName) throws IOException; } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/DefaultAssetResolver.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/DefaultAssetResolver.java index 8449a9451b..61b3f067d3 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/DefaultAssetResolver.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/assets/DefaultAssetResolver.java @@ -34,24 +34,23 @@ public DefaultAssetResolver(String appId) { } @Override - public byte[] getAsset(ClassLoader classLoader, - String assetName) throws IOException { - return getResourceFile(classLoader, assetName).readAllBytes(); + public byte[] getAsset(String assetName) throws IOException { + return getResourceFile(assetName).readAllBytes(); } @Override - public Properties getLocale(ClassLoader classLoader, - String localeName) throws IOException { + public Properties getLocale(String localeName) throws IOException { Properties props = new Properties(); - props.load(new InputStreamReader(getResourceFile(classLoader, localeName), StandardCharsets.UTF_8)); + props.load(new InputStreamReader(getResourceFile(localeName), StandardCharsets.UTF_8)); return props; } - protected InputStream getResourceFile(ClassLoader classLoader, - String filename) throws IOException { + protected InputStream getResourceFile(String filename) throws IOException { if (existsFile(filename)) { var path = makePath(filename); - return classLoader.getResourceAsStream(path); + return this.getClass() + .getClassLoader() + .getResourceAsStream(path); } else { throw new IOException(String.format("Could not read file %s", filename)); } diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/connect/StreamPipesAdapter.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/connect/StreamPipesAdapter.java index 8651b05ee5..4a5fb1d362 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/connect/StreamPipesAdapter.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/connect/StreamPipesAdapter.java @@ -22,26 +22,11 @@ import org.apache.streampipes.extensions.api.connect.context.IAdapterGuessSchemaContext; import org.apache.streampipes.extensions.api.connect.context.IAdapterRuntimeContext; import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor; -import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.connect.guess.GuessSchema; public interface StreamPipesAdapter { IAdapterConfiguration declareConfig(); - /** - * Preprocesses the adapter description before the adapter is invoked. - * - *

This method is designed to allow adapters to modify the adapter description prior to invocation. - * It is particularly useful for adapters that need to manipulate certain values internally, - * e.g. bypassing the adapter preprocessing pipeline. An example of such an adapter is the FileReplayAdapter, - * which manipulates timestamp values.

- * - *

This is a default method and does not need to be overridden unless specific preprocessing is required.

- * - * @param adapterDescription The adapter description to be preprocessed. - */ - default void preprocessAdapterDescription(AdapterDescription adapterDescription) {}; - void onAdapterStarted(IAdapterParameterExtractor extractor, IEventCollector collector, IAdapterRuntimeContext adapterRuntimeContext) throws AdapterException; diff --git a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java index 2fd8e40f76..447aad86fe 100644 --- a/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java +++ b/streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/extractor/IParameterExtractor.java @@ -60,15 +60,6 @@ public interface IParameterExtractor { List selectedMultiValues(String internalName, Class targetClass); - List selectedMultiValuesInternalNames(String internalName, Class targetClass); - - List selectedTreeNodesInternalNames(String internalName, - Class targetClass); - - /** - * @deprecated use {@link #selectedTreeNodesInternalNames(String, Class)} instead - */ - @Deprecated(since = "0.97.0", forRemoval = true) List selectedTreeNodesInternalNames(String internalName, Class targetClass, boolean onlyDataNodes); @@ -86,6 +77,9 @@ W getStaticPropertyByName(String internalName, Class< String propertyDatatype(String runtimeName); + V supportedOntologyPropertyValue(String domainPropertyInternalId, String + propertyId, Class targetClass); + List getEventPropertiesBySelector(List selectors) throws SpRuntimeException; @@ -104,6 +98,4 @@ List getEventPropertiesBySelector(List selectors) throws List getEventPropertiesSelectorByScope(PropertyScope scope); List getEventPropertiesByScope(PropertyScope scope); - - List getInputEventProperties(int streamIndex); } diff --git a/streampipes-extensions-management/pom.xml b/streampipes-extensions-management/pom.xml index 9c0427023b..c45114fe98 100644 --- a/streampipes-extensions-management/pom.xml +++ b/streampipes-extensions-management/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-extensions-management @@ -31,53 +31,58 @@ org.apache.streampipes streampipes-extensions-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-client - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-connect-shared - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-dataformat - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT + + + org.apache.streampipes + streampipes-dataformat-smile + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-measurement-units - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-sdk - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-serializers-json - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-test-utils - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT test org.apache.streampipes streampipes-vocabulary - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT @@ -94,11 +99,6 @@ com.opencsv opencsv - - org.apache.avro - avro - 1.11.4 - diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagement.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagement.java index 0d9cfd31f2..958d413f1f 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagement.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagement.java @@ -63,11 +63,6 @@ public void invokeAdapter(AdapterDescription adapterDescription) throws AdapterE newAdapterInstance, adapterDescription); - // This method allows adapters to modify the adapter description prior to invocation. - // It is particularly useful for adapters like FileReplayAdapter that need to manipulate timestamp values - // internally, bypassing the adapter preprocessing pipeline. - newAdapterInstance.preprocessAdapterDescription(adapterDescription); - var registeredParsers = newAdapterInstance.declareConfig().getSupportedParsers(); var extractor = AdapterParameterExtractor.from(adapterDescription, registeredParsers); var eventCollector = EventCollector.from(adapterDescription); diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/PullAdapterScheduler.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/PullAdapterScheduler.java index 7795d40bfb..de9a1b4146 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/PullAdapterScheduler.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/PullAdapterScheduler.java @@ -33,39 +33,27 @@ public class PullAdapterScheduler { - protected static final Logger LOG = LoggerFactory.getLogger(PullAdapterScheduler.class); + protected static final Logger LOGGER = LoggerFactory.getLogger(PullAdapterScheduler.class); private ScheduledExecutorService scheduler; public void schedule(IPullAdapter pullAdapter, String adapterElementId) { - scheduler = Executors.newSingleThreadScheduledExecutor(); - final Runnable task = new Runnable() { - @Override - public void run() { - try { - pullAdapter.pullData(); - } catch (ExecutionException | InterruptedException | TimeoutException e) { - LOG.error("Error while pulling data", e); - SpMonitoringManager.INSTANCE.addErrorMessage( - adapterElementId, - SpLogEntry.from(System.currentTimeMillis(), SpLogMessage.from(e)) - ); - } finally { - scheduler.schedule( - this, - pullAdapter.getPollingInterval().value(), - pullAdapter.getPollingInterval().timeUnit() - ); - } + final Runnable task = () -> { + try { + pullAdapter.pullData(); + } catch (ExecutionException | InterruptedException e) { + SpMonitoringManager.INSTANCE.addErrorMessage( + adapterElementId, + SpLogEntry.from(System.currentTimeMillis(), SpLogMessage.from(e))); + } catch (TimeoutException e) { + LOGGER.warn("Timeout occurred", e); } }; - scheduler.schedule( - task, - pullAdapter.getPollingInterval().value(), - pullAdapter.getPollingInterval().timeUnit() - ); + scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(task, 1, + pullAdapter.getPollingInterval().value(), pullAdapter.getPollingInterval().timeUnit()); } public void shutdown() { diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java index 9661b7705c..ef357e4c48 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/TransformationRuleUpdateVisitor.java @@ -31,7 +31,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.model.schema.EventProperty; @@ -81,13 +80,6 @@ public void visit(RenameRuleDescription rule) { } } - @Override - public void visit(RegexTransformationRuleDescription rule) { - if (containsKey(rule.getRuntimeKey())) { - validRules.add(rule); - } - } - @Override public void visit(EventRateTransformationRuleDescription rule) { // Do nothing diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/BrokerEventProcessor.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/BrokerEventProcessor.java index b3ed8c6fd1..2ad113e8a7 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/BrokerEventProcessor.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/BrokerEventProcessor.java @@ -23,10 +23,11 @@ import org.apache.streampipes.extensions.api.connect.IParser; import org.apache.streampipes.messaging.InternalEventProcessor; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; public record BrokerEventProcessor( IParser parser, @@ -38,7 +39,7 @@ public record BrokerEventProcessor( @Override public void onEvent(byte[] payload) { try { - parser.parse(new ByteArrayInputStream(payload), collector::collect); + parser.parse(IOUtils.toInputStream(new String(payload), StandardCharsets.UTF_8), collector::collect); } catch (ParseException e) { LOG.error("Error while parsing: " + e.getMessage()); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/Parsers.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/Parsers.java index 1dc2937792..3c8a0110da 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/Parsers.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/Parsers.java @@ -30,8 +30,7 @@ public static List defaultParsers() { new JsonParsers(), new CsvParser(), new XmlParser(), - new ImageParser(), - new AvroParser() + new ImageParser() ); } } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/json/GeoJsonParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/json/GeoJsonParser.java index f028ccfd12..0405a87ca2 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/json/GeoJsonParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/parser/json/GeoJsonParser.java @@ -23,6 +23,7 @@ import org.apache.streampipes.extensions.management.connect.adapter.parser.util.JsonEventProperty; import org.apache.streampipes.model.connect.guess.GuessSchema; import org.apache.streampipes.model.schema.EventProperty; +import org.apache.streampipes.model.schema.EventPropertyPrimitive; import org.apache.streampipes.sdk.builder.adapter.GuessSchemaBuilder; import org.apache.streampipes.vocabulary.Geo; import org.apache.streampipes.vocabulary.SO; @@ -40,6 +41,9 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -143,8 +147,12 @@ public void parse(InputStream inputStream, IParserEventHandler handler) throws P private EventProperty getEventPropertyGeoJson(String name, Object value, String domain) { EventProperty eventProperty = JsonEventProperty.getEventProperty(name, value); - eventProperty.setSemanticType(domain); + try { + ((EventPropertyPrimitive) eventProperty).setDomainProperties(Arrays.asList(new URI(domain))); + } catch (URISyntaxException e) { + LOG.error(e.getMessage()); + } return eventProperty; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java index d63cddc8f1..66983df357 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/preprocessing/elements/SendToBrokerAdapterSink.java @@ -20,14 +20,15 @@ import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.dataformat.SpDataFormatDefinition; -import org.apache.streampipes.dataformat.SpDataFormatManager; import org.apache.streampipes.extensions.api.connect.IAdapterPipelineElement; import org.apache.streampipes.extensions.api.monitoring.SpMonitoringManager; +import org.apache.streampipes.extensions.management.connect.adapter.util.TransportFormatSelector; import org.apache.streampipes.extensions.management.monitoring.ExtensionsLogger; import org.apache.streampipes.messaging.EventProducer; import org.apache.streampipes.messaging.SpProtocolManager; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.grounding.KafkaTransportProtocol; +import org.apache.streampipes.model.grounding.TransportFormat; import org.apache.streampipes.model.grounding.TransportProtocol; import java.util.Map; @@ -53,7 +54,13 @@ public SendToBrokerAdapterSink(AdapterDescription adapterDescription) { if (producerOpt.isPresent()) { this.producer = producerOpt.get().getProducer(this.protocol); - this.dataFormatDefinition = SpDataFormatManager.getFormatDefinition(); + TransportFormat transportFormat = adapterDescription + .getEventGrounding() + .getTransportFormats() + .get(0); + + this.dataFormatDefinition = + new TransportFormatSelector(transportFormat).getDataFormatDefinition(); producer.connect(); } else { diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatSelector.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatSelector.java new file mode 100644 index 0000000000..40d985c6a6 --- /dev/null +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/TransportFormatSelector.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.extensions.management.connect.adapter.util; + +import org.apache.streampipes.dataformat.SpDataFormatDefinition; +import org.apache.streampipes.dataformat.cbor.CborDataFormatDefinition; +import org.apache.streampipes.dataformat.fst.FstDataFormatDefinition; +import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.apache.streampipes.dataformat.smile.SmileDataFormatDefinition; +import org.apache.streampipes.model.grounding.TransportFormat; +import org.apache.streampipes.vocabulary.MessageFormat; + +public class TransportFormatSelector { + + private TransportFormat transportFormat; + + public TransportFormatSelector(TransportFormat format) { + this.transportFormat = format; + } + + public SpDataFormatDefinition getDataFormatDefinition() { + if (isJsonFormat(transportFormat)) { + return new JsonDataFormatDefinition(); + } else if (isCborFormat(transportFormat)) { + return new CborDataFormatDefinition(); + } else if (isFstFormat(transportFormat)) { + return new FstDataFormatDefinition(); + } else if (isSmileFormat(transportFormat)) { + return new SmileDataFormatDefinition(); + } else { + throw new IllegalArgumentException("Wrong transport format: " + makeError(transportFormat)); + } + } + + private boolean isSmileFormat(TransportFormat transportFormat) { + return isFormat(MessageFormat.SMILE, transportFormat); + } + + private boolean isFstFormat(TransportFormat transportFormat) { + return isFormat(MessageFormat.FST, transportFormat); + } + + private boolean isCborFormat(TransportFormat transportFormat) { + return isFormat(MessageFormat.CBOR, transportFormat); + } + + private Boolean isJsonFormat(TransportFormat transportFormat) { + return isFormat(MessageFormat.JSON, transportFormat); + } + + private boolean isFormat(String format, TransportFormat transportFormat) { + return transportFormat.getRdfType().stream().anyMatch(tf -> tf.toString().equals(format)); + } + + private String makeError(TransportFormat transportFormat) { + StringBuilder builder = new StringBuilder(); + transportFormat.getRdfType().forEach(type -> builder.append(type.toString()).append(", ")); + return builder.toString(); + } +} diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java index 311e15ced7..fed5a0baa4 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/init/DeclarersSingleton.java @@ -18,6 +18,8 @@ package org.apache.streampipes.extensions.management.init; +import org.apache.streampipes.dataformat.SpDataFormatFactory; +import org.apache.streampipes.dataformat.SpDataFormatManager; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; import org.apache.streampipes.extensions.api.pe.IStreamPipesDataProcessor; @@ -28,10 +30,15 @@ import org.apache.streampipes.extensions.management.model.SpServiceDefinition; import org.apache.streampipes.messaging.SpProtocolDefinitionFactory; import org.apache.streampipes.messaging.SpProtocolManager; +import org.apache.streampipes.model.grounding.TransportFormat; import org.apache.streampipes.model.grounding.TransportProtocol; import org.apache.streampipes.model.util.Cloner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -41,6 +48,7 @@ public class DeclarersSingleton implements IDeclarersSingleton { + private static final Logger LOG = LoggerFactory.getLogger(DeclarersSingleton.class); private static final String Http = "http://"; private static final String Colon = ":"; private static final String Slash = "/"; @@ -53,6 +61,8 @@ public class DeclarersSingleton implements IDeclarersSingleton { private final Map functions; private final Map supportedProtocols; + private final Map supportedFormats; + private final Map adapters; private List runtimeProviders; @@ -70,6 +80,7 @@ private DeclarersSingleton() { this.dataSinks = new HashMap<>(); this.dataStreams = new HashMap<>(); this.supportedProtocols = new HashMap<>(); + this.supportedFormats = new HashMap<>(); this.adapters = new HashMap<>(); this.functions = new HashMap<>(); this.runtimeProviders = new ArrayList<>(); @@ -92,6 +103,7 @@ public void populate(String host, Integer port, SpServiceDefinition serviceDef) this.serviceId = serviceDef.getServiceId(); this.serviceGroup = serviceDef.getServiceGroup(); this.registerProtocols(serviceDef.getProtocolDefinitionFactories()); + this.registerDataFormats(serviceDef.getDataFormatFactories()); this.runtimeProviders = serviceDef.getRuntimeProviders(); serviceDef.getAdapters().forEach(a -> this.adapters.put(a.declareConfig().getAdapterDescription().getAppId(), a)); serviceDef.getFunctions().forEach(f -> this.functions.put(f.getFunctionConfig().getFunctionId().getId(), f)); @@ -122,6 +134,7 @@ public Map> getDeclarers() { result.putAll(dataProcessors); result.putAll(dataStreams); result.putAll(dataSinks); + //result.putAll(pipelineTemplateDeclarers); return result; } @@ -135,6 +148,20 @@ public void registerProtocols(List> protocols) { protocols.forEach(this::registerProtocol); } + public void registerDataFormat(SpDataFormatFactory dataFormatDefinition) { + SpDataFormatManager.INSTANCE.register(dataFormatDefinition); + this.supportedFormats.put(dataFormatDefinition.getTransportFormatRdfUri(), + dataFormatDefinition.getTransportFormat()); + } + + public void registerDataFormats(SpDataFormatFactory... dataFormatDefinitions) { + registerDataFormats(Arrays.asList(dataFormatDefinitions)); + } + + public void registerDataFormats(List dataFormatDefinitions) { + dataFormatDefinitions.forEach(this::registerDataFormat); + } + private void addDataProcessor(IStreamPipesDataProcessor dataProcessor) { dataProcessors.put(dataProcessor.declareConfig().getDescription().getAppId(), dataProcessor); } @@ -168,6 +195,13 @@ public Collection getSupportedProtocols() { .collect(Collectors.toList()); } + public Collection getSupportedFormats() { + return this.supportedFormats.values() + .stream() + .map(TransportFormat::new) + .collect(Collectors.toList()); + } + public int getPort() { return this.port; } @@ -226,6 +260,15 @@ private void checkAndStartExecutableStreams(IStreamPipesDataStream declarer) { } } + public SpServiceDefinition toServiceDefinition(String serviceId) { + SpServiceDefinition serviceDef = new SpServiceDefinition(); + serviceDef.setServiceId(serviceId); + serviceDef.setDefaultPort(this.getPort()); + + // TODO create complete service definition + return serviceDef; + } + public String getServiceGroup() { return serviceGroup; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java index 7eb83a99bc..56a151a13b 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/locales/LabelGenerator.java @@ -20,14 +20,11 @@ import org.apache.streampipes.extensions.api.assets.AssetResolver; import org.apache.streampipes.extensions.api.assets.DefaultAssetResolver; import org.apache.streampipes.model.base.ConsumableStreamPipesEntity; -import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.base.NamedStreamPipesEntity; import org.apache.streampipes.model.connect.adapter.AdapterDescription; import org.apache.streampipes.model.graph.DataProcessorDescription; -import org.apache.streampipes.model.graph.DataProcessorInvocation; import org.apache.streampipes.model.output.AppendOutputStrategy; import org.apache.streampipes.model.output.FixedOutputStrategy; -import org.apache.streampipes.model.output.OutputStrategy; import org.apache.streampipes.model.staticproperty.CollectionStaticProperty; import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative; @@ -35,7 +32,6 @@ import org.apache.streampipes.model.staticproperty.StaticPropertyGroup; import java.io.IOException; -import java.util.List; import java.util.Properties; public class LabelGenerator { @@ -88,51 +84,40 @@ public T generateLabels() throws IOException { }); } - if (isInvocable()) { - ((InvocableStreamPipesEntity) desc).getStaticProperties().forEach(sp -> generateLabels(props, sp)); - if (isDataProcessorInvocation()) { - applyOutputStrategies(((DataProcessorInvocation) desc).getOutputStrategies(), props); - } - } - - if (isDataProcessorDescription()) { - applyOutputStrategies(((DataProcessorDescription) desc).getOutputStrategies(), props); + if (isDataProcessor()) { + ((DataProcessorDescription) desc).getOutputStrategies() + .forEach(os -> { + if (os instanceof AppendOutputStrategy) { + ((AppendOutputStrategy) os).getEventProperties() + .forEach(ep -> { + ep.setLabel(getTitle( + props, + ep.getRuntimeId() + )); + ep.setDescription(getDescription( + props, + ep.getRuntimeId() + )); + }); + } else if (os instanceof FixedOutputStrategy) { + ((FixedOutputStrategy) os).getEventProperties() + .forEach(ep -> { + ep.setLabel(getTitle( + props, + ep.getRuntimeId() + )); + ep.setDescription(getDescription( + props, + ep.getRuntimeId() + )); + }); + } + }); } } return desc; } - private void applyOutputStrategies(List outputStrategies, - Properties props) { - outputStrategies.forEach(os -> { - if (os instanceof AppendOutputStrategy) { - ((AppendOutputStrategy) os).getEventProperties() - .forEach(ep -> { - ep.setLabel(getTitle( - props, - ep.getRuntimeId() - )); - ep.setDescription(getDescription( - props, - ep.getRuntimeId() - )); - }); - } else if (os instanceof FixedOutputStrategy) { - ((FixedOutputStrategy) os).getEventProperties() - .forEach(ep -> { - ep.setLabel(getTitle( - props, - ep.getRuntimeId() - )); - ep.setDescription(getDescription( - props, - ep.getRuntimeId() - )); - }); - } - }); - } - /** * Returns the tile of the element description based on the data of the resource files @@ -200,18 +185,10 @@ private boolean isConsumable() { return desc instanceof ConsumableStreamPipesEntity; } - private boolean isDataProcessorDescription() { + private boolean isDataProcessor() { return desc instanceof DataProcessorDescription; } - private boolean isDataProcessorInvocation() { - return desc instanceof DataProcessorInvocation; - } - - private boolean isInvocable() { - return desc instanceof InvocableStreamPipesEntity; - } - private boolean isAdapter() { return desc instanceof AdapterDescription; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java index 42bd2dcc18..3c7f3af96e 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinition.java @@ -17,6 +17,7 @@ */ package org.apache.streampipes.extensions.management.model; +import org.apache.streampipes.dataformat.SpDataFormatFactory; import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; import org.apache.streampipes.extensions.api.declarer.IStreamPipesFunctionDeclarer; import org.apache.streampipes.extensions.api.migration.IModelMigrator; @@ -40,6 +41,7 @@ public class SpServiceDefinition { private Integer defaultPort; private List> pipelineElements; + private List dataFormatFactories; private List> protocolDefinitionFactories; private List functions; @@ -53,6 +55,7 @@ public class SpServiceDefinition { public SpServiceDefinition() { this.serviceId = UUID.randomUUID().toString(); this.pipelineElements = new ArrayList<>(); + this.dataFormatFactories = new ArrayList<>(); this.protocolDefinitionFactories = new ArrayList<>(); this.kvConfigs = new ArrayList<>(); this.functions = new ArrayList<>(); @@ -117,6 +120,14 @@ public void setDeclarers(List> pipelineElements) this.pipelineElements = pipelineElements; } + public void addDataFormatFactory(SpDataFormatFactory factory) { + this.dataFormatFactories.add(factory); + } + + public void addDataFormatFactories(List factories) { + this.dataFormatFactories.addAll(factories); + } + public void addConfig(ConfigItem configItem) { this.kvConfigs.add(configItem); } @@ -129,6 +140,14 @@ public void addAdapters(List adapters) { this.adapters.addAll(adapters); } + public List getDataFormatFactories() { + return dataFormatFactories; + } + + public void setDataFormatFactories(List dataFormatFactories) { + this.dataFormatFactories = dataFormatFactories; + } + public void addProtocolDefinitionFactory(SpProtocolDefinitionFactory factory) { this.protocolDefinitionFactories.add(factory); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java index ac02d4533b..1cb2bb004d 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/model/SpServiceDefinitionBuilder.java @@ -117,19 +117,13 @@ public SpServiceDefinitionBuilder registerAdapter(StreamPipesAdapter adapter) { } - /** - * @deprecated data format registration is no longer required - */ - @Deprecated(since = "0.97.0", forRemoval = true) public SpServiceDefinitionBuilder registerMessagingFormat(SpDataFormatFactory dataFormatDefinition) { + this.serviceDefinition.addDataFormatFactory(dataFormatDefinition); return this; } - /** - * @deprecated data format registration is no longer required - */ - @Deprecated(since = "0.97.0", forRemoval = true) public SpServiceDefinitionBuilder registerMessagingFormats(SpDataFormatFactory... dataFormatDefinitions) { + this.serviceDefinition.addDataFormatFactories(Arrays.asList(dataFormatDefinitions)); return this; } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java index 0f9a46b210..f0a7020447 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/util/EventSchemaUtils.java @@ -24,6 +24,7 @@ import org.apache.streampipes.model.schema.EventSchema; import org.apache.streampipes.vocabulary.SO; +import java.net.URI; import java.util.List; import java.util.Optional; @@ -46,7 +47,7 @@ public static Optional getTimestampProperty(EventSchema private static Optional getTimstampProperty(List eventProperties) { for (EventProperty ep : eventProperties) { - if (ep instanceof EventPropertyPrimitive && SO.DATE_TIME.equalsIgnoreCase(ep.getSemanticType())) { + if (ep instanceof EventPropertyPrimitive && ep.getDomainProperties().contains(URI.create(SO.DATE_TIME))) { return Optional.of((EventPropertyPrimitive) ep); } diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagementTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagementTest.java index 33f49575e3..ff7eb0294e 100644 --- a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagementTest.java +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/AdapterWorkerManagementTest.java @@ -36,7 +36,7 @@ public class AdapterWorkerManagementTest { @Test public void invokeAdapterNotPresent() throws AdapterException { var adapterDescription = AdapterConfigurationBuilder - .create("id", 0, null) + .create("id", null) .build(); var declarerSingleton = mock(IDeclarersSingleton.class); diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/ConnectWorkerDescriptionProviderTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/ConnectWorkerDescriptionProviderTest.java index 9ca0911886..602b0a4316 100644 --- a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/ConnectWorkerDescriptionProviderTest.java +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/ConnectWorkerDescriptionProviderTest.java @@ -44,7 +44,7 @@ public void initializeProvider() { var testAdapter = mock(StreamPipesAdapter.class); doAnswer(invocation -> AdapterConfigurationBuilder - .create(adapterId, 0, null) + .create(adapterId, null) .buildConfiguration()) .when(testAdapter) .declareConfig(); diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/init/DeclarersSingletonTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/init/DeclarersSingletonTest.java index e632a80cf9..1a8c2504a3 100644 --- a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/init/DeclarersSingletonTest.java +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/init/DeclarersSingletonTest.java @@ -36,18 +36,16 @@ public void getAdapterTest() { var id = "id"; var testAdapter = mock(StreamPipesAdapter.class); doAnswer(invocation -> - AdapterConfigurationBuilder - .create(id, 0, null) - .buildConfiguration()) + AdapterConfigurationBuilder + .create(id, null) + .buildConfiguration()) .when(testAdapter) .declareConfig(); - DeclarersSingleton.getInstance() - .setAdapters(List.of(testAdapter)); + DeclarersSingleton.getInstance().setAdapters(List.of(testAdapter)); - var result = DeclarersSingleton.getInstance() - .getAdapter(id); + var result = DeclarersSingleton.getInstance().getAdapter(id); Assertions.assertTrue(result.isPresent()); Assertions.assertEquals(testAdapter, result.get()); diff --git a/streampipes-extensions/pom.xml b/streampipes-extensions/pom.xml index e092103817..dbcbd4a94f 100644 --- a/streampipes-extensions/pom.xml +++ b/streampipes-extensions/pom.xml @@ -23,7 +23,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT ../pom.xml @@ -63,7 +63,6 @@ streampipes-sinks-notifications-jvm streampipes-connectors-plc streampipes-extensions-iiot-minimal - streampipes-connectors-ros @@ -77,7 +76,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.4.0 + 3.3.0 org.apache.maven.plugins @@ -114,7 +113,7 @@ org.apache.streampipes streampipes-maven-plugin - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/pom.xml b/streampipes-extensions/streampipes-connect-adapters-iiot/pom.xml index 972d37d963..46fa8a646f 100644 --- a/streampipes-extensions/streampipes-connect-adapters-iiot/pom.xml +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/pom.xml @@ -24,7 +24,7 @@ org.apache.streampipes streampipes-extensions - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-connect-adapters-iiot @@ -37,25 +37,29 @@ org.apache.streampipes streampipes-connectors-mqtt - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-pipeline-elements-shared - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-client - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-service-extensions - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT + + edu.wpi.rail + jrosbridge + org.apache.httpcomponents fluent-hc diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/IIoTAdaptersExtensionModuleExport.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/IIoTAdaptersExtensionModuleExport.java index ef2f44b7ec..99b4ccbca7 100644 --- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/IIoTAdaptersExtensionModuleExport.java +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/IIoTAdaptersExtensionModuleExport.java @@ -19,7 +19,9 @@ package org.apache.streampipes.connect.iiot; import org.apache.streampipes.connect.iiot.adapters.iolink.IfmAlMqttAdapter; +import org.apache.streampipes.connect.iiot.adapters.migrations.RosBridgeAdapterMigrationV1; import org.apache.streampipes.connect.iiot.adapters.oi4.Oi4Adapter; +import org.apache.streampipes.connect.iiot.adapters.ros.RosBridgeAdapter; import org.apache.streampipes.connect.iiot.adapters.simulator.machine.MachineDataSimulatorAdapter; import org.apache.streampipes.connect.iiot.protocol.stream.FileReplayAdapter; import org.apache.streampipes.connect.iiot.protocol.stream.HttpServerProtocol; @@ -40,6 +42,7 @@ public List adapters() { new FileReplayAdapter(), new IfmAlMqttAdapter(), new Oi4Adapter(), + new RosBridgeAdapter(), new HttpStreamProtocol(), new HttpServerProtocol() ); @@ -52,6 +55,8 @@ public List> pipelineElements() { @Override public List> migrators() { - return List.of(); + return List.of( + new RosBridgeAdapterMigrationV1() + ); } } diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/iolink/IfmAlMqttAdapter.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/iolink/IfmAlMqttAdapter.java index 7729dbba96..9c9fc05caf 100644 --- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/iolink/IfmAlMqttAdapter.java +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/iolink/IfmAlMqttAdapter.java @@ -36,11 +36,11 @@ import org.apache.streampipes.extensions.management.connect.adapter.parser.json.JsonObjectParser; import org.apache.streampipes.model.AdapterType; import org.apache.streampipes.model.connect.guess.GuessSchema; -import org.apache.streampipes.model.extensions.ExtensionAssetType; import org.apache.streampipes.sdk.builder.adapter.AdapterConfigurationBuilder; import org.apache.streampipes.sdk.helpers.Labels; import org.apache.streampipes.sdk.helpers.Locales; import org.apache.streampipes.sdk.helpers.Options; +import org.apache.streampipes.sdk.utils.Assets; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -79,7 +79,7 @@ public IAdapterConfiguration declareConfig() { return AdapterConfigurationBuilder .create(ID, 0, IfmAlMqttAdapter::new) .withLocales(Locales.EN) - .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON) + .withAssets(Assets.DOCUMENTATION, Assets.ICON) .withCategory(AdapterType.Generic, AdapterType.Manufacturing) .requiredTextParameter(MqttConnectUtils.getBrokerUrlLabel()) .requiredAlternatives(MqttConnectUtils.getAccessModeLabel(), MqttConnectUtils.getAlternativesOne(), diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/migrations/RosBridgeAdapterMigrationV1.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/migrations/RosBridgeAdapterMigrationV1.java new file mode 100644 index 0000000000..3d41e7fbf5 --- /dev/null +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/migrations/RosBridgeAdapterMigrationV1.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.iiot.adapters.migrations; + +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; +import org.apache.streampipes.extensions.api.migration.IAdapterMigrator; +import org.apache.streampipes.model.connect.adapter.AdapterDescription; +import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.migration.MigrationResult; +import org.apache.streampipes.model.migration.ModelMigratorConfig; +import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.apache.streampipes.model.staticproperty.StaticProperty; +import org.apache.streampipes.vocabulary.XSD; + +import static org.apache.streampipes.connect.iiot.adapters.ros.RosBridgeAdapter.ROS_PORT_KEY; + +public class RosBridgeAdapterMigrationV1 implements IAdapterMigrator { + + @Override + public ModelMigratorConfig config() { + return new ModelMigratorConfig( + "org.apache.streampipes.connect.iiot.adapters.ros", + SpServiceTagPrefix.ADAPTER, + 0, + 1); + } + + @Override + public MigrationResult migrate(AdapterDescription element, + IStaticPropertyExtractor extractor) throws RuntimeException { + + var portProperty = extractPortProperty(element); + portProperty.setRequiredDatatype(XSD.INTEGER); + + return MigrationResult.success(element); + } + + protected FreeTextStaticProperty extractPortProperty(AdapterDescription adapterDescription) { + return (FreeTextStaticProperty) adapterDescription.getConfig().stream() + .filter(this::isPortConfig) + .findFirst() + .orElseThrow(); + } + + private boolean isPortConfig(StaticProperty config) { + return config.getInternalName().equals(ROS_PORT_KEY); + } +} \ No newline at end of file diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/oi4/Oi4Adapter.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/oi4/Oi4Adapter.java index aa24c3ec5d..06bd974c6c 100644 --- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/oi4/Oi4Adapter.java +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/oi4/Oi4Adapter.java @@ -149,7 +149,7 @@ public void onAdapterStarted( InputStream in = convertByte(mqttEvent); var networkMessage = mapper.readValue(in, NetworkMessage.class); var payload = extractPayload(networkMessage); - payload.forEach(collector::collect); + collector.collect(payload); } catch (ParseException e) { LOG.debug("Message parsing failed - this might be caused by messages from a different sensor type"); } catch (IOException e) { @@ -253,7 +253,7 @@ private GuessSchema guessSchemaFromSampleMessage(byte[] sampleMessage) { var networkMessage = mapper.readValue(sampleMessage, NetworkMessage.class); var payload = extractPayload(networkMessage); - String plainPayload = mapper.writeValueAsString(payload.get(0)); + String plainPayload = mapper.writeValueAsString(payload); return new JsonParsers(new JsonObjectParser()) .getGuessSchema(convertByte(plainPayload.getBytes(StandardCharsets.UTF_8))); } catch (IOException e) { @@ -321,10 +321,9 @@ private MqttConsumer getGuessMqttConsumer(List sampleMessages) { return new MqttConsumer(this.mqttConfig, eventProcessor); } - private List> extractPayload(NetworkMessage message) throws ParseException { + private Map extractPayload(NetworkMessage message) throws ParseException { var dataMessages = findProcessDataInputMessage(message); - var result = new ArrayList>(); if (!dataMessages.isEmpty()) { @@ -339,16 +338,12 @@ private List> extractPayload(NetworkMessage message) throws // an empty list of selected sensors means that we want to collect data from all sensors available if (selectedSensors.isEmpty() || selectedSensors.contains(sensorId)) { - result.add(extractAndEnrichMessagePayload(dataMessage, sensorId)); + return extractAndEnrichMessagePayload(dataMessage, sensorId); } } } } - if (!result.isEmpty()) { - return result; - } else { - throw new ParseException(String.format("No sensor of type %s found in message", givenSensorType)); - } + throw new ParseException(String.format("No sensor of type %s found in message", givenSensorType)); } private List findProcessDataInputMessage(NetworkMessage message) { diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/ros/RosBridgeAdapter.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/ros/RosBridgeAdapter.java new file mode 100644 index 0000000000..9ff1d9ecb2 --- /dev/null +++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/adapters/ros/RosBridgeAdapter.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.connect.iiot.adapters.ros; + + +import org.apache.streampipes.commons.exceptions.connect.AdapterException; +import org.apache.streampipes.extensions.api.connect.IAdapterConfiguration; +import org.apache.streampipes.extensions.api.connect.IEventCollector; +import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter; +import org.apache.streampipes.extensions.api.connect.context.IAdapterGuessSchemaContext; +import org.apache.streampipes.extensions.api.connect.context.IAdapterRuntimeContext; +import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor; +import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor; +import org.apache.streampipes.extensions.api.runtime.ResolvesContainerProvidedOptions; +import org.apache.streampipes.extensions.management.connect.adapter.parser.ParserUtils; +import org.apache.streampipes.model.AdapterType; +import org.apache.streampipes.model.connect.guess.GuessSchema; +import org.apache.streampipes.model.staticproperty.Option; +import org.apache.streampipes.sdk.builder.adapter.AdapterConfigurationBuilder; +import org.apache.streampipes.sdk.helpers.Labels; +import org.apache.streampipes.sdk.helpers.Locales; +import org.apache.streampipes.sdk.utils.Assets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import edu.wpi.rail.jrosbridge.Ros; +import edu.wpi.rail.jrosbridge.Service; +import edu.wpi.rail.jrosbridge.Topic; +import edu.wpi.rail.jrosbridge.services.ServiceRequest; +import edu.wpi.rail.jrosbridge.services.ServiceResponse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class RosBridgeAdapter implements StreamPipesAdapter, ResolvesContainerProvidedOptions { + + public static final String ID = "org.apache.streampipes.connect.iiot.adapters.ros"; + + public static final String ROS_HOST_KEY = "ROS_HOST_KEY"; + public static final String ROS_PORT_KEY = "ROS_PORT_KEY"; + public static final String TOPIC_KEY = "TOPIC_KEY"; + + private String topic; + private String host; + private int port; + + + private final ObjectMapper mapper = new ObjectMapper(); + + private Ros ros; + + public RosBridgeAdapter() { + } + + @Override + public List @@ -126,7 +126,7 @@ org.apache.streampipes streampipes-service-extensions - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java index 1fa3c01a42..b36abb767b 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java @@ -23,7 +23,7 @@ import org.apache.streampipes.extensions.api.pe.config.IDataSinkConfiguration; import org.apache.streampipes.extensions.api.pe.config.IPipelineElementConfiguration; import org.apache.streampipes.extensions.management.model.SpServiceDefinition; -import org.apache.streampipes.service.extensions.StreamPipesExtensionsServiceBase; +import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter; import org.apache.streampipes.smp.constants.PeType; import org.apache.streampipes.smp.model.AssetModel; @@ -39,10 +39,8 @@ public class ExtensionsFinder { private final ClassLoader loader; private final String initClass; - public ExtensionsFinder( - ClassLoader loader, - String initClass - ) { + public ExtensionsFinder(ClassLoader loader, + String initClass) { this.loader = loader; this.initClass = initClass; } @@ -51,11 +49,7 @@ public List findExtensions() throws MalformedURLException, DependencyResolutionRequiredException, ClassNotFoundException, InstantiationException, IllegalAccessException { var extensions = new ArrayList(); - var serviceDef = ( - (StreamPipesExtensionsServiceBase) loader - .loadClass(initClass) - .newInstance() - ).provideServiceDefinition(); + var serviceDef = ((ExtensionsModelSubmitter) loader.loadClass(initClass).newInstance()).provideServiceDefinition(); extensions.addAll(findAdapters(serviceDef)); extensions.addAll(findPipelineElements(serviceDef, IDataProcessorConfiguration.class, PeType.PROCESSOR)); @@ -64,32 +58,20 @@ public List findExtensions() return extensions; } - private List findPipelineElements( - SpServiceDefinition serviceDef, - Class> configType, - PeType peType - ) { + private List findPipelineElements(SpServiceDefinition serviceDef, + Class> configType, + PeType peType) { return serviceDef.getDeclarers() .stream() .map(IStreamPipesPipelineElement::declareConfig) .filter(configType::isInstance) - .map(config -> new AssetModel(config.getDescription() - .getAppId(), peType)) - .toList(); + .map(config -> new AssetModel(config.getDescription().getAppId(), peType)).toList(); } private Collection findAdapters(SpServiceDefinition serviceDef) { - return serviceDef.getAdapters() - .stream() - .map(adapter -> { - var config = adapter.declareConfig(); - var resolver = config.getAssetResolver(); - return new AssetModel( - config.getAdapterDescription().getAppId(), - PeType.ADAPTER, - resolver - ); - }) - .toList(); + return serviceDef.getAdapters().stream().map(adapter -> { + var config = adapter.declareConfig(); + return new AssetModel(config.getAdapterDescription().getAppId(), PeType.ADAPTER); + }).toList(); } } diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java index 49e58707ce..7688c2e64f 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java @@ -18,7 +18,6 @@ package org.apache.streampipes.smp.extractor; -import org.apache.streampipes.extensions.api.assets.DefaultAssetResolver; import org.apache.streampipes.smp.model.AssetModel; import org.apache.maven.plugin.logging.Log; @@ -46,14 +45,7 @@ public LocalesExtractor(Log log, public void applyLocales(AssetModel assetModel) { var localeFile = loader.getResourceAsStream(assetModel.getAppId() + "/" + LOCALES_FILE_EN); try { - Properties props; - if (assetModel.getAssetResolver() != null - && !(assetModel.getAssetResolver().getClass() == DefaultAssetResolver.class)) { - props = assetModel.getAssetResolver().getLocale(loader, LOCALES_FILE_EN); - } else { - props = loadProperties(localeFile); - } - + var props = loadProperties(localeFile); assetModel.setPipelineElementName(extractKey(props, assetModel, TITLE)); assetModel.setPipelineElementDescription(extractKey(props, assetModel, DESCRIPTION)); } catch (IOException e) { diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java index d21984773e..9816ed785d 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java @@ -18,10 +18,8 @@ package org.apache.streampipes.smp.generator; -import org.apache.streampipes.extensions.api.assets.DefaultAssetResolver; import org.apache.streampipes.smp.model.AssetModel; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; @@ -40,13 +38,8 @@ public ResourceGenerator(ClassLoader loader, this.targetPath = targetPath; } - protected InputStream getResourceInputStream(String resourceName) throws IOException { - if (extensionsElement.getAssetResolver() != null - && !(extensionsElement.getAssetResolver().getClass() == DefaultAssetResolver.class)) { - return new ByteArrayInputStream(extensionsElement.getAssetResolver().getAsset(loader, resourceName)); - } else { - return loader.getResourceAsStream(extensionsElement.getAppId() + "/" + resourceName); - } + protected InputStream getResourceInputStream(String resourceName) { + return loader.getResourceAsStream(extensionsElement.getAppId() + "/" + resourceName); } public abstract void generate() throws IOException; diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java index 96dfef8282..ecb77de77b 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java @@ -18,7 +18,6 @@ package org.apache.streampipes.smp.model; -import org.apache.streampipes.extensions.api.assets.AssetResolver; import org.apache.streampipes.smp.constants.PeType; public class AssetModel implements Comparable { @@ -27,18 +26,11 @@ public class AssetModel implements Comparable { private String pipelineElementName; private String pipelineElementDescription; private PeType peType = PeType.PROCESSOR; - private AssetResolver assetResolver; public AssetModel() { } - public AssetModel(String appId, PeType peType, AssetResolver resolver) { - this.appId = appId; - this.peType = peType; - this.assetResolver = resolver; - } - public AssetModel(String appId, PeType peType) { this.appId = appId; this.peType = peType; @@ -71,10 +63,6 @@ public void setPipelineElementDescription(String pipelineElementDescription) { this.pipelineElementDescription = pipelineElementDescription; } - public AssetResolver getAssetResolver() { - return assetResolver; - } - public PeType getPeType() { return peType; } diff --git a/streampipes-measurement-units/pom.xml b/streampipes-measurement-units/pom.xml index 477129a0dc..84e761d626 100644 --- a/streampipes-measurement-units/pom.xml +++ b/streampipes-measurement-units/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-measurement-units @@ -31,7 +31,7 @@ org.apache.streampipes streampipes-commons - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-measurement-units/src/main/java/com/github/jqudt/Multiplier.java b/streampipes-measurement-units/src/main/java/com/github/jqudt/Multiplier.java index f815e9c6a0..46654c479c 100644 --- a/streampipes-measurement-units/src/main/java/com/github/jqudt/Multiplier.java +++ b/streampipes-measurement-units/src/main/java/com/github/jqudt/Multiplier.java @@ -11,6 +11,11 @@ public class Multiplier implements Serializable{ private double offset; private double multiplier; + public Multiplier(double offset, double multiplier) { + this.offset = offset; + this.multiplier = multiplier; + } + public Multiplier() {} public double getOffset() { diff --git a/streampipes-measurement-units/src/main/java/com/github/jqudt/Quantity.java b/streampipes-measurement-units/src/main/java/com/github/jqudt/Quantity.java index 1a77989188..e84d66dc5c 100644 --- a/streampipes-measurement-units/src/main/java/com/github/jqudt/Quantity.java +++ b/streampipes-measurement-units/src/main/java/com/github/jqudt/Quantity.java @@ -52,16 +52,11 @@ public Quantity convertTo(Unit newUnit) throws IllegalArgumentException, Illegal if (unit.getResource().equals(newUnit.getResource())) return this; // nothing to be done if (!unit.getType().equals(newUnit.getType())) { - LOG.error( - "The new unit does not have the same parent type (source: {}; target: {})", - unit.getType(), - newUnit.getType() - ); + LOG.error("The new unit does not have the same parent type " + + "(source: " + unit.getType() + "; target: " + newUnit.getType() + ")"); throw new IllegalAccessException( - "The new unit does not have the same parent type (source: %s; target: %s)".formatted( - unit.getType(), - newUnit.getType() - ) + "The new unit does not have the same parent type " + + "(source: " + unit.getType() + "; target: " + newUnit.getType() + ")" ); } @@ -78,11 +73,8 @@ public Quantity convertTo(Unit newUnit) throws IllegalArgumentException, Illegal return newMeasurement; } - @Override public String toString() { - return "Quantity{" + - "value=" + value + - ", unit=" + unit + - '}'; + return "" + getValue() + " " + getUnit().toString(); } + } diff --git a/streampipes-measurement-units/src/main/java/com/github/jqudt/uo/UnitOntologyFactory.java b/streampipes-measurement-units/src/main/java/com/github/jqudt/uo/UnitOntologyFactory.java new file mode 100644 index 0000000000..76ac520524 --- /dev/null +++ b/streampipes-measurement-units/src/main/java/com/github/jqudt/uo/UnitOntologyFactory.java @@ -0,0 +1,156 @@ +/* Copyright (C) 2012 Egon Willighagen + * + * License: new BSD + */ +package com.github.jqudt.uo; + +import com.github.jqudt.Unit; +import com.github.jqudt.onto.UnitFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class UnitOntologyFactory { + + private static UnitOntologyFactory factory = null; + + private static final Logger LOG = LoggerFactory.getLogger(UnitOntologyFactory.class); + + @SuppressWarnings("serial") + private Map uo2qudt = new HashMap() { + // a helper method + String longURI(String shortened) { + if (shortened.startsWith("uo:")) { + return "http://purl.obolibrary.org/obo/" + shortened.substring(3); + } + if (shortened.startsWith("ops:")) { + return "http://www.openphacts.org/units/" + shortened.substring(4); + } + return null; + } + + // the next defines all mappings from the Unit Ontology to the QUDT + { + put(longURI("uo:EFO_0004374"), longURI("ops:MilligramPerDeciliter")); + put(longURI("uo:EFO_0004385"), longURI("ops:PicogramPerMilliliter")); + put(longURI("uo:UO_0000009"), longURI("qudt:Kilogram")); + put(longURI("uo:UO_0000010"), longURI("qudt:SecondTime")); + put(longURI("uo:UO_0000015"), longURI("qudt:Centimeter")); + put(longURI("uo:UO_0000016"), longURI("qudt:Millimeter")); + put(longURI("uo:UO_0000017"), longURI("qudt:Micrometer")); + put(longURI("uo:UO_0000018"), longURI("ops:Nanometer")); + put(longURI("uo:UO_0000021"), longURI("qudt:Gram")); + put(longURI("uo:UO_0000022"), longURI("ops:Milligram")); + put(longURI("uo:UO_0000023"), longURI("ops:Microgram")); + put(longURI("uo:UO_0000024"), longURI("ops:Nanogram")); + put(longURI("uo:UO_0000025"), longURI("ops:Picogram")); + put(longURI("uo:UO_0000026"), longURI("ops:Femtogram")); + put(longURI("uo:UO_0000027"), longURI("qudt:DegreeCelsius")); + put(longURI("uo:UO_0000028"), longURI("qudt:Millisecond")); + put(longURI("uo:UO_0000031"), longURI("qudt:MinuteTime")); + put(longURI("uo:UO_0000032"), longURI("qudt:Hour")); + put(longURI("uo:UO_0000033"), longURI("qudt:Day")); + put(longURI("uo:UO_0000039"), longURI("qudt:Micromole")); + put(longURI("uo:UO_0000040"), longURI("qudt:Millimole")); + put(longURI("uo:UO_0000041"), longURI("qudt:Nanomole")); + put(longURI("uo:UO_0000042"), longURI("qudt:Picomole")); + put(longURI("uo:UO_0000043"), longURI("qudt:Femtomole")); + put(longURI("uo:UO_0000062"), longURI("ops:Molar")); + put(longURI("uo:UO_0000063"), longURI("ops:Millimolar")); + put(longURI("uo:UO_0000064"), longURI("ops:Micromolar")); + put(longURI("uo:UO_0000065"), longURI("ops:Nanomolar")); + put(longURI("uo:UO_0000066"), longURI("ops:Picomolar")); + put(longURI("uo:UO_0000073"), longURI("ops:Femtomolar")); + put(longURI("uo:UO_0000098"), longURI("ops:Milliliter")); + put(longURI("uo:UO_0000099"), longURI("qudt:Liter")); + put(longURI("uo:UO_0000101"), longURI("ops:Microliter")); + put(longURI("uo:UO_0000169"), longURI("ops:PartsPerMillion")); + put(longURI("uo:UO_0000173"), longURI("ops:GramPerMilliliter")); + put(longURI("uo:UO_0000175"), longURI("ops:GramPerLiter")); + put(longURI("uo:UO_0000176"), longURI("ops:MilligramPerMilliliter")); + put(longURI("uo:UO_0000187"), longURI("qudt:Percent")); + put(longURI("uo:UO_0000197"), longURI("ops:LiterPerKilogram")); + put(longURI("uo:UO_0000198"), longURI("ops:MilliliterPerKilogram")); + put(longURI("uo:UO_0000271"), longURI("ops:MicroliterPerMinute")); + put(longURI("uo:UO_0000272"), longURI("qudt:MillimeterOfMercury")); + put(longURI("uo:UO_0000274"), longURI("ops:MicrogramPerMilliliter")); + put(longURI("uo:UO_0000275"), longURI("ops:NanogramPerMilliliter")); + put(longURI("uo:UO_0000308"), longURI("ops:MilligramPerKilogram")); +// put(longURI("uo:UO_0000311"), longURI("")); + } + }; + private Map qudt2uo = null; + + private UnitOntologyFactory() { + // not backed up by a formal ontology (at this moment; see UnitFactory's constructor) + // instead, it uses defined mappings in uo2qudt + + // also make the reverse mapping table + qudt2uo = new HashMap<>(); + for (String keyURI : uo2qudt.keySet()) { + qudt2uo.put(uo2qudt.get(keyURI), keyURI); + } + } + + public static UnitOntologyFactory getInstance() { + if (factory == null) { + factory = new UnitOntologyFactory(); + } + return factory; + } + + private static URI asURI(String resource) { + try { + return new URI(resource); + } catch (URISyntaxException exception) { + return null; + } + } + + public Unit getUnit(String resource) { + LOG.info("resource:" + resource); + return getUnit(asURI(resource)); + } + + public Unit getUnit(URI resource) { + if (resource == null) { + throw new IllegalArgumentException("The URI cannot be null"); + } + + URI mappedURI = asURI(uo2qudt.get(resource.toString())); + if (mappedURI != null) { + return UnitFactory.getInstance().getUnit(mappedURI); + } else { + return null; + } + } + + public List getURIs(String type) { + URI uri; + try { + uri = new URI(type); + } catch (URISyntaxException exception) { + throw new IllegalStateException("Invalid URI: " + type, exception); + } + return getURIs(uri); + } + + public List getURIs(URI type) { + List uris = new ArrayList<>(); + + List qudtURIs = UnitFactory.getInstance().getURIs(type); + for (String qudtString : qudtURIs) { + String uoURI = qudt2uo.get(qudtString); + if (uoURI != null) { + uris.add(uoURI); + } + } + return uris; + } +} diff --git a/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitCollector.java b/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitCollector.java index f0ae3f4dd1..fd542e68c1 100644 --- a/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitCollector.java +++ b/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitCollector.java @@ -98,8 +98,8 @@ public class UnitCollector { "AngularVelocityUnit" }; - private final Set availableUnits = new HashSet<>(); - private final Set availableUnitTypes = new HashSet<>(); + private Set availableUnits = new HashSet<>(); + private Set availableUnitTypes = new HashSet<>(); public UnitCollector() { collect(); diff --git a/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitProvider.java b/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitProvider.java index b2feca9330..33473b4636 100644 --- a/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitProvider.java +++ b/streampipes-measurement-units/src/main/java/org/apache/streampipes/units/UnitProvider.java @@ -20,20 +20,25 @@ import com.github.jqudt.Unit; import com.github.jqudt.onto.UnitFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.NoSuchElementException; import java.util.stream.Collectors; public enum UnitProvider { INSTANCE; - private final List availableUnitTypes = new ArrayList<>(); - private final List availableUnits = new ArrayList<>(); + private static final Logger LOG = LoggerFactory.getLogger(UnitCollector.class); - private final UnitFactory factory; + private List availableUnitTypes = new ArrayList<>(); + private List availableUnits = new ArrayList<>(); + + private UnitFactory factory; UnitProvider() { factory = UnitFactory.getInstance(); @@ -54,6 +59,18 @@ public Unit getUnit(String resourceUri) { return factory.getUnit(resourceUri); } + public Unit getUnitByLabel(String label) { + try { + return availableUnits.stream() + .filter((Unit unit) -> unit.getLabel().equals(label)) + .findFirst() + .get(); + } catch (NoSuchElementException e) { + LOG.error("No unit with label \"" + label + "\" found"); + return null; + } + } + public List getUnitsByType(URI type) { return availableUnits .stream() diff --git a/streampipes-measurement-units/src/test/java/org/apache/streampipes/units/test/TestUnitProvider.java b/streampipes-measurement-units/src/test/java/org/apache/streampipes/units/test/TestUnitProvider.java index a21ba5c3cd..2c2c1172aa 100644 --- a/streampipes-measurement-units/src/test/java/org/apache/streampipes/units/test/TestUnitProvider.java +++ b/streampipes-measurement-units/src/test/java/org/apache/streampipes/units/test/TestUnitProvider.java @@ -20,16 +20,16 @@ import org.apache.streampipes.units.UnitProvider; +import com.github.jqudt.Unit; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class TestUnitProvider { @Test public void testUnitProvider() { - var unit = UnitProvider.INSTANCE.getUnit("http://qudt.org/vocab/unit#KilometerPerHour"); - assertEquals("Kilometer per Hour", unit.getLabel()); - assertEquals("km/hr", unit.getAbbreviation()); + Unit unit = UnitProvider.INSTANCE.getUnit("http://qudt.org/vocab/unit#KilometerPerHour"); + Assertions.assertEquals("Kilometer per Hour", unit.getLabel()); + Assertions.assertEquals("km/hr", unit.getAbbreviation()); } } diff --git a/streampipes-messaging-jms/pom.xml b/streampipes-messaging-jms/pom.xml index 550d903f3a..65b4171797 100644 --- a/streampipes-messaging-jms/pom.xml +++ b/streampipes-messaging-jms/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,7 +32,7 @@ org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging-kafka/pom.xml b/streampipes-messaging-kafka/pom.xml index 2964ca7697..c54b5ca00e 100644 --- a/streampipes-messaging-kafka/pom.xml +++ b/streampipes-messaging-kafka/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,12 +32,12 @@ org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-commons - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaConsumer.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaConsumer.java index fcbd38224f..e999cfbc4a 100644 --- a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaConsumer.java +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaConsumer.java @@ -104,6 +104,7 @@ public void onPartitionsAssigned(Collection partitions) { ConsumerRecords records = consumer.poll(duration); records.forEach(record -> eventProcessor.onEvent(record.value())); } + LOG.info("Closing Kafka Consumer."); consumer.close(); } diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaProducer.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaProducer.java index eb07b66c6b..7938ecf657 100644 --- a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaProducer.java +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/SpKafkaProducer.java @@ -96,11 +96,12 @@ public void connect() { LOG.info("Kafka producer: Connecting to " + protocol.getTopicDefinition().getActualTopicName()); this.brokerUrl = protocol.getBrokerHostname() + ":" + protocol.getKafkaPort(); this.topic = protocol.getTopicDefinition().getActualTopicName(); + String zookeeperHost = protocol.getZookeeperHost() + ":" + protocol.getZookeeperPort(); try { createKafkaTopic(protocol); } catch (ExecutionException | InterruptedException e) { - LOG.error("Could not create topic: " + topic + " on broker " + brokerUrl); + LOG.error("Could not create topic: " + topic + " on broker " + zookeeperHost); } this.producer = new KafkaProducer<>(makeProperties(protocol, Collections.emptyList())); diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/KafkaConfigAppender.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/KafkaConfigAppender.java index 77c5cb5afa..1e53ebd225 100644 --- a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/KafkaConfigAppender.java +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/config/KafkaConfigAppender.java @@ -18,11 +18,9 @@ package org.apache.streampipes.messaging.kafka.config; -import org.apache.streampipes.commons.exceptions.SpRuntimeException; - import java.util.Properties; public interface KafkaConfigAppender { - void appendConfig(Properties props) throws SpRuntimeException; + void appendConfig(Properties props); } diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityConfig.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityConfig.java new file mode 100644 index 0000000000..646033f437 --- /dev/null +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityConfig.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.messaging.kafka.security; + +import org.apache.streampipes.messaging.kafka.config.KafkaConfigAppender; + +public abstract class KafkaSecurityConfig implements KafkaConfigAppender { + +} diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslPlainConfig.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslPlainConfig.java new file mode 100644 index 0000000000..a7e5fa94fc --- /dev/null +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslPlainConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.messaging.kafka.security; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.security.auth.SecurityProtocol; + +import java.util.Properties; + +public class KafkaSecuritySaslPlainConfig extends KafkaSecurityConfig { + + private final String username; + private final String password; + + public KafkaSecuritySaslPlainConfig(String username, String password) { + this.username = username; + this.password = password; + } + + @Override + public void appendConfig(Properties props) { + + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.toString()); + + String saslJaasConfig = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + + username + + "\" password=\"" + + password + + "\";"; + + props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); + } +} diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslSSLConfig.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslSSLConfig.java new file mode 100644 index 0000000000..2c5ef691e3 --- /dev/null +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecuritySaslSSLConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.messaging.kafka.security; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.security.auth.SecurityProtocol; + +import java.util.Properties; + +public class KafkaSecuritySaslSSLConfig extends KafkaSecurityConfig { + + private final String username; + private final String password; + + public KafkaSecuritySaslSSLConfig(String username, String password) { + this.username = username; + this.password = password; + } + + @Override + public void appendConfig(Properties props) { + + props.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString()); + + String saslJaasConfig = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + + username + + "\" password=\"" + + password + + "\";"; + + props.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); + } +} diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedPlainConfig.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedPlainConfig.java new file mode 100644 index 0000000000..235a6eda39 --- /dev/null +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedPlainConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.messaging.kafka.security; + +import java.util.Properties; + +public class KafkaSecurityUnauthenticatedPlainConfig extends KafkaSecurityConfig { + + @Override + public void appendConfig(Properties props) { + + } +} diff --git a/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedSSLConfig.java b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedSSLConfig.java new file mode 100644 index 0000000000..8fdd02fd12 --- /dev/null +++ b/streampipes-messaging-kafka/src/main/java/org/apache/streampipes/messaging/kafka/security/KafkaSecurityUnauthenticatedSSLConfig.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.messaging.kafka.security; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.common.security.auth.SecurityProtocol; + +import java.util.Properties; + +public class KafkaSecurityUnauthenticatedSSLConfig extends KafkaSecurityConfig { + + @Override + public void appendConfig(Properties props) { + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SSL.toString()); + } +} diff --git a/streampipes-messaging-mqtt/pom.xml b/streampipes-messaging-mqtt/pom.xml index 17385dc9cc..5696f2436a 100644 --- a/streampipes-messaging-mqtt/pom.xml +++ b/streampipes-messaging-mqtt/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -31,7 +31,7 @@ org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging-nats/pom.xml b/streampipes-messaging-nats/pom.xml index fb2db9c41c..93e7ce2eb9 100644 --- a/streampipes-messaging-nats/pom.xml +++ b/streampipes-messaging-nats/pom.xml @@ -19,7 +19,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -35,7 +35,7 @@ org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging-pulsar/pom.xml b/streampipes-messaging-pulsar/pom.xml index 37ad9bd58a..75dcd3c445 100644 --- a/streampipes-messaging-pulsar/pom.xml +++ b/streampipes-messaging-pulsar/pom.xml @@ -19,7 +19,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -35,7 +35,7 @@ org.apache.streampipes streampipes-messaging - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging/pom.xml b/streampipes-messaging/pom.xml index 40747b9aa1..d6552e383e 100644 --- a/streampipes-messaging/pom.xml +++ b/streampipes-messaging/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,7 +32,7 @@ org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-messaging/src/main/java/org/apache/streampipes/messaging/SpProtocolManager.java b/streampipes-messaging/src/main/java/org/apache/streampipes/messaging/SpProtocolManager.java index 279c522eb9..35e2d9b6e6 100644 --- a/streampipes-messaging/src/main/java/org/apache/streampipes/messaging/SpProtocolManager.java +++ b/streampipes-messaging/src/main/java/org/apache/streampipes/messaging/SpProtocolManager.java @@ -43,6 +43,7 @@ public List> getAvailab } public Optional> findDefinition(T transportProtocol) { + // TODO add RDF URI for protocol in model return this.availableProtocols .stream() .filter diff --git a/streampipes-model-client/pom.xml b/streampipes-model-client/pom.xml index 7674d6dc42..30f1afff4a 100644 --- a/streampipes-model-client/pom.xml +++ b/streampipes-model-client/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT streampipes-model-client @@ -32,12 +32,12 @@ org.apache.streampipes streampipes-commons - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model-shared - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT @@ -88,9 +88,6 @@ asClasses true true - - import { Storable } from '@streampipes/platform-services' - cz.habarta.typescript.generator.ext.JsonDeserializationExtension diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/AssetDashboardConfig.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/AssetDashboardConfig.java new file mode 100644 index 0000000000..d8956e577c --- /dev/null +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/AssetDashboardConfig.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.client.assetdashboard; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; +import java.util.Map; + +public class AssetDashboardConfig { + + private @SerializedName("_id") String dashboardId; + private @SerializedName("_rev") String rev; + + private String dashboardName; + private String dashboardDescription; + private ImageInfo imageInfo; + private Map attrs; + private String className; + private List children; + + public AssetDashboardConfig() { + } + + public String getDashboardName() { + return dashboardName; + } + + public void setDashboardName(String dashboardName) { + this.dashboardName = dashboardName; + } + + public String getDashboardDescription() { + return dashboardDescription; + } + + public void setDashboardDescription(String dashboardDescription) { + this.dashboardDescription = dashboardDescription; + } + + public Map getAttrs() { + return attrs; + } + + public void setAttrs(Map attrs) { + this.attrs = attrs; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + public ImageInfo getImageInfo() { + return imageInfo; + } + + public void setImageInfo(ImageInfo imageInfo) { + this.imageInfo = imageInfo; + } + + public String getDashboardId() { + return dashboardId; + } + + public void setDashboardId(String dashboardId) { + this.dashboardId = dashboardId; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } +} diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/CanvasElement.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/CanvasElement.java new file mode 100644 index 0000000000..a0cefd20ce --- /dev/null +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/CanvasElement.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.client.assetdashboard; + +import java.util.List; +import java.util.Map; + +public class CanvasElement { + + private Map attrs; + private String className; + private List children; + + public CanvasElement() { + } + + public Map getAttrs() { + return attrs; + } + + public void setAttrs(Map attrs) { + this.attrs = attrs; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } +} diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/ImageInfo.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/ImageInfo.java new file mode 100644 index 0000000000..b80aeaf58b --- /dev/null +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/assetdashboard/ImageInfo.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.client.assetdashboard; + +public class ImageInfo { + + private String imageName; + private float x; + private float y; + private float width; + private float height; + private float scaleX = 1; + private float scaleY = 1; + private float rotation = 0; + + public ImageInfo() { + } + + public float getX() { + return x; + } + + public void setX(float x) { + this.x = x; + } + + public float getY() { + return y; + } + + public void setY(float y) { + this.y = y; + } + + public float getWidth() { + return width; + } + + public void setWidth(float width) { + this.width = width; + } + + public float getHeight() { + return height; + } + + public void setHeight(float height) { + this.height = height; + } + + public float getScaleX() { + return scaleX; + } + + public void setScaleX(float scaleX) { + this.scaleX = scaleX; + } + + public float getScaleY() { + return scaleY; + } + + public void setScaleY(float scaleY) { + this.scaleY = scaleY; + } + + public String getImageName() { + return imageName; + } + + public void setImageName(String imageName) { + this.imageName = imageName; + } + + public float getRotation() { + return rotation; + } + + public void setRotation(float rotation) { + this.rotation = rotation; + } +} diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/matching/MatchingResultType.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/matching/MatchingResultType.java index 225fe1180d..225a899533 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/matching/MatchingResultType.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/matching/MatchingResultType.java @@ -22,8 +22,8 @@ public enum MatchingResultType { DATATYPE_MATCH("Datatype Match", "A required datatype is not present in the input event property."), STREAM_MATCH("Stream Match", ""), - SEMANTIC_TYPE_MATCH("Semantic Type Match", - "A required semantic type is not present in the input event property."), + DOMAIN_PROPERTY_MATCH("Domain Property Match", + "A required domain property is not present in the input event property."), FORMAT_MATCH("Format Match", "No supported transport format found."), GROUNDING_MATCH("Grounding Match", ""), PROTOCOL_MATCH("Protocol Match", "No supported communication protocol found."), diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/AbstractMailToken.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/AbstractMailToken.java index 0434031dc0..372ad5de98 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/AbstractMailToken.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/AbstractMailToken.java @@ -17,11 +17,9 @@ */ package org.apache.streampipes.model.client.user; -import org.apache.streampipes.model.shared.api.Storable; - import com.google.gson.annotations.SerializedName; -public abstract class AbstractMailToken implements Storable { +public abstract class AbstractMailToken { protected @SerializedName("_id") String token; protected @SerializedName("_rev") String rev; @@ -51,14 +49,4 @@ public String getUsername() { public void setUsername(String username) { this.username = username; } - - @Override - public String getElementId() { - return this.token; - } - - @Override - public void setElementId(String elementId) { - this.token = elementId; - } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ChangePasswordRequest.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ChangePasswordRequest.java index ba32b0f2c8..36dbb1ccae 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ChangePasswordRequest.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ChangePasswordRequest.java @@ -18,5 +18,28 @@ package org.apache.streampipes.model.client.user; -public record ChangePasswordRequest(String existingPassword, String newPassword) { +public class ChangePasswordRequest { + + private String existingPassword; + private String newPassword; + + public ChangePasswordRequest() { + + } + + public String getNewPassword() { + return newPassword; + } + + public void setNewPassword(String newPassword) { + this.newPassword = newPassword; + } + + public String getExistingPassword() { + return existingPassword; + } + + public void setExistingPassword(String existingPassword) { + this.existingPassword = existingPassword; + } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Group.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Group.java index 4ce6d272c9..8829add8a3 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Group.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Group.java @@ -18,7 +18,6 @@ package org.apache.streampipes.model.client.user; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.gson.annotations.SerializedName; @@ -27,19 +26,19 @@ import java.util.Set; @TsModel -public class Group implements Storable { +public class Group { protected @SerializedName("_id") String groupId; protected @SerializedName("_rev") String rev; // This field should be called $type since this is the identifier used in the CouchDB view - @SerializedName("$type") + @SuppressWarnings("checkstyle:MemberName") @JsonIgnore - private String type = "group"; + private String $type = "group"; private String groupName; - private Set roles; + private Set roles; public Group() { this.roles = new HashSet<>(); @@ -61,16 +60,6 @@ public void setRev(String rev) { this.rev = rev; } - @Override - public String getElementId() { - return this.groupId; - } - - @Override - public void setElementId(String elementId) { - this.groupId = elementId; - } - public String getGroupName() { return groupName; } @@ -79,19 +68,21 @@ public void setGroupName(String groupName) { this.groupName = groupName; } - public Set getRoles() { + public Set getRoles() { return roles; } - public void setRoles(Set roles) { + public void setRoles(Set roles) { this.roles = roles; } - public String getType() { - return type; + @SuppressWarnings("checkstyle:MethodName") + public String get$type() { + return $type; } - public void setType(String type) { - this.type = type; + @SuppressWarnings({"checkstyle:MethodName", "checkstyle:ParameterName"}) + public void set$type(String $type) { + this.$type = $type; } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/JwtAuthenticationResponse.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/JwtAuthenticationResponse.java index cb77abff92..a86f68039f 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/JwtAuthenticationResponse.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/JwtAuthenticationResponse.java @@ -18,5 +18,24 @@ package org.apache.streampipes.model.client.user; -public record JwtAuthenticationResponse(String accessToken) { +public class JwtAuthenticationResponse { + + private String accessToken; + + private JwtAuthenticationResponse(String accessToken) { + this.accessToken = accessToken; + } + + public static JwtAuthenticationResponse from(String accessToken) { + return new JwtAuthenticationResponse(accessToken); + } + + public String getAccessToken() { + return accessToken; + } + + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } + } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/LoginRequest.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/LoginRequest.java index 878d4d0ce9..6615f6e7b8 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/LoginRequest.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/LoginRequest.java @@ -18,5 +18,36 @@ package org.apache.streampipes.model.client.user; -public record LoginRequest(String username, String password) { +public class LoginRequest { + + private String username; + private String password; + + public LoginRequest() { + + } + + public LoginRequest(String username, String password) { + super(); + this.username = username; + this.password = password; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PasswordRecoveryToken.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PasswordRecoveryToken.java index 8c0ea623e7..cb8ae692ea 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PasswordRecoveryToken.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PasswordRecoveryToken.java @@ -18,12 +18,9 @@ package org.apache.streampipes.model.client.user; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gson.annotations.SerializedName; public class PasswordRecoveryToken extends AbstractMailToken { - // This field should be called $type since this is the identifier used in the CouchDB view - @SerializedName("$type") @JsonIgnore private String type = "password-recovery"; diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Permission.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Permission.java index 4e20ca0daa..72b9a1b31e 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Permission.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Permission.java @@ -18,7 +18,6 @@ package org.apache.streampipes.model.client.user; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.gson.annotations.SerializedName; @@ -27,7 +26,7 @@ import java.util.List; @TsModel -public class Permission implements Storable { +public class Permission { protected @SerializedName("_id") String permissionId; protected @SerializedName("_rev") String rev; @@ -65,16 +64,6 @@ public void setRev(String rev) { this.rev = rev; } - @Override - public String getElementId() { - return this.permissionId; - } - - @Override - public void setElementId(String elementId) { - this.permissionId = elementId; - } - public String getObjectInstanceId() { return objectInstanceId; } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionBuilder.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionBuilder.java index e032f1e984..b7857caf7d 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionBuilder.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionBuilder.java @@ -19,7 +19,7 @@ public class PermissionBuilder { - private final Permission permission; + private Permission permission; private PermissionBuilder(String objectInstanceId, Class objectInstanceClass, diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionEntry.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionEntry.java index 561a1713ca..2dcc436bcc 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionEntry.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/PermissionEntry.java @@ -19,7 +19,34 @@ import java.util.Objects; -public record PermissionEntry(String sid, PrincipalType principalType) { +public class PermissionEntry { + + private String sid; + private PrincipalType principalType; + + public PermissionEntry() { + } + + public PermissionEntry(String sid, PrincipalType principalType) { + this.sid = sid; + this.principalType = principalType; + } + + public String getSid() { + return sid; + } + + public void setSid(String sid) { + this.sid = sid; + } + + public PrincipalType getPrincipalType() { + return principalType; + } + + public void setPrincipalType(PrincipalType principalType) { + this.principalType = principalType; + } @Override public boolean equals(Object o) { diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Principal.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Principal.java index 435451b787..384267f75e 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Principal.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Principal.java @@ -30,7 +30,7 @@ public abstract class Principal { protected @SerializedName("_rev") String rev; protected String username; protected Set objectPermissions; - protected Set roles; + protected Set roles; protected Set groups; @JsonIgnore private String type = "principal"; @@ -87,11 +87,11 @@ public void setAccountExpired(boolean accountExpired) { this.accountExpired = accountExpired; } - public Set getRoles() { + public Set getRoles() { return roles; } - public void setRoles(Set roles) { + public void setRoles(Set roles) { this.roles = roles; } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Privilege.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Privilege.java index 7786484d66..9b139f86b0 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Privilege.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Privilege.java @@ -15,57 +15,115 @@ * limitations under the License. * */ - package org.apache.streampipes.model.client.user; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gson.annotations.SerializedName; @TsModel -public class Privilege implements Storable { +public enum Privilege { + // Pipelines + PRIVILEGE_READ_PIPELINE(Constants.PRIVILEGE_READ_PIPELINE_VALUE), + PRIVILEGE_WRITE_PIPELINE(Constants.PRIVILEGE_WRITE_PIPELINE_VALUE), + PRIVILEGE_DELETE_PIPELINE(Constants.PRIVILEGE_DELETE_PIPELINE_VALUE), - protected @SerializedName("_id") String elementId; - protected @SerializedName("_rev") String rev; + // Adapters + PRIVILEGE_READ_ADAPTER(Constants.PRIVILEGE_READ_ADAPTER_VALUE), + PRIVILEGE_WRITE_ADAPTER(Constants.PRIVILEGE_WRITE_ADAPTER_VALUE), + PRIVILEGE_DELETE_ADAPTER(Constants.PRIVILEGE_DELETE_ADAPTER_VALUE), - // document type should be persisted to CouchDB with Gson serialization, but not via Jackson to the UI - @JsonIgnore - @SerializedName("$type") - private String type = "privilege"; + // Pipeline Elements + PRIVILEGE_READ_PIPELINE_ELEMENT(Constants.PRIVILEGE_READ_PIPELINE_ELEMENT_VALUE), + PRIVILEGE_WRITE_PIPELINE_ELEMENT(Constants.PRIVILEGE_WRITE_PIPELINE_ELEMENT_VALUE), + PRIVILEGE_DELETE_PIPELINE_ELEMENT(Constants.PRIVILEGE_DELETE_PIPELINE_ELEMENT_VALUE), - public static Privilege create(String id) { - Privilege privilege = new Privilege(); - privilege.setElementId(id); - return privilege; - } + // Dashboard + PRIVILEGE_READ_DASHBOARD(Constants.PRIVILEGE_READ_DASHBOARD_VALUE), + PRIVILEGE_WRITE_DASHBOARD(Constants.PRIVILEGE_WRITE_DASHBOARD_VALUE), + PRIVILEGE_DELETE_DASHBOARD(Constants.PRIVILEGE_DELETE_DASHBOARD_VALUE), - @Override - public String getRev() { - return rev; - } + // Dashboard widget + PRIVILEGE_READ_DASHBOARD_WIDGET(Constants.PRIVILEGE_READ_DASHBOARD_WIDGET_VALUE), + PRIVILEGE_WRITE_DASHBOARD_WIDGET(Constants.PRIVILEGE_WRITE_DASHBOARD_WIDGET_VALUE), + PRIVILEGE_DELETE_DASHBOARD_WIDGET(Constants.PRIVILEGE_DELETE_DASHBOARD_WIDGET_VALUE), - @Override - public void setRev(String rev) { - this.rev = rev; - } + // Data Explorer view + PRIVILEGE_READ_DATA_EXPLORER_VIEW(Constants.PRIVILEGE_READ_DATA_EXPLORER_VIEW_VALUE), + PRIVILEGE_WRITE_DATA_EXPLORER_VIEW(Constants.PRIVILEGE_WRITE_DATA_EXPLORER_VIEW_VALUE), + PRIVILEGE_DELETE_DATA_EXPLORER_VIEW(Constants.PRIVILEGE_DELETE_DATA_EXPLORER_VIEW_VALUE), - @Override - public String getElementId() { - return elementId; - } + // Data Explorer widget + PRIVILEGE_READ_DATA_EXPLORER_WIDGET(Constants.PRIVILEGE_READ_DATA_EXPLORER_WIDGET_VALUE), + PRIVILEGE_WRITE_DATA_EXPLORER_WIDGET(Constants.PRIVILEGE_WRITE_DATA_EXPLORER_WIDGET_VALUE), + PRIVILEGE_DELETE_DATA_EXPLORER_WIDGET(Constants.PRIVILEGE_DELETE_DATA_EXPLORER_WIDGET_VALUE), - @Override - public void setElementId(String elementId) { - this.elementId = elementId; - } + // Apps + PRIVILEGE_READ_APPS(Constants.PRIVILEGE_READ_APPS_VALUE), + PRIVILEGE_WRITE_APPS(Constants.PRIVILEGE_WRITE_APPS_VALUE), + + // NOTIFICATIONS + PRIVILEGE_READ_NOTIFICATIONS(Constants.PRIVILEGE_READ_NOTIFICATIONS_VALUE), + + // FILES + PRIVILEGE_READ_FILES(Constants.PRIVILEGE_READ_FILES_VALUE), + PRIVILEGE_WRITE_FILES(Constants.PRIVILEGE_WRITE_FILES_VALUE), + PRIVILEGE_DELETE_FILES(Constants.PRIVILEGE_DELETE_FILES_VALUE), + + // ASSETS + PRIVILEGE_READ_ASSETS(Constants.PRIVILEGE_READ_ASSETS_VALUE), + PRIVILEGE_WRITE_ASSETS(Constants.PRIVILEGE_WRITE_ASSETS_VALUE), + + // GENERIC STORAGE + PRIVILEGE_READ_GENERIC_STORAGE(Constants.PRIVILEGE_READ_GENERIC_STORAGE_VALUE), + PRIVILEGE_WRITE_GENERIC_STORAGE(Constants.PRIVILEGE_WRITE_GENERIC_STORAGE_VALUE); + + private String privilegeString; - public String getType() { - return type; + Privilege(String privilegeString) { + this.privilegeString = privilegeString; } - public void setType(String type) { - this.type = type; + public static final class Constants { + public static final String PRIVILEGE_READ_PIPELINE_VALUE = "PRIVILEGE_READ_PIPELINE"; + public static final String PRIVILEGE_WRITE_PIPELINE_VALUE = "PRIVILEGE_WRITE_PIPELINE"; + public static final String PRIVILEGE_DELETE_PIPELINE_VALUE = "PRIVILEGE_DELETE_PIPELINE"; + + public static final String PRIVILEGE_READ_ADAPTER_VALUE = "PRIVILEGE_READ_ADAPTER"; + public static final String PRIVILEGE_WRITE_ADAPTER_VALUE = "PRIVILEGE_WRITE_ADAPTER"; + public static final String PRIVILEGE_DELETE_ADAPTER_VALUE = "PRIVILEGE_DELETE_ADAPTER"; + + public static final String PRIVILEGE_READ_PIPELINE_ELEMENT_VALUE = "PRIVILEGE_READ_PIPELINE_ELEMENT"; + public static final String PRIVILEGE_WRITE_PIPELINE_ELEMENT_VALUE = "PRIVILEGE_WRITE_PIPELINE_ELEMENT"; + public static final String PRIVILEGE_DELETE_PIPELINE_ELEMENT_VALUE = "PRIVILEGE_DELETE_PIPELINE_ELEMENT"; + + public static final String PRIVILEGE_READ_DASHBOARD_VALUE = "PRIVILEGE_READ_DASHBOARD"; + public static final String PRIVILEGE_WRITE_DASHBOARD_VALUE = "PRIVILEGE_WRITE_DASHBOARD"; + public static final String PRIVILEGE_DELETE_DASHBOARD_VALUE = "PRIVILEGE_DELETE_DASHBOARD"; + + public static final String PRIVILEGE_READ_DASHBOARD_WIDGET_VALUE = "PRIVILEGE_READ_DASHBOARD_WIDGET"; + public static final String PRIVILEGE_WRITE_DASHBOARD_WIDGET_VALUE = "PRIVILEGE_WRITE_DASHBOARD_WIDGET"; + public static final String PRIVILEGE_DELETE_DASHBOARD_WIDGET_VALUE = "PRIVILEGE_DELETE_DASHBOARD_WIDGET"; + + public static final String PRIVILEGE_READ_DATA_EXPLORER_VIEW_VALUE = "PRIVILEGE_READ_DATA_EXPLORER_VIEW"; + public static final String PRIVILEGE_WRITE_DATA_EXPLORER_VIEW_VALUE = "PRIVILEGE_WRITE_DATA_EXPLORER_VIEW"; + public static final String PRIVILEGE_DELETE_DATA_EXPLORER_VIEW_VALUE = "PRIVILEGE_DELETE_DATA_EXPLORER_VIEW"; + + public static final String PRIVILEGE_READ_DATA_EXPLORER_WIDGET_VALUE = "PRIVILEGE_READ_DATA_EXPLORER_WIDGET"; + public static final String PRIVILEGE_WRITE_DATA_EXPLORER_WIDGET_VALUE = "PRIVILEGE_WRITE_DATA_EXPLORER_WIDGET"; + public static final String PRIVILEGE_DELETE_DATA_EXPLORER_WIDGET_VALUE = "PRIVILEGE_DELETE_DATA_EXPLORER_WIDGET"; + + public static final String PRIVILEGE_READ_APPS_VALUE = "PRIVILEGE_READ_APPS"; + public static final String PRIVILEGE_WRITE_APPS_VALUE = "PRIVILEGE_WRITE_APPS"; + + public static final String PRIVILEGE_READ_NOTIFICATIONS_VALUE = "PRIVILEGE_READ_NOTIFICATIONS"; + + public static final String PRIVILEGE_READ_FILES_VALUE = "PRIVILEGE_READ_FILES"; + public static final String PRIVILEGE_WRITE_FILES_VALUE = "PRIVILEGE_WRITE_FILES"; + public static final String PRIVILEGE_DELETE_FILES_VALUE = "PRIVILEGE_DELETE_FILES"; + + public static final String PRIVILEGE_READ_ASSETS_VALUE = "PRIVILEGE_READ_ASSETS"; + public static final String PRIVILEGE_WRITE_ASSETS_VALUE = "PRIVILEGE_WRITE_ASSETS"; + + public static final String PRIVILEGE_READ_GENERIC_STORAGE_VALUE = "PRIVILEGE_READ_GENERIC_STORAGE"; + public static final String PRIVILEGE_WRITE_GENERIC_STORAGE_VALUE = "PRIVILEGE_WRITE_GENERIC_STORAGE"; } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/RawUserApiToken.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/RawUserApiToken.java index ccf936dd33..d70df9e897 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/RawUserApiToken.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/RawUserApiToken.java @@ -20,5 +20,46 @@ import org.apache.streampipes.model.shared.annotation.TsModel; @TsModel -public record RawUserApiToken(String rawToken, String hashedToken, String tokenName, String tokenId) { +public class RawUserApiToken { + + private String rawToken; + private String hashedToken; + private String tokenName; + private String tokenId; + + public RawUserApiToken() { + + } + + public String getRawToken() { + return rawToken; + } + + public void setRawToken(String rawToken) { + this.rawToken = rawToken; + } + + public String getTokenName() { + return tokenName; + } + + public void setTokenName(String tokenName) { + this.tokenName = tokenName; + } + + public String getTokenId() { + return tokenId; + } + + public void setTokenId(String tokenId) { + this.tokenId = tokenId; + } + + public String getHashedToken() { + return hashedToken; + } + + public void setHashedToken(String hashedToken) { + this.hashedToken = hashedToken; + } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Role.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Role.java index 84542b9a30..26f5d1c003 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Role.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/Role.java @@ -19,88 +19,112 @@ package org.apache.streampipes.model.client.user; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gson.annotations.SerializedName; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; @TsModel -public class Role implements Storable { - - protected @SerializedName("_id") String elementId; - protected @SerializedName("_rev") String rev; - - private String label; - private boolean defaultRole; - private List privilegeIds; - - // document type should be persisted to CouchDB with Gson serialization, but not via Jackson to the UI - @JsonIgnore - @SerializedName("$type") - private String type = "role"; - - public static Role createDefaultRole(String id, - String label, - List privilegeIds) { - Role role = new Role(); - role.elementId = id; - role.label = label; - role.defaultRole = true; - role.privilegeIds = privilegeIds; - return role; - } - - @Override - public String getRev() { - return rev; - } - - @Override - public void setRev(String rev) { - this.rev = rev; - } - - @Override - public String getElementId() { - return elementId; - } - - @Override - public void setElementId(String elementId) { - this.elementId = elementId; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public boolean isDefaultRole() { - return defaultRole; +public enum Role { + ROLE_ADMIN(Constants.ROLE_ADMIN_VALUE), + + ROLE_SERVICE_ADMIN(Constants.ROLE_SERVICE_ADMIN_VALUE), + + ROLE_PIPELINE_ADMIN( + Constants.ROLE_PIPELINE_ADMIN_VALUE, + Privilege.PRIVILEGE_READ_PIPELINE, + Privilege.PRIVILEGE_WRITE_PIPELINE, + Privilege.PRIVILEGE_DELETE_PIPELINE, + Privilege.PRIVILEGE_READ_PIPELINE_ELEMENT + ), + + ROLE_DASHBOARD_ADMIN( + Constants.ROLE_DASHBOARD_ADMIN_VALUE, + Privilege.PRIVILEGE_READ_DASHBOARD, + Privilege.PRIVILEGE_WRITE_DASHBOARD, + Privilege.PRIVILEGE_DELETE_DASHBOARD + ), + + ROLE_DATA_EXPLORER_ADMIN( + Constants.ROLE_DATA_EXPLORER_ADMIN_VALUE, + Privilege.PRIVILEGE_READ_DATA_EXPLORER_VIEW, + Privilege.PRIVILEGE_READ_DATA_EXPLORER_WIDGET, + Privilege.PRIVILEGE_WRITE_DATA_EXPLORER_VIEW, + Privilege.PRIVILEGE_WRITE_DATA_EXPLORER_WIDGET, + Privilege.PRIVILEGE_DELETE_DATA_EXPLORER_VIEW, + Privilege.PRIVILEGE_DELETE_DATA_EXPLORER_WIDGET, + Privilege.PRIVILEGE_READ_PIPELINE + ), + ROLE_CONNECT_ADMIN( + Constants.ROLE_CONNECT_ADMIN_VALUE, + Privilege.PRIVILEGE_WRITE_ADAPTER, + Privilege.PRIVILEGE_READ_ADAPTER, + Privilege.PRIVILEGE_DELETE_ADAPTER + ), + + ROLE_DASHBOARD_USER( + Constants.ROLE_DASHBOARD_USER_VALUE, + Privilege.PRIVILEGE_READ_DASHBOARD, + Privilege.PRIVILEGE_READ_DASHBOARD_WIDGET, + Privilege.PRIVILEGE_READ_PIPELINE + ), + + ROLE_DATA_EXPLORER_USER( + Constants.ROLE_DATA_EXPLORER_USER_VALUE, + Privilege.PRIVILEGE_READ_DATA_EXPLORER_VIEW, + Privilege.PRIVILEGE_READ_DATA_EXPLORER_WIDGET, + Privilege.PRIVILEGE_READ_PIPELINE + ), + + ROLE_PIPELINE_USER( + Constants.ROLE_PIPELINE_USER_VALUE, + Privilege.PRIVILEGE_READ_PIPELINE, + Privilege.PRIVILEGE_READ_PIPELINE_ELEMENT + ), + + ROLE_APP_USER(Constants.ROLE_APP_USER_VALUE, + Privilege.PRIVILEGE_READ_PIPELINE, + Privilege.PRIVILEGE_READ_PIPELINE_ELEMENT), + + ROLE_ASSET_USER(Constants.ROLE_ASSET_USER_VALUE, + Privilege.PRIVILEGE_READ_GENERIC_STORAGE, + Privilege.PRIVILEGE_READ_ASSETS), + + ROLE_ASSET_ADMIN(Constants.ROLE_ASSET_ADMIN_VALUE, + Privilege.PRIVILEGE_READ_ASSETS, + Privilege.PRIVILEGE_READ_GENERIC_STORAGE, + Privilege.PRIVILEGE_WRITE_GENERIC_STORAGE, + Privilege.PRIVILEGE_WRITE_ASSETS); + + private List privileges; + private String roleString; + + Role(String roleString, + Privilege... privileges) { + this.roleString = roleString; + this.privileges = Arrays.asList(privileges); } - public void setDefaultRole(boolean defaultRole) { - this.defaultRole = defaultRole; + public List getPrivilegesAsString() { + return this.privileges.stream().map(Enum::name).collect(Collectors.toList()); } - public List getPrivilegeIds() { - return privilegeIds; + public List getPrivileges() { + return privileges; } - public void setPrivilegeIds(List privilegeIds) { - this.privilegeIds = privilegeIds; + public static final class Constants { + public static final String ROLE_ADMIN_VALUE = "ROLE_ADMIN"; + public static final String ROLE_SERVICE_ADMIN_VALUE = "ROLE_SERVICE_ADMIN"; + public static final String ROLE_PIPELINE_ADMIN_VALUE = "ROLE_PIPELINE_ADMIN"; + public static final String ROLE_DASHBOARD_ADMIN_VALUE = "ROLE_DASHBOARD_ADMIN"; + public static final String ROLE_DATA_EXPLORER_ADMIN_VALUE = "ROLE_DATA_EXPLORER_ADMIN"; + public static final String ROLE_CONNECT_ADMIN_VALUE = "ROLE_CONNECT_ADMIN"; + public static final String ROLE_DASHBOARD_USER_VALUE = "ROLE_DASHBOARD_USER"; + public static final String ROLE_DATA_EXPLORER_USER_VALUE = "ROLE_DATA_EXPLORER_USER"; + public static final String ROLE_PIPELINE_USER_VALUE = "ROLE_PIPELINE_USER"; + public static final String ROLE_APP_USER_VALUE = "ROLE_APP_USER"; + public static final String ROLE_ASSET_USER_VALUE = "ROLE_ASSET_USER"; + public static final String ROLE_ASSET_ADMIN_VALUE = "ROLE_ASSET_ADMIN"; } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ServiceAccount.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ServiceAccount.java index 611bb03727..acc87e7998 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ServiceAccount.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/ServiceAccount.java @@ -33,7 +33,7 @@ public ServiceAccount() { public static ServiceAccount from(String serviceAccountName, String clientSecret, - Set roles) { + Set roles) { ServiceAccount account = new ServiceAccount(); account.setUsername(serviceAccountName); account.setClientSecret(clientSecret); diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserAccount.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserAccount.java index 98bcfbca5c..3c6f5d96eb 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserAccount.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserAccount.java @@ -27,8 +27,6 @@ @TsModel public class UserAccount extends Principal { - public static final String LOCAL = "local"; - protected String fullName; protected String password; @@ -41,11 +39,6 @@ public class UserAccount extends Principal { protected boolean hideTutorial; protected boolean darkMode = false; - /** - * The authentication provider (LOCAL or one of the configured OAuth providers - */ - protected String provider; - public UserAccount() { super(PrincipalType.USER_ACCOUNT); this.hideTutorial = false; @@ -53,12 +46,11 @@ public UserAccount() { this.preferredDataProcessors = new ArrayList<>(); this.preferredDataSinks = new ArrayList<>(); this.preferredDataStreams = new ArrayList<>(); - this.provider = UserAccount.LOCAL; } public static UserAccount from(String username, String encryptedPassword, - Set roles) { + Set roles) { UserAccount account = new UserAccount(); account.setUsername(username); account.setPassword(encryptedPassword); @@ -69,11 +61,11 @@ public static UserAccount from(String username, return account; } - public Set getRoles() { + public Set getRoles() { return roles; } - public void setRoles(Set roles) { + public void setRoles(Set roles) { this.roles = roles; } @@ -164,12 +156,4 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } - - public String getProvider() { - return provider; - } - - public void setProvider(String provider) { - this.provider = provider; - } } diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserActivationToken.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserActivationToken.java index 17d96c777e..1fe6ab0288 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserActivationToken.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserActivationToken.java @@ -18,12 +18,9 @@ package org.apache.streampipes.model.client.user; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gson.annotations.SerializedName; public class UserActivationToken extends AbstractMailToken { - // This field should be called $type since this is the identifier used in the CouchDB view - @SerializedName("$type") @JsonIgnore private String type = "user-activation"; diff --git a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserRegistrationData.java b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserRegistrationData.java index 92b1a0edd7..5457645e38 100644 --- a/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserRegistrationData.java +++ b/streampipes-model-client/src/main/java/org/apache/streampipes/model/client/user/UserRegistrationData.java @@ -20,38 +20,5 @@ import java.util.List; -public class UserRegistrationData { - - private String username; - private String password; - private List roles; - private String provider; - - public UserRegistrationData(String username, - String password, - List roles) { - this.username = username; - this.password = password; - this.roles = roles; - } - - public String getUsername() { - return username; - } - - public String getPassword() { - return password; - } - - public List getRoles() { - return roles; - } - - public String getProvider() { - return provider; - } - - public void setProvider(String provider) { - this.provider = provider; - } +public record UserRegistrationData(String username, String password, List roles) { } diff --git a/streampipes-model-shared/pom.xml b/streampipes-model-shared/pom.xml index 81a76ca9ef..36066b8d06 100644 --- a/streampipes-model-shared/pom.xml +++ b/streampipes-model-shared/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.apache.streampipes - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 diff --git a/streampipes-model/pom.xml b/streampipes-model/pom.xml index 678ff5c5fb..a7326880d1 100644 --- a/streampipes-model/pom.xml +++ b/streampipes-model/pom.xml @@ -23,24 +23,24 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-commons - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model-shared - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-vocabulary - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/SpDataStream.java b/streampipes-model/src/main/java/org/apache/streampipes/model/SpDataStream.java index 118a6ebbb3..57510ffa01 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/SpDataStream.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/SpDataStream.java @@ -40,11 +40,25 @@ public class SpDataStream extends NamedStreamPipesEntity { private int index; private String correspondingAdapterId; + public SpDataStream(String uri, String name, String description, String iconUrl, + EventGrounding eventGrounding, + EventSchema eventSchema) { + super(uri, name, description, iconUrl); + this.eventGrounding = eventGrounding; + this.eventSchema = eventSchema; + } + + public SpDataStream(String uri, String name, String description, EventSchema eventSchema) { + super(uri, name, description); + this.eventSchema = eventSchema; + } + public SpDataStream() { super(prefix + RandomStringUtils.randomAlphabetic(6)); this.eventSchema = new EventSchema(); } + public SpDataStream(SpDataStream other) { super(other); this.index = other.getIndex(); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/VirtualSensor.java b/streampipes-model/src/main/java/org/apache/streampipes/model/VirtualSensor.java new file mode 100644 index 0000000000..0034a96f84 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/VirtualSensor.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model; + +import org.apache.streampipes.model.pipeline.ElementComposition; + +public class VirtualSensor extends ElementComposition { + + protected String createdBy; + + protected int outputIndex; + + public VirtualSensor() { + super(); + } + + public String getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(String createdBy) { + this.createdBy = createdBy; + } + + public int getOutputIndex() { + return outputIndex; + } + + public void setOutputIndex(int outputIndex) { + this.outputIndex = outputIndex; + } + + +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLinkType.java b/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLinkType.java index 1b34655a1e..f2b045e97e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLinkType.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/assets/AssetLinkType.java @@ -33,9 +33,6 @@ public class AssetLinkType { @JsonProperty("_id") private @SerializedName("_id") String id; - @JsonProperty("_rev") - private @SerializedName("_rev") String rev; - private String linkType; private String linkLabel; private String linkColor; @@ -131,12 +128,4 @@ public boolean isNavigationActive() { public void setNavigationActive(boolean navigationActive) { this.navigationActive = navigationActive; } - - public String getRev() { - return rev; - } - - public void setRev(String rev) { - this.rev = rev; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java index 3c763c15eb..e2fcb83f7d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/ConsumableStreamPipesEntity.java @@ -42,6 +42,12 @@ public ConsumableStreamPipesEntity() { this.staticProperties = new ArrayList<>(); } + public ConsumableStreamPipesEntity(String elementId, String name, String description, String iconUrl) { + super(elementId, name, description, iconUrl); + this.spDataStreams = new ArrayList<>(); + this.staticProperties = new ArrayList<>(); + } + public ConsumableStreamPipesEntity(ConsumableStreamPipesEntity other) { super(other); if (other.getSpDataStreams() != null) { @@ -69,6 +75,10 @@ public void setStaticProperties(List staticProperties) { this.staticProperties = staticProperties; } + public boolean addEventStream(SpDataStream spDataStream) { + return spDataStreams.add(spDataStream); + } + public EventGrounding getSupportedGrounding() { return supportedGrounding; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java index c18335797b..2f1fc8fa12 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/InvocableStreamPipesEntity.java @@ -23,6 +23,7 @@ import org.apache.streampipes.model.api.EndpointSelectable; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.grounding.EventGrounding; +import org.apache.streampipes.model.monitoring.ElementStatusInfoSettings; import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.Cloner; @@ -40,6 +41,8 @@ public abstract class InvocableStreamPipesEntity private String belongsTo; + private ElementStatusInfoSettings statusInfoSettings; + private EventGrounding supportedGrounding; private String correspondingPipeline; @@ -81,6 +84,22 @@ public InvocableStreamPipesEntity(InvocableStreamPipesEntity other) { } } + public InvocableStreamPipesEntity( + String uri, + String name, + String description, + String iconUrl, + SpServiceTagPrefix serviceTagPrefix + ) { + super(uri, name, description, iconUrl); + this.serviceTagPrefix = serviceTagPrefix; + this.configured = false; + } + + public boolean addStaticProperty(StaticProperty staticProperty) { + return staticProperties.add(staticProperty); + } + public List getInputStreams() { return inputStreams; } @@ -139,6 +158,14 @@ public void setConfigured(boolean configured) { this.configured = configured; } + public ElementStatusInfoSettings getStatusInfoSettings() { + return statusInfoSettings; + } + + public void setStatusInfoSettings(ElementStatusInfoSettings statusInfoSettings) { + this.statusInfoSettings = statusInfoSettings; + } + public String getCorrespondingUser() { return correspondingUser; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/NamedStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/NamedStreamPipesEntity.java index db57bf577e..6ce1b32115 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/NamedStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/NamedStreamPipesEntity.java @@ -23,10 +23,9 @@ import org.apache.streampipes.model.extensions.ExtensionAssetType; import org.apache.streampipes.model.extensions.ExtensionItemDescription; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import org.apache.streampipes.model.util.ServiceDefinitionUtil; -import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.gson.annotations.SerializedName; @@ -41,19 +40,20 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "@class") @TsModel -public abstract class NamedStreamPipesEntity implements Storable, Serializable { +public abstract class NamedStreamPipesEntity implements Serializable { private static final long serialVersionUID = -98951691820519795L; protected @SerializedName("_id") String elementId; - @JsonAlias("_rev") + @JsonProperty("_rev") protected @SerializedName("_rev") String rev; protected String dom; protected List connectedTo; private String name; private String description; + private String iconUrl; private String appId; private boolean includesAssets; private boolean includesLocales; @@ -73,11 +73,26 @@ public NamedStreamPipesEntity(String elementId) { this.elementId = elementId; } + public NamedStreamPipesEntity(String elementId, String name, String description, String iconUrl) { + this(elementId, name, description); + this.iconUrl = iconUrl; + } + + public NamedStreamPipesEntity(String elementId, String name, String description) { + super(); + this.elementId = elementId; + this.name = name; + this.description = description; + this.includedAssets = new ArrayList<>(); + this.includedLocales = new ArrayList<>(); + } + public NamedStreamPipesEntity(NamedStreamPipesEntity other) { this.elementId = other.getElementId(); this.rev = other.getRev(); this.description = other.getDescription(); this.name = other.getName(); + this.iconUrl = other.getIconUrl(); this.elementId = other.getElementId(); this.dom = other.getDom(); this.internallyManaged = other.isInternallyManaged(); @@ -109,6 +124,14 @@ public void setDescription(String description) { this.description = description; } + public String getIconUrl() { + return iconUrl; + } + + public void setIconUrl(String iconUrl) { + this.iconUrl = iconUrl; + } + public String getDom() { return dom; } @@ -173,22 +196,18 @@ public void setInternallyManaged(boolean internallyManaged) { this.internallyManaged = internallyManaged; } - @Override public String getRev() { return rev; } - @Override public void setRev(String rev) { this.rev = rev; } - @Override public String getElementId() { return elementId; } - @Override public void setElementId(String elementId) { this.elementId = elementId; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java index 43354f9764..4d533eaf3e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/base/VersionedNamedStreamPipesEntity.java @@ -31,6 +31,14 @@ public VersionedNamedStreamPipesEntity(VersionedNamedStreamPipesEntity other) { this.version = other.version; } + public VersionedNamedStreamPipesEntity(String elementId, String name, String description){ + super(elementId, name, description); + } + + public VersionedNamedStreamPipesEntity(String elementId, String name, String description, String iconUrl) { + super(elementId, name, description, iconUrl); + } + public int getVersion(){ return version; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java index 6ba4dd0865..457ec36b14 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultMessagingSettings.java @@ -50,6 +50,7 @@ public MessagingSettings make() { 5000012, 20, 2, + Arrays.asList(SpDataFormat.JSON, SpDataFormat.CBOR, SpDataFormat.FST, SpDataFormat.SMILE), protocolList); defaultSettings.setJmsHost("activemq"); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java index b669b3e4b7..4f654bd992 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java @@ -29,7 +29,7 @@ public class DefaultSpCoreConfiguration { public SpCoreConfiguration make() { var coreCfg = new SpCoreConfiguration(); - coreCfg.setGeneralConfig(new DefaultGeneralConfig().make()); + coreCfg.setGeneralConfig(new GeneralConfig()); coreCfg.setMessagingSettings(new DefaultMessagingSettings().make()); coreCfg.setEmailConfig(EmailConfig.fromDefaults()); coreCfg.setEmailTemplateConfig(new DefaultEmailTemplateConfiguration().getDefaultTemplates()); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java index 188590a196..2f3968d966 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/GeneralConfig.java @@ -31,7 +31,6 @@ public class GeneralConfig { private boolean allowPasswordRecovery; private List defaultUserRoles; - private LinkSettings linkSettings; public GeneralConfig() { } @@ -105,12 +104,4 @@ public String getAppName() { public void setAppName(String appName) { this.appName = appName; } - - public LinkSettings getLinkSettings() { - return linkSettings; - } - - public void setLinkSettings(LinkSettings linkSettings) { - this.linkSettings = linkSettings; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java index 874c200750..edad5c1a50 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/MessagingSettings.java @@ -30,6 +30,7 @@ public class MessagingSettings { private Integer lingerMs; private Integer acks; + private List prioritizedFormats; private List prioritizedProtocols; private String jmsHost; @@ -55,11 +56,13 @@ public MessagingSettings(Integer batchSize, Integer messageMaxBytes, Integer lingerMs, Integer acks, + List prioritizedFormats, List prioritizedProtocols) { this.batchSize = batchSize; this.messageMaxBytes = messageMaxBytes; this.lingerMs = lingerMs; this.acks = acks; + this.prioritizedFormats = prioritizedFormats; this.prioritizedProtocols = prioritizedProtocols; this.supportedProtocols = new ArrayList<>(); } @@ -100,6 +103,14 @@ public void setAcks(Integer acks) { this.acks = acks; } + public List getPrioritizedFormats() { + return prioritizedFormats; + } + + public void setPrioritizedFormats(List prioritizedFormats) { + this.prioritizedFormats = prioritizedFormats; + } + public List getPrioritizedProtocols() { return prioritizedProtocols; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java index b5766bd3f6..6aad62a172 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java @@ -32,7 +32,6 @@ public class SpCoreConfiguration { private EmailConfig emailConfig; private EmailTemplateConfig emailTemplateConfig; private GeneralConfig generalConfig; - private LocationConfig locationConfig; private boolean isConfigured; private SpCoreConfigurationStatus serviceStatus; @@ -41,7 +40,6 @@ public class SpCoreConfiguration { private String filesDir; public SpCoreConfiguration() { - this.locationConfig = new LocationConfig(false, "", ""); } public String getRev() { @@ -131,12 +129,4 @@ public SpCoreConfigurationStatus getServiceStatus() { public void setServiceStatus(SpCoreConfigurationStatus serviceStatus) { this.serviceStatus = serviceStatus; } - - public LocationConfig getLocationConfig() { - return locationConfig; - } - - public void setLocationConfig(LocationConfig locationConfig) { - this.locationConfig = locationConfig; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java new file mode 100644 index 0000000000..a2c5b6bf38 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpDataFormat.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.configuration; + + +import org.apache.streampipes.vocabulary.MessageFormat; + +public enum SpDataFormat { + + CBOR("Cbor", MessageFormat.CBOR), + JSON("JSON", MessageFormat.JSON), + FST("Fast-Serializer", MessageFormat.FST), + SMILE("Smile", MessageFormat.SMILE); + + private String name; + private String messageFormat; + + SpDataFormat(String name, String messageFormat) { + this.name = name; + this.messageFormat = messageFormat; + } + + public String getName() { + return name; + } + + public String getMessageFormat() { + return messageFormat; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java index 04c2d57be3..5fd6b21b3f 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/AdapterDescription.java @@ -25,7 +25,6 @@ import org.apache.streampipes.model.connect.rules.stream.StreamTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; import org.apache.streampipes.model.connect.rules.value.ValueTransformationRuleDescription; -import org.apache.streampipes.model.deployment.ExtensionDeploymentConfiguration; import org.apache.streampipes.model.grounding.EventGrounding; import org.apache.streampipes.model.schema.EventSchema; import org.apache.streampipes.model.shared.annotation.TsModel; @@ -56,8 +55,6 @@ public class AdapterDescription extends VersionedNamedStreamPipesEntity { // Is used to store where the adapter is running to stop it private String selectedEndpointUrl; - private ExtensionDeploymentConfiguration deploymentConfiguration; - /** * This is used to identify all the service within the service group the adapter can be invoked in */ @@ -75,7 +72,6 @@ public AdapterDescription() { this.config = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); - this.deploymentConfiguration = new ExtensionDeploymentConfiguration(); } public AdapterDescription(int version) { @@ -85,10 +81,17 @@ public AdapterDescription(int version) { this.config = new ArrayList<>(); this.category = new ArrayList<>(); this.dataStream = new SpDataStream(); - this.deploymentConfiguration = new ExtensionDeploymentConfiguration(); this.setVersion(version); } + public AdapterDescription(String elementId, String name, String description) { + super(elementId, name, description); + this.rules = new ArrayList<>(); + this.category = new ArrayList<>(); + this.dataStream = new SpDataStream(); + } + + public AdapterDescription(AdapterDescription other) { super(other); this.config = new Cloner().staticProperties(other.getConfig()); @@ -106,7 +109,6 @@ public AdapterDescription(AdapterDescription other) { this.dataStream = new SpDataStream(other.getDataStream()); } this.running = other.isRunning(); - this.deploymentConfiguration = other.getDeploymentConfiguration(); } public String getRev() { @@ -257,12 +259,4 @@ public boolean isRunning() { public void setRunning(boolean running) { this.running = running; } - - public ExtensionDeploymentConfiguration getDeploymentConfiguration() { - return deploymentConfiguration; - } - - public void setDeploymentConfiguration(ExtensionDeploymentConfiguration deploymentConfiguration) { - this.deploymentConfiguration = deploymentConfiguration; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ParserDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ParserDescription.java index b45ee1e9fb..87d7eca331 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ParserDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ParserDescription.java @@ -20,8 +20,11 @@ import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.staticproperty.StaticPropertyGroup; +import java.util.List; + public class ParserDescription extends NamedStreamPipesEntity { private StaticPropertyGroup config; @@ -31,6 +34,25 @@ public ParserDescription() { this.config = new StaticPropertyGroup(); } + public ParserDescription(String uri, String name, String description) { + super(uri, name, description); + this.config = new StaticPropertyGroup(uri, name, description); + } + + public ParserDescription(String uri, + String name, + String description, + List config) { + super(uri, name, description); + + this.config = new StaticPropertyGroup(uri, name, description, config); + } + + public ParserDescription(ParserDescription other) { + super(other); + this.config = other.getConfig(); + } + public StaticPropertyGroup getConfig() { return config; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ProtocolDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ProtocolDescription.java new file mode 100644 index 0000000000..502b650d1b --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/grounding/ProtocolDescription.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.connect.grounding; + +import org.apache.streampipes.model.base.NamedStreamPipesEntity; +import org.apache.streampipes.model.shared.annotation.TsModel; +import org.apache.streampipes.model.staticproperty.StaticProperty; +import org.apache.streampipes.model.util.Cloner; + +import java.util.ArrayList; +import java.util.List; + +@TsModel +@Deprecated(since = "0.93.0", forRemoval = true) +public class ProtocolDescription extends NamedStreamPipesEntity { + + @Deprecated + String sourceType; + + List config; + + private List category; + + public ProtocolDescription() { + this.config = new ArrayList<>(); + } + + public ProtocolDescription(String uri, String name, String description) { + super(uri, name, description); + this.config = new ArrayList<>(); + this.category = new ArrayList<>(); + } + + public ProtocolDescription(String uri, String name, String description, List config) { + super(uri, name, description); + this.config = config; + this.category = new ArrayList<>(); + } + + public ProtocolDescription(ProtocolDescription other) { + super(other); + + this.config = new Cloner().staticProperties(other.getConfig()); + if (other.getCategory() != null) { + this.category = new Cloner().epaTypes(other.getCategory()); + } + } + + public void addConfig(StaticProperty sp) { + this.config.add(sp); + } + + public List getConfig() { + return config; + } + + public void setConfig(List config) { + this.config = config; + } + + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public List getCategory() { + return category; + } + + public void setCategory(List category) { + this.category = category; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java index 78dbd37da1..bb45ff0eaf 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/ITransformationRuleVisitor.java @@ -28,7 +28,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; @@ -42,8 +41,6 @@ public interface ITransformationRuleVisitor { void visit(RenameRuleDescription rule); - void visit(RegexTransformationRuleDescription rule); - void visit(EventRateTransformationRuleDescription rule); void visit(RemoveDuplicatesTransformationRuleDescription rule); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java index 94b1817e8f..0ea2194456 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRuleDescription.java @@ -28,7 +28,6 @@ import org.apache.streampipes.model.connect.rules.value.AddValueTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.ChangeDatatypeTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.CorrectionValueTransformationRuleDescription; -import org.apache.streampipes.model.connect.rules.value.RegexTransformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription; import org.apache.streampipes.model.connect.rules.value.UnitTransformRuleDescription; import org.apache.streampipes.model.shared.annotation.TsModel; @@ -48,7 +47,6 @@ @JsonSubTypes.Type(CreateNestedRuleDescription.class), @JsonSubTypes.Type(DeleteRuleDescription.class), @JsonSubTypes.Type(RenameRuleDescription.class), - @JsonSubTypes.Type(RegexTransformationRuleDescription.class), @JsonSubTypes.Type(MoveRuleDescription.class), @JsonSubTypes.Type(ChangeDatatypeTransformationRuleDescription.class), @JsonSubTypes.Type(CorrectionValueTransformationRuleDescription.class), diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRulePriority.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRulePriority.java index 28458d81c8..3fcb8ba429 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRulePriority.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/TransformationRulePriority.java @@ -31,7 +31,6 @@ public enum TransformationRulePriority { CHANGE_UNIT(310), TIMESTAMP_TRANSFORMATION(320), CORRECTION_VALUE(330), - REGEX_TRANSFORMATION(331), CHANGE_DATATYPE(340), REMOVE_DUPLICATES(410), diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java index 3ed301f1e2..f03669a35d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/schema/CreateNestedRuleDescription.java @@ -21,11 +21,6 @@ import org.apache.streampipes.model.connect.rules.ITransformationRuleVisitor; import org.apache.streampipes.model.connect.rules.TransformationRulePriority; -/** - * @deprecated The functionlality to add nested rules was removed in version 0.97.0 form the UI - * For the next release we can also remove the functionality from the backend - */ -@Deprecated(since = "0.97.0", forRemoval = true) public class CreateNestedRuleDescription extends SchemaTransformationRuleDescription { private String runtimeKey; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ValueTransformationRuleDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ValueTransformationRuleDescription.java index e1f6e495bb..dee07a2151 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ValueTransformationRuleDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/rules/value/ValueTransformationRuleDescription.java @@ -28,7 +28,6 @@ @JsonSubTypes.Type(TimestampTranfsformationRuleDescription.class), @JsonSubTypes.Type(UnitTransformRuleDescription.class), @JsonSubTypes.Type(CorrectionValueTransformationRuleDescription.class), - @JsonSubTypes.Type(RegexTransformationRuleDescription.class), }) public abstract class ValueTransformationRuleDescription extends TransformationRuleDescription { diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardEntity.java b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardEntity.java index b2ec89845a..5ef0f8e095 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardEntity.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardEntity.java @@ -19,9 +19,8 @@ package org.apache.streampipes.model.dashboard; import org.apache.streampipes.model.datalake.DataExplorerWidgetModel; -import org.apache.streampipes.model.shared.api.Storable; -import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.google.gson.annotations.SerializedName; @@ -29,37 +28,31 @@ @JsonSubTypes.Type(DashboardWidgetModel.class), @JsonSubTypes.Type(DataExplorerWidgetModel.class) }) -public abstract class DashboardEntity implements Storable { +public abstract class DashboardEntity { - @JsonAlias("_id") - @SerializedName("_id") - private String elementId; + @JsonProperty("_id") + private @SerializedName("_id") String id; - @JsonAlias("_rev") - @SerializedName("_rev") - private String rev; + @JsonProperty("_rev") + private @SerializedName("_rev") String rev; public DashboardEntity() { super(); } - @Override - public String getRev() { - return rev; + public String getId() { + return id; } - @Override - public void setRev(String rev) { - this.rev = rev; + public void setId(String id) { + this.id = id; } - @Override - public String getElementId() { - return elementId; + public String getRev() { + return rev; } - @Override - public void setElementId(String elementId) { - this.elementId = elementId; + public void setRev(String rev) { + this.rev = rev; } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardItem.java b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardItem.java index 81759642cb..c7962b0411 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardItem.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardItem.java @@ -18,7 +18,6 @@ package org.apache.streampipes.model.dashboard; import java.util.List; -import java.util.Map; public class DashboardItem { @@ -32,7 +31,6 @@ public class DashboardItem { private Integer rows; private Integer x; private Integer y; - private Map timeSettings; public DashboardItem() { @@ -101,12 +99,4 @@ public Integer getY() { public void setY(Integer y) { this.y = y; } - - public Map getTimeSettings() { - return timeSettings; - } - - public void setTimeSettings(Map timeSettings) { - this.timeSettings = timeSettings; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardModel.java b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardModel.java index cf9ee00ea2..5c001011c9 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardModel.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/dashboard/DashboardModel.java @@ -19,9 +19,8 @@ package org.apache.streampipes.model.dashboard; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; -import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.gson.annotations.SerializedName; import java.util.HashMap; @@ -29,15 +28,13 @@ import java.util.Map; @TsModel -public class DashboardModel implements Storable { +public class DashboardModel { - @JsonAlias("_id") - @SerializedName("_id") - private String elementId; + @JsonProperty("_id") + private @SerializedName("_id") String couchDbId; - @JsonAlias("_rev") - @SerializedName("_rev") - private String rev; + @JsonProperty("_rev") + private @SerializedName("_rev") String couchDbRev; private String id; private String name; @@ -46,14 +43,28 @@ public class DashboardModel implements Storable { private Map dashboardTimeSettings; private Map dashboardGeneralSettings; - private Map dashboardLiveSettings; private List widgets; public DashboardModel() { this.dashboardTimeSettings = new HashMap<>(); this.dashboardGeneralSettings = new HashMap<>(); - this.dashboardLiveSettings = new HashMap<>(); + } + + public String getCouchDbId() { + return couchDbId; + } + + public void setCouchDbId(String couchDbId) { + this.couchDbId = couchDbId; + } + + public String getCouchDbRev() { + return couchDbRev; + } + + public void setCouchDbRev(String couchDbRev) { + this.couchDbRev = couchDbRev; } public String getId() { @@ -111,36 +122,4 @@ public Map getDashboardGeneralSettings() { public void setDashboardGeneralSettings(Map dashboardGeneralSettings) { this.dashboardGeneralSettings = dashboardGeneralSettings; } - - @Override - public String getRev() { - return this.rev; - } - - @Override - public void setRev(String rev) { - this.rev = rev; - } - - @Override - public String getElementId() { - return this.elementId; - } - - @Override - public void setElementId(String elementId) { - this.elementId = elementId; - } - - public String getCouchDbId() { - return this.elementId; - } - - public Map getDashboardLiveSettings() { - return dashboardLiveSettings; - } - - public void setDashboardLiveSettings(Map dashboardLiveSettings) { - this.dashboardLiveSettings = dashboardLiveSettings; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataExplorerWidgetModel.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataExplorerWidgetModel.java index 02b244d22c..2371363acc 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataExplorerWidgetModel.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataExplorerWidgetModel.java @@ -40,9 +40,6 @@ public class DataExplorerWidgetModel extends DashboardEntity { @JsonSerialize(using = CustomMapSerializer.class, as = Map.class) private Map dataConfig; - @JsonSerialize(using = CustomMapSerializer.class, as = Map.class) - private Map timeSettings; - private String pipelineId; private String measureName; @@ -52,7 +49,6 @@ public DataExplorerWidgetModel() { this.baseAppearanceConfig = new HashMap<>(); this.visualizationConfig = new HashMap<>(); this.dataConfig = new HashMap<>(); - this.timeSettings = new HashMap<>(); } public String getWidgetId() { @@ -111,12 +107,4 @@ public void setMeasureName(String measureName) { this.measureName = measureName; } - public Map getTimeSettings() { - return this.timeSettings; - } - - public void setTimeSettings(Map timeSettings) { - this.timeSettings = timeSettings; - } - } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java index a7029fa520..2d9441b283 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java @@ -21,24 +21,24 @@ import org.apache.streampipes.model.schema.EventSchema; import org.apache.streampipes.model.shared.annotation.TsIgnore; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; -import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.gson.annotations.SerializedName; @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "@class") @TsModel -public class DataLakeMeasure implements Storable { +public class DataLakeMeasure { public static final String CURRENT_SCHEMA_VERSION = "1.1"; public static final String ASSERTION_ERROR_MESSAGE = "timestamp field requires a stream prefix (e.g. s0::timestamp)"; private static final String STREAM_PREFIX_DELIMITER = "::"; + @JsonProperty("elementId") protected @SerializedName("_id") String elementId; - @JsonAlias("_rev") + @JsonProperty("_rev") private @SerializedName("_rev") String rev; private String measureName; @@ -150,22 +150,18 @@ public String getTimestampFieldName() { return timestampField.split(STREAM_PREFIX_DELIMITER)[1]; } - @Override public String getRev() { return rev; } - @Override public void setRev(String rev) { this.rev = rev; } - @Override public String getElementId() { return elementId; } - @Override public void setElementId(String elementId) { this.elementId = elementId; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/PageResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/PageResult.java new file mode 100644 index 0000000000..118d23076a --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/PageResult.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.datalake; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import java.util.List; +import java.util.Map; + +@TsModel +@Deprecated(forRemoval = true, since = "0.92.0") +public class PageResult extends DataSeries { + + private int page; + + private int pageSum; + + public PageResult(int total, List headers, List> rows, int page, int pageSum, + Map tags) { + super(total, rows, headers, tags); + this.page = page; + this.pageSum = pageSum; + } + + public int getPage() { + return page; + } + + public void setPage(int page) { + this.page = page; + } + + public int getPageSum() { + return pageSum; + } + + public void setPageSum(int pageSum) { + this.pageSum = pageSum; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java index 268eb04cd9..9cc4319d9a 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/configuration/SpServiceConfiguration.java @@ -19,14 +19,13 @@ package org.apache.streampipes.model.extensions.configuration; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.google.gson.annotations.SerializedName; import java.util.List; @TsModel -public class SpServiceConfiguration implements Storable { +public class SpServiceConfiguration { protected @SerializedName("_rev") String rev; private @SerializedName("_id") String serviceGroup; @@ -76,14 +75,4 @@ public String getRev() { public void setRev(String rev) { this.rev = rev; } - - @Override - public String getElementId() { - return this.serviceGroup; - } - - @Override - public void setElementId(String elementId) { - this.serviceGroup = elementId; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java index 7fb9b8d1e3..fc01712a8c 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceRegistration.java @@ -19,14 +19,13 @@ import org.apache.streampipes.model.extensions.ExtensionItemDescription; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.google.gson.annotations.SerializedName; import java.util.Set; @TsModel -public class SpServiceRegistration implements Storable { +public class SpServiceRegistration { private String svcType; private String svcGroup; @@ -131,16 +130,6 @@ public void setRev(String rev) { this.rev = rev; } - @Override - public String getElementId() { - return this.svcId; - } - - @Override - public void setElementId(String elementId) { - this.svcId = elementId; - } - public String getScheme() { return scheme; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java index 572fc4a7f6..71b7fe428e 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTag.java @@ -17,11 +17,6 @@ */ package org.apache.streampipes.model.extensions.svcdiscovery; -import org.apache.streampipes.model.shared.annotation.TsModel; - -import java.util.Objects; - -@TsModel public class SpServiceTag { private static final String COLON = ":"; @@ -61,21 +56,4 @@ public String getValue() { public void setValue(String value) { this.value = value; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SpServiceTag that = (SpServiceTag) o; - return prefix == that.prefix && Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hash(prefix, value); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java index cea9c39da1..f412bdbe6f 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/extensions/svcdiscovery/SpServiceTagPrefix.java @@ -19,7 +19,6 @@ public enum SpServiceTagPrefix { SP_GROUP, - CUSTOM, // Is used for user-defined service tags provided via Envs.SP_SERVICE_TAGS ADAPTER, DATA_STREAM, DATA_PROCESSOR, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java b/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java index d7193d0608..4696179903 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/file/FileMetadata.java @@ -18,12 +18,11 @@ package org.apache.streampipes.model.file; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.google.gson.annotations.SerializedName; @TsModel -public class FileMetadata implements Storable { +public class FileMetadata { private @SerializedName("_id") String fileId; @@ -53,16 +52,6 @@ public void setRev(String rev) { this.rev = rev; } - @Override - public String getElementId() { - return this.fileId; - } - - @Override - public void setElementId(String elementId) { - this.fileId = elementId; - } - public String getFilename() { return filename; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorDescription.java index 9de8809bf8..0899eea570 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorDescription.java @@ -18,8 +18,10 @@ package org.apache.streampipes.model.graph; +import org.apache.streampipes.model.SpDataStream; import org.apache.streampipes.model.base.ConsumableStreamPipesEntity; import org.apache.streampipes.model.output.OutputStrategy; +import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.Cloner; import java.util.ArrayList; @@ -31,11 +33,14 @@ public class DataProcessorDescription extends ConsumableStreamPipesEntity { private List outputStrategies; + private String pathName; + private List category; public DataProcessorDescription(DataProcessorDescription other) { super(other); this.outputStrategies = new Cloner().strategies(other.getOutputStrategies()); + this.pathName = other.getPathName(); this.category = new Cloner().epaTypes(other.getCategory()); } @@ -45,6 +50,31 @@ public DataProcessorDescription() { this.category = new ArrayList<>(); } + public DataProcessorDescription(String uri, String name, String description, String iconUrl, + List spDataStreams, List staticProperties, + List outputStrategies) { + super(uri, name, description, iconUrl); + this.pathName = uri; + this.spDataStreams = spDataStreams; + this.staticProperties = staticProperties; + this.outputStrategies = outputStrategies; + } + + public DataProcessorDescription(String pathName, String name, String description, String iconUrl) { + super(pathName, name, description, iconUrl); + this.pathName = pathName; + spDataStreams = new ArrayList<>(); + staticProperties = new ArrayList<>(); + } + + public DataProcessorDescription(String pathName, String name, String description) { + super(pathName, name, description, ""); + this.pathName = pathName; + spDataStreams = new ArrayList<>(); + staticProperties = new ArrayList<>(); + } + + public List getCategory() { return category; } @@ -53,6 +83,14 @@ public void setCategory(List category) { this.category = category; } + public String getPathName() { + return pathName; + } + + public void setPathName(String pathName) { + this.pathName = pathName; + } + public List getOutputStrategies() { return outputStrategies; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java index 4414f2b749..64aed3b49b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataProcessorInvocation.java @@ -22,6 +22,7 @@ import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; import org.apache.streampipes.model.output.OutputStrategy; +import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.Cloner; import org.apache.streampipes.model.util.ElementIdGenerator; @@ -37,12 +38,15 @@ public class DataProcessorInvocation extends InvocableStreamPipesEntity implemen private List outputStrategies; + private String pathName; + private List category; public DataProcessorInvocation(DataProcessorDescription other) { super(); this.setName(other.getName()); this.setDescription(other.getDescription()); + this.setIconUrl(other.getIconUrl()); this.setInputStreams(other.getSpDataStreams()); this.setSupportedGrounding(other.getSupportedGrounding()); this.setStaticProperties(other.getStaticProperties()); @@ -52,11 +56,8 @@ public DataProcessorInvocation(DataProcessorDescription other) { this.setStreamRequirements(other.getSpDataStreams()); this.setAppId(other.getAppId()); this.setIncludesAssets(other.isIncludesAssets()); - this.setIncludesLocales(other.isIncludesLocales()); this.setIncludedAssets(other.getIncludedAssets()); - this.setIncludedLocales(other.getIncludedLocales()); this.setElementId(ElementIdGenerator.makeElementId(this)); - this.setVersion(other.getVersion()); this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } @@ -66,16 +67,51 @@ public DataProcessorInvocation(DataProcessorInvocation other) { if (other.getOutputStream() != null) { this.outputStream = new Cloner().stream(other.getOutputStream()); } + this.pathName = other.getPathName(); this.category = new Cloner().epaTypes(other.getCategory()); this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } + public DataProcessorInvocation(DataProcessorDescription sepa, String domId) { + this(sepa); + this.dom = domId; + this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; + } + public DataProcessorInvocation() { super(); inputStreams = new ArrayList<>(); this.serviceTagPrefix = SpServiceTagPrefix.DATA_PROCESSOR; } + public DataProcessorInvocation(String uri, String name, String description, String iconUrl, String pathName, + List spDataStreams, List staticProperties) { + super(uri, name, description, iconUrl, SpServiceTagPrefix.DATA_PROCESSOR); + this.pathName = pathName; + this.inputStreams = spDataStreams; + this.staticProperties = staticProperties; + } + + public DataProcessorInvocation(String uri, String name, String description, String iconUrl, String pathName) { + super(uri, name, description, iconUrl, SpServiceTagPrefix.DATA_PROCESSOR); + this.pathName = pathName; + inputStreams = new ArrayList<>(); + staticProperties = new ArrayList<>(); + } + + public boolean addInputStream(SpDataStream spDataStream) { + return inputStreams.add(spDataStream); + } + + + public String getPathName() { + return pathName; + } + + public void setPathName(String pathName) { + this.pathName = pathName; + } + public SpDataStream getOutputStream() { return outputStream; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkDescription.java index d491d62d20..40aa13ddd4 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkDescription.java @@ -30,11 +30,22 @@ public class DataSinkDescription extends ConsumableStreamPipesEntity { private List category; + public DataSinkDescription(String uri, String name, String description, String iconUrl) { + super(uri, name, description, iconUrl); + this.spDataStreams = new ArrayList<>(); + this.category = new ArrayList<>(); + } + public DataSinkDescription(DataSinkDescription other) { super(other); this.category = new Cloner().ecTypes(other.getCategory()); } + public DataSinkDescription(String uri, String name, String description) { + this(uri, name, description, ""); + this.category = new ArrayList<>(); + } + public DataSinkDescription() { super(); this.category = new ArrayList<>(); @@ -48,4 +59,5 @@ public void setCategory(List category) { this.category = category; } + } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java index bc95f269c9..4476c6651b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/graph/DataSinkInvocation.java @@ -20,6 +20,7 @@ import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix; +import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.util.ElementIdGenerator; import java.util.ArrayList; @@ -41,6 +42,7 @@ public DataSinkInvocation(DataSinkDescription other) { super(); this.setName(other.getName()); this.setDescription(other.getDescription()); + this.setIconUrl(other.getIconUrl()); this.setInputStreams(other.getSpDataStreams()); this.setSupportedGrounding(other.getSupportedGrounding()); this.setStaticProperties(other.getStaticProperties()); @@ -49,11 +51,14 @@ public DataSinkInvocation(DataSinkDescription other) { this.setStreamRequirements(other.getSpDataStreams()); this.setAppId(other.getAppId()); this.setIncludesAssets(other.isIncludesAssets()); - this.setIncludesLocales(other.isIncludesLocales()); this.setElementId(ElementIdGenerator.makeElementId(this)); this.setIncludedAssets(other.getIncludedAssets()); - this.setIncludedLocales(other.getIncludedLocales()); - this.setVersion(other.getVersion()); + this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; + } + + public DataSinkInvocation(DataSinkDescription sec, String domId) { + this(sec); + this.setDom(domId); this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } @@ -63,6 +68,14 @@ public DataSinkInvocation() { this.serviceTagPrefix = SpServiceTagPrefix.DATA_SINK; } + public List getStaticProperties() { + return staticProperties; + } + + public void setStaticProperties(List staticProperties) { + this.staticProperties = staticProperties; + } + public List getCategory() { return category; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/grounding/EventGrounding.java b/streampipes-model/src/main/java/org/apache/streampipes/model/grounding/EventGrounding.java index f22a632b11..ccadf9652c 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/grounding/EventGrounding.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/grounding/EventGrounding.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -32,18 +33,24 @@ public class EventGrounding { private List transportProtocols; + private List transportFormats; + public EventGrounding() { super(); + this.transportFormats = new ArrayList<>(); this.transportProtocols = new ArrayList<>(); } - public EventGrounding(TransportProtocol transportProtocol) { + public EventGrounding(TransportProtocol transportProtocol, TransportFormat transportFormat) { this(); - this.transportProtocols = Collections.singletonList(transportProtocol); + this.transportFormats = new ArrayList<>(); + this.transportFormats.add(transportFormat); + this.transportProtocols = Arrays.asList(transportProtocol); } public EventGrounding(EventGrounding other) { this.transportProtocols = new Cloner().protocols(other.getTransportProtocols()); + this.transportFormats = new Cloner().transportFormats(other.getTransportFormats()); } public List getTransportProtocols() { @@ -56,7 +63,7 @@ public void setTransportProtocols(List transportProtocols) { @JsonIgnore public TransportProtocol getTransportProtocol() { - if (transportProtocols.isEmpty()) { + if (transportProtocols.size() == 0) { return null; } else { return transportProtocols.get(0); @@ -66,4 +73,13 @@ public TransportProtocol getTransportProtocol() { public void setTransportProtocol(TransportProtocol transportProtocol) { this.transportProtocols = Collections.singletonList(transportProtocol); } + + public List getTransportFormats() { + return transportFormats; + } + + public void setTransportFormats(List transportFormats) { + this.transportFormats = transportFormats; + } + } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Category.java b/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Category.java new file mode 100644 index 0000000000..b5b3b45e40 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Category.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.labeling; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; + +@TsModel +public class Category { + private String name; + private String internalName; + private String superLabelId; + + @JsonProperty("_id") + private @SerializedName("_id") String id; + + @JsonProperty("_rev") + private @SerializedName("_rev") String rev; + + public Category() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getInternalName() { + return internalName; + } + + public void setInternalName(String internalName) { + this.internalName = internalName; + } + + public String getSuperLabelId() { + return superLabelId; + } + + public void setSuperLabel(String superLabelId) { + this.superLabelId = superLabelId; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Label.java b/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Label.java new file mode 100644 index 0000000000..91b4c68faf --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/labeling/Label.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.labeling; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; + +@TsModel +public class Label { + private String name; + private String color; + private String internalName; + private String categoryId; + + @JsonProperty("_id") + private @SerializedName("_id") String id; + + @JsonProperty("_rev") + private @SerializedName("_rev") String rev; + + public Label() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public String getInternalName() { + return internalName; + } + + public void setInternalName(String internalName) { + this.internalName = internalName; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + public String getCategoryId() { + return categoryId; + } + + public void setCategoryId(String categoryId) { + this.categoryId = categoryId; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/message/Notifications.java b/streampipes-model/src/main/java/org/apache/streampipes/model/message/Notifications.java index 965a84257d..4e14af9262 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/message/Notifications.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/message/Notifications.java @@ -44,12 +44,8 @@ public static ErrorMessage error(NotificationType type) { return new ErrorMessage(new Notification(type.name(), type.description())); } - public static ErrorMessage error(String message, String description) { - return new ErrorMessage(new Notification(message, description)); - } - public static ErrorMessage error(String message) { - return error(message, ""); + return new ErrorMessage(new Notification(message, "")); } public static ErrorMessage error(NotificationType type, String info) { diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/ElementStatusInfoSettings.java b/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/ElementStatusInfoSettings.java new file mode 100644 index 0000000000..f3731d2a0d --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/ElementStatusInfoSettings.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.monitoring; + +public class ElementStatusInfoSettings { + + private String elementIdentifier; + + private String kafkaHost; + + private int kafkaPort; + + private String errorTopic; + + private String statsTopic; + + public ElementStatusInfoSettings() { + super(); + } + + public ElementStatusInfoSettings(ElementStatusInfoSettings other) { + this.kafkaHost = other.getKafkaHost(); + this.kafkaPort = other.getKafkaPort(); + this.errorTopic = other.getErrorTopic(); + this.statsTopic = other.getStatsTopic(); + } + + + public ElementStatusInfoSettings(String elementIdentifier, String kafkaHost, int kafkaPort, + String errorTopic, String + statsTopic) { + this.elementIdentifier = elementIdentifier; + this.kafkaHost = kafkaHost; + this.kafkaPort = kafkaPort; + this.errorTopic = errorTopic; + this.statsTopic = statsTopic; + } + + public String getKafkaHost() { + return kafkaHost; + } + + public void setKafkaHost(String kafkaHost) { + this.kafkaHost = kafkaHost; + } + + public int getKafkaPort() { + return kafkaPort; + } + + public void setKafkaPort(int kafkaPort) { + this.kafkaPort = kafkaPort; + } + + public String getErrorTopic() { + return errorTopic; + } + + public void setErrorTopic(String errorTopic) { + this.errorTopic = errorTopic; + } + + public String getStatsTopic() { + return statsTopic; + } + + public void setStatsTopic(String statsTopic) { + this.statsTopic = statsTopic; + } + + public String getElementIdentifier() { + return elementIdentifier; + } + + public void setElementIdentifier(String elementIdentifier) { + this.elementIdentifier = elementIdentifier; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/SpLogMessage.java b/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/SpLogMessage.java index d7b5f290c8..e1123a26cc 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/SpLogMessage.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/monitoring/SpLogMessage.java @@ -38,8 +38,7 @@ public static SpLogMessage from(Exception exception) { public static SpLogMessage from(Exception exception, String detail) { - var cause = exception.getCause() != null ? exception.getCause().getMessage() : exception.getMessage(); - + String cause = exception.getCause() != null ? exception.getCause().getMessage() : exception.getMessage(); return new SpLogMessage( SpLogLevel.ERROR, exception.getMessage(), @@ -78,7 +77,7 @@ public SpLogMessage(SpLogMessage other) { this.fullStackTrace = other.getFullStackTrace(); } - private SpLogMessage(SpLogLevel level, + public SpLogMessage(SpLogLevel level, String title, String detail) { this.level = level; @@ -86,7 +85,7 @@ private SpLogMessage(SpLogLevel level, this.detail = detail; } - private SpLogMessage(SpLogLevel level, + public SpLogMessage(SpLogLevel level, String title, String detail, String fullStackTrace, diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/AppendOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/AppendOutputStrategy.java index fa7ccca3c4..9d22313bc4 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/AppendOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/AppendOutputStrategy.java @@ -26,6 +26,8 @@ public class AppendOutputStrategy extends OutputStrategy { + private static final long serialVersionUID = 7202888911899551012L; + private List eventProperties; public AppendOutputStrategy() { @@ -33,11 +35,6 @@ public AppendOutputStrategy() { eventProperties = new ArrayList<>(); } - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } - public AppendOutputStrategy(AppendOutputStrategy other) { super(other); this.setEventProperties(new Cloner().properties(other.getEventProperties())); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomOutputStrategy.java index fa859aeef4..4468ba14f2 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomOutputStrategy.java @@ -23,6 +23,8 @@ public class CustomOutputStrategy extends OutputStrategy { + private static final long serialVersionUID = -5858193127308435472L; + private List selectedPropertyKeys; private boolean outputRight; @@ -75,9 +77,4 @@ public List getAvailablePropertyKeys() { public void setAvailablePropertyKeys(List availablePropertyKeys) { this.availablePropertyKeys = availablePropertyKeys; } - - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomTransformOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomTransformOutputStrategy.java index 426a8d29eb..91d1a6ae3d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomTransformOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/CustomTransformOutputStrategy.java @@ -32,6 +32,7 @@ public CustomTransformOutputStrategy() { this.eventProperties = new ArrayList<>(); } + public CustomTransformOutputStrategy(CustomTransformOutputStrategy other) { super(other); this.eventProperties = new Cloner().properties(other.getEventProperties()); @@ -44,9 +45,4 @@ public List getEventProperties() { public void setEventProperties(List eventProperties) { this.eventProperties = eventProperties; } - - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/FixedOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/FixedOutputStrategy.java index 1558aada49..e698348efd 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/FixedOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/FixedOutputStrategy.java @@ -50,9 +50,5 @@ public void setEventProperties(List eventProperties) { this.eventProperties = eventProperties; } - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/KeepOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/KeepOutputStrategy.java index 0c8b9b0e00..d455e7b4c2 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/KeepOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/KeepOutputStrategy.java @@ -63,9 +63,4 @@ public boolean isKeepBoth() { public void setKeepBoth(boolean keepBoth) { this.keepBoth = keepBoth; } - - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/ListOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/ListOutputStrategy.java index b2e1f4ba51..4040d9387c 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/ListOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/ListOutputStrategy.java @@ -46,9 +46,5 @@ public void setPropertyName(String propertyName) { this.propertyName = propertyName; } - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/OutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/OutputStrategy.java index ab19a8d89b..05484f1679 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/OutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/OutputStrategy.java @@ -75,6 +75,4 @@ public List getRenameRules() { public void setRenameRules(List renameRules) { this.renameRules = renameRules; } - - public abstract void accept(OutputStrategyVisitor visitor); } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/TransformOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/TransformOutputStrategy.java index 7e9de992ad..1ca951654b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/TransformOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/TransformOutputStrategy.java @@ -43,9 +43,4 @@ public List getTransformOperations() { public void setTransformOperations(List transformOperations) { this.transformOperations = transformOperations; } - - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/output/UserDefinedOutputStrategy.java b/streampipes-model/src/main/java/org/apache/streampipes/model/output/UserDefinedOutputStrategy.java index 622d1e25ec..7f8f725eef 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/output/UserDefinedOutputStrategy.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/output/UserDefinedOutputStrategy.java @@ -42,9 +42,4 @@ public List getEventProperties() { public void setEventProperties(List eventProperties) { this.eventProperties = eventProperties; } - - @Override - public void accept(OutputStrategyVisitor visitor) { - visitor.visit(this); - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ElementComposition.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ElementComposition.java new file mode 100644 index 0000000000..4e149a2aa7 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/ElementComposition.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.pipeline; + +import org.apache.streampipes.model.SpDataStream; +import org.apache.streampipes.model.graph.DataProcessorInvocation; + +import java.util.ArrayList; +import java.util.List; + + +public abstract class ElementComposition { + + protected List sepas; + + protected List streams; + + protected String name; + protected String description; + + public ElementComposition() { + this.sepas = new ArrayList<>(); + this.streams = new ArrayList<>(); + } + + public List getSepas() { + return sepas; + } + + public void setSepas(List sepas) { + this.sepas = sepas; + } + + public List getStreams() { + return streams; + } + + public void setStreams(List streams) { + this.streams = streams; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/Pipeline.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/Pipeline.java index 790faf3eb5..7055ca29aa 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/Pipeline.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/Pipeline.java @@ -18,11 +18,8 @@ package org.apache.streampipes.model.pipeline; -import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.graph.DataProcessorInvocation; import org.apache.streampipes.model.graph.DataSinkInvocation; import org.apache.streampipes.model.shared.annotation.TsModel; -import org.apache.streampipes.model.shared.api.Storable; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.gson.annotations.SerializedName; @@ -31,7 +28,7 @@ import java.util.List; @TsModel -public class Pipeline implements Storable { +public class Pipeline extends ElementComposition { private List actions; @@ -45,6 +42,8 @@ public class Pipeline implements Storable { private boolean publicElement; private String createdByUser; + + private List pipelineCategories; private List pipelineNotifications; private PipelineHealthStatus healthStatus; @@ -57,19 +56,11 @@ public class Pipeline implements Storable { private @SerializedName("_rev") String rev; - protected List sepas; - - protected List streams; - - protected String name; - protected String description; - public Pipeline() { super(); this.actions = new ArrayList<>(); this.pipelineNotifications = new ArrayList<>(); - this.sepas = new ArrayList<>(); - this.streams = new ArrayList<>(); + this.pipelineCategories = new ArrayList<>(); } public List getActions() { @@ -104,6 +95,7 @@ public void setPublicElement(boolean publicElement) { this.publicElement = publicElement; } + public String getCreatedByUser() { return createdByUser; } @@ -128,14 +120,12 @@ public void setRev(String rev) { this.rev = rev; } - @Override - public String getElementId() { - return pipelineId; + public List getPipelineCategories() { + return pipelineCategories; } - @Override - public void setElementId(String elementId) { - this.pipelineId = elementId; + public void setPipelineCategories(List pipelineCategories) { + this.pipelineCategories = pipelineCategories; } public long getCreatedAt() { @@ -178,38 +168,6 @@ public void setValid(boolean valid) { this.valid = valid; } - public List getSepas() { - return sepas; - } - - public void setSepas(List sepas) { - this.sepas = sepas; - } - - public List getStreams() { - return streams; - } - - public void setStreams(List streams) { - this.streams = streams; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - public Pipeline clone() { Pipeline pipeline = new Pipeline(); pipeline.setName(name); @@ -218,6 +176,7 @@ public Pipeline clone() { pipeline.setStreams(streams); pipeline.setActions(actions); pipeline.setCreatedByUser(createdByUser); + pipeline.setPipelineCategories(pipelineCategories); pipeline.setCreatedAt(createdAt); pipeline.setPipelineId(pipelineId); pipeline.setHealthStatus(healthStatus); @@ -227,4 +186,6 @@ public Pipeline clone() { return pipeline; } + + } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineCategory.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineCategory.java new file mode 100644 index 0000000000..95dd7d618a --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineCategory.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.pipeline; + +import org.apache.streampipes.model.shared.annotation.TsModel; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; + +@TsModel +public class PipelineCategory { + + private String categoryName; + private String categoryDescription; + + @JsonProperty("_id") + private @SerializedName("_id") String categoryId; + + @JsonProperty("_rev") + private @SerializedName("_rev") String rev; + + public PipelineCategory() { + + } + + public PipelineCategory(String categoryName, String categoryDescription) { + super(); + this.categoryName = categoryName; + this.categoryDescription = categoryDescription; + } + + public String getCategoryName() { + return categoryName; + } + + public void setCategoryName(String categoryName) { + this.categoryName = categoryName; + } + + public String getCategoryDescription() { + return categoryDescription; + } + + public void setCategoryDescription(String categoryDescription) { + this.categoryDescription = categoryDescription; + } + + public String getCategoryId() { + return categoryId; + } + + public void setCategoryId(String categoryId) { + this.categoryId = categoryId; + } + + public String getRev() { + return rev; + } + + public void setRev(String rev) { + this.rev = rev; + } + + +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendation.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendation.java index b202ace3e9..ae1d04cfcf 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendation.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendation.java @@ -23,6 +23,8 @@ public class PipelineElementRecommendation { private String elementId; private String name; private String description; + private Float weight; + private Integer count; public PipelineElementRecommendation() { @@ -58,4 +60,19 @@ public void setDescription(String description) { this.description = description; } + public Float getWeight() { + return weight; + } + + public void setWeight(Float weight) { + this.weight = weight; + } + + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendationMessage.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendationMessage.java index 4823bfe818..d57de8b7ca 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendationMessage.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineElementRecommendationMessage.java @@ -27,11 +27,13 @@ public class PipelineElementRecommendationMessage { private List possibleElements; + private List recommendedElements; private boolean success; public PipelineElementRecommendationMessage() { this.possibleElements = new ArrayList<>(); + this.recommendedElements = new ArrayList<>(); this.success = true; } @@ -54,4 +56,15 @@ public boolean isSuccess() { public void setSuccess(boolean success) { this.success = success; } + + public List getRecommendedElements() { + return recommendedElements; + } + + public void setRecommendedElements( + List recommendedElements) { + this.recommendedElements = recommendedElements; + } + + } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java index 585598890a..fc2ce9c6d3 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/pipeline/PipelineModification.java @@ -37,8 +37,19 @@ public class PipelineModification { private List inputStreams; private SpDataStream outputStream; + public PipelineModification(String domId, String elementId, + List staticProperties) { + super(); + this.domId = domId; + this.elementId = elementId; + this.staticProperties = staticProperties; + this.inputStreams = new ArrayList<>(); + this.outputStrategies = new ArrayList<>(); + this.validationInfos = new ArrayList<>(); + } + public PipelineModification() { - validationInfos = new ArrayList<>(); + } public String getDomId() { @@ -81,6 +92,10 @@ public void setInputStreams(List inputStreams) { this.inputStreams = inputStreams; } + public void addInputStream(SpDataStream inputStream) { + this.inputStreams.add(inputStream); + } + public boolean isPipelineElementValid() { return pipelineElementValid; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/preview/PipelinePreviewModel.java b/streampipes-model/src/main/java/org/apache/streampipes/model/preview/PipelinePreviewModel.java index 021c6d428f..b63db03d75 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/preview/PipelinePreviewModel.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/preview/PipelinePreviewModel.java @@ -19,18 +19,16 @@ import org.apache.streampipes.model.shared.annotation.TsModel; -import java.util.HashMap; -import java.util.Map; +import java.util.List; @TsModel public class PipelinePreviewModel { private String previewId; - private Map elementIdMappings; + private List supportedPipelineElementDomIds; public PipelinePreviewModel() { - this.elementIdMappings = new HashMap<>(); } public String getPreviewId() { @@ -41,11 +39,11 @@ public void setPreviewId(String previewId) { this.previewId = previewId; } - public Map getElementIdMappings() { - return elementIdMappings; + public List getSupportedPipelineElementDomIds() { + return supportedPipelineElementDomIds; } - public void setElementIdMappings(Map elementIdMappings) { - this.elementIdMappings = elementIdMappings; + public void setSupportedPipelineElementDomIds(List supportedPipelineElementDomIds) { + this.supportedPipelineElementDomIds = supportedPipelineElementDomIds; } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/ContainerProvidedOptionsParameterRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/ContainerProvidedOptionsParameterRequest.java new file mode 100644 index 0000000000..ced86d5a32 --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/ContainerProvidedOptionsParameterRequest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.streampipes.model.runtime; + +import org.apache.streampipes.model.SpDataStream; +import org.apache.streampipes.model.staticproperty.StaticProperty; + +import java.util.List; + +public class ContainerProvidedOptionsParameterRequest { + + private List staticProperties; + private List inputStreams; + + private String appId; + private String belongsTo; + private String runtimeResolvableInternalId; + + public ContainerProvidedOptionsParameterRequest(List staticProperties, + List inputStreams, String belongsTo) { + this.staticProperties = staticProperties; + this.inputStreams = inputStreams; + this.belongsTo = belongsTo; + } + + public ContainerProvidedOptionsParameterRequest() { + + } + + public List getStaticProperties() { + return staticProperties; + } + + public void setStaticProperties(List staticProperties) { + this.staticProperties = staticProperties; + } + + public List getInputStreams() { + return inputStreams; + } + + public void setInputStreams(List inputStreams) { + this.inputStreams = inputStreams; + } + + public String getBelongsTo() { + return belongsTo; + } + + public void setBelongsTo(String belongsTo) { + this.belongsTo = belongsTo; + } + + public String getRuntimeResolvableInternalId() { + return runtimeResolvableInternalId; + } + + public void setRuntimeResolvableInternalId(String runtimeResolvableInternalId) { + this.runtimeResolvableInternalId = runtimeResolvableInternalId; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/RuntimeOptionsRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/RuntimeOptionsRequest.java index 3e0417c8bb..0f5efe2411 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/RuntimeOptionsRequest.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/runtime/RuntimeOptionsRequest.java @@ -18,7 +18,6 @@ package org.apache.streampipes.model.runtime; import org.apache.streampipes.model.SpDataStream; -import org.apache.streampipes.model.deployment.ExtensionDeploymentConfiguration; import org.apache.streampipes.model.shared.annotation.TsModel; import org.apache.streampipes.model.staticproperty.StaticProperty; @@ -38,8 +37,6 @@ public class RuntimeOptionsRequest { protected List inputStreams; - protected ExtensionDeploymentConfiguration deploymentConfiguration; - private String belongsTo; public RuntimeOptionsRequest() { @@ -98,12 +95,4 @@ public String getBelongsTo() { public void setBelongsTo(String belongsTo) { this.belongsTo = belongsTo; } - - public ExtensionDeploymentConfiguration getDeploymentConfiguration() { - return deploymentConfiguration; - } - - public void setDeploymentConfiguration(ExtensionDeploymentConfiguration deploymentConfiguration) { - this.deploymentConfiguration = deploymentConfiguration; - } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventProperty.java b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventProperty.java index ceddacc02b..73ea4da700 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventProperty.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventProperty.java @@ -19,11 +19,15 @@ package org.apache.streampipes.model.schema; import org.apache.streampipes.model.util.ElementIdGenerator; +import org.apache.streampipes.model.util.ListUtils; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.net.URI; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -36,18 +40,31 @@ public abstract class EventProperty { protected static final String PREFIX = "urn:streampipes.org:spi:"; + private static final long serialVersionUID = 7079045979946059387L; private String elementId; private String label; + private String description; + private String runtimeName; - private String semanticType; + + private boolean required; + + private List domainProperties; + private String propertyScope; + + private int index = 0; + private String runtimeId; + private Map additionalMetadata; + public EventProperty() { this.elementId = ElementIdGenerator.makeElementId(EventProperty.class); + this.domainProperties = new ArrayList<>(); this.additionalMetadata = new HashMap<>(); } @@ -55,17 +72,24 @@ public EventProperty(EventProperty other) { this.elementId = other.getElementId(); this.label = other.getLabel(); this.description = other.getDescription(); + this.required = other.isRequired(); this.runtimeName = other.getRuntimeName(); + this.domainProperties = other.getDomainProperties(); this.propertyScope = other.getPropertyScope(); this.runtimeId = other.getRuntimeId(); + this.index = other.getIndex(); this.additionalMetadata = other.getAdditionalMetadata(); - this.semanticType = other.getSemanticType(); } - public EventProperty(String propertyName, String semanticType) { + public EventProperty(List subClassOf) { + this(); + this.domainProperties = subClassOf; + } + + public EventProperty(String propertyName, List subClassOf) { this(); this.runtimeName = propertyName; - this.semanticType = semanticType; + this.domainProperties = subClassOf; } public EventProperty(String propertyName) { @@ -86,6 +110,22 @@ public void setRuntimeName(String propertyName) { this.runtimeName = propertyName; } + public boolean isRequired() { + return required; + } + + public void setRequired(boolean required) { + this.required = required; + } + + public List getDomainProperties() { + return domainProperties; + } + + public void setDomainProperties(List subClassOf) { + this.domainProperties = subClassOf; + } + public String getLabel() { return label; } @@ -118,6 +158,14 @@ public void setRuntimeId(String runtimeId) { this.runtimeId = runtimeId; } + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + public String getElementId() { return elementId; } @@ -134,17 +182,9 @@ public void setAdditionalMetadata(Map additionalMetadata) { this.additionalMetadata = additionalMetadata; } - public String getSemanticType() { - return semanticType; - } - - public void setSemanticType(String semanticType) { - this.semanticType = semanticType; - } - @Override public int hashCode() { - return Objects.hash(elementId, label, description, runtimeName, semanticType, propertyScope, + return Objects.hash(elementId, label, description, runtimeName, required, domainProperties, propertyScope, index, runtimeId); } @@ -158,12 +198,14 @@ public boolean equals(Object o) { } EventProperty that = (EventProperty) o; - return Objects.equals(label, that.label) + return required == that.required + && index == that.index + && Objects.equals(label, that.label) && Objects.equals(description, that.description) && Objects.equals(runtimeName, that.runtimeName) && Objects.equals(propertyScope, that.propertyScope) && Objects.equals(runtimeId, that.runtimeId) - && Objects.equals(semanticType, that.semanticType); + && ListUtils.isEqualList(this.domainProperties, that.domainProperties); } @Override @@ -173,8 +215,10 @@ public String toString() { + ", label='" + label + '\'' + ", description='" + description + '\'' + ", runtimeName='" + runtimeName + '\'' - + ", semanticType=" + semanticType + + ", required=" + required + + ", domainProperties=" + domainProperties + ", propertyScope='" + propertyScope + '\'' + + ", index=" + index + ", runtimeId='" + runtimeId + '\'' + '}'; } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyList.java b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyList.java index 98af25f1d1..799a6ee568 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyList.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyList.java @@ -20,6 +20,8 @@ import org.apache.streampipes.model.util.Cloner; +import java.net.URI; +import java.util.List; import java.util.Objects; public class EventPropertyList extends EventProperty { @@ -49,9 +51,10 @@ public EventPropertyList(EventProperty listProperty) { this.eventProperty = listProperty; } - public EventPropertyList(String propertyName, EventProperty eventProperty, String semanticType) { - super(propertyName, semanticType); + public EventPropertyList(String propertyName, EventProperty eventProperty, List domainProperties) { + super(propertyName); this.eventProperty = eventProperty; + this.setDomainProperties(domainProperties); } public EventProperty getEventProperty() { diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyPrimitive.java b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyPrimitive.java index 0822c163a4..e116a67a02 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyPrimitive.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/schema/EventPropertyPrimitive.java @@ -21,12 +21,17 @@ import org.apache.streampipes.model.util.Cloner; import java.net.URI; +import java.util.List; import java.util.Objects; public class EventPropertyPrimitive extends EventProperty { + private static final long serialVersionUID = 665989638281665875L; + private String runtimeType; + private URI measurementUnit; + private ValueSpecification valueSpecification; public EventPropertyPrimitive() { @@ -43,11 +48,13 @@ public EventPropertyPrimitive(EventPropertyPrimitive other) { } } - public EventPropertyPrimitive(String runtimeType, - String runtimeName, - String measurementUnit, - String semanticType) { - super(runtimeName, semanticType); + public EventPropertyPrimitive(List subClassOf) { + super(subClassOf); + } + + public EventPropertyPrimitive(String runtimeType, String runtimeName, + String measurementUnit, List subClassOf) { + super(runtimeName, subClassOf); this.runtimeType = runtimeType; //this.measurementUnit = measurementUnit; } @@ -89,8 +96,8 @@ public boolean equals(Object o) { } EventPropertyPrimitive that = (EventPropertyPrimitive) o; return Objects.equals(runtimeType, that.runtimeType) - && Objects.equals(measurementUnit, that.measurementUnit) - && Objects.equals(valueSpecification, that.valueSpecification); + && Objects.equals(measurementUnit, that.measurementUnit) + && Objects.equals(valueSpecification, that.valueSpecification); } @Override @@ -101,9 +108,9 @@ public int hashCode() { @Override public String toString() { return "EventPropertyPrimitive{" - + "runtimeType='" + runtimeType + '\'' - + ", measurementUnit=" + measurementUnit - + ", valueSpecification=" + valueSpecification - + '}'; + + "runtimeType='" + runtimeType + '\'' + + ", measurementUnit=" + measurementUnit + + ", valueSpecification=" + valueSpecification + + '}'; } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java index 7a9400b17a..5e898c36e8 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DefaultStaticPropertyVisitor.java @@ -20,16 +20,6 @@ public abstract class DefaultStaticPropertyVisitor implements StaticPropertyVisitor { - protected boolean ignoreValidation; - - public DefaultStaticPropertyVisitor(boolean ignoreValidation) { - this.ignoreValidation = ignoreValidation; - } - - public DefaultStaticPropertyVisitor() { - this(false); - } - @Override public void visit(CollectionStaticProperty collectionStaticProperty) { collectionStaticProperty.getStaticPropertyTemplate().accept(this); diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DomainStaticProperty.java b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DomainStaticProperty.java new file mode 100644 index 0000000000..4c975b5a9f --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/DomainStaticProperty.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.staticproperty; + +import org.apache.streampipes.model.util.Cloner; + +import java.util.List; + +public class DomainStaticProperty extends StaticProperty { + + private static final long serialVersionUID = 1L; + + private String requiredClass; + + private List supportedProperties; + + public DomainStaticProperty() { + super(StaticPropertyType.DomainStaticProperty); + } + + public DomainStaticProperty(String internalName, String label, String description, + List supportedProperties) { + super(StaticPropertyType.DomainStaticProperty, internalName, label, description); + this.supportedProperties = supportedProperties; + } + + public DomainStaticProperty(String internalName, String label, String description, String requiredClass, + List supportedProperties) { + this(internalName, label, description, supportedProperties); + this.requiredClass = requiredClass; + } + + public DomainStaticProperty(DomainStaticProperty other) { + super(other); + this.requiredClass = other.getRequiredClass(); + this.supportedProperties = new Cloner().supportedProperties(other.getSupportedProperties()); + } + + public String getRequiredClass() { + return requiredClass; + } + + public void setRequiredClass(String requiredClass) { + this.requiredClass = requiredClass; + } + + public List getSupportedProperties() { + return supportedProperties; + } + + public void setSupportedProperties(List supportedProperties) { + this.supportedProperties = supportedProperties; + } + + @Override + public void accept(StaticPropertyVisitor visitor) { + visitor.visit(this); + } + +} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/RemoteOneOfStaticProperty.java b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/RemoteOneOfStaticProperty.java new file mode 100644 index 0000000000..b058d44c0b --- /dev/null +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/staticproperty/RemoteOneOfStaticProperty.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.model.staticproperty; + + +import org.apache.streampipes.model.util.Cloner; + +import java.util.ArrayList; +import java.util.List; + +@Deprecated +public class RemoteOneOfStaticProperty extends StaticProperty { + + private static final long serialVersionUID = 3483290363677184344L; + + private List diff --git a/streampipes-sdk/pom.xml b/streampipes-sdk/pom.xml index 103a2567c8..abf1f370b7 100644 --- a/streampipes-sdk/pom.xml +++ b/streampipes-sdk/pom.xml @@ -21,7 +21,7 @@ org.apache.streampipes streampipes-parent - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT 4.0.0 @@ -32,22 +32,22 @@ org.apache.streampipes streampipes-commons - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-extensions-api - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-model - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT org.apache.streampipes streampipes-vocabulary - 0.97.0-SNAPSHOT + 0.95.1-SNAPSHOT diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/StaticProperties.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/StaticProperties.java index 1793698efb..1c82339f6c 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/StaticProperties.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/StaticProperties.java @@ -19,7 +19,6 @@ package org.apache.streampipes.sdk; import org.apache.streampipes.model.schema.PropertyScope; -import org.apache.streampipes.model.staticproperty.AnyStaticProperty; import org.apache.streampipes.model.staticproperty.CodeInputStaticProperty; import org.apache.streampipes.model.staticproperty.CollectionStaticProperty; import org.apache.streampipes.model.staticproperty.FileStaticProperty; @@ -33,6 +32,7 @@ import org.apache.streampipes.model.staticproperty.RuntimeResolvableOneOfStaticProperty; import org.apache.streampipes.model.staticproperty.RuntimeResolvableTreeInputStaticProperty; import org.apache.streampipes.model.staticproperty.SecretStaticProperty; +import org.apache.streampipes.model.staticproperty.SelectionStaticProperty; import org.apache.streampipes.model.staticproperty.StaticProperty; import org.apache.streampipes.model.staticproperty.StaticPropertyAlternative; import org.apache.streampipes.model.staticproperty.StaticPropertyAlternatives; @@ -59,6 +59,10 @@ public static StaticPropertyAlternatives alternatives(Label label, List staticProperties = Arrays.asList(sp); + for (int i = 0; i < staticProperties.size(); i++) { + staticProperties.get(i).setIndex(i); + } return new StaticPropertyGroup(label.getInternalId(), label.getLabel(), label.getDescription(), staticProperties); } @@ -234,27 +241,19 @@ public static OneOfStaticProperty singleValueSelection(Label label, List