From 1082ac2844b12d0a94fbfa11bbabce82136d3177 Mon Sep 17 00:00:00 2001 From: Tim <50115603+bossenti@users.noreply.github.com> Date: Tue, 30 Apr 2024 09:04:04 +0200 Subject: [PATCH] refactor: restructure data explorer modules (#2795) * build: add dedicated influx storage module * refactor: move influx specific implementations to `DataExplorerInfluxQueryExecutor` * style: fix indentation * fix: improve result check * refactor: restructure module `streampipes-data-explorer` and `streampipes-data-explorer-commons` * refactor: add description tag for module `streampipes-data-explorer` * style: add missing headers * refactor: introduce module streampipes-data-explorer-export * refactor: add module description * chore: add missing file header * refactor: remove data explorer commons dependencies --- pom.xml | 7 +- streampipes-data-explorer-api/pom.xml | 61 +++++++++++++++ .../api/IDataExplorerSchemaManagement.java | 0 .../api}/IDataLakeQueryBuilder.java | 8 +- .../dataexplorer/api/IQueryStatement.java | 2 - streampipes-data-explorer-export/pom.xml | 56 ++++++++++++++ .../export}/ConfiguredCsvOutputWriter.java | 13 ++-- .../export}/ConfiguredJsonOutputWriter.java | 8 +- .../export}/ConfiguredOutputWriter.java | 4 +- .../dataexplorer/export}/OutputFormat.java | 2 +- .../export}/item/CsvItemGenerator.java | 2 +- .../export}/item/ItemGenerator.java | 2 +- .../export}/item/JsonItemGenerator.java | 2 +- .../TestConfiguredCsvOutputWriter.java | 4 +- .../TestConfiguredJsonOutputWriter.java | 4 +- .../export}/TestConfiguredOutputWriter.java | 2 +- .../export}/TestCsvItemGenerator.java | 4 +- .../export}/TestItemGenerator.java | 2 +- .../export}/TestJsonItemGenerator.java | 4 +- .../pom.xml | 75 ++++++++++++------- .../DataExplorerInfluxQueryExecutor.java | 10 ++- .../influx/DataLakeInfluxQueryBuilder.java | 8 +- .../influx/InfluxDbReservedKeywords.java | 2 +- .../dataexplorer}/influx/InfluxStore.java | 8 +- .../dataexplorer}/influx/PropertyHandler.java | 3 +- .../influx}/RawFieldSerializer.java | 2 +- .../influx}/auth/AuthInterceptor.java | 2 +- .../influx/auth}/InfluxAuthMode.java | 2 +- .../influx/client}/InfluxClientProvider.java | 3 +- .../influx/client}/InfluxClientUtils.java | 4 +- .../client}/InfluxConnectionSettings.java | 3 +- .../migrate}/AutoAggregationHandler.java | 38 ++++------ .../migrate}/DataExplorerQueryManagement.java | 9 +-- .../influx/migrate}/DataExplorerUtils.java | 6 +- .../migrate}/DataLakeMeasurementCount.java | 6 +- .../influx/migrate}/DeleteDataQuery.java | 4 +- .../IDataExplorerQueryManagement.java | 6 +- .../influx/migrate}/ImageStore.java | 2 +- .../influx/migrate}/ImageStoreUtils.java | 2 +- .../influx/migrate}/QueryResultProvider.java | 13 ++-- .../migrate}/StreamedQueryResultProvider.java | 20 +++-- .../influx/migrate}/TimeSeriesStore.java | 7 +- .../influx/migrate/package-info.java | 25 +++++++ .../influx/sanitize}/InfluxNameSanitizer.java | 4 +- .../sanitize}/MeasureNameSanitizer.java | 2 +- .../influx}/DataLakeQueryBuilderTest.java | 4 +- .../dataexplorer}/influx/InfluxStoreTest.java | 22 +++--- .../influx}/SelectQueryParamsTest.java | 7 +- .../influx}/TestRawFieldSerializer.java | 2 +- .../client}/InfluxClientProviderTest.java | 31 ++++---- .../sanitize}/MeasureNameSanitizerTest.java | 2 +- .../utils/ProvidedQueryParameterBuilder.java | 6 +- streampipes-data-explorer-management/pom.xml | 55 ++++++++++++++ streampipes-data-explorer/pom.xml | 21 +++--- .../ProvidedRestQueryParamConverter.java | 67 ++++++++--------- .../dataexplorer/param/SelectQueryParams.java | 2 +- .../param/model/FillClauseParams.java | 2 +- .../param/model/GroupByTagsClauseParams.java | 2 +- .../param/model/GroupByTimeClauseParams.java | 2 +- .../param/model/LimitClauseParams.java | 2 +- .../param/model/OffsetClauseParams.java | 2 +- .../param/model/OrderByClauseParams.java | 4 +- .../param/model/SelectClauseParams.java | 3 +- .../param/model/SelectColumn.java | 3 +- .../param/model/WhereClauseParams.java | 4 +- .../query/DataExplorerQueryExecutor.java | 8 +- .../DataExplorerSchemaManagementTest.java | 1 + .../param/WhereStatementParamsTest.java | 2 +- .../streampipes-connectors-influx/pom.xml | 2 +- .../influx/adapter/InfluxDbClient.java | 4 +- .../influx/shared/InfluxConfigs.java | 2 +- .../influx/shared/SharedInfluxClient.java | 4 +- .../influx/sink/InfluxDbClient.java | 4 +- .../streampipes-sinks-internal-jvm/pom.xml | 2 +- .../internal/jvm/datalake/DataLakeSink.java | 2 +- .../model/datalake}/AggregationFunction.java | 2 +- .../datalake}/DataLakeQueryOrdering.java | 2 +- .../model/datalake}/FilterCondition.java | 2 +- .../param/ProvidedRestQueryParams.java | 2 +- .../param/SupportedRestQueryParams.java | 2 +- streampipes-platform-services/pom.xml | 7 +- .../ps/DataLakeMeasureResourceV4.java | 2 +- .../streampipes/ps/DataLakeResourceV4.java | 42 +++++------ streampipes-rest/pom.xml | 2 +- .../streampipes/rest/ResetManagement.java | 2 +- .../datalake/PersistedDataStreamResource.java | 2 +- 86 files changed, 508 insertions(+), 286 deletions(-) create mode 100644 streampipes-data-explorer-api/pom.xml rename {streampipes-data-explorer => streampipes-data-explorer-api}/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java (100%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder => streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api}/IDataLakeQueryBuilder.java (92%) rename {streampipes-data-explorer => streampipes-data-explorer-api}/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java (92%) create mode 100644 streampipes-data-explorer-export/pom.xml rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/ConfiguredCsvOutputWriter.java (82%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/ConfiguredJsonOutputWriter.java (87%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/ConfiguredOutputWriter.java (93%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/OutputFormat.java (95%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/item/CsvItemGenerator.java (94%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/item/ItemGenerator.java (96%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export}/item/JsonItemGenerator.java (96%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestConfiguredCsvOutputWriter.java (93%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestConfiguredJsonOutputWriter.java (93%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestConfiguredOutputWriter.java (95%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestCsvItemGenerator.java (92%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestItemGenerator.java (94%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item => streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export}/TestJsonItemGenerator.java (91%) rename {streampipes-data-explorer-commons => streampipes-data-explorer-influx}/pom.xml (63%) rename {streampipes-data-explorer => streampipes-data-explorer-influx}/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java (95%) rename {streampipes-data-explorer => streampipes-data-explorer-influx}/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java (96%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer}/influx/InfluxDbReservedKeywords.java (97%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer}/influx/InfluxStore.java (96%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer}/influx/PropertyHandler.java (97%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx}/RawFieldSerializer.java (96%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx}/auth/AuthInterceptor.java (95%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth}/InfluxAuthMode.java (93%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client}/InfluxClientProvider.java (97%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client}/InfluxClientUtils.java (91%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client}/InfluxConnectionSettings.java (97%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/AutoAggregationHandler.java (79%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/DataExplorerQueryManagement.java (90%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/DataExplorerUtils.java (93%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/DataLakeMeasurementCount.java (92%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/DeleteDataQuery.java (92%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/IDataExplorerQueryManagement.java (89%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/ImageStore.java (98%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/ImageStoreUtils.java (96%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/QueryResultProvider.java (81%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/StreamedQueryResultProvider.java (85%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate}/TimeSeriesStore.java (90%) create mode 100644 streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/package-info.java rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize}/InfluxNameSanitizer.java (88%) rename {streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer => streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize}/MeasureNameSanitizer.java (95%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx}/DataLakeQueryBuilderTest.java (91%) rename {streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer}/influx/InfluxStoreTest.java (94%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx}/SelectQueryParamsTest.java (96%) rename {streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx}/TestRawFieldSerializer.java (97%) rename {streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/client}/InfluxClientProviderTest.java (76%) rename {streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/sanitize}/MeasureNameSanitizerTest.java (96%) rename {streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer => streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx}/utils/ProvidedQueryParameterBuilder.java (93%) create mode 100644 streampipes-data-explorer-management/pom.xml rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model => streampipes-model/src/main/java/org/apache/streampipes/model/datalake}/AggregationFunction.java (95%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder => streampipes-model/src/main/java/org/apache/streampipes/model/datalake}/DataLakeQueryOrdering.java (93%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder => streampipes-model/src/main/java/org/apache/streampipes/model/datalake}/FilterCondition.java (93%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer => streampipes-model/src/main/java/org/apache/streampipes/model/datalake}/param/ProvidedRestQueryParams.java (97%) rename {streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer => streampipes-model/src/main/java/org/apache/streampipes/model/datalake}/param/SupportedRestQueryParams.java (97%) diff --git a/pom.xml b/pom.xml index 85c6054d52..d8b09d4ba6 100644 --- a/pom.xml +++ b/pom.xml @@ -823,7 +823,10 @@ streampipes-extensions-api streampipes-extensions-management streampipes-data-explorer - streampipes-data-explorer-commons + streampipes-data-explorer-api + streampipes-data-explorer-export + streampipes-data-explorer-influx + streampipes-data-explorer-management streampipes-data-export streampipes-dataformat streampipes-dataformat-cbor @@ -862,8 +865,8 @@ streampipes-service-discovery-api streampipes-service-extensions streampipes-storage-api - streampipes-storage-management streampipes-storage-couchdb + streampipes-storage-management streampipes-test-utils streampipes-user-management streampipes-vocabulary diff --git a/streampipes-data-explorer-api/pom.xml b/streampipes-data-explorer-api/pom.xml new file mode 100644 index 0000000000..a433295db7 --- /dev/null +++ b/streampipes-data-explorer-api/pom.xml @@ -0,0 +1,61 @@ + + + + 4.0.0 + + org.apache.streampipes + streampipes-parent + 0.95.0-SNAPSHOT + + + streampipes-data-explorer-api + + This module establishes the API for data explorer elements, consolidating important high-level interfaces. + It serves as a centralized collection of key interfaces. + + + + + + org.apache.streampipes + streampipes-data-explorer-export + 0.95.0-SNAPSHOT + + + org.apache.streampipes + streampipes-model + 0.95.0-SNAPSHOT + + + + + org.slf4j + slf4j-api + + + + + 17 + 17 + UTF-8 + + + \ No newline at end of file diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java similarity index 100% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java rename to streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerSchemaManagement.java diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataLakeQueryBuilder.java similarity index 92% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java rename to streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataLakeQueryBuilder.java index 3298a8ed4d..372635a69c 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/IDataLakeQueryBuilder.java +++ b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataLakeQueryBuilder.java @@ -16,9 +16,13 @@ * */ -package org.apache.streampipes.dataexplorer.querybuilder; +package org.apache.streampipes.dataexplorer.api; -import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; + + +import org.apache.streampipes.model.datalake.AggregationFunction; +import org.apache.streampipes.model.datalake.DataLakeQueryOrdering; +import org.apache.streampipes.model.datalake.FilterCondition; import java.util.List; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java similarity index 92% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java rename to streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java index 1e1a0f39a2..3db11a97a5 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java +++ b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IQueryStatement.java @@ -18,8 +18,6 @@ package org.apache.streampipes.dataexplorer.api; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; - public interface IQueryStatement { void buildStatement(IDataLakeQueryBuilder builder); diff --git a/streampipes-data-explorer-export/pom.xml b/streampipes-data-explorer-export/pom.xml new file mode 100644 index 0000000000..35a106f111 --- /dev/null +++ b/streampipes-data-explorer-export/pom.xml @@ -0,0 +1,56 @@ + + + + 4.0.0 + + org.apache.streampipes + streampipes-parent + 0.95.0-SNAPSHOT + + + streampipes-data-explorer-export + + This module contains all components and functionalities related to exporting data from the data explorer storage. + + + + + + org.apache.streampipes + streampipes-model + 0.95.0-SNAPSHOT + + + + + org.junit.jupiter + junit-jupiter-api + test + + + + + 17 + 17 + UTF-8 + + + \ No newline at end of file diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java similarity index 82% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java index 3d37ff9230..0f78498656 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredCsvOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java @@ -16,18 +16,17 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.writer.item.CsvItemGenerator; +import org.apache.streampipes.dataexplorer.export.item.CsvItemGenerator; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import java.io.IOException; import java.io.OutputStream; import java.util.List; import java.util.StringJoiner; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_CSV_DELIMITER; - public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter { private static final String LINE_SEPARATOR = "\n"; @@ -40,8 +39,8 @@ public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter { @Override public void configure(ProvidedRestQueryParams params, boolean ignoreMissingValues) { - if (params.has(QP_CSV_DELIMITER)) { - delimiter = params.getAsString(QP_CSV_DELIMITER).equals("comma") ? COMMA : SEMICOLON; + if (params.has(SupportedRestQueryParams.QP_CSV_DELIMITER)) { + delimiter = params.getAsString(SupportedRestQueryParams.QP_CSV_DELIMITER).equals("comma") ? COMMA : SEMICOLON; } this.itemGenerator = new CsvItemGenerator(delimiter); } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java similarity index 87% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java index e14e866e05..d7d0b93c8c 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredJsonOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java @@ -16,11 +16,11 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.writer.item.ItemGenerator; -import org.apache.streampipes.dataexplorer.query.writer.item.JsonItemGenerator; +import org.apache.streampipes.dataexplorer.export.item.ItemGenerator; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.dataexplorer.export.item.JsonItemGenerator; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java similarity index 93% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java index ed60687ae5..c7ff305843 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/ConfiguredOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java @@ -16,9 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import java.io.IOException; import java.io.OutputStream; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java similarity index 95% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java index c855bf1c8f..2342f5879f 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/OutputFormat.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; import java.util.function.Supplier; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemGenerator.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java similarity index 94% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemGenerator.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java index f509338d3c..4afb7ffdbd 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/CsvItemGenerator.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/CsvItemGenerator.java @@ -17,7 +17,7 @@ */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export.item; public class CsvItemGenerator extends ItemGenerator { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/ItemGenerator.java similarity index 96% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/ItemGenerator.java index 31e9eb61d5..8ce39f516f 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/ItemGenerator.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/ItemGenerator.java @@ -17,7 +17,7 @@ */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export.item; import java.util.List; import java.util.StringJoiner; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemGenerator.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/JsonItemGenerator.java similarity index 96% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemGenerator.java rename to streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/JsonItemGenerator.java index 03d37197f0..404b84757c 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/writer/item/JsonItemGenerator.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/item/JsonItemGenerator.java @@ -17,7 +17,7 @@ */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export.item; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java similarity index 93% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java index 2aef0dbf37..0fc7cec228 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredCsvOutputWriter.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java @@ -16,9 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java similarity index 93% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java index e008f8847a..b67b65ea28 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredJsonOutputWriter.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java @@ -16,9 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredOutputWriter.java similarity index 95% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredOutputWriter.java index 14f453fcee..ea921e844e 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/TestConfiguredOutputWriter.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredOutputWriter.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer; +package org.apache.streampipes.dataexplorer.export; import org.junit.jupiter.api.BeforeEach; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemGenerator.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java similarity index 92% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemGenerator.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java index b24adf5838..d1b088bcd3 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestCsvItemGenerator.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestCsvItemGenerator.java @@ -16,7 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export; + +import org.apache.streampipes.dataexplorer.export.item.CsvItemGenerator; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemGenerator.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestItemGenerator.java similarity index 94% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemGenerator.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestItemGenerator.java index 3dd0fd7672..d5cdc01fb9 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestItemGenerator.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestItemGenerator.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export; import org.junit.jupiter.api.BeforeEach; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemGenerator.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestJsonItemGenerator.java similarity index 91% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemGenerator.java rename to streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestJsonItemGenerator.java index 786570bc83..e6d287816e 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/writer/item/TestJsonItemGenerator.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestJsonItemGenerator.java @@ -16,7 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.query.writer.item; +package org.apache.streampipes.dataexplorer.export; + +import org.apache.streampipes.dataexplorer.export.item.JsonItemGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer-commons/pom.xml b/streampipes-data-explorer-influx/pom.xml similarity index 63% rename from streampipes-data-explorer-commons/pom.xml rename to streampipes-data-explorer-influx/pom.xml index 759897f1a3..245db7a680 100644 --- a/streampipes-data-explorer-commons/pom.xml +++ b/streampipes-data-explorer-influx/pom.xml @@ -16,17 +16,23 @@ ~ limitations under the License. ~ --> - + + 4.0.0 - streampipes-parent org.apache.streampipes + streampipes-parent 0.95.0-SNAPSHOT - 4.0.0 - streampipes-data-explorer-commons + streampipes-data-explorer-influx + This module houses all InfluxDB-specific implementations necessary to fulfill data explorer storage + functionalities utilizing an underlying instance of InfluxDB. + + - + org.apache.streampipes streampipes-client-api @@ -34,34 +40,32 @@ org.apache.streampipes - streampipes-model + streampipes-commons 0.95.0-SNAPSHOT org.apache.streampipes - streampipes-commons + streampipes-data-explorer 0.95.0-SNAPSHOT org.apache.streampipes - streampipes-test-utils + streampipes-data-explorer-api 0.95.0-SNAPSHOT - test - - org.junit.jupiter - junit-jupiter-api - test + org.apache.streampipes + streampipes-data-explorer-export + 0.95.0-SNAPSHOT - org.mockito - mockito-core - test + org.apache.streampipes + streampipes-model + 0.95.0-SNAPSHOT - - + + org.lightcouch lightcouch @@ -69,13 +73,30 @@ org.influxdb influxdb-java + + + + org.apache.streampipes + streampipes-test-utils + 0.95.0-SNAPSHOT + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.mockito + mockito-core + test + - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - + + + 17 + 17 + UTF-8 + + + \ No newline at end of file diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java similarity index 95% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java index 822d5c9f23..a500a4bd11 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerInfluxQueryExecutor.java @@ -18,12 +18,12 @@ package org.apache.streampipes.dataexplorer.influx; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.migrate.DeleteDataQuery; import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.query.DataExplorerQueryExecutor; -import org.apache.streampipes.dataexplorer.query.DeleteDataQuery; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataSeries; import org.apache.streampipes.model.datalake.SpQueryResult; @@ -108,7 +108,7 @@ private IDataLakeQueryBuilder getQueryBuilder(String measurementId) { } @Override - protected QueryResult executeQuery(Query query) { + public QueryResult executeQuery(Query query) { try (final InfluxDB influxDB = InfluxClientProvider.getInfluxDBClient()) { return influxDB.query(query, TimeUnit.MILLISECONDS); } @@ -152,6 +152,7 @@ private String getDatabaseName() { return getEnvironment().getTsStorageBucket().getValueOrDefault(); } + @Override public Map getTagValues(String measurementId, String fields) { try (final InfluxDB influxDB = InfluxClientProvider.getInfluxDBClient()) { Map tags = new HashMap<>(); @@ -180,6 +181,7 @@ public Map getTagValues(String measurementId, String fields) { } } + @Override public boolean deleteData(DataLakeMeasure measure) { QueryResult queryResult = new DeleteDataQuery(measure).executeQuery(); diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java similarity index 96% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java index abed48684e..24f552ea24 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeInfluxQueryBuilder.java @@ -20,10 +20,10 @@ import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; -import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; -import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; -import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.AggregationFunction; +import org.apache.streampipes.model.datalake.DataLakeQueryOrdering; +import org.apache.streampipes.model.datalake.FilterCondition; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.influxdb.dto.Query; import org.influxdb.querybuilder.Ordering; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxDbReservedKeywords.java similarity index 97% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxDbReservedKeywords.java index 48d8c9e8d1..df4357eb6e 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxDbReservedKeywords.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxDbReservedKeywords.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx; import java.util.Arrays; import java.util.List; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxStore.java similarity index 96% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxStore.java index 1c16009ffa..0d10ac147f 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStore.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/InfluxStore.java @@ -16,10 +16,12 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.sanitize.InfluxNameSanitizer; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.runtime.Event; import org.apache.streampipes.model.schema.EventProperty; @@ -208,8 +210,8 @@ private void storeSanitizedRuntimeNames(DataLakeMeasure measure) { measure.getEventSchema() .getEventProperties() .forEach(ep -> sanitizedRuntimeNames.put( - ep.getRuntimeName(), - InfluxNameSanitizer.renameReservedKeywords(ep.getRuntimeName()) + ep.getRuntimeName(), + InfluxNameSanitizer.renameReservedKeywords(ep.getRuntimeName()) )); } diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java similarity index 97% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java index 9cc4a79d17..74d03ba3e2 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/PropertyHandler.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/PropertyHandler.java @@ -16,10 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.influx.serializer.RawFieldSerializer; import org.apache.streampipes.model.runtime.Event; import org.apache.streampipes.model.runtime.field.PrimitiveField; import org.apache.streampipes.model.schema.EventPropertyPrimitive; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/RawFieldSerializer.java similarity index 96% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/RawFieldSerializer.java index 05380df10c..4e6226babc 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/RawFieldSerializer.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/RawFieldSerializer.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx.serializer; +package org.apache.streampipes.dataexplorer.influx; import org.apache.streampipes.commons.exceptions.SpRuntimeException; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/AuthInterceptor.java similarity index 95% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/AuthInterceptor.java index 52c37c4b24..30d16dbba7 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/auth/AuthInterceptor.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/AuthInterceptor.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.auth; +package org.apache.streampipes.dataexplorer.influx.auth; import okhttp3.Interceptor; import okhttp3.Response; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/InfluxAuthMode.java similarity index 93% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/InfluxAuthMode.java index 781829260c..b2d4dae8a6 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxAuthMode.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/auth/InfluxAuthMode.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.auth; public enum InfluxAuthMode { diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProvider.java similarity index 97% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProvider.java index a95ecfac7e..f1b47285b4 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProvider.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProvider.java @@ -16,11 +16,12 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.client; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.commons.exceptions.SpRuntimeException; +import org.apache.streampipes.dataexplorer.influx.auth.InfluxAuthMode; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientUtils.java similarity index 91% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientUtils.java index 1f7b04d998..9b0d14d1ce 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientUtils.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientUtils.java @@ -16,9 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.client; -import org.apache.streampipes.dataexplorer.commons.auth.AuthInterceptor; +import org.apache.streampipes.dataexplorer.influx.auth.AuthInterceptor; import okhttp3.OkHttpClient; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxConnectionSettings.java similarity index 97% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxConnectionSettings.java index ac53b85730..80019e059f 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxConnectionSettings.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/client/InfluxConnectionSettings.java @@ -16,9 +16,10 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.client; import org.apache.streampipes.commons.environment.Environment; +import org.apache.streampipes.dataexplorer.influx.auth.InfluxAuthMode; public class InfluxConnectionSettings { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/AutoAggregationHandler.java similarity index 79% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/AutoAggregationHandler.java index 071d7a64d2..692e7412be 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/AutoAggregationHandler.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/AutoAggregationHandler.java @@ -15,14 +15,13 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.query; +package org.apache.streampipes.dataexplorer.influx.migrate; -import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement; import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement; -import org.apache.streampipes.dataexplorer.api.IDataExplorerQueryManagement; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import org.apache.streampipes.dataexplorer.param.model.SelectColumn; -import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; +import org.apache.streampipes.model.datalake.DataLakeQueryOrdering; import org.apache.streampipes.model.datalake.SpQueryResult; import org.apache.streampipes.storage.management.StorageDispatcher; @@ -36,14 +35,6 @@ import java.util.List; import java.util.stream.Collectors; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COLUMNS; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COUNT_ONLY; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_ORDER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_TIME_INTERVAL; - public class AutoAggregationHandler { private static final Logger LOG = LoggerFactory.getLogger(AutoAggregationHandler.class); @@ -86,7 +77,7 @@ public ProvidedRestQueryParams makeAutoAggregationQueryParams() throws IllegalAr int aggValue = getAggregationValue(newest, oldest); LOG.debug("Setting auto-aggregation value to {} ms", aggValue); - queryParams.update(QP_TIME_INTERVAL, aggValue + "ms"); + queryParams.update(SupportedRestQueryParams.QP_TIME_INTERVAL, aggValue + "ms"); return disableAutoAgg(queryParams); } } else { @@ -99,16 +90,16 @@ public ProvidedRestQueryParams makeAutoAggregationQueryParams() throws IllegalAr } private ProvidedRestQueryParams disableAutoAgg(ProvidedRestQueryParams params) { - params.remove(QP_AUTO_AGGREGATE); + params.remove(SupportedRestQueryParams.QP_AUTO_AGGREGATE); return params; } public Integer getCount(String fieldName) { ProvidedRestQueryParams countParams = disableAutoAgg(new ProvidedRestQueryParams(queryParams)); - countParams.remove(QP_TIME_INTERVAL); - countParams.remove(QP_AGGREGATION_FUNCTION); - countParams.update(QP_COUNT_ONLY, true); - countParams.update(QP_COLUMNS, fieldName); + countParams.remove(SupportedRestQueryParams.QP_TIME_INTERVAL); + countParams.remove(SupportedRestQueryParams.QP_AGGREGATION_FUNCTION); + countParams.update(SupportedRestQueryParams.QP_COUNT_ONLY, true); + countParams.update(SupportedRestQueryParams.QP_COLUMNS, fieldName); SpQueryResult result = dataLakeQueryManagement.getData(countParams, true); @@ -134,10 +125,11 @@ private int getAggregationValue(SpQueryResult newest, SpQueryResult oldest) thro private SpQueryResult getSingleRecord(DataLakeQueryOrdering order) throws ParseException { ProvidedRestQueryParams singleEvent = disableAutoAgg(new ProvidedRestQueryParams(queryParams)); - singleEvent.remove(QP_AGGREGATION_FUNCTION); - singleEvent.update(QP_LIMIT, 1); - singleEvent.update(QP_ORDER, order.name()); - singleEvent.update(QP_COLUMNS, transformColumns(singleEvent.getAsString(QP_COLUMNS))); + singleEvent.remove(SupportedRestQueryParams.QP_AGGREGATION_FUNCTION); + singleEvent.update(SupportedRestQueryParams.QP_LIMIT, 1); + singleEvent.update(SupportedRestQueryParams.QP_ORDER, order.name()); + singleEvent.update(SupportedRestQueryParams.QP_COLUMNS, transformColumns(singleEvent.getAsString( + SupportedRestQueryParams.QP_COLUMNS))); return fireQuery(singleEvent); } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerQueryManagement.java similarity index 90% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerQueryManagement.java index 426262f1cb..d9dcc5c284 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataExplorerQueryManagement.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerQueryManagement.java @@ -16,17 +16,14 @@ * */ -package org.apache.streampipes.dataexplorer; +package org.apache.streampipes.dataexplorer.influx.migrate; -import org.apache.streampipes.dataexplorer.api.IDataExplorerQueryManagement; import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement; +import org.apache.streampipes.dataexplorer.export.OutputFormat; import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.QueryResultProvider; -import org.apache.streampipes.dataexplorer.query.StreamedQueryResultProvider; -import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.SpQueryResult; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerUtils.java similarity index 93% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerUtils.java index 6c28f9c145..97d8b9089d 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/DataExplorerUtils.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataExplorerUtils.java @@ -15,12 +15,12 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.commons; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.client.api.IStreamPipesClient; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxNameSanitizer; -import org.apache.streampipes.dataexplorer.commons.sanitizer.MeasureNameSanitizer; +import org.apache.streampipes.dataexplorer.influx.sanitize.InfluxNameSanitizer; +import org.apache.streampipes.dataexplorer.influx.sanitize.MeasureNameSanitizer; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.schema.EventProperty; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataLakeMeasurementCount.java similarity index 92% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataLakeMeasurementCount.java index af8619fef5..aa21d051fa 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCount.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DataLakeMeasurementCount.java @@ -16,9 +16,11 @@ * */ -package org.apache.streampipes.dataexplorer.influx; +package org.apache.streampipes.dataexplorer.influx.migrate; -import org.apache.streampipes.dataexplorer.param.model.AggregationFunction; +import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; +import org.apache.streampipes.dataexplorer.influx.DataLakeInfluxQueryBuilder; +import org.apache.streampipes.model.datalake.AggregationFunction; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.SpQueryResult; import org.apache.streampipes.model.schema.EventProperty; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DeleteDataQuery.java similarity index 92% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DeleteDataQuery.java index 6ef1d68564..4885cd1ed9 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DeleteDataQuery.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/DeleteDataQuery.java @@ -15,11 +15,11 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.query; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.environment.Environments; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.influxdb.InfluxDB; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/IDataExplorerQueryManagement.java similarity index 89% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/IDataExplorerQueryManagement.java index 59620a21be..37e43002fc 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerQueryManagement.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/IDataExplorerQueryManagement.java @@ -16,11 +16,11 @@ * */ -package org.apache.streampipes.dataexplorer.api; +package org.apache.streampipes.dataexplorer.influx.migrate; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.dataexplorer.export.OutputFormat; import org.apache.streampipes.model.datalake.SpQueryResult; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import java.io.IOException; import java.io.OutputStream; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStore.java similarity index 98% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStore.java index e9323d111b..6cfd2087be 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStore.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStore.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.image; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.exceptions.SpRuntimeException; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStoreUtils.java similarity index 96% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStoreUtils.java index 42594827e4..6504303a5a 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/image/ImageStoreUtils.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/ImageStoreUtils.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.image; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.schema.EventProperty; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/QueryResultProvider.java similarity index 81% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/QueryResultProvider.java index ce88fd1b5d..85a23b188f 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/QueryResultProvider.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/QueryResultProvider.java @@ -16,16 +16,15 @@ * */ -package org.apache.streampipes.dataexplorer.query; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.dataexplorer.influx.DataExplorerInfluxQueryExecutor; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import org.apache.streampipes.model.datalake.SpQueryResult; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS; public class QueryResultProvider { @@ -40,13 +39,13 @@ public QueryResultProvider(ProvidedRestQueryParams queryParams, } public SpQueryResult getData() { - if (queryParams.has(QP_AUTO_AGGREGATE)) { + if (queryParams.has(SupportedRestQueryParams.QP_AUTO_AGGREGATE)) { queryParams = new AutoAggregationHandler(queryParams).makeAutoAggregationQueryParams(); } SelectQueryParams qp = ProvidedRestQueryParamConverter.getSelectQueryParams(queryParams); - if (queryParams.getProvidedParams().containsKey(QP_MAXIMUM_AMOUNT_OF_EVENTS)) { - int maximumAmountOfEvents = Integer.parseInt(queryParams.getProvidedParams().get(QP_MAXIMUM_AMOUNT_OF_EVENTS)); + if (queryParams.getProvidedParams().containsKey(SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS)) { + int maximumAmountOfEvents = Integer.parseInt(queryParams.getProvidedParams().get(SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS)); return new DataExplorerInfluxQueryExecutor(maximumAmountOfEvents).executeQuery(qp, ignoreMissingData); } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/StreamedQueryResultProvider.java similarity index 85% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/StreamedQueryResultProvider.java index e1a8f756e6..52ed5d7881 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/StreamedQueryResultProvider.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/StreamedQueryResultProvider.java @@ -16,12 +16,12 @@ * */ -package org.apache.streampipes.dataexplorer.query; +package org.apache.streampipes.dataexplorer.influx.migrate; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.writer.ConfiguredOutputWriter; -import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; +import org.apache.streampipes.dataexplorer.export.ConfiguredOutputWriter; +import org.apache.streampipes.dataexplorer.export.OutputFormat; import org.apache.streampipes.dataexplorer.utils.DataExplorerUtils; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.SpQueryResult; @@ -31,8 +31,6 @@ import java.util.List; import java.util.Optional; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; - public class StreamedQueryResultProvider extends QueryResultProvider { private static final int MAX_RESULTS_PER_QUERY = 200000; @@ -48,15 +46,15 @@ public StreamedQueryResultProvider(ProvidedRestQueryParams params, } public void getDataAsStream(OutputStream outputStream) throws IOException { - var usesLimit = queryParams.has(QP_LIMIT); + var usesLimit = queryParams.has(SupportedRestQueryParams.QP_LIMIT); var configuredWriter = ConfiguredOutputWriter .getConfiguredWriter(format, queryParams, ignoreMissingData); - if (!queryParams.has(QP_LIMIT)) { - queryParams.update(QP_LIMIT, MAX_RESULTS_PER_QUERY); + if (!queryParams.has(SupportedRestQueryParams.QP_LIMIT)) { + queryParams.update(SupportedRestQueryParams.QP_LIMIT, MAX_RESULTS_PER_QUERY); } - var limit = queryParams.getAsInt(QP_LIMIT); + var limit = queryParams.getAsInt(SupportedRestQueryParams.QP_LIMIT); var measurement = findByMeasurementName(queryParams.getMeasurementId()).get(); SpQueryResult dataResult; diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/TimeSeriesStore.java similarity index 90% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/TimeSeriesStore.java index 4bc6628696..ef8af9a24d 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/TimeSeriesStore.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/TimeSeriesStore.java @@ -16,14 +16,13 @@ * */ -package org.apache.streampipes.dataexplorer.commons; +package org.apache.streampipes.dataexplorer.influx.migrate; import org.apache.streampipes.client.api.IStreamPipesClient; import org.apache.streampipes.commons.environment.Environment; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.image.ImageStore; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxStore; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.InfluxStore; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.runtime.Event; diff --git a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/package-info.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/package-info.java new file mode 100644 index 0000000000..c96a83d257 --- /dev/null +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/migrate/package-info.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * This package contains classes that have direct dependencies on InfluxDB-specific implementations. + * To enhance code organization and promote agnostic design, these classes are slated for migration to other modules, + * such as 'streampipes-data-explorer' or 'streampipes-data-explorer-api'. + * Pending migration, they are temporarily housed within this package. + */ +package org.apache.streampipes.dataexplorer.influx.migrate; \ No newline at end of file diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/InfluxNameSanitizer.java similarity index 88% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/InfluxNameSanitizer.java index 353782ba8f..1d468e7703 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxNameSanitizer.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/InfluxNameSanitizer.java @@ -16,7 +16,9 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.sanitize; + +import org.apache.streampipes.dataexplorer.influx.InfluxDbReservedKeywords; public class InfluxNameSanitizer { diff --git a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizer.java similarity index 95% rename from streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java rename to streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizer.java index 1ffbd02fef..e597527868 100644 --- a/streampipes-data-explorer-commons/src/main/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizer.java +++ b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizer.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.sanitizer; +package org.apache.streampipes.dataexplorer.influx.sanitize; import java.util.List; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/DataLakeQueryBuilderTest.java similarity index 91% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/DataLakeQueryBuilderTest.java index 3c288e1ac0..48caeaa516 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/sdk/DataLakeQueryBuilderTest.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/DataLakeQueryBuilderTest.java @@ -16,9 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.sdk; - -import org.apache.streampipes.dataexplorer.influx.DataLakeInfluxQueryBuilder; +package org.apache.streampipes.dataexplorer.influx; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/InfluxStoreTest.java similarity index 94% rename from streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/InfluxStoreTest.java index 888e44e346..aec50f849d 100644 --- a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxStoreTest.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/InfluxStoreTest.java @@ -16,11 +16,11 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.influx.serializer.RawFieldSerializer; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.runtime.Event; import org.apache.streampipes.model.runtime.EventFactory; @@ -40,6 +40,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; import java.net.URI; import java.net.URISyntaxException; @@ -50,10 +52,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class InfluxStoreTest { @@ -68,7 +66,7 @@ public class InfluxStoreTest { @BeforeEach public void setUp() { - influxDBMock = mock(InfluxDB.class); + influxDBMock = Mockito.mock(InfluxDB.class); serializer = new RawFieldSerializer(); } @@ -306,7 +304,7 @@ private Point executeOnEvent(InfluxStore influxStore, Event event) { influxStore.onEvent(event); var pointArgumentCaptor = ArgumentCaptor.forClass(Point.class); - verify(influxDBMock).write(pointArgumentCaptor.capture()); + Mockito.verify(influxDBMock).write(pointArgumentCaptor.capture()); return pointArgumentCaptor.getValue(); } @@ -356,7 +354,7 @@ private Point testEventWithOneField( influxStore.onEvent(event); var pointArgumentCaptor = ArgumentCaptor.forClass(Point.class); - verify(influxDBMock).write(pointArgumentCaptor.capture()); + Mockito.verify(influxDBMock).write(pointArgumentCaptor.capture()); return pointArgumentCaptor.getValue(); } @@ -384,9 +382,9 @@ private InfluxStore getInfluxStore(EventSchema eventSchema) { eventSchema ); - var influxClientProviderMock = mock(InfluxClientProvider.class); - when(influxClientProviderMock.getInitializedInfluxDBClient(any())) - .thenReturn(influxDBMock); + var influxClientProviderMock = Mockito.mock(InfluxClientProvider.class); + Mockito.when(influxClientProviderMock.getInitializedInfluxDBClient(ArgumentMatchers.any())) + .thenReturn(influxDBMock); return new InfluxStore(measure, null, influxClientProviderMock); } diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/SelectQueryParamsTest.java similarity index 96% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/SelectQueryParamsTest.java index 5c254ee87b..828ed2304c 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/SelectQueryParamsTest.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/SelectQueryParamsTest.java @@ -16,10 +16,11 @@ * */ -package org.apache.streampipes.dataexplorer.param; +package org.apache.streampipes.dataexplorer.influx; -import org.apache.streampipes.dataexplorer.influx.DataLakeInfluxQueryBuilder; -import org.apache.streampipes.dataexplorer.utils.ProvidedQueryParameterBuilder; +import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; +import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.dataexplorer.influx.utils.ProvidedQueryParameterBuilder; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TestRawFieldSerializer.java similarity index 97% rename from streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TestRawFieldSerializer.java index 9708617418..48f6344443 100644 --- a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/serializer/TestRawFieldSerializer.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/TestRawFieldSerializer.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx.serializer; +package org.apache.streampipes.dataexplorer.influx; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProviderTest.java similarity index 76% rename from streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProviderTest.java index 79b0849812..563ab36d6e 100644 --- a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/influx/InfluxClientProviderTest.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/client/InfluxClientProviderTest.java @@ -16,28 +16,25 @@ * */ -package org.apache.streampipes.dataexplorer.commons.influx; +package org.apache.streampipes.dataexplorer.influx.client; import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.influxdb.InfluxDB; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class InfluxClientProviderTest { @@ -48,7 +45,7 @@ public class InfluxClientProviderTest { @BeforeEach public void setUp() { - influxDBMock = mock(InfluxDB.class); + influxDBMock = Mockito.mock(InfluxDB.class); influxClientProvider = new InfluxClientProvider(); } @@ -59,10 +56,10 @@ public void createDatabaseSuccess() { influxClientProvider.createDatabase(influxDBMock, "test"); var pointArgumentCaptor = ArgumentCaptor.forClass(Query.class); - verify(influxDBMock).query(pointArgumentCaptor.capture()); + Mockito.verify(influxDBMock).query(pointArgumentCaptor.capture()); var actualQuery = pointArgumentCaptor.getValue(); - assertEquals(actualQuery.getDatabase(), expectedQuery.getDatabase()); + Assertions.assertEquals(actualQuery.getDatabase(), expectedQuery.getDatabase()); } @Test @@ -85,20 +82,20 @@ public void dataBaseDoesNotExist() { private boolean testDataBaseExists(List databaseNames) { var queryResult = getQueryResultWithDatabaseNames(databaseNames); - when(influxDBMock.query(any())).thenReturn(queryResult); + Mockito.when(influxDBMock.query(ArgumentMatchers.any())).thenReturn(queryResult); return influxClientProvider.databaseExists(influxDBMock, DATABASE_NAME); } private QueryResult getQueryResultWithDatabaseNames(List databaseNames) { - var queryResult = mock(QueryResult.class); - var result = mock(QueryResult.Result.class); - when(queryResult.getResults()).thenReturn(List.of(result)); - var series = mock(QueryResult.Series.class); - when(result.getSeries()).thenReturn(List.of(series)); + var queryResult = Mockito.mock(QueryResult.class); + var result = Mockito.mock(QueryResult.Result.class); + Mockito.when(queryResult.getResults()).thenReturn(List.of(result)); + var series = Mockito.mock(QueryResult.Series.class); + Mockito.when(result.getSeries()).thenReturn(List.of(series)); List> values = List.of(databaseNames); - when(series.getValues()).thenReturn(values); + Mockito.when(series.getValues()).thenReturn(values); return queryResult; } } \ No newline at end of file diff --git a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizerTest.java similarity index 96% rename from streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizerTest.java index abbeb096f7..1f3a52f691 100644 --- a/streampipes-data-explorer-commons/src/test/java/org/apache/streampipes/dataexplorer/commons/sanitizer/MeasureNameSanitizerTest.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/sanitize/MeasureNameSanitizerTest.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.commons.sanitizer; +package org.apache.streampipes.dataexplorer.influx.sanitize; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/utils/ProvidedQueryParameterBuilder.java similarity index 93% rename from streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java rename to streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/utils/ProvidedQueryParameterBuilder.java index c4cd73cebc..8f55f57987 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/utils/ProvidedQueryParameterBuilder.java +++ b/streampipes-data-explorer-influx/src/test/java/org/apache/streampipes/dataexplorer/influx/utils/ProvidedQueryParameterBuilder.java @@ -16,10 +16,10 @@ * */ -package org.apache.streampipes.dataexplorer.utils; +package org.apache.streampipes.dataexplorer.influx.utils; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import java.util.HashMap; import java.util.List; diff --git a/streampipes-data-explorer-management/pom.xml b/streampipes-data-explorer-management/pom.xml new file mode 100644 index 0000000000..09c89dacff --- /dev/null +++ b/streampipes-data-explorer-management/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + org.apache.streampipes + streampipes-parent + 0.95.0-SNAPSHOT + + + streampipes-data-explorer-management + + This Maven module acts as the central access point for components to interact with the data explorer storage. + It provides querying and writing capabilities while abstracting the underlying implementation details. + It ensures easy integration for any component needing access to the data explorer or its storage (data lake). + + + + + org.apache.streampipes + streampipes-commons + 0.95.0-SNAPSHOT + + + org.apache.streampipes + streampipes-data-explorer-api + 0.95.0-SNAPSHOT + + + + + 17 + 17 + UTF-8 + + + \ No newline at end of file diff --git a/streampipes-data-explorer/pom.xml b/streampipes-data-explorer/pom.xml index 50443cf42c..3601ff3a8c 100644 --- a/streampipes-data-explorer/pom.xml +++ b/streampipes-data-explorer/pom.xml @@ -26,11 +26,15 @@ 4.0.0 streampipes-data-explorer + + This Maven module encompasses all StreamPipes-specific implementation endeavors concerning data explorer functionalities. + + org.apache.streampipes - streampipes-data-explorer-commons + streampipes-data-explorer-api 0.95.0-SNAPSHOT @@ -43,22 +47,17 @@ streampipes-storage-management 0.95.0-SNAPSHOT - - org.influxdb - influxdb-java - - - org.apache.streampipes - streampipes-test-utils - 0.95.0-SNAPSHOT + org.junit.jupiter + junit-jupiter-api test - org.junit.jupiter - junit-jupiter-api + org.apache.streampipes + streampipes-test-utils + 0.95.0-SNAPSHOT test diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java index 186a5eb74c..dd658d0575 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParamConverter.java @@ -26,24 +26,13 @@ import org.apache.streampipes.dataexplorer.param.model.OrderByClauseParams; import org.apache.streampipes.dataexplorer.param.model.SelectClauseParams; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COLUMNS; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COUNT_ONLY; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_END_DATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_FILTER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_GROUP_BY; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_OFFSET; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_ORDER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_PAGE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_START_DATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_TIME_INTERVAL; - public class ProvidedRestQueryParamConverter { @@ -55,56 +44,62 @@ public class ProvidedRestQueryParamConverter { public static SelectQueryParams getSelectQueryParams(ProvidedRestQueryParams params) { SelectQueryParams queryParameters = new SelectQueryParams(params.getMeasurementId()); - if (params.has(QP_COUNT_ONLY) && params.getAsBoolean(QP_COUNT_ONLY)) { - queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(QP_COLUMNS), true)); + if (params.has(SupportedRestQueryParams.QP_COUNT_ONLY) + && params.getAsBoolean(SupportedRestQueryParams.QP_COUNT_ONLY)) { + queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(SupportedRestQueryParams.QP_COLUMNS), + true) + ); } else { - queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(QP_COLUMNS), - params.getAsString(QP_AGGREGATION_FUNCTION))); + queryParameters.withSelectParams(SelectClauseParams.from(params.getAsString(SupportedRestQueryParams.QP_COLUMNS), + params.getAsString(SupportedRestQueryParams.QP_AGGREGATION_FUNCTION))); } - String filterConditions = params.getAsString(QP_FILTER); + String filterConditions = params.getAsString(SupportedRestQueryParams.QP_FILTER); if (hasTimeParams(params)) { queryParameters.withWhereParams(WhereClauseParams.from( - params.getAsLong(QP_START_DATE), - params.getAsLong(QP_END_DATE), + params.getAsLong(SupportedRestQueryParams.QP_START_DATE), + params.getAsLong(SupportedRestQueryParams.QP_END_DATE), filterConditions)); } else if (filterConditions != null) { queryParameters.withWhereParams(WhereClauseParams.from(filterConditions)); } - if (params.has(QP_TIME_INTERVAL)) { - String timeInterval = params.getAsString(QP_TIME_INTERVAL); - if (!params.has(QP_GROUP_BY)) { + if (params.has(SupportedRestQueryParams.QP_TIME_INTERVAL)) { + String timeInterval = params.getAsString(SupportedRestQueryParams.QP_TIME_INTERVAL); + if (!params.has(SupportedRestQueryParams.QP_GROUP_BY)) { queryParameters.withGroupByTimeParams(GroupByTimeClauseParams.from(timeInterval)); } else { - params.update(QP_GROUP_BY, params.getAsString(QP_GROUP_BY) + ",time(" + timeInterval + ")"); + params.update(SupportedRestQueryParams.QP_GROUP_BY, + params.getAsString(SupportedRestQueryParams.QP_GROUP_BY) + ",time(" + timeInterval + ")"); } queryParameters.withFillParams(FillClauseParams.from()); } - if (params.has(QP_GROUP_BY)) { - queryParameters.withGroupByTagsParams(GroupByTagsClauseParams.from(params.getAsString(QP_GROUP_BY))); + if (params.has(SupportedRestQueryParams.QP_GROUP_BY)) { + queryParameters.withGroupByTagsParams(GroupByTagsClauseParams.from( + params.getAsString(SupportedRestQueryParams.QP_GROUP_BY)) + ); } - if (params.has(QP_ORDER)) { - String order = params.getAsString(QP_ORDER); + if (params.has(SupportedRestQueryParams.QP_ORDER)) { + String order = params.getAsString(SupportedRestQueryParams.QP_ORDER); if (order.equals(ORDER_DESCENDING)) { queryParameters.withOrderByParams(OrderByClauseParams.from(order)); } } - if (params.has(QP_LIMIT)) { - queryParameters.withLimitParams(LimitClauseParams.from(params.getAsInt(QP_LIMIT))); + if (params.has(SupportedRestQueryParams.QP_LIMIT)) { + queryParameters.withLimitParams(LimitClauseParams.from(params.getAsInt(SupportedRestQueryParams.QP_LIMIT))); } - if (params.has(QP_OFFSET)) { - queryParameters.withOffsetParams(OffsetClauseParams.from(params.getAsInt(QP_OFFSET))); - } else if (params.has(QP_LIMIT) && params.has(QP_PAGE)) { + if (params.has(SupportedRestQueryParams.QP_OFFSET)) { + queryParameters.withOffsetParams(OffsetClauseParams.from(params.getAsInt(SupportedRestQueryParams.QP_OFFSET))); + } else if (params.has(SupportedRestQueryParams.QP_LIMIT) && params.has(SupportedRestQueryParams.QP_PAGE)) { queryParameters.withOffsetParams(OffsetClauseParams.from( - params.getAsInt(QP_PAGE) * params.getAsInt(QP_LIMIT))); + params.getAsInt(SupportedRestQueryParams.QP_PAGE) * params.getAsInt(SupportedRestQueryParams.QP_LIMIT))); } return queryParameters; @@ -117,8 +112,8 @@ public static DeleteQueryParams getDeleteQueryParams(String measurementId, } private static boolean hasTimeParams(ProvidedRestQueryParams params) { - return params.has(QP_START_DATE) - || params.has(QP_END_DATE); + return params.has(SupportedRestQueryParams.QP_START_DATE) + || params.has(SupportedRestQueryParams.QP_END_DATE); } public static List buildConditions(String queryPart) { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java index 4131d9bf22..cca5c1b024 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SelectQueryParams.java @@ -17,6 +17,7 @@ */ package org.apache.streampipes.dataexplorer.param; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.param.model.FillClauseParams; import org.apache.streampipes.dataexplorer.param.model.GroupByTagsClauseParams; import org.apache.streampipes.dataexplorer.param.model.GroupByTimeClauseParams; @@ -25,7 +26,6 @@ import org.apache.streampipes.dataexplorer.param.model.OrderByClauseParams; import org.apache.streampipes.dataexplorer.param.model.SelectClauseParams; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import java.util.Objects; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java index 5204ce5a0b..ccece054ba 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/FillClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class FillClauseParams implements IQueryStatement { String fill = "none"; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java index 833fb3363a..3a4c8fd84d 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTagsClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; import java.util.ArrayList; import java.util.Arrays; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java index 0d83953ceb..aa3f3f9422 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/GroupByTimeClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public class GroupByTimeClauseParams implements IQueryStatement { private final String timeInterval; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java index f1ecef0e77..b22292d3bc 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/LimitClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public record LimitClauseParams(Integer limit) implements IQueryStatement { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java index 6419b7a73e..5eb445d90e 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OffsetClauseParams.java @@ -18,8 +18,8 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; public record OffsetClauseParams(Integer offset) implements IQueryStatement { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java index 612a91674a..6c184084a1 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/OrderByClauseParams.java @@ -18,9 +18,9 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.DataLakeQueryOrdering; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.DataLakeQueryOrdering; public class OrderByClauseParams implements IQueryStatement { private final String ordering; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java index 02db7227f2..af87b0778d 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectClauseParams.java @@ -19,8 +19,9 @@ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.AggregationFunction; import java.util.Arrays; import java.util.List; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java index 1fb7661e90..3e4fd1048c 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/SelectColumn.java @@ -17,9 +17,10 @@ */ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.AggregationFunction; public class SelectColumn implements IQueryStatement { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java index ab5a6712b8..6e1f633622 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java @@ -17,10 +17,10 @@ */ package org.apache.streampipes.dataexplorer.param.model; +import org.apache.streampipes.dataexplorer.api.IDataLakeQueryBuilder; import org.apache.streampipes.dataexplorer.api.IQueryStatement; import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParamConverter; -import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; -import org.apache.streampipes.dataexplorer.querybuilder.IDataLakeQueryBuilder; +import org.apache.streampipes.model.datalake.FilterCondition; import org.apache.commons.lang3.math.NumberUtils; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java index f5e12bb247..d56856257f 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataExplorerQueryExecutor.java @@ -20,6 +20,7 @@ import org.apache.streampipes.dataexplorer.param.DeleteQueryParams; import org.apache.streampipes.dataexplorer.param.SelectQueryParams; +import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataSeries; import org.apache.streampipes.model.datalake.SpQueryResult; import org.apache.streampipes.model.datalake.SpQueryStatus; @@ -27,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + public abstract class DataExplorerQueryExecutor { private static final Logger LOG = LoggerFactory.getLogger(DataExplorerQueryExecutor.class); @@ -108,11 +111,14 @@ public SpQueryResult executeQuery(X query, protected abstract SpQueryResult postQuery(W queryResult, boolean ignoreMissingValues); - protected abstract W executeQuery(X query); + public abstract W executeQuery(X query); protected abstract String asQueryString(X query); protected abstract X makeDeleteQuery(DeleteQueryParams params); protected abstract X makeSelectQuery(SelectQueryParams params); + + public abstract Map getTagValues(String measurementId, String fields); + public abstract boolean deleteData(DataLakeMeasure measure); } diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java index 1c98f3422c..8484e3f326 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/DataExplorerSchemaManagementTest.java @@ -41,6 +41,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + public class DataExplorerSchemaManagementTest { public static final String NEW_PROPERTY = "newProperty"; public static final String OLD_PROPERTY = "oldProperty"; diff --git a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java index 984aee2bed..52b5552d07 100644 --- a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java +++ b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/param/WhereStatementParamsTest.java @@ -19,7 +19,7 @@ package org.apache.streampipes.dataexplorer.param; import org.apache.streampipes.dataexplorer.param.model.WhereClauseParams; -import org.apache.streampipes.dataexplorer.querybuilder.FilterCondition; +import org.apache.streampipes.model.datalake.FilterCondition; import org.junit.jupiter.api.Test; diff --git a/streampipes-extensions/streampipes-connectors-influx/pom.xml b/streampipes-extensions/streampipes-connectors-influx/pom.xml index daae9e8823..39a5a016b4 100644 --- a/streampipes-extensions/streampipes-connectors-influx/pom.xml +++ b/streampipes-extensions/streampipes-connectors-influx/pom.xml @@ -31,7 +31,7 @@ org.apache.streampipes - streampipes-data-explorer-commons + streampipes-data-explorer-influx 0.95.0-SNAPSHOT diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/adapter/InfluxDbClient.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/adapter/InfluxDbClient.java index b370f415f8..d23af15257 100644 --- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/adapter/InfluxDbClient.java +++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/adapter/InfluxDbClient.java @@ -20,8 +20,8 @@ import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.commons.exceptions.connect.AdapterException; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxConnectionSettings; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.client.InfluxConnectionSettings; import org.apache.streampipes.extensions.connectors.influx.shared.SharedInfluxClient; import org.apache.streampipes.model.connect.guess.GuessSchema; import org.apache.streampipes.model.schema.EventProperty; diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java index 4c6e1d4cd5..65bb2d3a62 100644 --- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java +++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/InfluxConfigs.java @@ -18,7 +18,7 @@ package org.apache.streampipes.extensions.connectors.influx.shared; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxConnectionSettings; +import org.apache.streampipes.dataexplorer.influx.client.InfluxConnectionSettings; import org.apache.streampipes.extensions.api.extractor.IParameterExtractor; import org.apache.streampipes.sdk.StaticProperties; import org.apache.streampipes.sdk.builder.AbstractConfigurablePipelineElementBuilder; diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/SharedInfluxClient.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/SharedInfluxClient.java index 8ab920aeef..c455f31aa9 100644 --- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/SharedInfluxClient.java +++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/shared/SharedInfluxClient.java @@ -19,8 +19,8 @@ package org.apache.streampipes.extensions.connectors.influx.shared; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxConnectionSettings; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.client.InfluxConnectionSettings; import org.influxdb.InfluxDB; import org.influxdb.dto.Pong; diff --git a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java index 1be9306661..9c6f49f411 100644 --- a/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java +++ b/streampipes-extensions/streampipes-connectors-influx/src/main/java/org/apache/streampipes/extensions/connectors/influx/sink/InfluxDbClient.java @@ -19,8 +19,8 @@ package org.apache.streampipes.extensions.connectors.influx.sink; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxClientProvider; -import org.apache.streampipes.dataexplorer.commons.influx.InfluxConnectionSettings; +import org.apache.streampipes.dataexplorer.influx.client.InfluxClientProvider; +import org.apache.streampipes.dataexplorer.influx.client.InfluxConnectionSettings; import org.apache.streampipes.extensions.connectors.influx.shared.SharedInfluxClient; import org.apache.streampipes.model.runtime.Event; diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml b/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml index 13afc03b90..d9b9f7d216 100644 --- a/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml +++ b/streampipes-extensions/streampipes-sinks-internal-jvm/pom.xml @@ -51,7 +51,7 @@ org.apache.streampipes - streampipes-data-explorer-commons + streampipes-data-explorer-influx 0.95.0-SNAPSHOT diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java index cbf1511334..2a8250cb82 100644 --- a/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java +++ b/streampipes-extensions/streampipes-sinks-internal-jvm/src/main/java/org/apache/streampipes/sinks/internal/jvm/datalake/DataLakeSink.java @@ -20,7 +20,7 @@ import org.apache.streampipes.commons.environment.Environments; import org.apache.streampipes.commons.exceptions.SpRuntimeException; -import org.apache.streampipes.dataexplorer.commons.TimeSeriesStore; +import org.apache.streampipes.dataexplorer.influx.migrate.TimeSeriesStore; import org.apache.streampipes.extensions.api.pe.context.EventSinkRuntimeContext; import org.apache.streampipes.model.DataSinkType; import org.apache.streampipes.model.datalake.DataLakeMeasure; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/AggregationFunction.java similarity index 95% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/AggregationFunction.java index 0d899c2b94..7e85b47e48 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/AggregationFunction.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/AggregationFunction.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.param.model; +package org.apache.streampipes.model.datalake; public enum AggregationFunction { diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeQueryOrdering.java similarity index 93% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeQueryOrdering.java index f8825c157f..7c126eaf56 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/DataLakeQueryOrdering.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeQueryOrdering.java @@ -16,7 +16,7 @@ * */ -package org.apache.streampipes.dataexplorer.querybuilder; +package org.apache.streampipes.model.datalake; public enum DataLakeQueryOrdering { ASC, DESC diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/FilterCondition.java similarity index 93% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/FilterCondition.java index ed2592d8d3..8519bbee01 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/querybuilder/FilterCondition.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/FilterCondition.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.querybuilder; +package org.apache.streampipes.model.datalake; public record FilterCondition(String field, String operator, Object condition) { } diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/ProvidedRestQueryParams.java similarity index 97% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/ProvidedRestQueryParams.java index d703eb934b..a976bb93dc 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/ProvidedRestQueryParams.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/ProvidedRestQueryParams.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.param; +package org.apache.streampipes.model.datalake.param; import java.util.HashMap; import java.util.Map; diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java similarity index 97% rename from streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java rename to streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java index 7c3e3f829c..881be1c659 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/SupportedRestQueryParams.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java @@ -15,7 +15,7 @@ * limitations under the License. * */ -package org.apache.streampipes.dataexplorer.param; +package org.apache.streampipes.model.datalake.param; import java.util.Arrays; import java.util.List; diff --git a/streampipes-platform-services/pom.xml b/streampipes-platform-services/pom.xml index 3c7b59e9af..13f92da128 100644 --- a/streampipes-platform-services/pom.xml +++ b/streampipes-platform-services/pom.xml @@ -30,7 +30,12 @@ org.apache.streampipes - streampipes-data-explorer + streampipes-data-explorer-export + 0.95.0-SNAPSHOT + + + org.apache.streampipes + streampipes-data-explorer-influx 0.95.0-SNAPSHOT diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java index f315571cff..64c009e7a8 100644 --- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java +++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeMeasureResourceV4.java @@ -20,7 +20,7 @@ import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement; import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement; -import org.apache.streampipes.dataexplorer.influx.DataLakeMeasurementCount; +import org.apache.streampipes.dataexplorer.influx.migrate.DataLakeMeasurementCount; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.storage.management.StorageDispatcher; diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java index d19e60738d..4716a9ec06 100644 --- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java +++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java @@ -18,13 +18,13 @@ package org.apache.streampipes.ps; -import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement; import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement; -import org.apache.streampipes.dataexplorer.param.ProvidedRestQueryParams; -import org.apache.streampipes.dataexplorer.query.writer.OutputFormat; +import org.apache.streampipes.dataexplorer.export.OutputFormat; +import org.apache.streampipes.dataexplorer.influx.migrate.DataExplorerQueryManagement; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.DataSeries; import org.apache.streampipes.model.datalake.SpQueryResult; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.apache.streampipes.model.message.Notifications; import org.apache.streampipes.model.monitoring.SpLogMessage; import org.apache.streampipes.rest.core.base.impl.AbstractRestResource; @@ -57,24 +57,24 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COLUMNS; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_COUNT_ONLY; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_CSV_DELIMITER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_END_DATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_FILTER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_FORMAT; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_GROUP_BY; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_LIMIT; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_MISSING_VALUE_BEHAVIOUR; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_OFFSET; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_ORDER; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_PAGE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_START_DATE; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.QP_TIME_INTERVAL; -import static org.apache.streampipes.dataexplorer.param.SupportedRestQueryParams.SUPPORTED_PARAMS; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_AGGREGATION_FUNCTION; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_AUTO_AGGREGATE; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_COLUMNS; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_COUNT_ONLY; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_CSV_DELIMITER; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_END_DATE; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FILTER; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FORMAT; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_GROUP_BY; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_LIMIT; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MISSING_VALUE_BEHAVIOUR; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_OFFSET; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_ORDER; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_PAGE; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_START_DATE; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_TIME_INTERVAL; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.SUPPORTED_PARAMS; @RestController @RequestMapping("/api/v4/datalake") diff --git a/streampipes-rest/pom.xml b/streampipes-rest/pom.xml index 81f79d526e..48b45acddc 100644 --- a/streampipes-rest/pom.xml +++ b/streampipes-rest/pom.xml @@ -42,7 +42,7 @@ org.apache.streampipes - streampipes-data-explorer + streampipes-data-explorer-influx 0.95.0-SNAPSHOT diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java index d5e89b9671..3c68acf0b9 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/ResetManagement.java @@ -22,8 +22,8 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException; import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager; import org.apache.streampipes.connect.management.management.AdapterMasterManagement; -import org.apache.streampipes.dataexplorer.DataExplorerQueryManagement; import org.apache.streampipes.dataexplorer.DataExplorerSchemaManagement; +import org.apache.streampipes.dataexplorer.influx.migrate.DataExplorerQueryManagement; import org.apache.streampipes.manager.file.FileManager; import org.apache.streampipes.manager.pipeline.PipelineCacheManager; import org.apache.streampipes.manager.pipeline.PipelineCanvasMetadataCacheManager; diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java index 63f86b34c9..72c0e9cf2c 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/PersistedDataStreamResource.java @@ -17,7 +17,7 @@ */ package org.apache.streampipes.rest.impl.datalake; -import org.apache.streampipes.dataexplorer.commons.sanitizer.MeasureNameSanitizer; +import org.apache.streampipes.dataexplorer.influx.sanitize.MeasureNameSanitizer; import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.graph.DataSinkInvocation; import org.apache.streampipes.model.pipeline.Pipeline;