feat(#3158): Harmonize data explorer and live dashboard (#3440)

* feat(#3439): Extract shared data explorer components to shared module * feat(#3441): Remove deprecated live dashboard (#3442) * feat(#3443): Move data explorer dashboard sources to dashboard module (#3444) * feat(#3443): Move data explorer dashboard to dashboard UI module * refactor(#3443): Rename data view to charts * Refactor asset links, export and import, add migration * Fix checkstyle * Refactor chart asset filter * Fix resource count * fix tests * Fix navigation in dashboard sidebar * Shorten component path * Fix tests, cleanup dashboard resource * Rename services * Fix import * Fix migration
apache · Jan 30, 2025 · 624da1e · 624da1e
1 parent 64624c6
commit 624da1e
Show file tree

Hide file tree

Showing 387 changed files with 1,463 additions and 10,170 deletions.
diff --git a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
@@ -20,9 +20,9 @@
 
 import org.apache.streampipes.export.constants.ResolvableAssetLinks;
 import org.apache.streampipes.export.resolver.AdapterResolver;
+import org.apache.streampipes.export.resolver.ChartResolver;
 import org.apache.streampipes.export.resolver.DashboardResolver;
 import org.apache.streampipes.export.resolver.DataSourceResolver;
-import org.apache.streampipes.export.resolver.DataViewResolver;
 import org.apache.streampipes.export.resolver.FileResolver;
 import org.apache.streampipes.export.resolver.MeasurementResolver;
 import org.apache.streampipes.export.resolver.PipelineResolver;
@@ -33,6 +33,8 @@
 import org.apache.streampipes.storage.management.StorageDispatcher;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Map;
@@ -41,6 +43,8 @@
 
 public class AssetLinkResolver {
 
+  private static final Logger LOG = LoggerFactory.getLogger(AssetLinkResolver.class);
+
   private final String assetId;
   private final ObjectMapper mapper;
 
@@ -58,8 +62,8 @@ public AssetExportConfiguration resolveResources() {
       exportConfig.setAssetId(this.assetId);
       exportConfig.setAssetName(asset.getAssetName());
       exportConfig.setAdapters(new AdapterResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.ADAPTER)));
+      exportConfig.setDataViews(new ChartResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.CHART)));
       exportConfig.setDashboards(new DashboardResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DASHBOARD)));
-      exportConfig.setDataViews(new DataViewResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_VIEW)));
       exportConfig.setDataSources(
           new DataSourceResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.DATA_SOURCE)));
       exportConfig.setPipelines(new PipelineResolver().resolve(getLinks(assetLinks, ResolvableAssetLinks.PIPELINE)));
@@ -69,7 +73,7 @@ public AssetExportConfiguration resolveResources() {
 
       return exportConfig;
     } catch (IOException e) {
-      e.printStackTrace();
+      LOG.error("IO Exception when writing export configuration", e);
       return new AssetExportConfiguration();
     }
   }

diff --git a/...ta-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java b/...ta-export/src/main/java/org/apache/streampipes/export/constants/ResolvableAssetLinks.java
@@ -20,7 +20,7 @@
 
 public class ResolvableAssetLinks {
 
-  public static final String DATA_VIEW = "data-view";
+  public static final String CHART = "chart";
   public static final String DASHBOARD = "dashboard";
   public static final String MEASUREMENT = "measurement";
   public static final String ADAPTER = "adapter";

diff --git a/...s-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java b/...s-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
@@ -67,19 +67,11 @@ public T generate(InputStream inputStream) throws IOException {
       }
     }
 
-    for (String dashboardId : manifest.getDashboards()) {
+    for (String chartId : manifest.getDataViewWidgets()) {
       try {
-        handleDashboard(asString(previewFiles.get(dashboardId)), dashboardId);
+        handleChart(asString(previewFiles.get(chartId)), chartId);
       } catch (DocumentConflictException e) {
-        LOG.warn("Skipping import of dashboard {} (already present with the same id)", dashboardId);
-      }
-    }
-
-    for (String dataViewId : manifest.getDataViews()) {
-      try {
-        handleDataView(asString(previewFiles.get(dataViewId)), dataViewId);
-      } catch (DocumentConflictException e) {
-        LOG.warn("Skipping import of data view {} (already present with the same id)", dataViewId);
+        LOG.warn("Skipping import of chart {} (already present with the same id)", chartId);
       }
     }
 
@@ -107,19 +99,11 @@ public T generate(InputStream inputStream) throws IOException {
       }
     }
 
-    for (String dashboardWidgetId : manifest.getDashboardWidgets()) {
-      try {
-        handleDashboardWidget(asString(previewFiles.get(dashboardWidgetId)), dashboardWidgetId);
-      } catch (DocumentConflictException e) {
-        LOG.warn("Skipping import of dashboard widget {} (already present with the same id)", dashboardWidgetId);
-      }
-    }
-
-    for (String dataViewWidgetId : manifest.getDataViewWidgets()) {
+    for (String dashboardId : manifest.getDashboards()) {
       try {
-        handleDataViewWidget(asString(previewFiles.get(dataViewWidgetId)), dataViewWidgetId);
+        handleDashboard(asString(previewFiles.get(dashboardId)), dashboardId);
       } catch (DocumentConflictException e) {
-        LOG.warn("Skipping import of data view widget {} (already present with the same id)", dataViewWidgetId);
+        LOG.warn("Skipping import of dashboard {} (already present with the same id)", dashboardId);
       }
     }
 
@@ -149,9 +133,9 @@ private StreamPipesApplicationPackage getManifest(Map<String, byte[]> previewFil
 
   protected abstract void handleAdapter(String document, String adapterId) throws JsonProcessingException;
 
-  protected abstract void handleDashboard(String document, String dashboardId) throws JsonProcessingException;
+  protected abstract void handleChart(String document, String dataViewId) throws JsonProcessingException;
 
-  protected abstract void handleDataView(String document, String dataViewId) throws JsonProcessingException;
+  protected abstract void handleDashboard(String document, String dashboardId) throws JsonProcessingException;
 
   protected abstract void handleDataSource(String document, String dataSourceId) throws JsonProcessingException;
 
@@ -160,11 +144,6 @@ private StreamPipesApplicationPackage getManifest(Map<String, byte[]> previewFil
   protected abstract void handleDataLakeMeasure(String document, String dataLakeMeasureId)
       throws JsonProcessingException;
 
-  protected abstract void handleDashboardWidget(String document, String dashboardWidgetId)
-      throws JsonProcessingException;
-
-  protected abstract void handleDataViewWidget(String document, String dataViewWidgetId) throws JsonProcessingException;
-
   protected abstract void handleFile(String document, String fileMetadataId, Map<String, byte[]> zipContent)
       throws IOException;
 

diff --git a/...export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java b/...export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
@@ -20,11 +20,9 @@
 
 import org.apache.streampipes.export.model.PermissionInfo;
 import org.apache.streampipes.export.resolver.AdapterResolver;
+import org.apache.streampipes.export.resolver.ChartResolver;
 import org.apache.streampipes.export.resolver.DashboardResolver;
-import org.apache.streampipes.export.resolver.DashboardWidgetResolver;
 import org.apache.streampipes.export.resolver.DataSourceResolver;
-import org.apache.streampipes.export.resolver.DataViewResolver;
-import org.apache.streampipes.export.resolver.DataViewWidgetResolver;
 import org.apache.streampipes.export.resolver.FileResolver;
 import org.apache.streampipes.export.resolver.MeasurementResolver;
 import org.apache.streampipes.export.resolver.PipelineResolver;
@@ -33,6 +31,7 @@
 import org.apache.streampipes.model.SpDataStream;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.model.dashboard.DashboardModel;
+import org.apache.streampipes.model.datalake.DataExplorerWidgetModel;
 import org.apache.streampipes.model.datalake.DataLakeMeasure;
 import org.apache.streampipes.model.export.AssetExportConfiguration;
 import org.apache.streampipes.model.export.ExportItem;
@@ -51,10 +50,10 @@
 
 public class PerformImportGenerator extends ImportGenerator<Void> {
 
-  private AssetExportConfiguration config;
-  private INoSqlStorage storage;
-  private Set<PermissionInfo> permissionsToStore = new HashSet<>();
-  private String ownerSid;
+  private final AssetExportConfiguration config;
+  private final INoSqlStorage storage;
+  private final Set<PermissionInfo> permissionsToStore = new HashSet<>();
+  private final String ownerSid;
 
   public PerformImportGenerator(AssetExportConfiguration config,
                                 String ownerSid) {
@@ -78,18 +77,18 @@ protected void handleAdapter(String document, String adapterId) throws JsonProce
   }
 
   @Override
-  protected void handleDashboard(String document, String dashboardId) throws JsonProcessingException {
-    if (shouldStore(dashboardId, config.getDashboards())) {
-      new DashboardResolver().writeDocument(document);
-      permissionsToStore.add(new PermissionInfo(dashboardId, DashboardModel.class));
+  protected void handleChart(String document, String chartId) throws JsonProcessingException {
+    if (shouldStore(chartId, config.getDataViews())) {
+      new ChartResolver().writeDocument(document);
+      permissionsToStore.add(new PermissionInfo(chartId, DataExplorerWidgetModel.class));
     }
   }
 
   @Override
-  protected void handleDataView(String document, String dataViewId) throws JsonProcessingException {
-    if (shouldStore(dataViewId, config.getDataViews())) {
-      new DataViewResolver().writeDocument(document);
-      permissionsToStore.add(new PermissionInfo(dataViewId, DashboardModel.class));
+  protected void handleDashboard(String document, String dashboardId) throws JsonProcessingException {
+    if (shouldStore(dashboardId, config.getDashboards())) {
+      new DashboardResolver().writeDocument(document);
+      permissionsToStore.add(new PermissionInfo(dashboardId, DashboardModel.class));
     }
   }
 
@@ -117,16 +116,6 @@ protected void handleDataLakeMeasure(String document, String dataLakeMeasureId)
     }
   }
 
-  @Override
-  protected void handleDashboardWidget(String document, String dashboardWidgetId) throws JsonProcessingException {
-    new DashboardWidgetResolver().writeDocument(document);
-  }
-
-  @Override
-  protected void handleDataViewWidget(String document, String dataViewWidget) throws JsonProcessingException {
-    new DataViewWidgetResolver().writeDocument(document);
-  }
-
   @Override
   protected void handleFile(String document,
                             String fileMetadataId,

diff --git a/...export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java b/...export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
@@ -20,9 +20,9 @@
 
 
 import org.apache.streampipes.export.resolver.AdapterResolver;
+import org.apache.streampipes.export.resolver.ChartResolver;
 import org.apache.streampipes.export.resolver.DashboardResolver;
 import org.apache.streampipes.export.resolver.DataSourceResolver;
-import org.apache.streampipes.export.resolver.DataViewResolver;
 import org.apache.streampipes.export.resolver.FileResolver;
 import org.apache.streampipes.export.resolver.MeasurementResolver;
 import org.apache.streampipes.export.resolver.PipelineResolver;
@@ -77,13 +77,17 @@ protected void handleAdapter(String document,
   }
 
   @Override
-  protected void handleDashboard(String document, String dashboardId) throws JsonProcessingException {
-    addExportItem(dashboardId, new DashboardResolver().readDocument(document).getName(), importConfig::addDashboard);
+  protected void handleChart(String document, String dataViewId) throws JsonProcessingException {
+    addExportItem(
+        dataViewId,
+        new ChartResolver().readDocument(document).getBaseAppearanceConfig().get("widgetTitle").toString(),
+        importConfig::addDataView
+    );
   }
 
   @Override
-  protected void handleDataView(String document, String dataViewId) throws JsonProcessingException {
-    addExportItem(dataViewId, new DataViewResolver().readDocument(document).getName(), importConfig::addDataView);
+  protected void handleDashboard(String document, String dashboardId) throws JsonProcessingException {
+    addExportItem(dashboardId, new DashboardResolver().readDocument(document).getName(), importConfig::addDashboard);
   }
 
   @Override
@@ -102,16 +106,6 @@ protected void handleDataLakeMeasure(String document, String measurementId) thro
         importConfig::addDataLakeMeasure);
   }
 
-  @Override
-  protected void handleDashboardWidget(String document, String dashboardWidgetId) {
-
-  }
-
-  @Override
-  protected void handleDataViewWidget(String document, String dataViewWidget) {
-
-  }
-
   @Override
   protected void handleFile(String document,
                             String fileMetadataId,

diff --git a/...-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java b/...-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -21,11 +21,9 @@
 import org.apache.streampipes.commons.exceptions.ElementNotFoundException;
 import org.apache.streampipes.export.resolver.AbstractResolver;
 import org.apache.streampipes.export.resolver.AdapterResolver;
+import org.apache.streampipes.export.resolver.ChartResolver;
 import org.apache.streampipes.export.resolver.DashboardResolver;
-import org.apache.streampipes.export.resolver.DashboardWidgetResolver;
 import org.apache.streampipes.export.resolver.DataSourceResolver;
-import org.apache.streampipes.export.resolver.DataViewResolver;
-import org.apache.streampipes.export.resolver.DataViewWidgetResolver;
 import org.apache.streampipes.export.resolver.FileResolver;
 import org.apache.streampipes.export.resolver.MeasurementResolver;
 import org.apache.streampipes.export.resolver.PipelineResolver;
@@ -55,12 +53,10 @@ public class ExportPackageGenerator {
 
   private final ExportConfiguration exportConfiguration;
   private ObjectMapper defaultMapper;
-  private ObjectMapper spMapper;
 
   public ExportPackageGenerator(ExportConfiguration exportConfiguration) {
     this.exportConfiguration = exportConfiguration;
     this.defaultMapper = SerializationUtils.getDefaultObjectMapper();
-    this.spMapper = SerializationUtils.getSpObjectMapper();
   }
 
   public byte[] generateExportPackage() throws IOException {
@@ -74,24 +70,11 @@ public byte[] generateExportPackage() throws IOException {
         .collect(Collectors.toList()), manifest);
 
     this.exportConfiguration.getAssetExportConfiguration().forEach(config -> {
-
       config.getAdapters().forEach(item -> addDoc(builder,
           item,
           new AdapterResolver(),
           manifest::addAdapter));
 
-      config.getDashboards().forEach(item -> {
-        var resolver = new DashboardResolver();
-        addDoc(builder,
-            item,
-            resolver,
-            manifest::addDashboard);
-
-        var widgets = resolver.getWidgets(item.getResourceId());
-        var widgetResolver = new DashboardWidgetResolver();
-        widgets.forEach(widgetId -> addDoc(builder, widgetId, widgetResolver, manifest::addDashboardWidget));
-      });
-
       config.getDataSources().forEach(item -> addDoc(builder,
           item,
           new DataSourceResolver(),
@@ -107,16 +90,22 @@ public byte[] generateExportPackage() throws IOException {
           new PipelineResolver(),
           manifest::addPipeline));
 
+      config.getDashboards().forEach(item -> {
+        var resolver = new DashboardResolver();
+        addDoc(builder,
+            item,
+            new DashboardResolver(),
+            manifest::addDashboard);
+        var charts = resolver.getCharts(item.getResourceId());
+        var chartResolver = new ChartResolver();
+        charts.forEach(widgetId -> addDoc(builder, widgetId, chartResolver, manifest::addDataViewWidget));
+      });
+
       config.getDataViews().forEach(item -> {
-        var resolver = new DataViewResolver();
         addDoc(builder,
             item,
-            resolver,
+            new ChartResolver(),
             manifest::addDataView);
-
-        var widgets = resolver.getWidgets(item.getResourceId());
-        var widgetResolver = new DataViewWidgetResolver();
-        widgets.forEach(widgetId -> addDoc(builder, widgetId, widgetResolver, manifest::addDataViewWidget));
       });
 
       config.getFiles().forEach(item -> {

diff --git a/...port/resolver/DataViewWidgetResolver.java → ...mpipes/export/resolver/ChartResolver.java b/...port/resolver/DataViewWidgetResolver.java → ...mpipes/export/resolver/ChartResolver.java
@@ -24,7 +24,7 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 
-public class DataViewWidgetResolver extends AbstractResolver<DataExplorerWidgetModel> {
+public class ChartResolver extends AbstractResolver<DataExplorerWidgetModel> {
 
   @Override
   public DataExplorerWidgetModel findDocument(String resourceId) {
@@ -39,12 +39,15 @@ public DataExplorerWidgetModel modifyDocumentForExport(DataExplorerWidgetModel d
 
   @Override
   public DataExplorerWidgetModel readDocument(String serializedDoc) throws JsonProcessingException {
-    return SerializationUtils.getSpObjectMapper().readValue(serializedDoc, DataExplorerWidgetModel.class);
+    return SerializationUtils.getDefaultObjectMapper().readValue(serializedDoc, DataExplorerWidgetModel.class);
   }
 
   @Override
   public ExportItem convert(DataExplorerWidgetModel document) {
-    return new ExportItem(document.getElementId(), document.getWidgetId(), true);
+    return new ExportItem(
+        document.getElementId(),
+        document.getBaseAppearanceConfig().get("widgetTitle").toString(), true
+    );
   }
 
   @Override

diff --git a/...s-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java b/...s-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
@@ -16,6 +16,7 @@
  *
  */
 
+
 package org.apache.streampipes.export.resolver;
 
 import org.apache.streampipes.export.utils.SerializationUtils;
@@ -33,7 +34,7 @@ public class DashboardResolver extends AbstractResolver<DashboardModel> {
 
   @Override
   public DashboardModel findDocument(String resourceId) {
-    return getNoSqlStore().getDashboardStorage().getElementById(resourceId);
+    return getNoSqlStore().getDataExplorerDashboardStorage().getElementById(resourceId);
   }
 
   @Override
@@ -59,15 +60,15 @@ public ExportItem convert(DashboardModel document) {
 
   @Override
   public void writeDocument(String document) throws JsonProcessingException {
-    getNoSqlStore().getDashboardStorage().persist(deserializeDocument(document));
+    getNoSqlStore().getDataExplorerDashboardStorage().persist(deserializeDocument(document));
   }
 
   @Override
   protected DashboardModel deserializeDocument(String document) throws JsonProcessingException {
     return this.spMapper.readValue(document, DashboardModel.class);
   }
 
-  public List<String> getWidgets(String resourceId) {
+  public List<String> getCharts(String resourceId) {
     var document = findDocument(resourceId);
     return document.getWidgets().stream().map(DashboardItem::getId).collect(Collectors.toList());
   }