Improve data explorer widgets (#1857)

* Improve rendering of echarts widgets in data explorer

* Migrate heatmap widget to echarts renderer

* Add revision to correlation charts

* Remove unused echarts-stats module

* Remove unused css files, add license header

* Improve rendering of tooltips and labels

* Fix reloading behaviour in time selector

* Fix format

ui/deployment/appng5.module.mst

@@ -54,6 +54,19 @@ import { AvailableRolesService } from './services/available-roles.service';
 
 import { MAT_FORM_FIELD_DEFAULT_OPTIONS } from '@angular/material/form-field';
 
+import { ValueDistributionTransform } from './core-ui/echarts-transform/value-distribution.transform';
+import { HistogramTransform } from './core-ui/echarts-transform/histogram.transform';
+import { RoundValuesTransform} from './core-ui/echarts-transform/round-values.transform';
+
+import * as echarts from 'echarts';
+import * as transform from 'echarts-simple-transform';
+
+echarts.registerTransform(transform.aggregate);
+echarts.registerTransform(ValueDistributionTransform);
+echarts.registerTransform(HistogramTransform);
+echarts.registerTransform(RoundValuesTransform);
+
+
 import * as $ from 'jquery';
 
 @NgModule({

ui/package.json

@@ -60,11 +60,13 @@
     "angular2-uuid": "1.1.1",
     "codemirror": "^5.65.11",
     "console-browserify": "^1.2.0",
+    "d3-array": "^3.2.4",
     "dagre": "0.8.5",
     "datatables.net": "^1.13.1",
     "datatables.net-dt": "^1.13.1",
     "echarts": "^5.4.1",
     "echarts-wordcloud": "^2.1.0",
+    "echarts-simple-transform": "^1.0.0",
     "file-saver": "2.0.5",
     "jquery": "^3.7.0",
     "jquery-ui-dist": "1.13.2",

ui/src/app/core-ui/echarts-transform/histogram.transform.ts

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import {
+    DataTransformOption,
+    ExternalDataTransform,
+    ExternalDataTransformResultItem,
+} from 'echarts/types/src/data/helper/transform';
+
+import { bin } from 'd3-array';
+import { OptionSourceDataArrayRows } from 'echarts/types/src/util/types';
+
+export interface HistogramConfig extends DataTransformOption {
+    field: string;
+    autoBin: boolean;
+    numberOfBins: number;
+    autoDomain: boolean;
+    domainMin: number;
+    domainMax: number;
+}
+
+export const HistogramTransform: ExternalDataTransform<HistogramConfig> = {
+    type: 'sp:histogram',
+
+    transform: function (
+        params,
+    ): ExternalDataTransformResultItem | ExternalDataTransformResultItem[] {
+        const upstream = params.upstream;
+        const clonedData = upstream.cloneRawData();
+        const field = params.config['field'];
+        const autoBin = params.config['autoBin'];
+        const numberOfBins = +params.config['numberOfBins'];
+        const autoDomain = params.config['autoDomain'];
+        const domainMin = +params.config['domainMin'];
+        const domainMax = +params.config['domainMax'];
+        const dimension = upstream.getDimensionInfo(field);
+        const values = (clonedData as any).map(row => row[dimension.index]);
+        values.shift();
+
+        let bins = bin();
+        if (!autoBin && numberOfBins) {
+            bins = bins.thresholds(numberOfBins);
+        }
+        if (!autoDomain && domainMin && domainMax) {
+            bins = bins.domain([domainMin, domainMax]);
+        }
+        const d3h = bins(values);
+        const source: OptionSourceDataArrayRows = [];
+        const hist: number[] = d3h.map(item => item.length);
+        const binEdges = d3h.map(item => item.x0);
+        if (d3h.length) {
+            binEdges.push(d3h.at(-1).x1);
+        }
+        hist.forEach((val, index) => {
+            source.push([binEdges[index] + '-' + binEdges[index + 1], val]);
+        });
+
+        return {
+            data: source,
+            dimensions: ['edge', 'hist'],
+        };
+    },
+};

ui/src/app/core-ui/echarts-transform/round-values.transform.ts

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import {
+    DataTransformOption,
+    ExternalDataTransform,
+    ExternalDataTransformResultItem,
+} from 'echarts/types/src/data/helper/transform';
+import { DataExplorerField } from '@streampipes/platform-services';
+import { OptionSourceDataArrayRows } from 'echarts/types/src/util/types';
+
+export interface RoundValuesConfig extends DataTransformOption {
+    fields: DataExplorerField[];
+    roundingValue: number;
+}
+
+export const RoundValuesTransform: ExternalDataTransform<RoundValuesConfig> = {
+    type: 'sp:round',
+
+    transform: function (
+        params,
+    ): ExternalDataTransformResultItem | ExternalDataTransformResultItem[] {
+        const fields: string[] = (
+            params.config['fields'] as DataExplorerField[]
+        )
+            .filter(f => f.fieldCharacteristics.numeric)
+            .map(f => f.fullDbName);
+        const roundingValue = +params.config['roundingValue'];
+        const upstream = params.upstream;
+
+        const dimsDef = upstream.cloneAllDimensionInfo();
+
+        const roundColumnIndices = dimsDef
+            .filter(dim => fields.indexOf(dim.name) > -1)
+            .map(dim => dim.index);
+
+        const data = upstream.cloneRawData() as OptionSourceDataArrayRows;
+        const result = data.map((row, index) => {
+            if (index == 0) {
+                return row;
+            } else {
+                return row.map((value, index) => {
+                    if (roundColumnIndices.indexOf(index) > -1) {
+                        return (
+                            Math.round((value as number) / roundingValue) *
+                            roundingValue
+                        );
+                    } else {
+                        return value;
+                    }
+                });
+            }
+        });
+        result.shift();
+        return {
+            dimensions: upstream.cloneAllDimensionInfo(),
+            data: result,
+        };
+    },
+};

ui/src/app/core-ui/echarts-transform/value-distribution.transform.ts

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import {
+    DataTransformOption,
+    ExternalDataTransform,
+    ExternalDataTransformResultItem,
+} from 'echarts/types/src/data/helper/transform';
+
+export interface ValueDistributionConfig extends DataTransformOption {
+    field: string;
+    resolution: number;
+}
+
+export const ValueDistributionTransform: ExternalDataTransform<ValueDistributionConfig> =
+    {
+        type: 'sp:value-distribution',
+
+        transform: function (
+            params,
+        ): ExternalDataTransformResultItem | ExternalDataTransformResultItem[] {
+            const upstream = params.upstream;
+            const clonedData = upstream.cloneRawData();
+            const field = params.config['field'];
+            const resolution = +params.config['resolution'];
+            const dimension = upstream.getDimensionInfo(field);
+
+            const dataResult = [];
+            const allValues = (clonedData as any).map(
+                row => row[dimension.index],
+            );
+            allValues.shift();
+            const total = allValues.length;
+            let currentCount = 0;
+            allValues.sort((a, b) => a - b);
+            let start = allValues[0];
+            for (let i = 0; i < allValues.length; i++) {
+                const value = allValues[i];
+                if (value < start + +resolution) {
+                    currentCount += 1;
+                }
+                if (value >= start + resolution || i + 1 === allValues.length) {
+                    const currentRange =
+                        '>' +
+                        start.toFixed(2) +
+                        (i + 1 < allValues.length
+                            ? '<' + allValues[i + 1].toFixed(2)
+                            : '');
+                    dataResult.push([currentRange, 0, currentCount / total]);
+                    currentCount = 0;
+                    start = allValues[i + 1];
+                }
+            }
+
+            return {
+                data: dataResult,
+                dimensions: ['category', 'group', field],
+            };
+        },
+    };

ui/src/app/data-explorer/components/designer-panel/visualisation-settings/data-explorer-visualisation-settings.component.ts

@@ -33,7 +33,7 @@ export class DataExplorerVisualisationSettingsComponent implements OnInit {
 
     constructor(private widgetTypeService: WidgetTypeService) {}
 
-    availableWidgets: IWidget[];
+    availableWidgets: IWidget<any>[];
 
     ngOnInit(): void {
         this.availableWidgets =

ui/src/app/data-explorer/components/time-selector/timeRangeSelector.component.html

@@ -86,8 +86,9 @@
             class="button-margin smaller-button-font-size"
             matTooltip="Reload"
             matTooltipPosition="above"
+            (click)="reloadData()"
         >
-            <i (click)="refreshData()" class="material-icons mr-0">autorenew</i>
+            <i class="material-icons mr-0">autorenew</i>
         </button>
     </div>
 </div>