Enhance analytics functionality by adding groupBy support in various …

…models and components

Common/Server/API/BaseAnalyticsAPI.ts

@@ -291,6 +291,18 @@ export default class BaseAnalyticsAPI<
       ) as any;
     }
 
+    let groupBy: GroupBy<AnalyticsDataModel> | null = req.body["groupBy"] || null;
+
+    if(groupBy && Object.keys(groupBy).length > 0) {
+      groupBy = JSONFunctions.deserialize(
+        groupBy as JSONObject,
+      ) as any;
+    }
+
+    if(groupBy && Object.keys(groupBy).length === 0) {
+      groupBy = null;
+    }
+
     if (!aggregateBy) {
       throw new BadRequestException("AggregateBy is required");
     }

Common/Server/Services/AnalyticsDatabaseService.ts

@@ -57,14 +57,14 @@ import ModelEventType from "../../Types/Realtime/ModelEventType";
 export default class AnalyticsDatabaseService<
   TBaseModel extends AnalyticsBaseModel,
 > extends BaseService {
-  public modelType!: { new (): TBaseModel };
+  public modelType!: { new(): TBaseModel };
   public database!: ClickhouseDatabase;
   public model!: TBaseModel;
   public databaseClient!: ClickhouseClient;
   public statementGenerator!: StatementGenerator<TBaseModel>;
 
   public constructor(data: {
-    modelType: { new (): TBaseModel };
+    modelType: { new(): TBaseModel };
     database?: ClickhouseDatabase | undefined;
   }) {
     super();
@@ -240,6 +240,8 @@ export default class AnalyticsDatabaseService<
         columns: Array<string>;
       } = this.toAggregateStatement(aggregateBy);
 
+      debugger;
+
       const dbResult: ExecResult<Stream> = await this.execute(
         findStatement.statement,
       );
@@ -259,24 +261,30 @@ export default class AnalyticsDatabaseService<
 
       // convert date column from string to date.
 
+      const groupByColumnName: keyof TBaseModel | undefined = aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0 ? Object.keys(aggregateBy.groupBy)[0] as keyof TBaseModel : undefined;
+
       for (const item of items) {
         if (
           !(item as JSONObject)[
-            aggregateBy.aggregationTimestampColumnName as string
+          aggregateBy.aggregationTimestampColumnName as string
           ]
         ) {
           continue;
         }
 
+
         const aggregatedModel: AggregatedModel = {
           timestamp: OneUptimeDate.fromString(
             (item as JSONObject)[
-              aggregateBy.aggregationTimestampColumnName as string
+            aggregateBy.aggregationTimestampColumnName as string
             ] as string,
           ),
           value: (item as JSONObject)[
             aggregateBy.aggregateColumnName as string
           ] as number,
+          [groupByColumnName as string]: (item as JSONObject)[
+            groupByColumnName as string
+          ],
         };
 
         aggregatedItems.push(aggregatedModel);
@@ -457,7 +465,7 @@ export default class AnalyticsDatabaseService<
                 count()
             FROM ${databaseName}.${this.model.tableName}
             WHERE TRUE `.append(whereStatement);
-    
+
 
     if (countBy.groupBy && Object.keys(countBy.groupBy).length > 0) {
       statement.append(
@@ -519,9 +527,11 @@ export default class AnalyticsDatabaseService<
     statement.append(SQL` FROM ${databaseName}.${this.model.tableName}`);
     statement.append(SQL` WHERE TRUE `).append(whereStatement);
 
-
-      statement.append(SQL` GROUP BY `).append(`${aggregateBy.aggregationTimestampColumnName.toString()}`);
-
+    statement.append(SQL` GROUP BY `).append(`${aggregateBy.aggregationTimestampColumnName.toString()}`);
+
+    if (aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0) {
+      statement.append(SQL` , `).append(this.statementGenerator.toGroupByStatement(aggregateBy.groupBy));
+    }
 
     statement.append(SQL` ORDER BY `).append(sortStatement);
 
@@ -538,7 +548,7 @@ export default class AnalyticsDatabaseService<
     }}
         `);
 
-    
+
 
     logger.debug(`${this.model.tableName} Aggregate Statement`);
     logger.debug(statement);
@@ -604,7 +614,7 @@ export default class AnalyticsDatabaseService<
     }}
         `);
 
-    
+
 
     logger.debug(`${this.model.tableName} Find Statement`);
     logger.debug(statement);
@@ -626,7 +636,7 @@ export default class AnalyticsDatabaseService<
     const statement: Statement = SQL`
             ALTER TABLE ${databaseName}.${this.model.tableName}
             DELETE WHERE TRUE `.append(whereStatement);
-    
+
 
     logger.debug(`${this.model.tableName} Delete Statement`);
     logger.debug(statement);

Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -548,11 +548,24 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       `${aggregationMethod}(${aggregateBy.aggregateColumnName.toString()}) as ${aggregateBy.aggregateColumnName.toString()}, date_trunc('${aggregationInterval.toLowerCase()}', toStartOfInterval(${aggregateBy.aggregationTimestampColumnName.toString()}, INTERVAL 1 ${aggregationInterval.toLowerCase()})) as ${aggregateBy.aggregationTimestampColumnName.toString()}`,
     );
 
+
+    const columns: Array<string> = [
+      aggregateBy.aggregateColumnName.toString(),
+      aggregateBy.aggregationTimestampColumnName.toString(),
+    ];
+
+    if(aggregateBy.groupBy && Object.keys(aggregateBy.groupBy).length > 0) {
+      const groupByStatement: Statement = this.toGroupByStatement(aggregateBy.groupBy);
+      selectStatement.append(SQL`, `).append(groupByStatement);
+
+      // add to columns. 
+      for (const key in aggregateBy.groupBy) {
+        columns.push(key);
+      }
+    }
+
     return {
-      columns: [
-        aggregateBy.aggregateColumnName.toString(),
-        aggregateBy.aggregationTimestampColumnName.toString(),
-      ],
+      columns: columns,
       statement: selectStatement,
     };
   }

Common/Types/BaseDatabase/AggregateBy.ts

@@ -1,3 +1,4 @@
+import GroupBy from "../../Server/Types/Database/GroupBy";
 import GenericObject from "../GenericObject";
 import AggregationType from "./AggregationType";
 import Query from "./Query";
@@ -14,4 +15,5 @@ export default interface AggregateBy<TBaseModel extends GenericObject> {
   limit: number;
   skip: number;
   sort?: Sort<TBaseModel> | undefined;
+  groupBy?: GroupBy<TBaseModel> | undefined;
 }

Common/Types/BaseDatabase/AggregatedModel.ts

@@ -1,4 +1,7 @@
+import { JSONValue } from "../JSON";
+
 export default interface AggregateModel {
+  [x: string]: JSONValue;
   timestamp: Date;
   value: number;
 }

Common/Types/Metrics/MetricsQuery.ts

@@ -6,4 +6,13 @@ export default interface MetricsQuery {
   attributes: Dictionary<string | boolean | number>;
   aggegationType: MetricsAggregationType;
   aggregateBy: Dictionary<boolean>;
+
+
+  // This is used for example for probes. 
+  // To display US probe and EU probe in chart for example. 
+  // In this case groupByAttribute is "probeId"
+  // and attributeValueToLegendMap is { "xx-xx-xx-xx": "US Probe", "yy-yyy-yyy-yy-yy": "EU Probe" }
+
+  groupByAttribute?: string | undefined;
+  attributeValueToLegendMap?: Dictionary<string>;
 }

Dashboard/src/Components/Metrics/MetricQuery.tsx

@@ -7,9 +7,12 @@ import MetricsAggregationType from "Common/Types/Metrics/MetricsAggregationType"
 import Query from "Common/Types/BaseDatabase/Query";
 import MetricsQuery from "Common/Types/Metrics/MetricsQuery";
 import MetricNameAndUnit from "./Types/MetricNameAndUnit";
+import GroupBy from "Common/UI/Utils/BaseDatabase/GroupBy";
+import Metric from "Common/Models/AnalyticsModels/Metric";
 
 export interface MetricQueryData {
   filterData: FilterData<MetricsQuery>;
+  groupBy?: GroupBy<Metric> | undefined; 
 }
 
 export interface ComponentProps {
@@ -31,6 +34,7 @@ const MetricFilter: FunctionComponent<ComponentProps> = (
           filterData={props.data.filterData}
           onFilterChanged={(filterData: Query<MetricsQuery>) => {
             props.onDataChanged({
+              ...props.data,
               filterData,
             });
           }}

Dashboard/src/Components/Metrics/MetricQueryConfig.tsx

@@ -8,10 +8,16 @@ import Button, {
   ButtonStyleType,
 } from "Common/UI/Components/Button/Button";
 import MetricNameAndUnit from "./Types/MetricNameAndUnit";
+import AggregatedModel from "Common/Types/BaseDatabase/AggregatedModel";
+
+export interface ChartSeries { 
+  title: string;
+}
 
 export interface MetricQueryConfigData {
   metricAliasData: MetricAliasData;
   metricQueryData: MetricQueryData;
+  getSeries?: ((data: AggregatedModel) => ChartSeries) | undefined;
 }
 
 export interface ComponentProps {

Dashboard/src/Components/Metrics/MetricView.tsx

@@ -5,7 +5,7 @@ import React, {
   useEffect,
   useState,
 } from "react";
-import MetricQueryConfig, { MetricQueryConfigData } from "./MetricQueryConfig";
+import MetricQueryConfig, { ChartSeries, MetricQueryConfigData } from "./MetricQueryConfig";
 import MetricGraphConfig, {
   MetricFormulaConfigData,
 } from "./MetricFormulaConfig";
@@ -53,6 +53,7 @@ import ChartCurve from "Common/UI/Components/Charts/Types/ChartCurve";
 import { XAxisAggregateType } from "Common/UI/Components/Charts/Types/XAxis/XAxis";
 import { YAxisPrecision } from "Common/UI/Components/Charts/Types/YAxis/YAxis";
 import MetricNameAndUnit from "./Types/MetricNameAndUnit";
+import SeriesPoint from "Common/UI/Components/Charts/Types/SeriesPoints";
 
 export interface MetricViewData {
   queryConfigs: Array<MetricQueryConfigData>;
@@ -198,6 +199,59 @@ const MetricView: FunctionComponent<ComponentProps> = (
         xAxisAggregationType = XAxisAggregateType.Average;
       }
 
+      let chartSeries: Array<SeriesPoint> = [
+
+      ];
+
+      if (queryConfig.getSeries) {
+
+        for (const item of metricResults[index]!.data) {
+          const series: ChartSeries = queryConfig.getSeries(item);
+          const seriesName: string = series.title;
+
+          //check if the series already exists if it does then add the data to the existing series
+
+          // if it does not exist then create a new series and add the data to it
+
+          const existingSeries: SeriesPoint | undefined = chartSeries.find((s: SeriesPoint) => {
+            return s.seriesName === seriesName;
+          });
+
+          if (existingSeries) {
+            existingSeries.data.push({
+              x: OneUptimeDate.fromString(item.timestamp),
+              y: item.value
+            });
+          } else {
+            const newSeries: SeriesPoint = {
+              seriesName: seriesName,
+              data: [{
+                x: OneUptimeDate.fromString(item.timestamp),
+                y: item.value
+              }]
+            };
+
+            chartSeries.push(newSeries);
+          }
+
+        }
+      } else {
+        chartSeries.push({
+          seriesName:
+            queryConfig.metricAliasData.title ||
+            queryConfig.metricQueryData.filterData.metricName?.toString() ||
+            "",
+          data: metricResults[index]!.data.map(
+            (result: AggregatedModel) => {
+              return {
+                x: OneUptimeDate.fromString(result.timestamp),
+                y: result.value,
+              };
+            },
+          ),
+        });
+      }
+
       const chart: Chart = {
         id: index.toString(),
         type: ChartType.LINE,
@@ -207,22 +261,7 @@ const MetricView: FunctionComponent<ComponentProps> = (
           "",
         description: queryConfig.metricAliasData.description,
         props: {
-          data: [
-            {
-              seriesName:
-                queryConfig.metricAliasData.title ||
-                queryConfig.metricQueryData.filterData.metricName?.toString() ||
-                "",
-              data: metricResults[index]!.data.map(
-                (result: AggregatedModel) => {
-                  return {
-                    x: OneUptimeDate.fromString(result.timestamp),
-                    y: result.value,
-                  };
-                },
-              ),
-            },
-          ],
+          data: chartSeries,
           xAxis: {
             legend: "Time",
             options: {
@@ -326,14 +365,14 @@ const MetricView: FunctionComponent<ComponentProps> = (
       const metricAttributesResponse:
         | HTTPResponse<JSONObject>
         | HTTPErrorResponse = await API.post(
-        URL.fromString(APP_API_URL.toString()).addRoute(
-          "/telemetry/metrics/get-attributes",
-        ),
-        {},
-        {
-          ...ModelAPI.getCommonHeaders(),
-        },
-      );
+          URL.fromString(APP_API_URL.toString()).addRoute(
+            "/telemetry/metrics/get-attributes",
+          ),
+          {},
+          {
+            ...ModelAPI.getCommonHeaders(),
+          },
+        );
 
       if (metricAttributesResponse instanceof HTTPErrorResponse) {
         throw metricAttributesResponse;
@@ -382,6 +421,7 @@ const MetricView: FunctionComponent<ComponentProps> = (
                 OneUptimeDate.getCurrentDate(),
               limit: LIMIT_PER_PROJECT,
               skip: 0,
+              groupBy: queryConfig.metricQueryData.groupBy
             },
           });
 

Dashboard/src/Components/Monitor/MonitorMetrics.tsx

@@ -113,8 +113,18 @@ const MonitorMetricsElement: FunctionComponent<ComponentProps> = (
                 MonitorMetricTypeUtil.getAggregationTypeByMonitorMetricType(
                   monitorMetricType,
                 ),
+
             },
+            groupBy: {
+              attributes: true
+            }
+
           },
+          getSeries: (data) => {
+            return {
+              title: data.attributes.monitorId,
+            };
+          }
         });
       }