Fix segment configuration heatmap (#777)

* Refactor update segment configuration

* Adjust endpoint to work better with frontend

* Adjust frontend

* Add explicit hover text

* handle case when no data is avaiable

* Adjust sql query

* Adjust query

* Fix linting errors

* Improce doc strings

* Adjust heatmap y axis

* Fix e2e test

* Set automargin for x axis to prevent overlapping

* remove pydocstyle ini

* extract execute sql to separate job

* Implement feedback

* Add gap between cells in heatmap

* remove zero line

* Fix liniting errors

* Improve code documentation

* Fix typo

* Rmove d ticks

* Adjust heat labels

* Add Chunk ID to hovertemplate

* handle non existing chunks

hyrisecockpit/database_manager/job/execute_sql.py

@@ -0,0 +1,25 @@
+from hyrisecockpit.database_manager.cursor import ConnectionFactory
+from psycopg2 import DatabaseError, InterfaceError
+
+
+def execute_sql(sql: str, connection_factory: ConnectionFactory):
+    """Execute sql on hyrise.
+
+    This function executes a sql query via a psycopg2 cursor in a hyrise. If an error occurs
+    in the hyrise, the methods returns an empty list. This is a very light error handling
+    to make sure the cockpit is still running even when the hyrise is not responsive for
+    a moment.
+
+    Args:
+        sql (str): SQL query that should be executed in the hyrise database.
+        connection_factory (ConnectionFactory): A factory that returns a wrapper object around a psycopg2
+            cursor that is connected to the hyrise database. All the attributes needed
+            to connect to the hyrise are already defined inside the factory.
+    """
+
+    try:
+        with connection_factory.create_cursor() as cur:
+            cur.execute(sql, None)
+            return cur.fetchall()
+    except (DatabaseError, InterfaceError):
+        return []

hyrisecockpit/database_manager/job/update_segment_configuration.py

@@ -1,77 +1,100 @@
-"""This job updates the segment configurations."""
 from json import dumps
 from time import time_ns
-from typing import Dict
-
-from pandas import DataFrame
+from typing import Dict, List, Tuple
 
 from hyrisecockpit.database_manager.cursor import StorageConnectionFactory
-from hyrisecockpit.database_manager.job.sql_to_data_frame import sql_to_data_frame
+from hyrisecockpit.database_manager.cursor import ConnectionFactory
+from .execute_sql import execute_sql
+
 
+def _format_results(results: List[Tuple]) -> Dict:
+    """Format psycopg2 cursor results.
 
-def _create_dictionary(
-    segments_configuration: DataFrame, configuration_type: str
-) -> Dict:  # TODO refactoring
-    segment_configuration_data: Dict = {}
-    grouped_tables = segments_configuration.reset_index().groupby("table_name")
+    This function iterates over each row and creates a dictionary where
+    the keys are the table names. For every table name the value is a
+    dictionary where the keys are the column names. For every column name
+    the value is a list where the indices are the chunk_id and the value is
+    the name of the property. The name of the property can be for example "Dictionary",
+    The name of the property is represented by an integer.
+    The integer is the index of the name of the property in the mode_id_mapping list.
+    """
+    formatted_results: Dict = {}
+    mode_id_mapping: List = []
 
-    for table_name in grouped_tables.groups:
-        segment_configuration_data[table_name] = {}
-        table = grouped_tables.get_group(table_name)
-        grouped_columns = table.reset_index().groupby("chunk_id")
+    for row in results:
+        table_name, column_name, chunk_id, chunk_property = row
+        if table_name not in formatted_results:
+            formatted_results[table_name] = {}
+        if column_name not in formatted_results[table_name]:
+            formatted_results[table_name][column_name] = []
+        if chunk_property not in mode_id_mapping:
+            mode_id_mapping.append(chunk_property)
+        # We need to fill the missing chunks with None
+        if len(formatted_results[table_name][column_name]) < chunk_id:
+            missing_chunks = chunk_id - len(formatted_results[table_name][column_name])
+            formatted_results[table_name][column_name] += [
+                None for _ in range(missing_chunks)
+            ]
+        # We can append the property in every iteration
+        # since the chunks are in ascending order and so the indices
+        # of the list are the chunk id's with the property.
+        formatted_results[table_name][column_name].append(
+            mode_id_mapping.index(chunk_property)
+        )
 
-        for column_name in grouped_columns.groups:
-            segment_configuration_data[table_name][column_name] = {}
-            column = grouped_columns.get_group(column_name)
-            for _, row in column.iterrows():
-                segment_configuration_data[table_name][column_name][
-                    configuration_type
-                ] = row[configuration_type]
-    return segment_configuration_data
+    return {"columns": formatted_results, "mode_mapping": mode_id_mapping}
 
 
 def update_segment_configuration(
     database_blocked,
-    connection_factory,
+    connection_factory: ConnectionFactory,
     storage_connection_factory: StorageConnectionFactory,
 ) -> None:
-    """Update segment configuration data for database instance."""
+    """Update segment configuration data for database instance.
+
+    First the needed information is extracted from the Hyrise. After that the raw SQL results
+    are formatted and written to the influx.
+    """
+    sql_segments_encoding: str = """SELECT
+                                    table_name,
+                                    column_name,
+                                    chunk_id,
+                                    encoding_type
+                                    FROM meta_segments
+                                    ORDER BY chunk_id ASC;"""
+    sql_segments_order: str = """SELECT
+                                meta_chunk_sort_orders.table_name,
+                                meta_segments.column_name,
+                                meta_chunk_sort_orders.chunk_id,
+                                meta_chunk_sort_orders.order_mode
+                                FROM meta_chunk_sort_orders
+                                JOIN meta_segments
+                                    ON meta_segments.table_name = meta_chunk_sort_orders.table_name
+                                    AND meta_segments.chunk_id = meta_chunk_sort_orders.chunk_id
+                                ORDER BY meta_chunk_sort_orders.chunk_id ASC;"""
     time_stamp = time_ns()
 
-    segments_encodings = sql_to_data_frame(
-        database_blocked,
-        connection_factory,
-        """SELECT table_name, chunk_id, encoding_type FROM meta_segments;""",
-        None,
+    sql_segments_encoding_results: List[Tuple] = execute_sql(
+        sql_segments_encoding, connection_factory
     )
-    segments_orders = sql_to_data_frame(
-        database_blocked,
-        connection_factory,
-        """SELECT table_name, chunk_id, order_mode FROM meta_chunk_sort_orders;""",
-        None,
+    sql_segments_order_results: List[Tuple] = execute_sql(
+        sql_segments_order, connection_factory
     )
 
-    segment_configuration_encoding_type = {}
-    if not (segments_encodings.empty):
-        segment_configuration_encoding_type = _create_dictionary(
-            segments_encodings, "encoding_type"
-        )
-
-    segment_configuration_order_mode = {}
-    if not segments_orders.empty:
-        segment_configuration_order_mode = _create_dictionary(
-            segments_orders, "order_mode"
-        )
+    formatted_sql_segments_encoding_results = _format_results(
+        sql_segments_encoding_results
+    )
+    formatted_sql_segments_order_results = _format_results(sql_segments_order_results)
 
     with storage_connection_factory.create_cursor() as log:
         log.log_meta_information(
             "segment_configuration",
             {
                 "segment_configuration_encoding_type": dumps(
-                    segment_configuration_encoding_type
+                    formatted_sql_segments_encoding_results
                 ),
                 "segment_configuration_order_mode": dumps(
-                    segment_configuration_order_mode
+                    formatted_sql_segments_order_results
                 ),
             },
             time_stamp,

hyrisecockpit/frontend/src/components/charts/Heatmap.vue

@@ -88,6 +88,8 @@ function useHeatMapConfiguration(
     return {
       xaxis: {
         rangemode: "tozero",
+        tickangle: 45,
+        automargin: true,
         title: {
           text: props.chartConfiguration.xaxis,
           font: {
@@ -98,7 +100,12 @@ function useHeatMapConfiguration(
         fixedrange: true,
       },
       yaxis: {
+        // Removes the line  which is drawn along the 0 value of the y axis
+        zeroline: false,
         rangemode: "tozero",
+        // Sets the tick label formatting rule using d3 formatting mini-languages
+        // ",d" means comma separated decimal
+        tickformat: ",d",
         title: {
           text: props.chartConfiguration.yaxis,
           font: {
@@ -112,7 +119,6 @@ function useHeatMapConfiguration(
       autosize: props.autosize,
       width: props.autosize ? 0 : 1300,
       height: props.autosize ? 0 : 600,
-
       margin: {
         l: 60,
         r: 10,
@@ -129,15 +135,19 @@ function useHeatMapConfiguration(
       descriptions: [],
       chunks: [],
       columns: [],
+      text: [],
     }
   ): Object {
     return {
       z: data.dataByChunks,
       x: data.columns,
       y: data.chunks,
+      // xgap and ygap set the horizontal and vertical gap (in pixels) between bricks.
+      ygap: 1,
+      xgap: 1,
       zmin: 0,
       zmax: maxValue,
-      text: data.descriptions,
+      text: data.text,
       type: "heatmap",
       colorscale: props.colorScale,
       cauto: false,
@@ -147,7 +157,7 @@ function useHeatMapConfiguration(
     };
   }
   function getOptions(): Object {
-    return { displayModeBar: false };
+    return { displayModeBar: false, ygap: 10, xgap: 10 };
   }
   return { getDataset, getLayout, getOptions };
 }

hyrisecockpit/frontend/src/components/metrics/SegmentConfiguration.vue

@@ -45,7 +45,7 @@
               :hover-template="hoverTemplate"
               :color-scale="colorScale"
               :color-bar="colorBar"
-              :show-y="false"
+              :show-y="true"
             />
           </template>
         </metric-detailed-view>
@@ -89,7 +89,7 @@
       :hover-template="hoverTemplate"
       :color-scale="colorScale"
       :color-bar="colorBar"
-      :show-y="false"
+      :show-y="true"
     />
   </div>
 </template>
@@ -184,14 +184,11 @@ export default defineComponent({
           autotick: false,
           tick0: !0,
           dtick: 1,
-          ticktext: newData.valueToId.map((id) =>
-            id.length > 7 ? id.substring(0, 7) + ".." : id
-          ),
+          ticktext: newData.valueToId,
           tickvals: [...Array(scaleLength).keys()],
         };
       }
     );
-
     return {
       chartConfiguration: getMetricChartConfiguration(props.metric),
       maxValue,
@@ -201,7 +198,7 @@ export default defineComponent({
       ...selection,
       hoverTemplate: computed(
         () =>
-          `<b>segment: %{text.column}</b> <br>${selection.selectedType.value}: %{text.value} <extra></extra>`
+          `<b>segment: %{x}</b><br>Chunk ID: %{y}<br>${selection.selectedType.value}: %{text} <extra></extra>`
       ),
       types: [
         { name: "encoding_type", icon: "mdi-barcode" },

hyrisecockpit/frontend/src/meta/metrics.ts

@@ -166,11 +166,11 @@ const metricValueStateOrder: Record<
 const timeLabel = "Timestamps";
 const queryLabel = "Number of queries";
 
-const metricsChartConfiguration: Record<Metric, ChartConfiguration> = {
+const metricsChartConfiguration: Record<Metric, any> = {
   access: {
     title: "Segment Access Frequencies", //Access Frequency
     xaxis: "Columns",
-    yaxis: "Chunks",
+    yaxis: "Chunk ID",
   },
   cpu: {
     title: "CPU Utilization", //CPU (from hyrise utilization table)
@@ -215,7 +215,7 @@ const metricsChartConfiguration: Record<Metric, ChartConfiguration> = {
   segmentConfiguration: {
     title: "Segment Configurations",
     xaxis: "Segments",
-    yaxis: "",
+    yaxis: "Chunk ID",
   },
   storage: {
     title: "Data Size - Overview", //Storage