Introduction of the application score, new Plotter class and update t…

…o seaborn 0.13.0 (#97) Co-authored-by: Philipp Ross <[email protected]> Co-authored-by: Marvin Erdmann <[email protected]>
QUARK-framework · Dec 8, 2023 · 4648cb9 · 4648cb9
1 parent e5f826d
commit 4648cb9
Show file tree

Hide file tree

Showing 11 changed files with 305 additions and 92 deletions.
diff --git a/.settings/module_db.json b/.settings/module_db.json
@@ -1,7 +1,7 @@
 {
-  "build_number": 5,
-  "build_date": "22-11-2023 09:57:53",
-  "git_revision_number": "aa2f578cbc65cda03fd6a743505a14049bd260d6",
+  "build_number": 6,
+  "build_date": "22-11-2023 13:47:42",
+  "git_revision_number": "5ea1fcf56c438df4b32fcc318dcb6e2c8e58447b",
   "modules": [
     {
       "name": "PVC",

diff --git a/.settings/requirements_full.txt b/.settings/requirements_full.txt
@@ -1,4 +1,4 @@
-seaborn==0.12.2
+seaborn==0.13.0
 networkx==2.8.8
 inquirer==3.1.2
 packaging==23.1

diff --git a/docs/developer.rst b/docs/developer.rst
@@ -69,6 +69,18 @@ In addition, there are some special flags you can set for each parameter:
     - :code:`custom_input`: Enabling this feature for your parameter will give the user the option to enter a custom input (text or numbers) for this value. Keep in mind that there is no validation of this user input!
     - :code:`postproc`: Specifies a function that can be called and applied to the parameters, which must be callable.
 
+Application Score
+"""""""""""""""""
+
+For applications, there is the option to provide a set of fields representing the application score, which defines the
+overall quality of the benchmark run in the view of the application.
+If these three fields are present in the :code:`Metrics` object of the application, the :code:`Plotter` class will generate some plots visualizing the application score:
+
+    - "application_score_value": Value of the application score. None if there is no valid application score for this run.
+    - "application_score_unit": Unit of the application will be used for the y-axis of the application score plot.
+    - "application_score_type": The type of the application score needs to be a string. For example :code:`str(float)`.
+
+
 Example for an application, which should reside under ``src/modules/applications/myApplication/``:
 
 .. code-block:: python
@@ -141,9 +153,12 @@ Example for an application, which should reside under ``src/modules/applications
                     input_data)
                 if solution_validity and processed_solution:
                    solution_quality, time_to_evaluation = self.evaluate(processed_solution)
+                else:
+                   solution_quality = None
 
-                self.metrics.add_metric_batch({"solution_validity": solution_validity, "solution_quality": solution_quality,
-                           "solution": input_data})
+                self.metrics.add_metric_batch({"solution_validity": solution_validity,
+                          "application_score_value": solution_quality, "application_score_unit": "score",
+                          "application_score_type": str(float), "solution": input_data})
 
                 return solution_validity, sum(time_to_validation, time_to_evaluation))
 

diff --git a/docs/modules.rst b/docs/modules.rst
@@ -16,3 +16,4 @@
    ConfigManager
    BenchmarkManager
    Installer
+   Plotter
diff --git a/docs/reference.rst b/docs/reference.rst
@@ -10,4 +10,5 @@ Reference
    BenchmarkRecord <_autosummary/BenchmarkRecord>
    Config Manager <_autosummary/ConfigManager>
    Metrics <_autosummary/Metrics>
-   Installer <_autosummary/Installer>
+   Installer <_autosummary/Installer>
+   Plotter <_autosummary/Plotter>
diff --git a/src/BenchmarkManager.py b/src/BenchmarkManager.py
@@ -22,22 +22,18 @@
 from pathlib import Path
 from typing import List, Dict
 
-import matplotlib
-import matplotlib.pyplot as plt
-import seaborn as sns
-import pandas as pd
 import numpy as np
 
 from ConfigManager import ConfigManager
 from BenchmarkRecord import BenchmarkRecord
+from Plotter import Plotter
 from modules.Core import Core
 from utils import get_git_revision
 
 from utils_mpi import get_comm
 
 comm = get_comm()
 
-matplotlib.rcParams['savefig.dpi'] = 300
 
 
 class BenchmarkManager:
@@ -142,7 +138,8 @@ def run_benchmark(self, benchmark_backlog: list, repetitions: int):
                                  f" Iteration {i}/{repetitions}:")
                     try:
 
-                        self.benchmark_record_template = BenchmarkRecord(datetime.today().strftime('%Y-%m-%d-%H-%M-%S'),
+                        self.benchmark_record_template = BenchmarkRecord(idx_backlog,
+                                                                         datetime.today().strftime('%Y-%m-%d-%H-%M-%S'),
                                                                          git_revision_number, git_uncommitted_changes,
                                                                          i, repetitions)
                         self.application.metrics.set_module_config(backlog_item["config"])
@@ -266,7 +263,7 @@ def summarize_results(self, input_dirs: list) -> None:
         logging.info(f"Summarizing {len(input_dirs)} benchmark directories")
         results = self.load_results(input_dirs)
         self._save_as_json(results)
-        BenchmarkManager.visualize_results(results, self.store_dir)
+        Plotter.visualize_results(results, self.store_dir)
 
     def load_results(self, input_dirs: list = None) -> list:
         """
@@ -289,77 +286,6 @@ def load_results(self, input_dirs: list = None) -> list:
 
         return results
 
-    @staticmethod
-    def _extract_columns(config, rest_result):
-
-        if rest_result:
-            module_name = rest_result["module_name"]
-            for key, value in sorted(rest_result["module_config"].items(),
-                                     key=lambda key_value_pair: key_value_pair[0]):
-                module_name += f", {key}: {value}"
-
-            config_combo = config.pop("config_combo") + "\n" + module_name if "config_combo" in config else ""
-            return BenchmarkManager._extract_columns(
-                {
-                    **config,
-                    "config_combo": config_combo,
-                    module_name: rest_result["total_time"] if module_name not in config else config[module_name] +
-                                                                                             rest_result["total_time"]
-                },
-                rest_result["submodule"]
-            )
-
-        return config
-
-    @staticmethod
-    def visualize_results(results: List[Dict], store_dir: str):
-        """
-        Function to plot the execution times of the benchmark.
-
-        :param results: Dict containing the results
-        :type results: List[Dict]
-        :param store_dir: directory where the plots are stored
-        :type store_dir:  str
-        :return:
-        :rtype: None
-        """
-        processed = []
-        app_name = None
-        for x in results:
-            app_name = x["module"]["module_name"]
-            app_config = ', '.join([f"{key}: {value}" for (key, value) in sorted(x["module"]["module_config"].items(),
-                                                                                 key=lambda key_value_pair:
-                                                                                 key_value_pair[0])])
-            processed.append(
-                BenchmarkManager._extract_columns({"config_hash": x["config_hash"], "total_time": x["total_time"],
-                                                   "app_config": app_config}, x["module"]))
-
-        df = pd.DataFrame.from_dict(processed)
-        df = df.fillna(0.0)
-        df_melt = df.drop(["app_config", "config_combo", "total_time"], axis=1)
-        df_melt = pd.melt(frame=df_melt, id_vars='config_hash', var_name='module_config', value_name='time')
-
-        # This plot shows the execution time of each module
-        ax = sns.barplot(x='config_hash', y='time', data=df_melt, hue='module_config')
-        plt.title(app_name)
-        # Put the legend out of the figure
-        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
-        ax.set(xlabel="benchmark config hash", ylabel='execution time of module (ms)')
-        ax.set_xticklabels(ax.get_xticklabels(), rotation=90, size=6)
-        plt.savefig(f"{store_dir}/time_by_module.pdf", dpi=300, bbox_inches='tight')
-        plt.clf()
-
-        # This plot shows the total time of a benchmark run
-        ax = sns.barplot(x='app_config', y='total_time', data=df, hue='config_combo')
-        ax.set(xlabel=f"{app_name} config", ylabel='total execution time (ms)')
-        # Put the legend out of the figure
-        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
-        plt.title(app_name)
-        plt.savefig(f"{store_dir}/total_time.pdf", dpi=300, bbox_inches='tight')
-        plt.clf()
-
-        logging.info("Finished creating plots.")
-
 
 class NumpyEncoder(json.JSONEncoder):
     """
@@ -371,6 +297,8 @@ def default(self, o: any):
             return int(o)
         if isinstance(o, np.floating):
             return float(o)
+        if isinstance(o, np.bool_):
+            return bool(o)
         if isinstance(o, np.ndarray):
             return o.tolist()
         return super().default(o)
diff --git a/src/BenchmarkRecord.py b/src/BenchmarkRecord.py
@@ -26,9 +26,11 @@ class BenchmarkRecord:
     benchmark run.
     """
 
-    def __init__(self, timestamp: str, git_revision_number: str, git_uncommitted_changes: str, repetition: int,
-                 total_repetitions: int):
+    def __init__(self, benchmark_backlog_item_number: int, timestamp: str, git_revision_number: str,
+                 git_uncommitted_changes: str, repetition: int, total_repetitions: int):
         """
+        :param benchmark_backlog_item_number: Number of the item in the benchmark backlog
+        :type benchmark_backlog_item_number: int
         :param timestamp: Timestamp of the benchmark run
         :type timestamp: str
         :param git_revision_number: Git revision number during the benchmark run
@@ -40,6 +42,7 @@ def __init__(self, timestamp: str, git_revision_number: str, git_uncommitted_cha
         :param total_repetitions: Number of total repetitions of the benchmark run
         :type total_repetitions: int
         """
+        self.benchmark_backlog_item_number = benchmark_backlog_item_number
         self.timestamp = timestamp
         self.git_revision_number = git_revision_number
         self.git_uncommitted_changes = git_uncommitted_changes
@@ -154,6 +157,7 @@ def get(self) -> dict:
         :rtype: dict
         """
         return {
+            "benchmark_backlog_item_number": self.benchmark_backlog_item_number,
             "timestamp": self.timestamp,
             "config_hash": self.start_hash_config(),
             "total_time": self.total_time,

diff --git a/src/Installer.py b/src/Installer.py
@@ -43,12 +43,12 @@ def __init__(self):
             {"name": "PVC", "class": "PVC", "module": "modules.applications.optimization.PVC.PVC"},
             {"name": "SAT", "class": "SAT", "module": "modules.applications.optimization.SAT.SAT"},
             {"name": "TSP", "class": "TSP", "module": "modules.applications.optimization.TSP.TSP"},
-            {"name": "GenerativeModeling", "class": "GenerativeModeling", \
+            {"name": "GenerativeModeling", "class": "GenerativeModeling",
              "module": "modules.applications.QML.generative_modeling.GenerativeModeling"}
         ]
 
         self.core_requirements = [
-            {"name": "seaborn", "version": "0.12.2"},
+            {"name": "seaborn", "version": "0.13.0"},
             {"name": "networkx", "version": "2.8.8"},
             {"name": "inquirer", "version": "3.1.2"},
             {"name": "packaging", "version": "23.1"},