uncore,power in system view and html improvements (#57)

README.md

@@ -59,29 +59,16 @@ Modify the template [deamonset.yml](docs/daemonset.yml) to deploy in kubernetes
 
 ## Requirements
 
-### Packages:
-
-- **perf** - PerfSpect uses the Linux perf tool to collect PMU counters
-
-### Minimum supported kernels
-
-| Xeon Generation | CentOS 7+ | Ubuntu 16.04+ |
-| --------------- | --------- | ------------- |
-| Broadwell       | 3.10      | 4.15          |
-| Skylake         | 3.10      | 4.15          |
-| Cascade Lake    | 3.10      | 4.15          |
-| Ice Lake        | 3.10      | 4.15          |
-| Sapphire Rapids | 5.12      | 5.12          |
-
-### Supported Operating Systems:
-
-- Ubuntu 16.04 and newer
-- CentOS 7 and newer
-- Amazon Linux 2
-- RHEL 9
-- Debian 11
-
-_Note: PerfSpect may work on other Linux distributions, but has not been thoroughly tested_
+**perf** - PerfSpect uses the Linux perf tool to collect PMU counters
+
+Different events require different minimum kernels (PerfSpect will automatically collect only supported events)
+1. Base (CPU util, CPI, Cache misses, etc.)
+    - 3.10
+2. Uncore (NUMA traffic, DRAM traffic, etc.)
+    - 4.9
+3. TMA (Micro-architecture boundness breakdown)
+    - ICX, SPR: 5.10
+    - BDX, SKX, CLX: 3.10
 
 ## Build from source
 

_version.txt

@@ -1 +1 @@
-1.3.5
+1.3.6

perf-collect.py

@@ -369,14 +369,7 @@ def validate_file(fname):
             )
     events, collection_events = prep_events.prepare_perf_events(
         eventfile,
-        (
-            args.pid is not None
-            or args.cid is not None
-            or args.cpu
-            or args.socket
-            or not have_uncore
-        ),
-        args.pid is not None or args.cid is not None,
+        (args.pid is not None or args.cid is not None or not have_uncore),
         include_tma,
         not have_uncore,
     )
@@ -388,14 +381,17 @@ def validate_file(fname):
 
     mux_intervals = perf_helpers.get_perf_event_mux_interval()
     if args.muxinterval > 0:
+        logging.info(
+            "changing default perf mux interval to " + str(args.muxinterval) + "ms"
+        )
         perf_helpers.set_perf_event_mux_interval(False, args.muxinterval, mux_intervals)
 
     # parse cgroups
     cgroups = []
     if args.cid is not None:
         cgroups = perf_helpers.get_cgroups(args.cid)
 
-    if args.cpu or args.socket or args.pid is not None or args.cid is not None:
+    if args.pid is not None or args.cid is not None:
         logging.info("Not collecting uncore events in this run mode")
 
     # log some metadata
@@ -481,6 +477,7 @@ def validate_file(fname):
     if nmi_watchdog != 0:
         perf_helpers.enable_nmi_watchdog()
 
+    logging.info("changing perf mux interval back to default")
     perf_helpers.set_perf_event_mux_interval(True, 1, mux_intervals)
 
     logging.info("perf stat dumped to %s" % args.outcsv)

perf-postprocess.py

@@ -515,15 +515,14 @@ def extract_dataframe(perf_data_lines, meta_data, perf_mode):
             axis=1,
         )
 
-    if perf_mode != Mode.CPU and perf_mode != Mode.Socket:
-        # fix metric name X.1, X.2, etc -> just X
-        # we don't need this in cpu/socket modes
-        perf_data_df["metric"] = perf_data_df.apply(
-            lambda x: ".".join(x["metric"].split(".")[:-1])
-            if len(re.findall(r"^[0-9]*$", x["metric"].split(".")[-1])) > 0
-            else x["metric"],
-            axis=1,
-        )
+    # fix metric name X.1, X.2, etc -> just X
+    perf_data_df["metric"] = perf_data_df.apply(
+        lambda x: ".".join(x["metric"].split(".")[:-1])
+        if len(re.findall(r"^[0-9]*$", x["metric"].split(".")[-1])) > 0
+        else x["metric"],
+        axis=1,
+    )
+
     # set data frame types
     perf_data_df["value"] = pd.to_numeric(
         perf_data_df["value"], errors="coerce"
@@ -930,6 +929,21 @@ def generate_metrics(
     verbose=False,
     fail_postprocessing=False,
 ):
+    # filter out uncore metrics if in cpu or socket mode
+    filtered_metrics = []
+    for m in metrics:
+        if perf_mode == Mode.CPU or perf_mode == Mode.Socket:
+            if any(
+                [
+                    e.startswith("power/")
+                    or e.startswith("cstate_")
+                    or e.startswith("UNC_")
+                    for e in m["events"]
+                ]
+            ):
+                continue
+        filtered_metrics.append(m)
+
     time_slice_groups = perf_data_df.groupby("ts", sort=False)
     time_metrics_result = {}
     errors = {
@@ -940,7 +954,19 @@ def generate_metrics(
         "NOT COUNTED GROUPS": set(),
     }
     prev_time_slice = 0
-    logging.info("processing " + str(time_slice_groups.ngroups) + " samples")
+    logging.info(
+        "processing "
+        + str(time_slice_groups.ngroups)
+        + " samples in "
+        + (
+            "System"
+            if perf_mode == Mode.System
+            else "CPU"
+            if perf_mode == Mode.CPU
+            else "Socket"
+        )
+        + " mode"
+    )
     group_start_end_index_dict = {}
     for time_slice, item in time_slice_groups:
         time_slice_float = float(time_slice)
@@ -965,7 +991,7 @@ def generate_metrics(
         )
 
         time_metrics_result[time_slice] = evaluate_metrics(
-            verbose, metrics, metadata, group_to_event, group_to_df, errors
+            verbose, filtered_metrics, metadata, group_to_event, group_to_df, errors
         )
 
     time_series_df = pd.DataFrame(time_metrics_result).reindex(
@@ -1159,9 +1185,6 @@ def generate_raw_events(perf_data_df, out_file_path, perf_mode):
                 args.verbose,
                 args.fail_postprocessing,
             )
-            logging.info(
-                "Generated results file(s) in: " + out_file_path.rsplit("/", 1)[0]
-            )
     else:
         generate_metrics(
             perf_data_df,
@@ -1188,5 +1211,5 @@ def generate_raw_events(perf_data_df, out_file_path, perf_mode):
                 args.fail_postprocessing,
             )
 
-        logging.info("Generated results file(s) in: " + out_file_path.rsplit("/", 1)[0])
+    logging.info("Generated results file(s) in: " + out_file_path.rsplit("/", 1)[0])
     logging.info("Done!")

similarity-analyzer/dopca.py

@@ -104,7 +104,7 @@ def handle_nan(data, comp_size):
             if len(deleted_row_indices) in (comp_size, comp_size - 1):
                 # too many workload profiles have NaN greater than threshold, must quit similarity analysis
                 logger.error(
-                    "Attempted dropping of NaNs resulted in fewer #input profiles without NaN....quiting similarity analysis"
+                    "Attempted dropping of NaNs resulted in fewer #input profiles without NaN....quitting similarity analysis"
                 )
                 sys.exit(1)
             logger.warning(

src/base.html

@@ -785,24 +785,24 @@
                         key={row.metrics}
                         sx={{ '&:last-child td, &:last-child th': { border: 0 } }}
                       >
-                        <TableCell component="th" scope="row" >
+                        <TableCell sx={{ fontFamily: 'Monospace' }} component="th" scope="row" >
                           <Tooltip title={description.hasOwnProperty(row.metrics) ? description[row.metrics] : ""}>
-                            {description.hasOwnProperty(row.metrics) && <IconButton>
+                            {description.hasOwnProperty(row.metrics) && <IconButton sx={{ padding: "0 8px 0 0" }}>
                               <Icon>help</Icon>
                             </IconButton>}
-                            {!description.hasOwnProperty(row.metrics) && <IconButton disabled>
+                            {!description.hasOwnProperty(row.metrics) && <IconButton sx={{ padding: "0 8px 0 0" }} disabled>
                               <Icon>help</Icon>
                             </IconButton>}
                           </Tooltip>
-                          {row.metrics}
+                          {(row.metrics.startsWith("metric_") ? row.metrics.replace("metric_", "") : row.metrics).replaceAll("_", " ")}
                         </TableCell>
-                        <TableCell align="right">
-                          {Number(row["0"]).toFixed(1)}
+                        <TableCell sx={{ fontFamily: 'Monospace' }} align="right">
+                          {Number(row["0"]).toFixed(4)}
                         </TableCell>
-                        {row.hasOwnProperty("other") && <TableCell align="right">
-                          {Number(row["other"]).toFixed(1)}
+                        {row.hasOwnProperty("other") && <TableCell sx={{ fontFamily: 'Monospace' }} align="right">
+                          {Number(row["other"]).toFixed(4)}
                         </TableCell>}
-                        {row.hasOwnProperty("other") && <TableCell align="right" sx={{ backgroundColor: (row.diff > 0 ? "rgba(255,0,0," + (row.diff / maxdiff * .5) + ")" : "rgba(0,0,255," + (row.diff / mindiff * .5) + ")") }}>
+                        {row.hasOwnProperty("other") && <TableCell align="right" sx={{ fontFamily: 'Monospace', backgroundColor: (row.diff > 0 ? "rgba(255,0,0," + (row.diff / maxdiff * .5) + ")" : "rgba(0,0,255," + (row.diff / mindiff * .5) + ")") }}>
                           {Math.round(Number(row["diff"]))}%
                         </TableCell>}
                       </TableRow>
@@ -827,4 +827,4 @@
   </script>
 </body>
 
-</html>
+</html>

src/prepare_perf_events.py

@@ -115,7 +115,7 @@ def get_cgroup_events_format(cgroups, events, num_events):
     return perf_format
 
 
-def filter_events(event_file, cpu_only, PID_CID_mode, TMA_supported, in_vm):
+def filter_events(event_file, cpu_only, TMA_supported, in_vm):
     if not os.path.isfile(event_file):
         crash("event file not found")
     collection_events = []
@@ -135,8 +135,6 @@ def process(line):
         line = line.strip()
         if line == "" or line.startswith("#") or (cpu_only and not is_cpu_event(line)):
             return
-        if PID_CID_mode and line.startswith("cstate_"):
-            return
         if not TMA_supported and (
             "name='TOPDOWN.SLOTS'" in line or "name='PERF_METRICS." in line
         ):
@@ -165,15 +163,15 @@ def process(line):
     return collection_events, unsupported_events
 
 
-def prepare_perf_events(event_file, cpu_only, PID_CID_mode, TMA_supported, in_vm):
+def prepare_perf_events(event_file, cpu_only, TMA_supported, in_vm):
     start_group = "'{"
     end_group = "}'"
     group = ""
     prev_group = ""
     new_group = True
 
     collection_events, unsupported_events = filter_events(
-        event_file, cpu_only, PID_CID_mode, TMA_supported, in_vm
+        event_file, cpu_only, TMA_supported, in_vm
     )
     core_event = []
     uncore_event = []