Merge pull request #722 from egraphs-good/oflatt-new-macros

New macros in nightly

infra/graphs.py

@@ -233,29 +233,6 @@ def format_latex_macro_percent(name, percent_as_ratio):
   percent = percent_as_ratio * 100
   return format_latex_macro(name, f"{percent:.2f}")
 
-def make_macros(profile, output_file):
-  number_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0 = 0
-  benchmarks = dedup([b.get('benchmark') for b in profile])
-
-  for benchmark in benchmarks:
-    baseline_cycles = get_baseline_cycles(profile, benchmark)
-    llvm_O3_O0_cycles = get_cycles(profile, benchmark, 'llvm-O3-O0')
-    eggcc_O0_O0_cycles = get_cycles(profile, benchmark, 'llvm-eggcc-O0-O0')
-
-    perf_improvement_llvm = mean(baseline_cycles) - mean(llvm_O3_O0_cycles)
-    perf_improvement_eggcc = mean(baseline_cycles) - mean(eggcc_O0_O0_cycles)
-    if perf_improvement_llvm < 0:
-      number_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0 += 1
-    else:
-      if perf_improvement_eggcc > 0.8 * perf_improvement_llvm:
-        number_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0 += 1
-  ratio_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0 = number_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0 / len(benchmarks)
-
-  with open(output_file, 'w') as f:
-    f.write(format_latex_macro_percent('percentRecoverEightyPercentPerformanceImprovementEggccVsLlvmOThreeOZero', ratio_recover_80_percent_performance_improvement_eggcc_vs_llvm_O3_O0))
-
-
-
 def benchmarks_in_folder(folder):
   # recursively find all files
   files = []
@@ -264,6 +241,25 @@ def benchmarks_in_folder(folder):
       files.append(os.path.join(root, filename))
   # just get file name without extension
   return [os.path.splitext(os.path.basename(f))[0] for f in files]
+
+
+# given a profile.json, list of suite paths, and an output file
+def make_macros(profile, benchmark_suites, output_file):
+  with open(output_file, 'a') as out:
+    # report number of benchmarks in each benchmark suite
+    for suite in benchmark_suites:
+      suite_name = os.path.basename(suite)
+      benchmarks = benchmarks_in_folder(suite)
+      macro_name = f"Num{suite_name}Benchmarks"
+      out.write(format_latex_macro(macro_name, len(benchmarks)))
+
+    # report the number of benchmarks in the profile
+    out.write(format_latex_macro("NumBenchmarksAllSuites", len(dedup([b.get('benchmark') for b in profile]))))
+
+
+
+
+
 
 
 def get_code_size(benchmark, suites_path):
@@ -344,16 +340,17 @@ def make_code_size_vs_compile_time(profile, output, suites_path):
 
   # folders in 
   benchmark_suites = [f for f in os.listdir(benchmark_suite_folder) if os.path.isdir(os.path.join(benchmark_suite_folder, f))]
+  benchmark_suites = [os.path.join(benchmark_suite_folder, f) for f in benchmark_suites]
 
   make_jitter(profile, 4, f'{graphs_folder}/jitter_plot_max_4.png')
 
-  for suite in benchmark_suites:
-    suite_path = os.path.join(benchmark_suite_folder, suite)
+  for suite_path in benchmark_suites:
+    suite = os.path.basename(suite_path)
     suite_benchmarks = benchmarks_in_folder(suite_path)
     profile_for_suite = [b for b in profile if b.get('benchmark') in suite_benchmarks]
     make_bar_chart(profile_for_suite, f'{graphs_folder}/{suite}_bar_chart.png')
 
-  make_macros(profile, f'{output_folder}/nightlymacros.tex')
+  make_macros(profile, benchmark_suites, f'{output_folder}/nightlymacros.tex')
 
   make_code_size_vs_compile_time(profile, f'{graphs_folder}/code_size_vs_compile_time.png', benchmark_suite_folder)
 

infra/nightly-resources/index.html

@@ -82,15 +82,6 @@ <h2> Latex Macros </h2>
     <textarea id="latex-macros-text" rows="10" cols="50" readonly></textarea>
   </div>
 
-  <h2>Code Size vs Compile Time</h2>
-  <button type="button" class="collapsible" id="code-size-toggle"
-    onclick="toggle(this, `\u25B6 Show`, `\u25BC Hide`)">
-    &#9654; Show
-  </button>
-  <div class="content collapsed" id="code-size">
-    <image id="code-size-plot" src="code_size_vs_compile_time.png"/>
-  </div>
-
 
 
   <h2>Overall Stats</h2>

infra/nightly-resources/index.js

@@ -47,7 +47,16 @@ function addTableTo(element, data) {
     navigator.clipboard.writeText(table);
   };
 
+  // add a button that copies latex macros for table
+  const copyMacrosButton = document.createElement("button");
+  copyMacrosButton.innerText = "Copy Latex Macros";
+  copyMacrosButton.onclick = () => {
+    const macros = jsonToLatexMacros(data, "runMethod");
+    navigator.clipboard.writeText(macros);
+  };
+
   element.appendChild(copyButton);
+  element.appendChild(copyMacrosButton);
 
   // add a new div for the table
   const tableDiv = document.createElement("div");
@@ -72,8 +81,6 @@ function refreshView() {
   }));
   tableData.sort((l, r) => l.name - r.name);
 
-  document.getElementById("profile").innerHTML = ConvertJsonToTable(tableData);
-
   addTableTo(document.getElementById("profile"), tableData);
 
   // fill in the overall stats table

infra/nightly-resources/latex.js

@@ -7,23 +7,48 @@ function formatNumber(num) {
   return num;
 }
 
-// convert a javascript array object to a latex table
-function jsonToLatexTable(json) {
-  var res = "\\begin{tabular}{|";
+function jsonHeaders(json) {
   var headers = [];
-  var header = "";
-  var rows = json.length;
-  var cols = 0;
 
-  // get the headers
-  for (var i = 0; i < rows; i++) {
+  for (var i = 0; i < json.length; i++) {
     for (var key in json[i]) {
       if (headers.indexOf(key) == -1) {
         headers.push(key);
       }
     }
   }
 
+  return headers;
+}
+
+function jsonColumns(json, rowIndex) {
+  var cols = [];
+
+  for (var i = 0; i < json.length; i++) {
+    cols.push(json[i][rowIndex]);
+
+    // throw an error if undefined
+    if (json[i][rowIndex] === undefined) {
+      throw new Error("undefined value for " + rowIndex + " in row " + i);
+    }
+  }
+
+  return cols;
+}
+
+// convert a javascript array object to a latex table
+// each element of the array is a dictionary associating a header with a value
+// example element: { geoMeanNormalized : "0.619", meanEggccCompileTimeSecs : "0.001", meanLlvmCompileTimeSecs : "0.447",
+// runMethod : "llvm-O1-O0" }
+function jsonToLatexTable(json) {
+  console.log(json);
+  var res = "\\begin{tabular}{|";
+  var header = "";
+  var rows = json.length;
+  var cols = 0;
+
+  var headers = jsonHeaders(json);
+
   // create the header
   cols = headers.length;
   for (var i = 0; i < cols; i++) {
@@ -58,3 +83,46 @@ function jsonToLatexTable(json) {
 
   return res;
 }
+
+// change dashes to underscores
+function convertStringToValidLatexVar(str) {
+  return str.replace(/-/g, "_");
+}
+
+// convert a javascript array object storing a table
+// to a bunch of latex macros, one for each row, column pair
+// each element of the array is a dictionary associating a header with a value
+// example element: { geoMeanNormalized : "0.619", meanEggccCompileTimeSecs : "0.001", meanLlvmCompileTimeSecs : "0.447",
+// runMethod : "llvm-O1-O0" }
+// rowIndex is the name of the header of the left column, in this case "runMethod"
+function jsonToLatexMacros(json, rowIndex) {
+  var names = [];
+  var res = "";
+
+  // get the headers
+  var headers = jsonHeaders(json);
+  var cols = jsonColumns(json, rowIndex);
+
+  // for each header, col pair create a macro
+  for (var i = 0; i < cols.length; i++) {
+    for (var j = 0; j < headers.length; j++) {
+      var name = `${convertStringToValidLatexVar(cols[i])}${convertStringToValidLatexVar(headers[j])}`;
+      res +=
+        "\\newcommand{\\" +
+        name +
+        "}{" +
+        formatNumber(json[i][headers[j]]) +
+        "}\n";
+
+      names.push(name);
+    }
+  }
+
+  // if there are duplicate names then throw an error
+  var uniqueNames = new Set(names);
+  if (uniqueNames.size !== names.length) {
+    throw new Error("Duplicate names in jsonToLatexMacros");
+  }
+
+  return res;
+}