report: add bug-specific table (#654)

This is to make it easier to traverse bugs in larger experiments --------- Signed-off-by: David Korczynski <[email protected]>
google · Sep 27, 2024 · d26a523 · d26a523
1 parent 43bf4db
commit d26a523
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 3 deletions.
diff --git a/report/templates/index.html b/report/templates/index.html
@@ -128,6 +128,28 @@ <h2>Project summary</h2>
     </tbody>
 </table>
 
+<h2>Crashes found by generated fuzz harnesses</h2>
+<table class="sortable-table" id="bug-summary-table">
+    <thead>
+        <tr>
+            <th></th>
+            <th data-sort-string>Project</th>
+            <th data-sorted="asc">Benchmark</th>
+            <th data-sort-string>AI bug validation</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for bug_sample in samples_with_bugs %}
+        <tr>
+            <td class="table-index">{{ loop.index }}</td>
+            <td data-sort-value="{{bug_sample.benchmark.project}}">{{bug_sample.benchmark.project}}</td>
+            <td data-sort-value="{{bug_sample.benchmark.id}}"><a href="sample/{{ bug_sample.benchmark.id|urlencode }}/{{ bug_sample.sample.id }}.html">{{bug_sample.benchmark.id}}-{{ bug_sample.sample.id }} </a> </td>
+            <td style="color: {{ 'red' if bug_sample.sample.result.crashes and not bug_sample.sample.result.is_semantic_error else 'green' }}" data-sort-value="{{'True positive' if bug_sample.sample.result.is_semantic_error else 'False positive'}}">{{'True positive' if bug_sample.sample.result.is_semantic_error else 'False positive'}}</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+
 <h2>Accumulated results</h2>
 <table>
   <thead>

diff --git a/report/web.py b/report/web.py
@@ -139,6 +139,7 @@ def _copy_and_set_coverage_report(self, benchmark, sample):
   def generate(self):
     """Generate and write every report file."""
     benchmarks = []
+    samples_with_bugs = []
     for benchmark_id in self._results.list_benchmark_ids():
       results, targets = self._results.get_results(benchmark_id)
       benchmark = self._results.match_benchmark(benchmark_id, results, targets)
@@ -154,6 +155,8 @@ def generate(self):
       self._write_benchmark_crash(benchmark, samples)
 
       for sample in samples:
+        if sample.result.crashes:
+          samples_with_bugs.append({'benchmark': benchmark, 'sample': sample})
         sample_targets = self._results.get_targets(benchmark.id, sample.id)
         self._write_benchmark_sample(benchmark, sample, sample_targets)
 
@@ -163,7 +166,7 @@ def generate(self):
     time_results = self.read_timings()
 
     self._write_index_html(benchmarks, accumulated_results, time_results,
-                           projects)
+                           projects, samples_with_bugs)
     self._write_index_json(benchmarks)
 
   def _write(self, output_path: str, content: str):
@@ -183,13 +186,15 @@ def _write(self, output_path: str, content: str):
 
   def _write_index_html(self, benchmarks: List[Benchmark],
                         accumulated_results: AccumulatedResult,
-                        time_results: dict[str, Any], projects: list[Project]):
+                        time_results: dict[str, Any], projects: list[Project],
+                        samples_with_bugs: list[dict[str, Any]]):
     """Generate the report index.html and write to filesystem."""
     rendered = self._jinja.render('index.html',
                                   benchmarks=benchmarks,
                                   accumulated_results=accumulated_results,
                                   time_results=time_results,
-                                  projects=projects)
+                                  projects=projects,
+                                  samples_with_bugs=samples_with_bugs)
     self._write('index.html', rendered)
 
   def _write_index_json(self, benchmarks: List[Benchmark]):