feat(bug report): add detailed property checking statistics bug report

kea/input_policy.py

@@ -1,11 +1,10 @@
-from dataclasses import dataclass
 import os
 import logging
 import random
 import copy
 import re
 import time
-from .utils import Time, generate_report, save_log
+from .utils import Time, generate_report, save_log, RULE_STATE
 from abc import abstractmethod
 from .input_event import (
     KEY_RotateDeviceToPortraitEvent,
@@ -57,14 +56,6 @@
 POLICY_NONE = "none"
 POLICY_LLM = "llm"
 
-
-@dataclass
-class RULE_STATE:
-    PRECONDITION_SATISFIED = "#satisfy pre"
-    PROPERTY_CHECKED = "#check property"
-    POSTCONDITION_VIOLATED = "#postcondition is violated"
-
-
 class InputInterruptedException(Exception):
     pass
 
@@ -89,6 +80,7 @@ def __init__(self, device: "Device", app: "App", allow_to_generate_utg=False):
         self.allow_to_generate_utg = allow_to_generate_utg
         self.triggered_bug_information = []
         self.time_needed_to_satisfy_precondition = []
+        self.statistics_of_rules = {}
 
         self._num_restarts = 0
         self._num_steps_outside = 0
@@ -148,6 +140,7 @@ def start(self, input_manager: "InputManager"):
                     self.time_needed_to_satisfy_precondition,
                     self.device.cur_event_count,
                     self.time_recoder.get_time_duration(),
+                    self.statistics_of_rules
                 )
 
             except KeyboardInterrupt:
@@ -262,12 +255,12 @@ def __init__(self, device, app, kea: "Kea" = None, allow_to_generate_utg=False):
         # self.to_state = None
 
         # retrive all the rules from the provided properties
-        self.statistics_of_rules = {}
         for rule in self.kea.all_rules:
-            self.statistics_of_rules[str(rule)] = {
+            self.statistics_of_rules[str(rule.function.__name__)] = {
                 RULE_STATE.PRECONDITION_SATISFIED: 0,
                 RULE_STATE.PROPERTY_CHECKED: 0,
                 RULE_STATE.POSTCONDITION_VIOLATED: 0,
+                RULE_STATE.UI_OBJECT_NOT_FOUND: 0
             }
 
     def run_initializer(self):
@@ -298,16 +291,12 @@ def check_rule_whose_precondition_are_satisfied(self):
             return
 
         candidate_rules_list = list(rules_ready_to_be_checked.keys())
-        for candidate_rule in candidate_rules_list:
-            self.statistics_of_rules[str(candidate_rule)][
-                RULE_STATE.PRECONDITION_SATISFIED
-            ] += 1
         # randomly select a rule to check
         rule_to_check = random.choice(candidate_rules_list)
 
         if rule_to_check is not None:
             self.logger.info(f"-------Check Property : {rule_to_check}------")
-            self.statistics_of_rules[str(rule_to_check)][
+            self.statistics_of_rules[str(rule_to_check.function.__name__)][
                 RULE_STATE.PROPERTY_CHECKED
             ] += 1
             precondition_page_index = self.device.cur_event_count
@@ -323,7 +312,7 @@ def check_rule_whose_precondition_are_satisfied(self):
                     "-------time from start : %s-----------"
                     % str(self.time_recoder.get_time_duration())
                 )
-                self.statistics_of_rules[str(rule_to_check)][
+                self.statistics_of_rules[str(rule_to_check.function.__name__)][
                     RULE_STATE.POSTCONDITION_VIOLATED
                 ] += 1
                 postcondition_page__index = self.device.cur_event_count
@@ -347,6 +336,9 @@ def check_rule_whose_precondition_are_satisfied(self):
                 self.logger.error(
                     f"-------Execution failed: UiObjectNotFound during exectution. Property:{rule_to_check}-----------"
                 )
+                self.statistics_of_rules[str(rule_to_check.function.__name__)][
+                    RULE_STATE.UI_OBJECT_NOT_FOUND
+                ] += 1
             elif result == CHECK_RESULT.PRECON_NOT_SATISFIED:
                 self.logger.info("-------Precondition not satisfied-----------")
             else:
@@ -419,6 +411,10 @@ def generate_event(self):
             return KillAndRestartAppEvent(app=self.app)
 
         rules_to_check = self.kea.get_rules_whose_preconditions_are_satisfied()
+        for rule_to_check in rules_to_check:
+            self.statistics_of_rules[str(rule_to_check.function.__name__)][
+                RULE_STATE.PRECONDITION_SATISFIED
+            ] += 1
 
         if len(rules_to_check) > 0:
             t = self.time_recoder.get_time_duration()
@@ -579,9 +575,11 @@ def mutate_the_main_path(self):
                     self.logger.info(
                         "reach the end of the main path that could satisfy the precondition"
                     )
-                    rules_to_check = (
-                        self.kea.get_rules_whose_preconditions_are_satisfied()
-                    )
+                    rules_to_check = self.kea.get_rules_whose_preconditions_are_satisfied()
+                    for rule_to_check in rules_to_check:
+                        self.statistics_of_rules[str(rule_to_check.function.__name__)][
+                            RULE_STATE.PRECONDITION_SATISFIED
+                        ] += 1
                     if len(rules_to_check) > 0:
                         t = self.time_recoder.get_time_duration()
                         self.time_needed_to_satisfy_precondition.append(t)
@@ -861,6 +859,10 @@ def generate_llm_event(self):
             )
             return ReInstallAppEvent(self.app)
         rules_to_check = self.kea.get_rules_whose_preconditions_are_satisfied()
+        for rule_to_check in rules_to_check:
+            self.statistics_of_rules[str(rule_to_check.function.__name__)][
+                RULE_STATE.PRECONDITION_SATISFIED
+            ] += 1
 
         if len(rules_to_check) > 0:
             t = self.time_recoder.get_time_duration()

kea/kea.py

@@ -344,7 +344,7 @@ def get_rules_whose_preconditions_are_satisfied(self) -> Dict["Rule", "KeaTest"]
 
     def single_thread_precondition_checker(self):
         """
-        check precondition with sigle-thread
+        check precondition with single-thread
         """
         rules_passed_precondition: Dict["Rule", "KeaTest"] = dict()
 

kea/resources/style/style.html

@@ -112,7 +112,7 @@ <h2 class="panel-title", style="font-size: 30px">Runtime Statistics</h2>
             <td>
                 <div class="panel panel-success", style="display: block;width:100%;height: 200px">
                     <div class="panel-heading">
-                        <h2 class="panel-title", style="font-size: 30px">Property Checking Statistics</h2>
+                        <h2 class="panel-title", style="font-size: 30px">Comprehensive Statistics</h2>
                     </div>
                     <div class="panel-body">
                         <div>Num</div>
@@ -130,6 +130,7 @@ <h2 style="font-size: 30px">Property Violations</h2>
     <table>
             <thead>
                 <tr>
+                    <th>Index</th>
                     <th>Property Name</th>
                     <th>Page of Precondition</th>
                     <th>Pages of Interaction Scenario</th>
@@ -141,6 +142,27 @@ <h2 style="font-size: 30px">Property Violations</h2>
             </tbody>
     </table>
   </div>
+
+    <hr>
+
+  <h2 style="font-size: 30px">Property Checking Statistics</h2>
+  <div class="table-container">
+    <table>
+            <thead>
+                <tr>
+                    <th>Index</th>
+                    <th>Property Name</th>
+                    <th>Precondition Satisfied Times</th>
+                    <th>Precondition Checked Times</th>
+                    <th>Postcondition Violated Times</th>
+                    <th>Ui Object Not Found Times</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr><td>property statistic</td><td>property statistic</td><td>property statistic</td><td>property statistic</td></tr>
+            </tbody>
+    </table>
+  </div>
 </body>
 
 </html>

kea/utils.py

@@ -8,7 +8,7 @@
 import warnings
 import pkg_resources  # TODO: warning pkg_resources has been depreciated
 import yaml
-
+from dataclasses import dataclass
 # logcat regex, which will match the log message generated by `adb logcat -v threadtime`
 LOGCAT_THREADTIME_RE = re.compile(
     '^(?P<date>\S+)\s+(?P<time>\S+)\s+(?P<pid>[0-9]+)\s+(?P<tid>[0-9]+)\s+'
@@ -27,6 +27,13 @@
 DEFAULT_EVENT_COUNT = 100000000
 DEFAULT_TIMEOUT = 3600
 
+@dataclass
+class RULE_STATE:
+    PRECONDITION_SATISFIED = "#satisfy pre"
+    PROPERTY_CHECKED = "#check property"
+    POSTCONDITION_VIOLATED = "#postcondition is violated"
+    UI_OBJECT_NOT_FOUND = "#UI object not found"
+
 def lazy_property(func):
     attribute = '_lazy_' + func.__name__
 
@@ -111,29 +118,40 @@ def safe_get_dict(view_dict, key, default=None):
     return view_dict[key] if (key in view_dict) else default
 
 
-def generate_report(img_path, html_path, bug_information=None, precondition_information=None, total_count = 0, total_time = 0):
+def generate_report(img_path, html_path, bug_information=None, precondition_information=None, total_count = 0, total_time = 0, statistics_of_rules = None):
     '''Generate report for the test based on the executed events'''
     line_list = []
-
     bug_link_list = []
     # user can click to jump to the corresponding event, contains the event index of each bug
     bug_set = set()
+    statistics = []
     json_dir = os.path.join(html_path, "report_screenshot.json")
     with open(json_dir, 'r') as json_file:
         report_screens = json.load(json_file)
     if bug_information is not None:
-        for bug in bug_information:
+        for index, bug in enumerate(bug_information):
             property_name = "<p>" + bug[2] + "</p>"
             for report_screen in report_screens:
                 if str(bug[0]) + "." in report_screen['event_index']:
                     interaction_end = report_screen['event_index']
-            bug_link = ("<tr><td>" + property_name + "</td>" +
+            bug_link = ("<tr><td>" + str(index + 1) + "</td>" +
+                        "<td>" + property_name + "</td>" +
                         "<td><a href=\"#"+str(bug[0][0] + 1)+"\">"+str(bug[0][0] + 1)+"</a></td>" +
                         "<td><a href=\"#"+str(bug[0][1] + 1) + "\">"+str(bug[0][0] + 1)+ " ~ " + str(bug[0][1] + 1) + "</a></td>" +
                         "<td><a href=\"#"+str(bug[0][1] + 1) + "\">"+str(bug[0][1] + 1)+"</a></td></tr>")
             bug_link_list.append(bug_link)
             bug_set.add(str(bug[0][0] + 1))
             bug_set.add(str(bug[0][1] + 1))
+    if statistics_of_rules is not None:
+        for index, statistic_rule in enumerate(statistics_of_rules):
+            statistic = ("<tr><td>" + str(index + 1) + "</td>" +
+                         "<td>" + statistic_rule + "</td>" +
+                         "<td>" + str(statistics_of_rules[statistic_rule][RULE_STATE.PRECONDITION_SATISFIED]) + "</td>" +
+                         "<td>" + str(statistics_of_rules[statistic_rule][RULE_STATE.PROPERTY_CHECKED]) + "</td>" +
+                         "<td>" + str(statistics_of_rules[statistic_rule][RULE_STATE.POSTCONDITION_VIOLATED]) + "</td>" +
+                         "<td>" + str(statistics_of_rules[statistic_rule][RULE_STATE.UI_OBJECT_NOT_FOUND]) + "</td></tr>"
+                         )
+            statistics.append(statistic)
     f_html = open(
         os.path.join(html_path,  "bug_report.html"), 'w', encoding='utf-8'
     )
@@ -144,6 +162,7 @@ def generate_report(img_path, html_path, bug_information=None, precondition_info
     # f_style = open("droidbot/resources/style/style.html", 'r', encoding='utf-8')
     new_str = "<ul id=\"menu\">" + '\n'
     new_bug_str = ""
+    new_statistic_str = ""
     for report_screen in report_screens:
         action_count = report_screen['event_index']
         event_name = report_screen['event']
@@ -176,6 +195,8 @@ def generate_report(img_path, html_path, bug_information=None, precondition_info
         new_str = new_str + item
     for item in bug_link_list:
         new_bug_str = new_bug_str + item
+    for item in statistics:
+        new_statistic_str = new_statistic_str + item
 
     if len(bug_information) > 0:
         first_bug_infor = ("<div style=\"color:red;\">Time needed to found the first bug: " + str(bug_information[0][1]) + " seconds</div><br>")
@@ -193,6 +214,7 @@ def generate_report(img_path, html_path, bug_information=None, precondition_info
     old_bug_str = "<tr><td>bug_link</td><td>bug_link</td><td>bug_link</td><td>bug_link</td></tr>"
     old_first_str="<div>Time</div>"
     old_num_str="<div>Num</div>"
+    old_statistic_str="<tr><td>property statistic</td><td>property statistic</td><td>property statistic</td><td>property statistic</td></tr>"
     if bug_information is None or len(bug_information) == 0:
         new_num_str = "<div style=\"color:green;\">Found 0 bugs.</div><br>"
     else:
@@ -215,6 +237,9 @@ def generate_report(img_path, html_path, bug_information=None, precondition_info
         if old_num_str in line:
             f_html.write(line.replace(old_num_str, new_num_str))
             continue
+        if old_statistic_str in line:
+            f_html.write(line.replace(old_statistic_str, new_statistic_str))
+            continue
         f_html.write(line)
 
 def get_yml_config()->dict[str,str]: