update

ecnusse · Dec 5, 2024 · 8765b0f · 8765b0f
1 parent abaff99
commit 8765b0f
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 9 deletions.
diff --git a/kea/device_state.py b/kea/device_state.py
@@ -10,7 +10,7 @@
     from .device import Device
 
 from .utils import md5, deprecated
-from .input_event import SearchEvent, SetTextAndSearchEvent, TouchEvent, LongTouchEvent, ScrollEvent, SetTextEvent, KeyEvent
+from .input_event import SearchEvent, SetTextAndSearchEvent, TouchEvent, LongTouchEvent, ScrollEvent, SetTextEvent, KeyEvent, UIEvent
 
 
 class DeviceState(object):
@@ -808,3 +808,105 @@ def is_view_exist(self, view_dict):
 
     def get_state_screen(self): 
         return self.screenshot_path
+
+    def get_view_desc(self, view):
+        content_description = self.__safe_dict_get(view, 'content_description', default='')
+        view_text = self.__safe_dict_get(view, 'text', default='')
+        scrollable = self.__safe_dict_get(view, 'scrollable')
+        view_desc = f'view'
+        if scrollable:
+            view_desc = f'scrollable view'
+        if content_description:
+            view_desc += f' "{content_description}"'
+        if view_text:
+            view_text = view_text.replace('\n', '  ')
+            view_text = f'{view_text[:20]}...' if len(view_text) > 20 else view_text
+            view_desc += f' with text "{view_text}"'
+        return view_desc
+
+    def get_described_actions(self):
+        """
+        Get a text description of current state
+        """
+        enabled_view_ids = []
+        for view_dict in self.views:
+            # exclude navigation bar if exists
+            if self.__safe_dict_get(view_dict, 'visible') and \
+                self.__safe_dict_get(view_dict, 'resource_id') not in \
+               ['android:id/navigationBarBackground',
+                'android:id/statusBarBackground']:
+                enabled_view_ids.append(view_dict['temp_id'])
+
+        view_descs = []
+        available_actions = []
+        for view_id in enabled_view_ids:
+            view = self.views[view_id]
+            clickable = self._get_self_ancestors_property(view, 'clickable')
+            scrollable = self.__safe_dict_get(view, 'scrollable')
+            checkable = self._get_self_ancestors_property(view, 'checkable')
+            long_clickable = self._get_self_ancestors_property(view, 'long_clickable')
+            editable = self.__safe_dict_get(view, 'editable')
+            actionable = clickable or scrollable or checkable or long_clickable or editable
+            checked = self.__safe_dict_get(view, 'checked')
+            selected = self.__safe_dict_get(view, 'selected')
+            content_description = self.__safe_dict_get(view, 'content_description', default='')
+            view_text = self.__safe_dict_get(view, 'text', default='')
+            if not content_description and not view_text and not scrollable:  # actionable?
+                continue
+
+            view_status = ''
+            if editable:
+                view_status += 'editable '
+            if checked or selected:
+                view_status += 'checked '
+            view_desc = f'- a {view_status}view'
+            if content_description:
+                content_description = content_description.replace('\n', '  ')
+                content_description = f'{content_description[:20]}...' if len(content_description) > 20 else content_description
+                view_desc += f' "{content_description}"'
+            if view_text:
+                view_text = view_text.replace('\n', '  ')
+                view_text = f'{view_text[:20]}...' if len(view_text) > 20 else view_text
+                view_desc += f' with text "{view_text}"'
+            if actionable:
+                view_actions = []
+                if editable:
+                    view_actions.append(f'edit ({len(available_actions)})')
+                    available_actions.append(SetTextEvent(view=view, text='HelloWorld'))
+                if clickable or checkable:
+                    view_actions.append(f'click ({len(available_actions)})')
+                    available_actions.append(TouchEvent(view=view))
+                # if checkable:
+                #     view_actions.append(f'check/uncheck ({len(available_actions)})')
+                #     available_actions.append(TouchEvent(view=view))
+                # if long_clickable:
+                #     view_actions.append(f'long click ({len(available_actions)})')
+                #     available_actions.append(LongTouchEvent(view=view))
+                if scrollable:
+                    view_actions.append(f'scroll up ({len(available_actions)})')
+                    available_actions.append(ScrollEvent(view=view, direction='UP'))
+                    view_actions.append(f'scroll down ({len(available_actions)})')
+                    available_actions.append(ScrollEvent(view=view, direction='DOWN'))
+                view_actions_str = ', '.join(view_actions)
+                view_desc += f' that can {view_actions_str}'
+            view_descs.append(view_desc)
+        view_descs.append(f'- a key to go back ({len(available_actions)})')
+        available_actions.append(KeyEvent(name='BACK'))
+        state_desc = 'The current state has the following UI views and corresponding actions, with action id in parentheses:\n '
+        state_desc += ';\n '.join(view_descs)
+        return state_desc, available_actions
+
+    def get_action_desc(self, action):
+        desc = action.event_type
+        if isinstance(action, KeyEvent):
+            desc = f'- go {action.name.lower()}'
+        if isinstance(action, UIEvent):
+            action_name = action.event_type
+            if isinstance(action, LongTouchEvent):
+                action_name = 'long click'
+            elif isinstance(action, SetTextEvent):
+                action_name = f'enter "{action.text}" into'
+            elif isinstance(action, ScrollEvent):
+                action_name = f'scroll {action.direction.lower()}'
+            desc = f'- {action_name} {self.get_view_desc(action.view)}'
+        return desc
diff --git a/kea/input_manager.py b/kea/input_manager.py
@@ -22,7 +22,7 @@
 DEFAULT_EVENT_COUNT = 100000000
 DEFAULT_TIMEOUT = 3600
 DEFAULT_DEVICE_SERIAL = "emulator-5554"
-DEFAULT_UI_TARPIT_NUM = 5
+DEFAULT_UI_TARPIT_NUM = 2
 
 class UnknownInputException(Exception):
     pass

diff --git a/kea/input_policy.py b/kea/input_policy.py
@@ -824,6 +824,9 @@ def __init__(self, device, app, random_input=True, kea=None, restart_app_after_c
         self.__action_history=[]
         self.__all_action_history=set()
         self.__activity_history = set()
+        self.from_state = None
+        self.__missed_states = set()
+        self.task = "You are an expert in App GUI testing. Please guide the testing tool to enhance the coverage of functional scenarios in testing the App based on your extensive App testing experience. "
 
     def start(self, input_manager:"InputManager"):
         """
@@ -859,7 +862,7 @@ def start(self, input_manager:"InputManager"):
                             input_manager.sim_calculator.sim_count = 0 
                         else:
                             # stop random policy, start query LLM
-                            event = self.generate_LLM_event()
+                            event = self.generate_llm_event()
                     else:
                         event = self.generate_event()
 
@@ -891,7 +894,7 @@ def start(self, input_manager:"InputManager"):
             self.action_count += 1
         self.tear_down()
 
-    def generate_LLM_event(self):
+    def generate_llm_event(self):
         """
         generate an LLM event
         @return:
@@ -928,7 +931,7 @@ def generate_LLM_event(self):
         event = None
 
         if event is None:
-            event = self.generate_event_based_on_utg()
+            event = self.generate_llm_event_based_on_utg()
 
         if isinstance(event, RotateDevice):
             if self.last_rotate_events == KEY_RotateDeviceNeutralEvent:
@@ -940,7 +943,7 @@ def generate_LLM_event(self):
 
         return event
 
-    def generate_event_based_on_utg(self):
+    def generate_llm_event_based_on_utg(self):
         """
         generate an event based on current UTG
         @return: InputEvent
@@ -1026,8 +1029,8 @@ def generate_event_based_on_utg(self):
     def _query_llm(self, prompt, model_name='gpt-3.5-turbo'):
         # TODO: replace with your own LLM
         from openai import OpenAI
-        gpt_url = 'https://api.chatanywhere.tech/v1' 
-        gpt_key = 'sk-G3dXD5UnEjiv1OVxwc5ZnRSFNccp2WiGOp2tJDjLM7WeDW8D' 
+        gpt_url = '' 
+        gpt_key = '' 
         client = OpenAI(
             base_url=gpt_url,
             api_key=gpt_key

diff --git a/kea/similarity.py b/kea/similarity.py
@@ -3,7 +3,7 @@
 import os
 import cv2
 
-THREGHOLD = 0.95
+THREGHOLD = 0.85
 
 class Similarity(object):
     def __init__(self, sim_k) -> None: