diff --git a/kea/device_state.py b/kea/device_state.py index 29672a5..6ce3d6b 100644 --- a/kea/device_state.py +++ b/kea/device_state.py @@ -10,7 +10,7 @@ from .device import Device from .utils import md5, deprecated -from .input_event import SearchEvent, SetTextAndSearchEvent, TouchEvent, LongTouchEvent, ScrollEvent, SetTextEvent, KeyEvent +from .input_event import SearchEvent, SetTextAndSearchEvent, TouchEvent, LongTouchEvent, ScrollEvent, SetTextEvent, KeyEvent, UIEvent class DeviceState(object): @@ -808,3 +808,105 @@ def is_view_exist(self, view_dict): def get_state_screen(self): return self.screenshot_path + + def get_view_desc(self, view): + content_description = self.__safe_dict_get(view, 'content_description', default='') + view_text = self.__safe_dict_get(view, 'text', default='') + scrollable = self.__safe_dict_get(view, 'scrollable') + view_desc = f'view' + if scrollable: + view_desc = f'scrollable view' + if content_description: + view_desc += f' "{content_description}"' + if view_text: + view_text = view_text.replace('\n', ' ') + view_text = f'{view_text[:20]}...' if len(view_text) > 20 else view_text + view_desc += f' with text "{view_text}"' + return view_desc + + def get_described_actions(self): + """ + Get a text description of current state + """ + enabled_view_ids = [] + for view_dict in self.views: + # exclude navigation bar if exists + if self.__safe_dict_get(view_dict, 'visible') and \ + self.__safe_dict_get(view_dict, 'resource_id') not in \ + ['android:id/navigationBarBackground', + 'android:id/statusBarBackground']: + enabled_view_ids.append(view_dict['temp_id']) + + view_descs = [] + available_actions = [] + for view_id in enabled_view_ids: + view = self.views[view_id] + clickable = self._get_self_ancestors_property(view, 'clickable') + scrollable = self.__safe_dict_get(view, 'scrollable') + checkable = self._get_self_ancestors_property(view, 'checkable') + long_clickable = self._get_self_ancestors_property(view, 'long_clickable') + editable = self.__safe_dict_get(view, 'editable') + actionable = clickable or scrollable or checkable or long_clickable or editable + checked = self.__safe_dict_get(view, 'checked') + selected = self.__safe_dict_get(view, 'selected') + content_description = self.__safe_dict_get(view, 'content_description', default='') + view_text = self.__safe_dict_get(view, 'text', default='') + if not content_description and not view_text and not scrollable: # actionable? + continue + + view_status = '' + if editable: + view_status += 'editable ' + if checked or selected: + view_status += 'checked ' + view_desc = f'- a {view_status}view' + if content_description: + content_description = content_description.replace('\n', ' ') + content_description = f'{content_description[:20]}...' if len(content_description) > 20 else content_description + view_desc += f' "{content_description}"' + if view_text: + view_text = view_text.replace('\n', ' ') + view_text = f'{view_text[:20]}...' if len(view_text) > 20 else view_text + view_desc += f' with text "{view_text}"' + if actionable: + view_actions = [] + if editable: + view_actions.append(f'edit ({len(available_actions)})') + available_actions.append(SetTextEvent(view=view, text='HelloWorld')) + if clickable or checkable: + view_actions.append(f'click ({len(available_actions)})') + available_actions.append(TouchEvent(view=view)) + # if checkable: + # view_actions.append(f'check/uncheck ({len(available_actions)})') + # available_actions.append(TouchEvent(view=view)) + # if long_clickable: + # view_actions.append(f'long click ({len(available_actions)})') + # available_actions.append(LongTouchEvent(view=view)) + if scrollable: + view_actions.append(f'scroll up ({len(available_actions)})') + available_actions.append(ScrollEvent(view=view, direction='UP')) + view_actions.append(f'scroll down ({len(available_actions)})') + available_actions.append(ScrollEvent(view=view, direction='DOWN')) + view_actions_str = ', '.join(view_actions) + view_desc += f' that can {view_actions_str}' + view_descs.append(view_desc) + view_descs.append(f'- a key to go back ({len(available_actions)})') + available_actions.append(KeyEvent(name='BACK')) + state_desc = 'The current state has the following UI views and corresponding actions, with action id in parentheses:\n ' + state_desc += ';\n '.join(view_descs) + return state_desc, available_actions + + def get_action_desc(self, action): + desc = action.event_type + if isinstance(action, KeyEvent): + desc = f'- go {action.name.lower()}' + if isinstance(action, UIEvent): + action_name = action.event_type + if isinstance(action, LongTouchEvent): + action_name = 'long click' + elif isinstance(action, SetTextEvent): + action_name = f'enter "{action.text}" into' + elif isinstance(action, ScrollEvent): + action_name = f'scroll {action.direction.lower()}' + desc = f'- {action_name} {self.get_view_desc(action.view)}' + return desc \ No newline at end of file diff --git a/kea/input_manager.py b/kea/input_manager.py index 8903584..75a8749 100644 --- a/kea/input_manager.py +++ b/kea/input_manager.py @@ -22,7 +22,7 @@ DEFAULT_EVENT_COUNT = 100000000 DEFAULT_TIMEOUT = 3600 DEFAULT_DEVICE_SERIAL = "emulator-5554" -DEFAULT_UI_TARPIT_NUM = 5 +DEFAULT_UI_TARPIT_NUM = 2 class UnknownInputException(Exception): pass diff --git a/kea/input_policy.py b/kea/input_policy.py index 65a4f38..485ca20 100644 --- a/kea/input_policy.py +++ b/kea/input_policy.py @@ -824,6 +824,9 @@ def __init__(self, device, app, random_input=True, kea=None, restart_app_after_c self.__action_history=[] self.__all_action_history=set() self.__activity_history = set() + self.from_state = None + self.__missed_states = set() + self.task = "You are an expert in App GUI testing. Please guide the testing tool to enhance the coverage of functional scenarios in testing the App based on your extensive App testing experience. " def start(self, input_manager:"InputManager"): """ @@ -859,7 +862,7 @@ def start(self, input_manager:"InputManager"): input_manager.sim_calculator.sim_count = 0 else: # stop random policy, start query LLM - event = self.generate_LLM_event() + event = self.generate_llm_event() else: event = self.generate_event() @@ -891,7 +894,7 @@ def start(self, input_manager:"InputManager"): self.action_count += 1 self.tear_down() - def generate_LLM_event(self): + def generate_llm_event(self): """ generate an LLM event @return: @@ -928,7 +931,7 @@ def generate_LLM_event(self): event = None if event is None: - event = self.generate_event_based_on_utg() + event = self.generate_llm_event_based_on_utg() if isinstance(event, RotateDevice): if self.last_rotate_events == KEY_RotateDeviceNeutralEvent: @@ -940,7 +943,7 @@ def generate_LLM_event(self): return event - def generate_event_based_on_utg(self): + def generate_llm_event_based_on_utg(self): """ generate an event based on current UTG @return: InputEvent @@ -1026,8 +1029,8 @@ def generate_event_based_on_utg(self): def _query_llm(self, prompt, model_name='gpt-3.5-turbo'): # TODO: replace with your own LLM from openai import OpenAI - gpt_url = 'https://api.chatanywhere.tech/v1' - gpt_key = 'sk-G3dXD5UnEjiv1OVxwc5ZnRSFNccp2WiGOp2tJDjLM7WeDW8D' + gpt_url = '' + gpt_key = '' client = OpenAI( base_url=gpt_url, api_key=gpt_key diff --git a/kea/similarity.py b/kea/similarity.py index 91bf1ee..5aa576b 100644 --- a/kea/similarity.py +++ b/kea/similarity.py @@ -3,7 +3,7 @@ import os import cv2 -THREGHOLD = 0.95 +THREGHOLD = 0.85 class Similarity(object): def __init__(self, sim_k) -> None: