Add OverSight Mode (#11)

* - Add more attribute to XML - Add invisible_nodes settings to Latte - Add event logs to visulization * Add Node model * Add OverSight Static Analysis * Add Snapshot Search Tab * Add OverSight Tab with info * Add BS_Snapshot row, fix some bugs in search, refactor app page * Add TalkBack APK, Update Readme * Add Prev navigation * Add RQ1 RQ2 Latex * Add TalkBack apk * Fix some bugs * Add explore and action modes
noidsirius · Apr 1, 2022 · ca19510 · ca19510
1 parent ad171cf
commit ca19510
Show file tree

Hide file tree

Showing 42 changed files with 2,068 additions and 535 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,16 @@
 # BlindMonkey
 ## Setup
--  Install python packages `pip install -r requirements.txt`
+- For OS X
+  - Install coreutils "brew install coreutils"
+  - Use Java8, if there are multiple Java versions use [jenv](https://www.jenv.be/)
+  - Set `ANDROID_HOME` environment varilable (usually `export ANDROID_HOME=~/Library/Android/sdk`)
+  - Add emulator and platform tools to `PATH` (if it's not already added). `export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/emulator:${PATH}"`
+- For other UNIX system: it's not tested
+
+- Optional: create a virtual environment in `.env`
+
+- run `source env`
+- Install python packages `pip install -r requirements.txt`
 - Initialize an Android Virtual Device (AVD) with SDK +28 and name it `testAVD_1`
 -	Disable soft main keys and virtual keyboard by adding `hw.mainKeys=yes` and `hw.kayboard=yes`  to `~/.android/avd/testAVD_1.avd/config.ini`
 	- If virtual device is not disabled, please follow this [link](https://support.honeywellaidc.com/s/article/CN51-Android-How-to-prevent-virtual-keyboard-from-popping-up)
@@ -23,6 +33,7 @@ You can interact with Latte by sending commands to its Broadcast Receiver or rec
 - **General**
 	- `log`: Prints the current layout's xpaths in Android logs.
 	- `is_live`: Given a string as the extra, creates a file `is_live_<extra>.txt`. It can be used to determine Latte is alive.
+	- `invisible_nodes`: Make Latte does (not) consider invisible nodes. Extra can be 'true' or 'false'
 	- `capture_layout`: Dumps the XML file of the current layout. Output's file name: `a11y_layout.xml`
 	- `report_a11y_issues`: Prints the accessibility issues (reported by Accessibility Testing Framework) in Android logs.
 	- `sequence`: Execute a sequence of commands which is given in input as a JSON string. Example: [{'command': 'log', 'extra': 'NONE'}]

diff --git a/Setup/API30/com.google.android.marvin.talkback_12.1.apk b/Setup/API30/com.google.android.marvin.talkback_12.1.apk
diff --git a/Setup/API32/com.google.android.marvin.talkback_12.1.apk b/Setup/API32/com.google.android.marvin.talkback_12.1.apk
diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/ActionUtils.java b/UseCaseExecutor/src/main/java/dev/navids/latte/ActionUtils.java
@@ -20,6 +20,9 @@
 import java.util.Map;
 import java.util.Random;
 
+import dev.navids.latte.UseCase.FocusStep;
+import dev.navids.latte.UseCase.StepState;
+
 public class ActionUtils {
     private static Map<Integer, String> pendingActions = new HashMap<>();
     private static int pendingActionId = 0;
@@ -280,4 +283,17 @@ public void onCancelled(GestureDescription gestureDescription) {
         new Handler().postDelayed(() -> { executeSwipeGesture(direction, secondDirection, callback);}, 10);
         return true;
     }
+
+    public static boolean a11yFocusOnNode(AccessibilityNodeInfo node){
+        AccessibilityNodeInfo currentFocusedNode = LatteService.getInstance().getAccessibilityFocusedNode();
+        if (currentFocusedNode != null)
+            currentFocusedNode.performAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
+        ActualWidgetInfo focusableWidget = ActualWidgetInfo.createFromA11yNode(node);
+        if (focusableWidget == null) {
+            Log.e(LatteService.TAG, "The requested focusing  widget is null!");
+            return false;
+        }
+        Log.i(LatteService.TAG, "Focusing on widget: " + focusableWidget.completeToString(true));
+        return node.performAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
+    }
 }
diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/ActualWidgetInfo.java b/UseCaseExecutor/src/main/java/dev/navids/latte/ActualWidgetInfo.java
@@ -70,7 +70,7 @@ public String getXpath() {
                 if (child == null)
                     continue;
                 String childClsName = String.valueOf(child.getClassName());
-                if (!child.isVisibleToUser())
+                if (!LatteService.considerInvisibleNodes && !child.isVisibleToUser())
                     continue;
                 if (itClsName.equals(childClsName))
                     length++;

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/CommandReceiver.java b/UseCaseExecutor/src/main/java/dev/navids/latte/CommandReceiver.java
@@ -33,18 +33,19 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
 import dev.navids.latte.UseCase.RegularStepExecutor;
+import dev.navids.latte.UseCase.SightedTalkBackStepExecutor;
 import dev.navids.latte.UseCase.StepExecutor;
 import dev.navids.latte.UseCase.UseCaseExecutor;
 
 public class CommandReceiver extends BroadcastReceiver {
     static final String ACTION_COMMAND_INTENT = "dev.navids.latte.COMMAND";
     static final String ACTION_COMMAND_CODE = "command";
     static final String ACTION_COMMAND_EXTRA = "extra";
+
     interface CommandEvent {
         void doAction(String extra);
     }
@@ -55,6 +56,7 @@ public CommandReceiver() {
         // ----------- General ----------------
         commandEventMap.put("is_live", (extra) -> Utils.createFile(String.format(Config.v().IS_LIVE_FILE_PATH_PATTERN, extra), "I'm alive " + extra));
         commandEventMap.put("log", (extra) -> Utils.getAllA11yNodeInfo(true));
+        commandEventMap.put("invisible_nodes", (extra) -> LatteService.considerInvisibleNodes = (extra.equals("true")));
         commandEventMap.put("report_a11y_issues", (extra) -> {
             Context context2 = LatteService.getInstance().getApplicationContext();
             Set<AccessibilityHierarchyCheck> contrastChecks = new HashSet<>(Arrays.asList(
@@ -112,11 +114,13 @@ else if(extra.equals("touch"))
             }
         });
         // ---------------------------- TalkBack Navigation -----------
-        commandEventMap.put("nav_next", (extra) -> TalkBackNavigator.v().nextFocus(null));
+        commandEventMap.put("nav_next", (extra) -> TalkBackNavigator.v().changeFocus(null, false));
+        commandEventMap.put("nav_prev", (extra) -> TalkBackNavigator.v().changeFocus(null, true));
         commandEventMap.put("nav_select", (extra) -> TalkBackNavigator.v().selectFocus(null));
         commandEventMap.put("nav_current_focus", (extra) -> TalkBackNavigator.v().currentFocus());
         commandEventMap.put("tb_a11y_tree", (extra) -> TalkBackNavigator.v().logTalkBackTreeNodeList(null));
         commandEventMap.put("nav_clear_history", (extra) -> TalkBackNavigator.v().clearHistory());
+        commandEventMap.put("nav_api_focus", (extra) -> SightedTalkBackStepExecutor.apiFocus = (extra.equals("true")));
         commandEventMap.put("nav_interrupt", (extra) -> TalkBackNavigator.v().interrupt());
         // --------------------------- UseCase Executor ----------------
         commandEventMap.put("enable", (extra) -> UseCaseExecutor.v().enable());

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/LatteService.java b/UseCaseExecutor/src/main/java/dev/navids/latte/LatteService.java
@@ -43,6 +43,7 @@ public boolean isConnected() {
     private boolean connected = false;
     public static String TAG = "LATTE_SERVICE";
     private String A11Y_EVENT_TAG = "LATTE_A11Y_EVENT_TAG";
+    public static boolean considerInvisibleNodes = true;
 
     public static LatteService getInstance() {
         return instance;
@@ -111,8 +112,9 @@ else if(event.getEventType() == AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED)
                     break;
                 }
             AccessibilityWindowInfo activeWindow = null;
-            if(getRootInActiveWindow() != null)
-                activeWindow = getRootInActiveWindow().getWindow();
+            AccessibilityNodeInfo rootInActiveWindow = getRootInActiveWindow();
+            if(rootInActiveWindow != null)
+                activeWindow = rootInActiveWindow.getWindow();
             JSONObject jsonWindowContentChange = null;
             int activeWindowId = activeWindow != null ? activeWindow.getId() : -1;
             String activeWindowTitle = activeWindow != null && activeWindow.getTitle() != null ? activeWindow.getTitle().toString() : "null";

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/TalkBackNavigator.java b/UseCaseExecutor/src/main/java/dev/navids/latte/TalkBackNavigator.java
@@ -45,6 +45,16 @@ public boolean performNext(Navigator.DoneCallback doneCallback){
         return true;
     }
 
+    public boolean performPrev(Navigator.DoneCallback doneCallback){
+        Log.i(LatteService.TAG, "performPrev");
+        if (!ActionUtils.swipeLeft(doneCallback))
+        {
+            Log.i(LatteService.TAG, "There is a problem with swiping left");
+            return false;
+        }
+        return true;
+    }
+
     public boolean logTalkBackTreeNodeList(Navigator.DoneCallback doneCallback){
         Log.i(LatteService.TAG, "perform Up then Left");
         if (!ActionUtils.swipeUpThenLeft(doneCallback))
@@ -117,11 +127,11 @@ public void clearHistory(){
         new File(dir, Config.v().FINISH_ACTION_FILE_PATH).delete();
     }
 
-    public boolean nextFocus(Navigator.DoneCallback callback) {
+    public boolean changeFocus(Navigator.DoneCallback callback, boolean prev) {
         Utils.deleteFile(Config.v().FINISH_ACTION_FILE_PATH);
         WidgetInfo widgetInfo = ActualWidgetInfo.createFromA11yNode(LatteService.getInstance().getAccessibilityFocusedNode());
         Log.i(LatteService.TAG, String.format("Widget %s is visited XPATH: %s.", widgetInfo, widgetInfo != null ? widgetInfo.getAttr("xpath") : "NONE"));
-        boolean result = performNext(new Navigator.DoneCallback() {
+        Navigator.DoneCallback afterChangeCallback = new Navigator.DoneCallback() {
             @Override
             public void onCompleted(AccessibilityNodeInfo nodeInfo) {
                 WidgetInfo newWidgetNodeInfo = ActualWidgetInfo.createFromA11yNode(nodeInfo, true);
@@ -146,7 +156,12 @@ public void onError(String message) {
                 if(callback != null)
                     callback.onError(message);
             }
-        });
+        };
+        boolean result = false;
+        if(prev)
+            result = performPrev(afterChangeCallback);
+        else
+            result = performNext(afterChangeCallback);
         return result;
     }
 

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/RegularStepExecutor.java b/UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/RegularStepExecutor.java
@@ -111,12 +111,7 @@ private boolean executeClick(ClickStep clickStep, AccessibilityNodeInfo node){
     }
 
     private boolean executeFocus(FocusStep focusStep, AccessibilityNodeInfo node){
-        AccessibilityNodeInfo currentFocusedNode = LatteService.getInstance().getFocusedNode();
-        if (currentFocusedNode != null)
-            currentFocusedNode.performAction(AccessibilityNodeInfo.ACTION_CLEAR_FOCUS);
-        ActualWidgetInfo focusableWidget = ActualWidgetInfo.createFromA11yNode(node);
-        boolean result = node.performAction(AccessibilityNodeInfo.ACTION_FOCUS);
-        Log.i(LatteService.TAG, "Focusing on widget: " + focusableWidget.completeToString(true));
+        boolean result = ActionUtils.a11yFocusOnNode(node);
         focusStep.setState(result ? StepState.COMPLETED : StepState.FAILED);
         return result;
     }

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/SightedTalkBackStepExecutor.java b/UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/SightedTalkBackStepExecutor.java
@@ -12,7 +12,7 @@
 import dev.navids.latte.Utils;
 
 public class SightedTalkBackStepExecutor implements StepExecutor {
-
+    public static boolean apiFocus = false;
     @Override
     public boolean executeStep(StepCommand step) {
         Log.i(LatteService.TAG, "STB Executing Step " + step);
@@ -49,10 +49,16 @@ public boolean executeStep(StepCommand step) {
             else {
                 AccessibilityNodeInfo node = similarNodes.get(0);
                 if(!ActionUtils.isFocusedNodeTarget(similarNodes)){
-                    Pair<Integer, Integer> clickableCoordinate = ActionUtils.getClickableCoordinate(node, false);
-                    int x =clickableCoordinate.first, y = clickableCoordinate.second;
-                    Log.e(LatteService.TAG, String.format("Physically clicking on (%d, %d)", x, y));
-                    ActionUtils.performTap(x, y);
+                    if(apiFocus){
+                        Log.e(LatteService.TAG, String.format("API Focusing on %s", node));
+                        ActionUtils.a11yFocusOnNode(node);
+                    }
+                    else {
+                        Pair<Integer, Integer> clickableCoordinate = ActionUtils.getClickableCoordinate(node, false);
+                        int x = clickableCoordinate.first, y = clickableCoordinate.second;
+                        Log.e(LatteService.TAG, String.format("Physically clicking on (%d, %d)", x, y));
+                        ActionUtils.performTap(x, y);
+                    }
                     locatableStep.increaseLocatingAttempts();
                     return false;
                 }

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/Utils.java b/UseCaseExecutor/src/main/java/dev/navids/latte/Utils.java
@@ -76,20 +76,14 @@ public static List<AccessibilityNodeInfo> findSimilarNodes(ConceivedWidgetInfo t
 
     public static List<AccessibilityNodeInfo> findSimilarNodes(ConceivedWidgetInfo target, List<String> myMaskedAttributes){
         List<AccessibilityNodeInfo> result = new ArrayList<>();
-        List<AccessibilityNodeInfo> nonVisibleResult = new ArrayList<>();
         for(AccessibilityNodeInfo node : getAllA11yNodeInfo(false)) {
-            if(!node.isVisibleToUser())
+            if(!LatteService.considerInvisibleNodes && !node.isVisibleToUser())
                 continue;
             ActualWidgetInfo currentNodeInfo = ActualWidgetInfo.createFromA11yNode(node); // TODO: Use Cache
             if (target.isSimilar(currentNodeInfo, myMaskedAttributes)) {
-                if(node.isVisibleToUser())
-                    result.add(node);
-                else
-                    nonVisibleResult.add(node);
+                result.add(node);
             }
         }
-        if(result.size() == 0)
-            result.addAll(nonVisibleResult);
         return result;
     }
 
@@ -176,10 +170,12 @@ private static void dumpNodeRec(AccessibilityNodeInfo node, XmlSerializer serial
         // Extra Attributes
         serializer.attribute("", "importantForAccessibility", Boolean.toString(node.isImportantForAccessibility()));
         serializer.attribute("", "supportsWebAction", Boolean.toString(supportsWebAction));
-        serializer.attribute("", "z-a11y-actions", String.join("-", actions));
+        serializer.attribute("", "actionList", String.join("-", actions));
         serializer.attribute("", "clickableSpan", Boolean.toString(hasClickableSpan));
         serializer.attribute("", "drawingOrder", Integer.toString(node.getDrawingOrder()));
-//        serializer.attribute("", "visible", Boolean.toString(node.isVisibleToUser()));
+        serializer.attribute("", "visible", Boolean.toString(node.isVisibleToUser()));
+        serializer.attribute("", "invalid", Boolean.toString(node.isContentInvalid()));
+        serializer.attribute("", "contextClickable", Boolean.toString(node.isContextClickable()));
         // Regular Attributes
         serializer.attribute("", "index", Integer.toString(index));
         serializer.attribute("", "text", safeCharSeqToString(node.getText()));
@@ -202,7 +198,7 @@ private static void dumpNodeRec(AccessibilityNodeInfo node, XmlSerializer serial
         for (int i = 0; i < count; i++) {
             AccessibilityNodeInfo child = node.getChild(i);
             if (child != null) {
-                if (child.isVisibleToUser()) {
+                if (LatteService.considerInvisibleNodes || child.isVisibleToUser()) {
                 dumpNodeRec(child, serializer, i, width, height);
                 child.recycle();
                 } else {

diff --git a/UseCaseExecutor/src/main/java/dev/navids/latte/WidgetInfo.java b/UseCaseExecutor/src/main/java/dev/navids/latte/WidgetInfo.java
@@ -98,10 +98,10 @@ public JSONObject getJSONCommand(String located_by, boolean skip, String action)
         JSONObject jsonCommand = null;
         try {
             jsonCommand = new JSONObject()
-                    .put("resourceId", this.getAttr("resourceId"))
-                    .put("contentDescription", this.getAttr("contentDescription"))
+                    .put("resource_id", this.getAttr("resourceId"))
+                    .put("content_desc", this.getAttr("contentDescription"))
                     .put("text", this.getAttr("text"))
-                    .put("class", this.getAttr("class"))
+                    .put("class_name", this.getAttr("class"))
                     .put("xpath", this.getAttr("xpath"))
                     .put("located_by", located_by)
                     .put("skip", skip)

diff --git a/env b/env
@@ -0,0 +1,2 @@
+export LATTE_PATH="$(realpath $(dirname $0))"
+source $LATTE_PATH/.env/bin/activate
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		export LATTE_PATH="$(realpath $(dirname $0))"
		source $LATTE_PATH/.env/bin/activate