Skip to content

Commit

Permalink
Bug Fixes (#37)
Browse files Browse the repository at this point in the history 
* Add Fallback mechanism

* Add JumpCommands

* Add jump support

* Add interative mode to replyaer

* Fix Talkback locate issue

* Add Text Description, Add ExecuteSingleAction task to main.py

* Fix Focusable Bug in TalkBack

* Fix dir bug

* Add Talkback Search Controller

* Fix TextDescription

* Add blind fold mode
  • Loading branch information
@noidsirius
noidsirius authored Sep 22, 2022
1 parent 4415185 commit ea5b0cb
Show file tree
Hide file tree
Showing 37 changed files with 755 additions and 160 deletions.
Binary file modified Setup/Sugilite.apk
View file
Binary file not shown.
93 changes: 86 additions & 7 deletions UseCaseExecutor/src/main/java/dev/navids/latte/ActionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,34 @@ public void onCancelled(GestureDescription gestureDescription) {
}
};

public static ActualWidgetInfo findActualWidget(ConceivedWidgetInfo targetWidget){
if (targetWidget == null)
return null;
List<AccessibilityNodeInfo> similarNodes = Utils.findSimilarNodes(targetWidget);
if(similarNodes.size() != 1){
if(similarNodes.size() == 0) {
Log.e(LatteService.TAG, "The target widget could not be found in current screen.");
// TODO: For debugging
Log.d(LatteService.TAG, "The target XPATH: \n\t" + targetWidget.getXpath());
List<AccessibilityNodeInfo> allNodes = Utils.getAllA11yNodeInfo(false);
for(AccessibilityNodeInfo nodeInfo : allNodes){
ActualWidgetInfo actualWidgetInfo = ActualWidgetInfo.createFromA11yNode(nodeInfo);
if (actualWidgetInfo != null)
Log.d(LatteService.TAG, "\t" + actualWidgetInfo.getXpath());
}
}
else{
Log.d(LatteService.TAG, "There are more than one candidates for the target.");
for(AccessibilityNodeInfo node : similarNodes){
Log.d(LatteService.TAG, " Node: " + node);
}
}
return null;
}
AccessibilityNodeInfo node = similarNodes.get(0);
return ActualWidgetInfo.createFromA11yNode(node);
}

public static boolean isFocusedNodeTarget(AccessibilityNodeInfo similarNode) {
return isFocusedNodeTarget(Collections.singletonList(similarNode));
}
Expand All @@ -68,15 +96,35 @@ public static boolean isFocusedNodeTarget(List<AccessibilityNodeInfo> similarNod
return false;
AccessibilityNodeInfo targetNode = similarNodes.get(0); // TODO: This strategy works even we found multiple similar widgets
AccessibilityNodeInfo firstReachableNode = targetNode;
boolean isSimilar = firstReachableNode != null && firstReachableNode.equals(LatteService.getInstance().getAccessibilityFocusedNode());
AccessibilityNodeInfo focusedNode = LatteService.getInstance().getAccessibilityFocusedNode();
boolean isSimilar = firstReachableNode != null && firstReachableNode.equals(focusedNode);
if(!isSimilar) {
AccessibilityNodeInfo it = targetNode;
while (it != null) {
if (it.isClickable()) {
firstReachableNode = it;
break;
if (focusedNode != null && focusedNode.getChildCount() == 0){
Log.i(LatteService.TAG, "-- TalkBack is focused on a leaf node" + focusedNode);
AccessibilityNodeInfo it = focusedNode;
while(it != null){
if (it.equals(targetNode)){
firstReachableNode = focusedNode;
break;
}
it = it.getParent();
}
}
else{
Log.i(LatteService.TAG, "-- TalkBack is focused on a parent node" + focusedNode);
AccessibilityNodeInfo it = targetNode;
while (it != null) {
if (it.isClickable() && !targetNode.isClickable()) {
firstReachableNode = it;
break;
}
if (it.isImportantForAccessibility() && !targetNode.isImportantForAccessibility() && targetNode.isClickable())
{
firstReachableNode = it;
break;
}
it = it.getParent();
}
it = it.getParent();
}
Log.i(LatteService.TAG, "-- FIRST REACHABLE NODE IS " + firstReachableNode);
isSimilar = firstReachableNode != null && firstReachableNode.equals(LatteService.getInstance().getAccessibilityFocusedNode());
Expand Down Expand Up @@ -161,6 +209,28 @@ public static boolean performTap(int x, int y, int startTime, int duration, Acce
return LatteService.getInstance().dispatchGesture(gestureDescription, callback, null);
}

public static boolean performThreeFingerTap(AccessibilityService.GestureResultCallback callback){
GestureDescription.Builder gestureBuilder = new GestureDescription.Builder();
int BASE = 50;
WindowManager wm = (WindowManager) LatteService.getInstance().getApplicationContext().getSystemService(Context.WINDOW_SERVICE);
Display display = wm.getDefaultDisplay();
Point size = new Point();
display.getSize(size);
int width = size.x;
int height = size.y;
int centerX = width/2;
int centerY = height/2;

for(int i=0; i<3; i++) {
Path fingerPath = new Path();
fingerPath.moveTo(centerX + (i-1) * BASE, centerY + (int) Math.pow(-1,i+1) * BASE);
gestureBuilder.addStroke(new GestureDescription.StrokeDescription(fingerPath, 0, Config.v().TAP_DURATION));
}
GestureDescription gestureDescription = gestureBuilder.build();
Log.i(LatteService.TAG, "Execute ThreeFingerTap Gesture " + gestureDescription.toString());
return LatteService.getInstance().dispatchGesture(gestureDescription, callback, null);
}

public static boolean performType(AccessibilityNodeInfo node, String message){
Log.i(LatteService.TAG, "performType");
Bundle arguments = new Bundle();
Expand Down Expand Up @@ -261,6 +331,11 @@ private static void executeSwipeGesture(String direction, String secondDirection
Log.i(LatteService.TAG, "Add left direction to " + Integer.toString(100+dx1) + " " +Integer.toString(y2-dy1));
swipePath.lineTo(100+dx1, y2-dy1);
}
if (direction.equals("left") && secondDirection.equals("down"))
{
Log.i(LatteService.TAG, "Add down direction to " + Integer.toString(100+dx1) + " " +Integer.toString(y2-dy1));
swipePath.lineTo(100+dx2, height-100+dy1);
}
}
gestureBuilder.addStroke(new GestureDescription.StrokeDescription(swipePath, 0, Config.v().GESTURE_DURATION));
GestureDescription gestureDescription = gestureBuilder.build();
Expand All @@ -278,6 +353,9 @@ public static boolean swipeToDirection(String direction, ActionCallback doneCall
public static boolean swipeUpThenLeft(ActionCallback doneCallback){
return swipeToDirection("up", "left", doneCallback);
}
public static boolean swipeLeftThenDown(ActionCallback doneCallback){
return swipeToDirection("left", "down", doneCallback);
}
public static boolean swipeToDirection(String direction, String secondDirection, ActionCallback doneCallback){
if(isActionPending()){
Log.i(LatteService.TAG, String.format("Do nothing since another action is pending! (Size:%d)", pendingActions.size()));
Expand All @@ -289,6 +367,7 @@ public static boolean swipeToDirection(String direction, String secondDirection,
AccessibilityService.GestureResultCallback callback = new AccessibilityService.GestureResultCallback() {
@Override
public void onCompleted(GestureDescription gestureDescription) {
Log.i(LatteService.TAG, "Gesture execution is completed!");
new Handler().postDelayed(() -> {
pendingActions.remove(thisActionId);
if(doneCallback != null)
Expand Down
4 changes: 4 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/ConceivedWidgetInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@

public class ConceivedWidgetInfo extends WidgetInfo {

public void setLocatedBy(String locatedBy) {
this.locatedBy = locatedBy;
}

String locatedBy = "";

public ConceivedWidgetInfo(String resourceId, String contentDescription, String text, String clsName, String xpath, String locatedBy) {
Expand Down
4 changes: 4 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/MessageReceiver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ public MessageReceiver() {
// ----------- General ----------------
messageEventMap.put("is_live", (extra) -> Utils.createFile(String.format(Config.v().IS_LIVE_FILE_PATH_PATTERN, extra), "I'm alive " + extra));
messageEventMap.put("log", (extra) -> Utils.getAllA11yNodeInfo(true));
messageEventMap.put("tb_search", (extra) -> {
Log.i(LatteService.TAG, "Opening TalkBack Search Page");
ActionUtils.swipeLeftThenDown(null);
});
messageEventMap.put("report_tb_nodes", (extra) -> {
Log.i(LatteService.TAG, "I'm reporting TalkBack Focusable Nodes!");
// Logs the ordered list of focusable nodes in TalkBack
Expand Down
4 changes: 4 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/Command.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ else if (NextCommand.isNextAction(action))
command = new NextCommand(commandJSON);
else if (PreviousCommand.isPreviousAction(action))
command = new PreviousCommand(commandJSON);
else if (JumpNextCommand.isJumpNextAction(action))
command = new JumpNextCommand(commandJSON);
else if (JumpPreviousCommand.isJumpPreviousAction(action))
command = new JumpPreviousCommand(commandJSON);
else if (SelectCommand.isSelectCommand(action))
command = new SelectCommand(commandJSON);
else if (BackCommand.isBackAction(action))
Expand Down
7 changes: 7 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/InfoCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ public class InfoCommand extends Command {


private String question = "";
private JSONObject extra = null;

InfoCommand(JSONObject stepJson) {
super(stepJson);
question = (String) stepJson.getOrDefault("question", "");
extra = (JSONObject) stepJson.getOrDefault("extra", null);
}
@Override
public JSONObject getJSON() {
Expand All @@ -21,6 +24,10 @@ public String getQuestion() {
return question;
}

public JSONObject getExtra() {
return extra;
}

public JSONObject getJsonResult() {
return jsonResult;
}
Expand Down
27 changes: 27 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/JumpNextCommand.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package dev.navids.latte.UseCase;

import android.util.Log;

import org.json.simple.JSONObject;

import dev.navids.latte.LatteService;

public class JumpNextCommand extends NavigateCommand {

JumpNextCommand(JSONObject stepJson) {
super(stepJson);
Log.i(LatteService.TAG, "Jump Next Step");
}

public static boolean isJumpNextAction(String action){
return action.equals("jump_next");
}


@Override
public String toString() {
return "JumpNextStep{" +
"State=" + getState().name() +
'}';
}
}
27 changes: 27 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/UseCase/JumpPreviousCommand.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package dev.navids.latte.UseCase;

import android.util.Log;

import org.json.simple.JSONObject;

import dev.navids.latte.LatteService;

public class JumpPreviousCommand extends NavigateCommand {

JumpPreviousCommand(JSONObject stepJson) {
super(stepJson);
Log.i(LatteService.TAG, "Jump Previous Step");
}

public static boolean isJumpPreviousAction(String action){
return action.equals("jump_previous");
}


@Override
public String toString() {
return "JumpPreviousStep{" +
"State=" + getState().name() +
'}';
}
}
8 changes: 8 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/controller/AbstractActionPerformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import dev.navids.latte.UseCase.BackCommand;
import dev.navids.latte.UseCase.ClickCommand;
import dev.navids.latte.UseCase.FocusCommand;
import dev.navids.latte.UseCase.JumpNextCommand;
import dev.navids.latte.UseCase.JumpPreviousCommand;
import dev.navids.latte.UseCase.LocatableCommand;
import dev.navids.latte.UseCase.NavigateCommand;
import dev.navids.latte.UseCase.NextCommand;
Expand All @@ -22,6 +24,10 @@ public void navigate(NavigateCommand navigateCommand, ExecutorCallback callback)
navigateNext((NextCommand) navigateCommand, callback);
else if (navigateCommand instanceof PreviousCommand)
navigatePrevious((PreviousCommand) navigateCommand, callback);
else if (navigateCommand instanceof JumpNextCommand)
navigateJumpNext((JumpNextCommand) navigateCommand, callback);
else if (navigateCommand instanceof JumpPreviousCommand)
navigateJumpPrevious((JumpPreviousCommand) navigateCommand, callback);
else if (navigateCommand instanceof SelectCommand)
navigateSelect((SelectCommand) navigateCommand, callback);
else if (navigateCommand instanceof BackCommand)
Expand Down Expand Up @@ -72,6 +78,8 @@ else if(locatableCommand instanceof FocusCommand){
public abstract boolean executeFocus(FocusCommand focusStep, ActualWidgetInfo actualWidgetInfo);
public abstract void navigateNext(NextCommand nextStep, ExecutorCallback callback);
public abstract void navigatePrevious(PreviousCommand previousStep, ExecutorCallback callback);
public abstract void navigateJumpNext(JumpNextCommand nextStep, ExecutorCallback callback);
public abstract void navigateJumpPrevious(JumpPreviousCommand previousStep, ExecutorCallback callback);
public abstract void navigateSelect(SelectCommand selectCommand, ExecutorCallback callback);
public abstract void navigateBack(BackCommand selectCommand, ExecutorCallback callback);
}
28 changes: 0 additions & 28 deletions UseCaseExecutor/src/main/java/dev/navids/latte/controller/AbstractLocator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,32 +109,4 @@ else if (locatorResult.status == LocatorStatus.WAITING){
}

protected abstract LocatorResult locateAttempt(ConceivedWidgetInfo targetWidget);

protected ActualWidgetInfo findActualWidget(ConceivedWidgetInfo targetWidget){
if (targetWidget == null)
return null;
List<AccessibilityNodeInfo> similarNodes = Utils.findSimilarNodes(targetWidget);
if(similarNodes.size() != 1){
if(similarNodes.size() == 0) {
Log.e(LatteService.TAG, "The target widget could not be found in current screen.");
// TODO: For debugging
Log.d(LatteService.TAG, "The target XPATH: \n\t" + targetWidget.getXpath());
List<AccessibilityNodeInfo> allNodes = Utils.getAllA11yNodeInfo(false);
for(AccessibilityNodeInfo nodeInfo : allNodes){
ActualWidgetInfo actualWidgetInfo = ActualWidgetInfo.createFromA11yNode(nodeInfo);
if (actualWidgetInfo != null)
Log.d(LatteService.TAG, "\t" + actualWidgetInfo.getXpath());
}
}
else{
Log.d(LatteService.TAG, "There are more than one candidates for the target.");
for(AccessibilityNodeInfo node : similarNodes){
Log.d(LatteService.TAG, " Node: " + node);
}
}
return null;
}
AccessibilityNodeInfo node = similarNodes.get(0);
return ActualWidgetInfo.createFromA11yNode(node);
}
}
12 changes: 12 additions & 0 deletions UseCaseExecutor/src/main/java/dev/navids/latte/controller/BaseActionPerformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import dev.navids.latte.UseCase.BackCommand;
import dev.navids.latte.UseCase.ClickCommand;
import dev.navids.latte.UseCase.FocusCommand;
import dev.navids.latte.UseCase.JumpNextCommand;
import dev.navids.latte.UseCase.JumpPreviousCommand;
import dev.navids.latte.UseCase.NextCommand;
import dev.navids.latte.UseCase.PreviousCommand;
import dev.navids.latte.UseCase.SelectCommand;
Expand Down Expand Up @@ -44,6 +46,16 @@ public void navigatePrevious(PreviousCommand previousStep, ExecutorCallback call

}

@Override
public void navigateJumpNext(JumpNextCommand nextStep, ExecutorCallback callback) {

}

@Override
public void navigateJumpPrevious(JumpPreviousCommand previousStep, ExecutorCallback callback) {

}

@Override
public void navigateSelect(SelectCommand selectCommand, ExecutorCallback callback) {

Expand Down
3 changes: 2 additions & 1 deletion UseCaseExecutor/src/main/java/dev/navids/latte/controller/BaseLocator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import android.util.Log;

import dev.navids.latte.ActionUtils;
import dev.navids.latte.ActualWidgetInfo;
import dev.navids.latte.ConceivedWidgetInfo;
import dev.navids.latte.LatteService;
Expand All @@ -10,7 +11,7 @@ public class BaseLocator extends AbstractLocator {
@Override
protected LocatorResult locateAttempt(ConceivedWidgetInfo targetWidget)
{
ActualWidgetInfo actualWidgetInfo = findActualWidget(targetWidget);
ActualWidgetInfo actualWidgetInfo = ActionUtils.findActualWidget(targetWidget);
if(actualWidgetInfo == null){
Log.i(LatteService.TAG, "The target widget could not be found at this moment!");
return new LocatorResult(LocatorStatus.WAITING);
Expand Down
Loading

0 comments on commit ea5b0cb

Please sign in to comment.