diff --git a/Android/app/src/main/AndroidManifest.xml b/Android/app/src/main/AndroidManifest.xml index 90c2dea..58242d1 100644 --- a/Android/app/src/main/AndroidManifest.xml +++ b/Android/app/src/main/AndroidManifest.xml @@ -21,11 +21,11 @@ android:usesCleartextTraffic="true"> + android:exported="true" + android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"> - + - + + + + + \ No newline at end of file diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Accessibility/HttpServer.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Accessibility/HttpServer.java index cae3859..53a79f2 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Accessibility/HttpServer.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Accessibility/HttpServer.java @@ -18,7 +18,6 @@ import cn.xjiangwei.RobotHelper.Tools.Robot; import fi.iki.elonen.NanoHTTPD; -import static android.support.v4.content.ContextCompat.getSystemService; public class HttpServer extends NanoHTTPD { @@ -63,7 +62,7 @@ public Response serve(IHTTPSession session) { } else if (parms.containsKey("id")) { ret = dumpHierarchyImpl(Accessibility.DOM.findAccessibilityNodeInfosByViewId(parms.get("id")), false).toString(); } else { - ret = dumpHierarchyImpl(new Node(Accessibility.DOM, 1440, 3120), false).toString(); + ret = dumpHierarchyImpl(new Node(Accessibility.DOM, MainApplication.sceenWidth, MainApplication.sceenHeight), false).toString(); } break; case "/sceenSize": @@ -71,7 +70,7 @@ public Response serve(IHTTPSession session) { break; case "/swipe": try { - Robot.swipe(Float.parseFloat(parms.get("start_x")), Float.parseFloat(parms.get("start_y")), Float.parseFloat(parms.get("end_x")), Float.parseFloat(parms.get("end_y")), Float.parseFloat(parms.get("duration"))); + Robot.swipe(Float.parseFloat(parms.get("start_x")), Float.parseFloat(parms.get("start_y")), Float.parseFloat(parms.get("end_x")), Float.parseFloat(parms.get("end_y")), Float.parseFloat(parms.get("duration")) * 1000); ret = "{\"code\":200,\"msg\":\"success\"}"; } catch (Exception e) { ret = "{\"code\":500,\"msg\":\"检查是否开启xposed\"}"; @@ -110,7 +109,7 @@ public Response serve(IHTTPSession session) { public JSONArray dumpHierarchyImpl(List nodes, boolean onlyVisibleNode) { JSONArray jsonArray = new JSONArray(); for (AccessibilityNodeInfo node : nodes) { - jsonArray.put(dumpHierarchyImpl(new Node(node, 3120, 1440), onlyVisibleNode)); + jsonArray.put(dumpHierarchyImpl(new Node(node, MainApplication.sceenWidth, MainApplication.sceenHeight), onlyVisibleNode)); } return jsonArray; diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/GamePackage/Main.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/GamePackage/Main.java index adfd414..5e3b8b7 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/GamePackage/Main.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/GamePackage/Main.java @@ -33,6 +33,10 @@ public class Main { public void start() { sleep(5000); //点击开始后等待5秒后再执行，因为状态栏收起有动画时间，建议保留这行代码 + //Robot.setExecType(Robot.ExecTypeXposed); //使用xposed权限执行模拟操作，建议优先使用此方式 + //Robot.setExecType(Robot.ExecTypeAccessibillty); //使用安卓无障碍接口执行模拟操作 + + /**************************** 模板匹配demo *******************************/ InputStream is = null; try { @@ -56,7 +60,8 @@ public void start() { e.printStackTrace(); } bitmap = BitmapFactory.decodeStream(is); - String res = TessactOcr.img2string(bitmap, "chi_sim", "", ""); + + String res = TessactOcr.img2string(ScreenCaptureUtil.getScreenCap(0,0,200,30), "chi_sim", "", ""); MLog.info("文字识别结果：" + res); diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainActivity.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainActivity.java index 24c951c..33cced8 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainActivity.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainActivity.java @@ -3,8 +3,6 @@ import android.Manifest; import android.content.ActivityNotFoundException; import android.content.Intent; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; import android.net.Uri; @@ -12,37 +10,23 @@ import android.os.Bundle; import android.os.Environment; import android.support.annotation.NonNull; +import android.support.v4.content.FileProvider; import android.support.v7.app.AppCompatActivity; import android.util.DisplayMetrics; import android.view.View; -import android.widget.EditText; import android.widget.TextView; import com.github.dfqin.grantor.PermissionListener; import com.github.dfqin.grantor.PermissionsUtil; -import com.googlecode.tesseract.android.TessBaseAPI; -import com.lahm.library.EasyProtectorLib; -import com.lahm.library.EmulatorCheckCallback; import org.opencv.android.BaseLoaderCallback; import org.opencv.android.LoaderCallbackInterface; import org.opencv.android.OpenCVLoader; -import org.opencv.android.Utils; -import org.opencv.core.Core; -import org.opencv.core.CvType; -import org.opencv.core.Mat; -import org.opencv.imgproc.Imgproc; import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import cn.xjiangwei.RobotHelper.Accessibility.HttpServer; -import cn.xjiangwei.RobotHelper.Service.Accessibility; import cn.xjiangwei.RobotHelper.Service.RunTime; import cn.xjiangwei.RobotHelper.Tools.MLog; -import cn.xjiangwei.RobotHelper.Tools.Robot; -import cn.xjiangwei.RobotHelper.Tools.ScreenCaptureUtil; import cn.xjiangwei.RobotHelper.Tools.ScreenCaptureUtilByMediaPro; import cn.xjiangwei.RobotHelper.Tools.TessactOcr; import cn.xjiangwei.RobotHelper.Tools.Toast; @@ -132,27 +116,25 @@ public void start(View view) { } - @Override protected void onResume() { super.onResume(); updateStatus(); } - private boolean checkXposedHook() { - return false; - } - public void openLog(View view) { - String file = Environment.getExternalStorageDirectory().toString() + "/RobotHelper.log"; + String filePath = Environment.getExternalStorageDirectory().toString() + "/RobotHelper.log"; + File file = new File(filePath); + Uri fileURI = FileProvider.getUriForFile(this, this.getApplicationContext().getPackageName() + ".provider", file); try { Intent intent = new Intent(); intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); intent.setAction(Intent.ACTION_VIEW); - intent.setDataAndType(Uri.fromFile(new File(file)), "text/plain"); + intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION); + intent.setDataAndType(fileURI, "text/plain"); startActivity(intent); Intent.createChooser(intent, "请选择对应的软件打开该附件！"); } catch (ActivityNotFoundException e) { @@ -183,9 +165,10 @@ private void updateStatus() { TextView asStatus = (TextView) findViewById(R.id.accessibility_status); TextView hsStatus = (TextView) findViewById(R.id.httpserver_status); - xpStatus.setText(checkXposedHook() ? "Xposed状态：已加载" : "Xposed状态：未加载"); - asStatus.setText(Accessibility.DOM == null ? "Accessibility状态：未加载" : "Accessibility状态：已加载"); - hsStatus.setText((RunTime.httpServer != null && RunTime.httpServer.runing) ? "HttpServer状态：已开启" : "HttpServer状态：未开启"); + xpStatus.setText(mainApplication.checkXposedHook() ? "Xposed状态：已加载" : "Xposed状态：未加载"); + asStatus.setText(mainApplication.checkAccessibilityService() ? "Accessibility状态：已加载" : "Accessibility状态：未加载"); + hsStatus.setText(mainApplication.checkHttpServer() ? "HttpServer状态：已开启" : "HttpServer状态：未开启"); } + } diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainApplication.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainApplication.java index 45b4e91..b91437b 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainApplication.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/MainApplication.java @@ -2,28 +2,20 @@ import android.app.Application; +import cn.xjiangwei.RobotHelper.Service.Accessibility; +import cn.xjiangwei.RobotHelper.Service.RunTime; public class MainApplication extends Application { public static int sceenWidth = 0; public static int sceenHeight = 0; public static int dpi; - private static String serverUrl = "http://192.168.199.241:8001/"; private static MainApplication instance; - public static MainApplication getInstance() { return instance; } - public static String getServerUrl() { - return serverUrl; - } - - public static void setServerUrl(String serverUrl) { - MainApplication.serverUrl = serverUrl; - } - @Override public void onCreate() { @@ -39,4 +31,16 @@ public static int sceenSize() { return 2; } + + public boolean checkXposedHook() { + return false; + } + + public boolean checkAccessibilityService() { + return Accessibility.DOM != null; + } + + public boolean checkHttpServer() { + return RunTime.httpServer != null && RunTime.httpServer.runing; + } } diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Service/Accessibility.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Service/Accessibility.java index c8b7790..3cf2248 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Service/Accessibility.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Service/Accessibility.java @@ -1,6 +1,10 @@ package cn.xjiangwei.RobotHelper.Service; import android.accessibilityservice.AccessibilityService; +import android.accessibilityservice.GestureDescription; +import android.graphics.Path; +import android.os.Build; +import android.support.annotation.RequiresApi; import android.util.Log; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityNodeInfo; @@ -10,31 +14,22 @@ public class Accessibility extends AccessibilityService { - + private static Accessibility instance; public static AccessibilityNodeInfo DOM; - - @Override - public void onAccessibilityEvent(AccessibilityEvent accessibilityEvent) { - DOM = getRootInActiveWindow(); + public Accessibility() { + instance = this; } - public static void logViewHierarchy(AccessibilityNodeInfo nodeInfo, final int depth) { - - if (nodeInfo == null) return; - - String spacerString = ""; - for (int i = 0; i < depth; ++i) { - spacerString += '-'; - } - //Log the info you care about here... I choce classname and view resource name, because they are simple, but interesting. - Log.d("TAG", spacerString + nodeInfo.getClassName() + " " + nodeInfo.getViewIdResourceName()); + public static Accessibility getInstance(){ + return instance; + } - for (int i = 0; i < nodeInfo.getChildCount(); ++i) { - logViewHierarchy(nodeInfo.getChild(i), depth + 1); - } + @Override + public void onAccessibilityEvent(AccessibilityEvent accessibilityEvent) { + DOM = getRootInActiveWindow(); } @@ -44,4 +39,6 @@ public void onInterrupt() { } + + } diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/AccessibilityInput.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/AccessibilityInput.java new file mode 100644 index 0000000..01c9623 --- /dev/null +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/AccessibilityInput.java @@ -0,0 +1,98 @@ +package cn.xjiangwei.RobotHelper.Tools.InputImp; + +import android.accessibilityservice.GestureDescription; +import android.graphics.Path; +import android.os.Build; +import android.support.annotation.RequiresApi; + +import cn.xjiangwei.RobotHelper.Service.Accessibility; +import cn.xjiangwei.RobotHelper.Tools.Point; + +@RequiresApi(api = Build.VERSION_CODES.N) +public class AccessibilityInput implements Input { + + + private static AccessibilityInput instance; + + public static AccessibilityInput getInstance() { + if (instance == null) { + synchronized (AccessibilityInput.class) { + if (instance == null) { + instance = new AccessibilityInput(); + } + } + } + + return instance; + } + + + @Override + public void tap(int x, int y) { + GestureDescription.Builder builder = new GestureDescription.Builder(); + Path path = new Path(); + path.moveTo((float) x, (float) y); + path.lineTo((float) x, (float) y); + /** + * 参数path：笔画路径 + * 参数startTime：时间 (以毫秒为单位)，从手势开始到开始笔划的时间，非负数 + * 参数duration：笔划经过路径的持续时间(以毫秒为单位)，非负数 + */ + builder.addStroke(new GestureDescription.StrokeDescription(path, 0, 50)); + final GestureDescription build = builder.build(); + + Accessibility.getInstance().dispatchGesture(build, null, null); + } + + @Override + public void tap(int x, int y, long delay) { + GestureDescription.Builder builder = new GestureDescription.Builder(); + Path path = new Path(); + path.moveTo((float) x, (float) y); + path.lineTo((float) x, (float) y); + /** + * 参数path：笔画路径 + * 参数startTime：时间 (以毫秒为单位)，从手势开始到开始笔划的时间，非负数 + * 参数duration：笔划经过路径的持续时间(以毫秒为单位)，非负数 + */ + builder.addStroke(new GestureDescription.StrokeDescription(path, 1, delay)); + final GestureDescription build = builder.build(); + Accessibility.getInstance().dispatchGesture(build, null, null); + } + + @Override + public void tap(Point p) { + tap(p.getX(), p.getY()); + } + + @Override + public void tap(Point p, long delay) { + tap(p.getX(), p.getY(), delay); + } + + @Override + public void swipe(int x1, int y1, int x2, int y2, int duration) { + swipe((float) x1, (float) y1, (float) x2, (float) y2, duration); + } + + @Override + public void swipe(float x1, float y1, float x2, float y2, float duration) { + GestureDescription.Builder builder = new GestureDescription.Builder(); + Path path = new Path(); + path.moveTo((float) x1, (float) y1); + path.lineTo((float) x2, (float) y2); + /** + * 参数path：笔画路径 + * 参数startTime：时间 (以毫秒为单位)，从手势开始到开始笔划的时间，非负数 + * 参数duration：笔划经过路径的持续时间(以毫秒为单位)，非负数 + */ + builder.addStroke(new GestureDescription.StrokeDescription(path, 1, (long) duration)); + final GestureDescription build = builder.build(); + Accessibility.getInstance().dispatchGesture(build, null, null); + } + + @Override + public void input(String str) { + + } +} diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/Input.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/Input.java new file mode 100644 index 0000000..95010c0 --- /dev/null +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/Input.java @@ -0,0 +1,55 @@ +package cn.xjiangwei.RobotHelper.Tools.InputImp; + + +import cn.xjiangwei.RobotHelper.Tools.Point; + +public interface Input { + + + + + /** + * 点击操作 + * + * @param x + * @param y + */ + void tap(final int x, final int y); + + /** + * 长按操作，可以自定义按下时间，单位为毫秒 + * + * @param x + * @param y + * @param delay + */ + void tap(final int x, final int y, final long delay); + + + void tap(Point p); + + + void tap(Point p, long delay); + + + /** + * 拖拽操作 + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param duration //单位为毫秒 + */ + void swipe(int x1, int y1, int x2, int y2, int duration); + + void swipe(float x1, float y1, float x2, float y2, float duration); + + + /** + * 往输入框输入文字 + * + * @param str + */ + void input(String str); +} diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/InstrumentationInput.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/InstrumentationInput.java new file mode 100644 index 0000000..536a836 --- /dev/null +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/InstrumentationInput.java @@ -0,0 +1,124 @@ +package cn.xjiangwei.RobotHelper.Tools.InputImp; + +import android.app.Instrumentation; +import android.os.SystemClock; +import android.view.MotionEvent; + +import cn.xjiangwei.RobotHelper.Tools.Point; + +import static android.os.SystemClock.sleep; + +public class InstrumentationInput implements Input { + + private static InstrumentationInput instance; + + public static InstrumentationInput getInstance() { + if (instance == null) { + synchronized (InstrumentationInput.class) { + if (instance == null) { + instance = new InstrumentationInput(); + } + } + } + + return instance; + } + + + private static Instrumentation mInst = new Instrumentation(); + + /** + * 点击操作 + * + * @param x + * @param y + */ + @Override + public void tap(final int x, final int y) { + if (x < 0 || y < 0) { + return; + } + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); //x,y 即是事件的坐标 + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); + + } + + /** + * 长按操作，可以自定义按下时间，单位为毫秒 + * + * @param x + * @param y + * @param delay + */ + @Override + public void tap(final int x, final int y, final long delay) { + if (x < 0 || y < 0) { + return; + } + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); //x,y 即是事件的坐标 + sleep(delay); + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); + } + + @Override + public void tap(Point p) { + tap(p.getX(), p.getY()); + } + + @Override + public void tap(Point p, long delay) { + tap(p.getX(), p.getY(), delay); + } + + @Override + public void swipe(int x1, int y1, int x2, int y2, int duration) { + swipe((float) x1, (float) y1, (float) x2, (float) y2, duration); + } + + + /** + * 拖拽操作 + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param duration //单位为毫秒 + */ + @Override + public void swipe(float x1, float y1, float x2, float y2, float duration) { + final int interval = 25; + int steps = (int) (duration / interval + 1); + float dx = (x2 - x1) / steps; + float dy = (y2 - y1) / steps; + down(x1, y1); + for (int step = 0; step < steps; step++) { + sleep(interval); + moveTo(x1 + step * dx, y1 + step * dy, 0); + } + sleep(interval); + up(x2, y2); + } + + @Override + public void input(String str) { + + } + + + private void down(float x, float y) { + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); + } + + + private void up(float x, float y) { + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); + } + + + private void moveTo(float x, float y, int contactId) { + mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_MOVE, x, y, 0)); + } + + +} diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/NullInput.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/NullInput.java new file mode 100644 index 0000000..30196c8 --- /dev/null +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/InputImp/NullInput.java @@ -0,0 +1,64 @@ +package cn.xjiangwei.RobotHelper.Tools.InputImp; + +import cn.xjiangwei.RobotHelper.Tools.MLog; +import cn.xjiangwei.RobotHelper.Tools.Point; +import cn.xjiangwei.RobotHelper.Tools.Toast; + +public class NullInput implements Input { + + private static NullInput instance; + + public static NullInput getInstance() { + if (instance == null) { + synchronized (NullInput.class) { + if (instance == null) { + instance = new NullInput(); + } + } + } + return instance; + } + + + @Override + public void tap(int x, int y) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void tap(int x, int y, long delay) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void tap(Point p) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void tap(Point p, long delay) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void swipe(int x1, int y1, int x2, int y2, int duration) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void swipe(float x1, float y1, float x2, float y2, float duration) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } + + @Override + public void input(String str) { + MLog.error("没有权限执行操作！请检查xposed或者无障碍权限！"); + Toast.show("没有权限执行操作！请检查xposed或者无障碍权限！"); + } +} diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/MLog.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/MLog.java index 33e2758..cd8ac21 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/MLog.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/MLog.java @@ -11,7 +11,7 @@ public class MLog { - private static final boolean debug = true; + private static final boolean debug = false; private static String Tag = "RobotHelper"; diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/OcrApi.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/OcrApi.java deleted file mode 100644 index 32118ef..0000000 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/OcrApi.java +++ /dev/null @@ -1,103 +0,0 @@ -package cn.xjiangwei.RobotHelper.Tools; - -import android.graphics.Bitmap; - -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import cn.xjiangwei.RobotHelper.MainApplication; - - -/** - * 建议使用TessactOcr替代 - */ -@Deprecated -public class OcrApi { - - private static String ocrApiUrl = MainApplication.getServerUrl() + "singleLineOcr"; - private static String multOcrApiUrl = MainApplication.getServerUrl() + "multLineOcr"; - private static String testApiUrl = MainApplication.getServerUrl() + "testImg"; - private static int timeOut = 5000; - - - - public static String singLineOcr(Bitmap img, int leftTopX, int leftTopY, int rightBottomX, int rightBottomY) { - img = Image.cropBitmap(img, leftTopX, leftTopY, rightBottomX, rightBottomY); - String base64 = Image.encodeImage(img); - String res = Post(ocrApiUrl, "img=" + base64); - Pattern p = Pattern.compile("\\s*|\t|\r|\n"); - Matcher m = p.matcher(res); - res = m.replaceAll(""); - return res; - } - - - public static String multLineOcr(Bitmap img, int leftTopX, int leftTopY, int rightBottomX, int rightBottomY) { - img = Image.cropBitmap(img, leftTopX, leftTopY, rightBottomX, rightBottomY); - String base64 = Image.encodeImage(img); - return Post(multOcrApiUrl, "img=" + base64); - } - - private static String Post(String path, String paras)//string POST参数,get 请求的URL地址,context 联系上下文 - { - - String html; - try { - URL url = new URL(path); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setConnectTimeout(timeOut);//超时时间 - conn.setRequestMethod("POST"); - conn.setDoOutput(true); -// conn.setRequestProperty("Content-Type", "application/json"); -// conn.setRequestProperty("User-Agent", Other.getUserAgent(context)); - OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream()); - out.write(paras); - out.flush(); - out.close(); - InputStream inputStream = conn.getInputStream(); - byte[] data = read(inputStream); - html = new String(data, StandardCharsets.UTF_8); - - } catch (Exception e) { - e.printStackTrace(); - System.out.println(e.toString()); - return ""; - } - - return html; - } - - private static byte[] read(InputStream inStream) throws Exception { - ByteArrayOutputStream outStream = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int len = 0; - while ((len = inStream.read(buffer)) != -1) { - outStream.write(buffer, 0, len); - } - inStream.close(); - return outStream.toByteArray(); - } - - - public static void testImage(Bitmap bitmap) { - Thread thread = new Thread() { - @Override - public void run() { - String base64 = Image.encodeImage(bitmap); - Post(testApiUrl, "img=" + base64); - } - }; - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } -} diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/Robot.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/Robot.java index f4956e2..e528ea0 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/Robot.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Tools/Robot.java @@ -1,126 +1,134 @@ package cn.xjiangwei.RobotHelper.Tools; -import android.app.Instrumentation; -import android.os.SystemClock; -import android.view.MotionEvent; +import android.os.Build; -public class Robot { - private static Instrumentation mInst = null; +import cn.xjiangwei.RobotHelper.MainApplication; +import cn.xjiangwei.RobotHelper.Tools.InputImp.AccessibilityInput; +import cn.xjiangwei.RobotHelper.Tools.InputImp.Input; +import cn.xjiangwei.RobotHelper.Tools.InputImp.InstrumentationInput; +import cn.xjiangwei.RobotHelper.Tools.InputImp.NullInput; - public static void tap(final int x, final int y) { - if (x < 0 || y < 0) { - return; - } - - if (Robot.mInst == null) { - mInst = new Instrumentation(); +/** + * 模拟操作的实现类 + *

+ * 目前只使用了xposed提权实现 + *

+ * 未来可能考虑加入root权限的实现 + */ +public class Robot { - } - Thread thread = new Thread() { - @Override - public void run() { - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); //x,y 即是事件的坐标 - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); - } - }; - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } + private static int execType; - } + public static final int ExecTypeXposed = 389; + public static final int ExecTypeAccessibillty = 18; + public static final int ExecTypeNull = 115; + private static Input getInput() { + MainApplication mainApplication = MainApplication.getInstance(); - public static void tap(final int x, final int y, final long delay) { - if (Robot.mInst == null) { - mInst = new Instrumentation(); - } - Thread thread = new Thread() { - @Override - public void run() { - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); //x,y 即是事件的坐标 - try { - sleep(delay); - } catch (InterruptedException e) { - e.printStackTrace(); - } - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); + // 没有手动设置模式的时候 + if (execType == 0) { + // 有xposed权限优先使用xposed提权方式 + if (mainApplication.checkXposedHook()) { + execType = ExecTypeXposed; + return InstrumentationInput.getInstance(); + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && mainApplication.checkAccessibilityService()) { + execType = ExecTypeAccessibillty; + return AccessibilityInput.getInstance(); + } else { + return NullInput.getInstance(); + } + } else { + switch (execType) { + case ExecTypeAccessibillty: + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && mainApplication.checkAccessibilityService()) { + return AccessibilityInput.getInstance(); + } + case ExecTypeXposed: + if (mainApplication.checkXposedHook()) { + return InstrumentationInput.getInstance(); + } + case ExecTypeNull: + return NullInput.getInstance(); + default: + return NullInput.getInstance(); } - }; - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); } } - - public static void tap(Point p) { - tap(p.getX(), p.getY()); + /** + * 设置以什么方式执行模拟操作 + *

+ * 目前支持无障碍和xposed提权操作 + * + * @param execType + */ + public static void setExecType(int execType) { + Robot.execType = execType; } - public static void tap(Point p, long delay) { - tap(p.getX(), p.getY(), delay); + /** + * 点击操作 + * + * @param x + * @param y + */ + public static void tap(final int x, final int y) { + getInput().tap(x, y); } - public static void swipe(float x1, float y1, float x2, float y2, float duration) { - final int interval = 25; - int steps = (int) (duration * 1000 / interval + 1); - float dx = (x2 - x1) / steps; - float dy = (y2 - y1) / steps; - down(x1, y1); - for (int step = 0; step < steps; step++) { - SystemClock.sleep(interval); - moveTo(x1 + step * dx, y1 + step * dy, 0); - } - SystemClock.sleep(interval); - up(x2, y2); + /** + * 长按操作，可以自定义按下时间，单位为毫秒 + * + * @param x + * @param y + * @param delay + */ + public static void tap(final int x, final int y, final long delay) { + getInput().tap(x, y, delay); } - private static void down(float x, float y) { - if (Robot.mInst == null) { - mInst = new Instrumentation(); - } - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_DOWN, x, y, 0)); + public static void tap(Point p) { + getInput().tap(p); } - private static void up(float x, float y) { - if (Robot.mInst == null) { - mInst = new Instrumentation(); - } - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0)); + public static void tap(Point p, long delay) { + getInput().tap(p, delay); } - private static void moveTo(float x, float y, int contactId) { - if (Robot.mInst == null) { - mInst = new Instrumentation(); - } - mInst.sendPointerSync(MotionEvent.obtain(SystemClock.uptimeMillis(), SystemClock.uptimeMillis(), MotionEvent.ACTION_MOVE, x, y, 0)); + /** + * 拖拽操作 + * + * @param x1 + * @param y1 + * @param x2 + * @param y2 + * @param duration //单位为毫秒 + */ + public static void swipe(int x1, int y1, int x2, int y2, int duration) { + getInput().swipe(x1, y1, x2, y2, duration); + } + + public static void swipe(float x1, float y1, float x2, float y2, float duration) { + getInput().swipe(x1, y1, x2, y2, duration); } /** - * TODO 待实现 + * 往输入框输入文字 + * * @param str */ public static void input(String str) { -// if (Robot.mInst == null) { -// mInst = new Instrumentation(); -// } - - + getInput().input(str); } - } diff --git a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Xposed/Hook4XposedCheck.java b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Xposed/Hook4XposedCheck.java index a6f7565..21772ab 100644 --- a/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Xposed/Hook4XposedCheck.java +++ b/Android/app/src/main/java/cn/xjiangwei/RobotHelper/Xposed/Hook4XposedCheck.java @@ -11,7 +11,7 @@ public class Hook4XposedCheck { public static void run(XC_LoadPackage.LoadPackageParam lpparam) { if (lpparam.packageName.equals("cn.xjiangwei.RobotHelper")) { // 检查xp框架加载是否成功 - XposedHelpers.findAndHookMethod("cn.xjiangwei.RobotHelper.MainActivity", lpparam.classLoader, "checkXposedHook", + XposedHelpers.findAndHookMethod("cn.xjiangwei.RobotHelper.MainApplication", lpparam.classLoader, "checkXposedHook", new XC_MethodHook() { @Override protected void afterHookedMethod(MethodHookParam param) throws Throwable { diff --git a/Android/app/src/main/res/xml/accessibility_service_config.xml b/Android/app/src/main/res/xml/accessibility_service_config.xml index 45d149b..7934692 100644 --- a/Android/app/src/main/res/xml/accessibility_service_config.xml +++ b/Android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,6 +1,7 @@ \ No newline at end of file + android:canPerformGestures="true" + android:canRetrieveWindowContent="true" /> \ No newline at end of file diff --git a/Android/app/src/main/res/xml/provider_paths.xml b/Android/app/src/main/res/xml/provider_paths.xml new file mode 100644 index 0000000..ffa74ab --- /dev/null +++ b/Android/app/src/main/res/xml/provider_paths.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/README.md b/README.md index 5c7201c..35f51bd 100644 --- a/README.md +++ b/README.md @@ -11,30 +11,9 @@ [wiki](https://github.com/Jinnrry/RobotHelper/wiki) +该框架主要是方便游戏、爬虫项目的快速开发。比按键精灵等商业软件扩展性强，比AutoJS等更轻量 -## V2.0版本新功能 - -> 这个版本主要更新了Android app爬虫开发的相关支持 - -1.新增了界面抓取功能，可以将当前窗口的全部Dom元素输出成Json数据 - -2.新增了Web Api功能，可以使用Web接口调用相关接口 - -3.新增拖动操作 - -[使用指南](https://github.com/Jinnrry/RobotHelper/wiki/%E5%9F%BA%E4%BA%8EWEB%E6%8E%A5%E5%8F%A3%E7%9A%84%E5%BC%80%E5%8F%91%E6%8C%87%E5%8D%97) - - - - -## V1.0版本简介 - -1.新增了图片模板匹配（相当于按键精灵找图功能的升级版），自动处理图像分辨率问题，一次抓图，多分辨率终端可用 - -2.内置了Tessact-Ocr，不再依赖服务端做文字识别，支持设置黑白名单 - -3.内置opencv，各种图像处理，轨迹追踪，再也不是难题 ## Demo @@ -46,18 +25,32 @@ Point point = Image.findPointByMulColor(ScreenCaptureUtil.getScreenCap(), "434FD Robot.tap(point); ``` -以上代码将会在屏幕中查找![chrome](./docs/chrome.png)图标，然后点击这个图标（你直接运行这段代码可能不会成功，因为你的手机屏幕尺寸跟我不一样） +以上代码将会在屏幕中查找 图标，然后点击这个图标（你直接运行这段代码可能不会成功，因为你的手机屏幕尺寸跟我不一样） ``` -String s = OcrApi.multLineOcr(ScreenCaptureUtil.getScreenCap(), 0, 0, 200, 30); -MLog.info(s); +String s = TessactOcr.img2string(ScreenCaptureUtil.getScreenCap(0,0,200,30), "chi_sim", "", ""); +MLog.info("文字识别结果：" + s); ``` 以上代码将输出(0,0)到(200,30)这个矩形区域内的文字。 +`chi_sim`表示语言为简体中文，默认语言包只有chi_sim和eng(英语) + +你可以自己引入TessactOcr所支持的任意语言。[语言包下载](https://github.com/tesseract-ocr/tessdata_best) + + +### [更新日志](./UPDATE.md) + ## 未来规划 -- [ ] 1.Hook系统相关api，修改系统相关常量，使游戏可以在模拟器中运行，并且让游戏无法识别模拟器 -- [ ] 2.没了，如有建议可以提issues或者pr \ No newline at end of file +- [ ] 1.添加Root权限实现底层点击操作 + +## Thanks + +Thanks for all these great works that make this project better. + +- [Airtest](https://github.com/AirtestProject/Airtest) +- [Tesseract](https://github.com/tesseract-ocr/tesseract) +- [AutoJS](https://github.com/hyb1996/Auto.js) \ No newline at end of file diff --git a/UPDATE.md b/UPDATE.md new file mode 100644 index 0000000..e9d4878 --- /dev/null +++ b/UPDATE.md @@ -0,0 +1,27 @@ +## V2.1版本新功能 + +1.底层点击实现新增了无障碍接口实现,你可以使用`Robot.setExecType(Robot.ExecTypeXposed);`或者`Robot.setExecType(Robot.ExecTypeAccessibillty);`切换实现方式 + +2.代码整理（But 整理后依然很乱） + + +## V2.0版本新功能 + +> 这个版本主要更新了Android app爬虫开发的相关支持 + +1.新增了界面抓取功能，可以将当前窗口的全部Dom元素输出成Json数据 + +2.新增了Web Api功能，可以使用Web接口调用相关接口 + +3.新增拖动操作 + +[使用指南](https://github.com/Jinnrry/RobotHelper/wiki/%E5%9F%BA%E4%BA%8EWEB%E6%8E%A5%E5%8F%A3%E7%9A%84%E5%BC%80%E5%8F%91%E6%8C%87%E5%8D%97) + + +## V1.0版本 + +1.新增了图片模板匹配（相当于按键精灵找图功能的升级版），自动处理图像分辨率问题，一次抓图，多分辨率终端可用 + +2.内置了Tessact-Ocr，不再依赖服务端做文字识别，支持设置黑白名单 + +3.内置opencv，各种图像处理，轨迹追踪，再也不是难题