From d0ef52e4187dd20f178707c5977d16086f7d8215 Mon Sep 17 00:00:00 2001 From: fufesou <13586388+fufesou@users.noreply.github.com> Date: Thu, 7 Nov 2024 21:23:41 +0800 Subject: [PATCH] fix: touch input, ensure message orders (#9855) Signed-off-by: fufesou --- flutter/lib/common/widgets/remote_input.dart | 93 +++++++++++--------- flutter/lib/models/input_model.dart | 26 +++--- flutter/lib/models/model.dart | 14 +-- 3 files changed, 70 insertions(+), 63 deletions(-) diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index c6fdaf75b84f..fa635222d47b 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -106,7 +106,7 @@ class _RawTouchGestureDetectorRegionState ); } - onTapDown(TapDownDetails d) { + onTapDown(TapDownDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -114,45 +114,49 @@ class _RawTouchGestureDetectorRegionState if (handleTouch) { _lastPosOfDoubleTapDown = d.localPosition; // Desktop or mobile "Touch mode" - if (ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy)) { - inputModel.tapDown(MouseButtons.left); + final isMoved = + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + if (isMoved) { + await inputModel.tapDown(MouseButtons.left); } } } - onTapUp(TapUpDetails d) { + onTapUp(TapUpDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (handleTouch) { - if (ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy)) { + final isMoved = + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + if (isMoved) { inputModel.tapUp(MouseButtons.left); } } } - onTap() { + onTap() async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (!handleTouch) { // Mobile, "Mouse mode" - inputModel.tap(MouseButtons.left); + await inputModel.tap(MouseButtons.left); } } - onDoubleTapDown(TapDownDetails d) { + onDoubleTapDown(TapDownDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (handleTouch) { _lastPosOfDoubleTapDown = d.localPosition; - ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); } } - onDoubleTap() { + onDoubleTap() async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } @@ -163,11 +167,11 @@ class _RawTouchGestureDetectorRegionState !ffi.cursorModel.isInRemoteRect(_lastPosOfDoubleTapDown)) { return; } - inputModel.tap(MouseButtons.left); - inputModel.tap(MouseButtons.left); + await inputModel.tap(MouseButtons.left); + await inputModel.tap(MouseButtons.left); } - onLongPressDown(LongPressDownDetails d) { + onLongPressDown(LongPressDownDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -175,39 +179,42 @@ class _RawTouchGestureDetectorRegionState if (handleTouch) { _lastPosOfDoubleTapDown = d.localPosition; _cacheLongPressPosition = d.localPosition; - if (!ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy)) { + final isMoved = + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + if (!isMoved) { return; } _cacheLongPressPositionTs = DateTime.now().millisecondsSinceEpoch; } } - onLongPressUp() { + onLongPressUp() async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (handleTouch) { - inputModel.tapUp(MouseButtons.left); + await inputModel.tapUp(MouseButtons.left); } } // for mobiles - onLongPress() { + onLongPress() async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (handleTouch) { - if (!ffi.cursorModel - .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy)) { + final isMoved = await ffi.cursorModel + .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); + if (!isMoved) { return; } } if (!ffi.ffiModel.isPeerMobile) { - inputModel.tap(MouseButtons.right); + await inputModel.tap(MouseButtons.right); } } - onDoubleFinerTapDown(TapDownDetails d) { + onDoubleFinerTapDown(TapDownDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -216,7 +223,7 @@ class _RawTouchGestureDetectorRegionState // ignore for desktop and mobile } - onDoubleFinerTap(TapDownDetails d) { + onDoubleFinerTap(TapDownDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -228,39 +235,39 @@ class _RawTouchGestureDetectorRegionState final isDesktopInRemoteRect = (isDesktop || isWebDesktop) && ffi.cursorModel.isInRemoteRect(_doubleFinerTapPosition); if (isMobileMouseMode || isDesktopInRemoteRect) { - inputModel.tap(MouseButtons.right); + await inputModel.tap(MouseButtons.right); } } - onHoldDragStart(DragStartDetails d) { + onHoldDragStart(DragStartDetails d) async { lastDeviceKind = d.kind; if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (!handleTouch) { - inputModel.sendMouse('down', MouseButtons.left); + await inputModel.sendMouse('down', MouseButtons.left); } } - onHoldDragUpdate(DragUpdateDetails d) { + onHoldDragUpdate(DragUpdateDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (!handleTouch) { - ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch); + await ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch); } } - onHoldDragEnd(DragEndDetails d) { + onHoldDragEnd(DragEndDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if (!handleTouch) { - inputModel.sendMouse('up', MouseButtons.left); + await inputModel.sendMouse('up', MouseButtons.left); } } - onOneFingerPanStart(BuildContext context, DragStartDetails d) { + onOneFingerPanStart(BuildContext context, DragStartDetails d) async { lastDeviceKind = d.kind ?? lastDeviceKind; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -285,11 +292,11 @@ class _RawTouchGestureDetectorRegionState // TODO: We should find a better way to send the first pan event as soon as possible. if (DateTime.now().millisecondsSinceEpoch - _cacheLongPressPositionTs < 500) { - ffi.cursorModel + await ffi.cursorModel .move(_cacheLongPressPosition.dx, _cacheLongPressPosition.dy); } - inputModel.sendMouse('down', MouseButtons.left); - ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); + await inputModel.sendMouse('down', MouseButtons.left); + await ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy); } else { final offset = ffi.cursorModel.offset; final cursorX = offset.dx; @@ -298,12 +305,12 @@ class _RawTouchGestureDetectorRegionState ffi.cursorModel.getVisibleRect().inflate(1); // extend edges final size = MediaQueryData.fromView(View.of(context)).size; if (!visible.contains(Offset(cursorX, cursorY))) { - ffi.cursorModel.move(size.width / 2, size.height / 2); + await ffi.cursorModel.move(size.width / 2, size.height / 2); } } } - onOneFingerPanUpdate(DragUpdateDetails d) { + onOneFingerPanUpdate(DragUpdateDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } @@ -313,10 +320,10 @@ class _RawTouchGestureDetectorRegionState if (handleTouch && !_touchModePanStarted) { return; } - ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch); + await ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch); } - onOneFingerPanEnd(DragEndDetails d) { + onOneFingerPanEnd(DragEndDetails d) async { _touchModePanStarted = false; if (lastDeviceKind != PointerDeviceKind.touch) { return; @@ -324,7 +331,7 @@ class _RawTouchGestureDetectorRegionState if (isDesktop || isWebDesktop) { ffi.cursorModel.clearRemoteWindowCoords(); } - inputModel.sendMouse('up', MouseButtons.left); + await inputModel.sendMouse('up', MouseButtons.left); } // scale + pan event @@ -334,7 +341,7 @@ class _RawTouchGestureDetectorRegionState } } - onTwoFingerScaleUpdate(ScaleUpdateDetails d) { + onTwoFingerScaleUpdate(ScaleUpdateDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } @@ -343,7 +350,7 @@ class _RawTouchGestureDetectorRegionState _scale = d.scale; if (scale != 0) { - bind.sessionSendPointer( + await bind.sessionSendPointer( sessionId: sessionId, msg: json.encode( PointerEventToRust(kPointerEventKindTouch, 'scale', scale) @@ -358,12 +365,12 @@ class _RawTouchGestureDetectorRegionState } } - onTwoFingerScaleEnd(ScaleEndDetails d) { + onTwoFingerScaleEnd(ScaleEndDetails d) async { if (lastDeviceKind != PointerDeviceKind.touch) { return; } if ((isDesktop || isWebDesktop)) { - bind.sessionSendPointer( + await bind.sessionSendPointer( sessionId: sessionId, msg: json.encode( PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson())); @@ -373,7 +380,7 @@ class _RawTouchGestureDetectorRegionState // No idea why we need to set the view style to "" here. // bind.sessionSetViewStyle(sessionId: sessionId, value: ""); } - inputModel.sendMouse('up', MouseButtons.left); + await inputModel.sendMouse('up', MouseButtons.left); } get onHoldDragCancel => null; diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 0cc0537d67bb..0692ba2df4d4 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -768,22 +768,22 @@ class InputModel { } /// Send a mouse tap event(down and up). - void tap(MouseButtons button) { - sendMouse('down', button); - sendMouse('up', button); + Future tap(MouseButtons button) async { + await sendMouse('down', button); + await sendMouse('up', button); } - void tapDown(MouseButtons button) { - sendMouse('down', button); + Future tapDown(MouseButtons button) async { + await sendMouse('down', button); } - void tapUp(MouseButtons button) { - sendMouse('up', button); + Future tapUp(MouseButtons button) async { + await sendMouse('up', button); } /// Send scroll event with scroll distance [y]. - void scroll(int y) { - bind.sessionSendMouse( + Future scroll(int y) async { + await bind.sessionSendMouse( sessionId: sessionId, msg: json .encode(modify({'id': id, 'type': 'wheel', 'y': y.toString()}))); @@ -804,9 +804,9 @@ class InputModel { } /// Send mouse press event. - void sendMouse(String type, MouseButtons button) { + Future sendMouse(String type, MouseButtons button) async { if (!keyboardPerm) return; - bind.sessionSendMouse( + await bind.sessionSendMouse( sessionId: sessionId, msg: json.encode(modify({'type': type, 'buttons': button.value}))); } @@ -830,11 +830,11 @@ class InputModel { } /// Send mouse movement event with distance in [x] and [y]. - void moveMouse(double x, double y) { + Future moveMouse(double x, double y) async { if (!keyboardPerm) return; var x2 = x.toInt(); var y2 = y.toInt(); - bind.sessionSendMouse( + await bind.sessionSendMouse( sessionId: sessionId, msg: json.encode(modify({'x': '$x2', 'y': '$y2'}))); } diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index f68801f583e4..d3c76457a8a9 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -2038,7 +2038,7 @@ class CursorModel with ChangeNotifier { } // For touch mode - move(double x, double y) { + Future move(double x, double y) async { if (shouldBlock(x, y)) { _lastIsBlocked = true; return false; @@ -2047,7 +2047,7 @@ class CursorModel with ChangeNotifier { if (!_moveLocalIfInRemoteRect(x, y)) { return false; } - parent.target?.inputModel.moveMouse(_x, _y); + await parent.target?.inputModel.moveMouse(_x, _y); return true; } @@ -2105,9 +2105,9 @@ class CursorModel with ChangeNotifier { notifyListeners(); } - updatePan(Offset delta, Offset localPosition, bool touchMode) { + updatePan(Offset delta, Offset localPosition, bool touchMode) async { if (touchMode) { - _handleTouchMode(delta, localPosition); + await _handleTouchMode(delta, localPosition); return; } double dx = delta.dx; @@ -2205,7 +2205,7 @@ class CursorModel with ChangeNotifier { return x >= 0 && y >= 0 && x <= w && y <= h; } - _handleTouchMode(Offset delta, Offset localPosition) { + _handleTouchMode(Offset delta, Offset localPosition) async { bool isMoved = false; if (_remoteWindowCoords.isNotEmpty && _windowRect != null && @@ -2221,7 +2221,7 @@ class CursorModel with ChangeNotifier { coords.canvas.scale; x2 += coords.cursor.offset.dx; y2 += coords.cursor.offset.dy; - parent.target?.inputModel.moveMouse(x2, y2); + await parent.target?.inputModel.moveMouse(x2, y2); isMoved = true; } } @@ -2264,7 +2264,7 @@ class CursorModel with ChangeNotifier { _x = movement.dx; _y = movement.dy; - parent.target?.inputModel.moveMouse(_x, _y); + await parent.target?.inputModel.moveMouse(_x, _y); } notifyListeners(); }