fix: ios timestamp mismatch

dogu-team · Nov 16, 2023 · 34c1e5c · 34c1e5c
1 parent 75ab5be
commit 34c1e5c
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 11 deletions.
diff --git a/projects/console-web-front/src/hooks/streaming/useDeviceInput.ts b/projects/console-web-front/src/hooks/streaming/useDeviceInput.ts
@@ -76,7 +76,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
         },
         buttons: mapMouseEventButtonToDeviceControlButtons(event.button),
         repeat: isDoubleClicked ? 2 : 0,
-        timeStamp: Date.now(),
       };
 
       try {
@@ -124,7 +123,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
           screenWidth: Math.trunc(event.currentTarget.clientWidth),
           screenHeight: Math.trunc(event.currentTarget.clientHeight),
         },
-        timeStamp: Date.now(),
       };
 
       try {
@@ -159,7 +157,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
           text: await navigator.clipboard.readText(),
           keycode: mapWebKeyboardToDeviceKeyboard(event),
           key: event.key,
-          timeStamp: Date.now(),
         };
 
         try {
@@ -183,7 +180,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
         type: DeviceControlType.DEVICE_CONTROL_TYPE_AOS_INJECT_KEYCODE,
         keycode: mapWebKeyboardToDeviceKeyboard(event),
         key: event.key,
-        timeStamp: Date.now(),
       };
 
       const result = await deviceRTCCaller.call('cfGdcDaControlParam', 'cfGdcDaControlResult', {
@@ -219,7 +215,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
           action: DeviceControlAction.DEVICE_CONTROL_ACTION_AOS_KEYEVENT_ACTION_DOWN_UNSPECIFIED,
           type: DeviceControlType.DEVICE_CONTROL_TYPE_AOS_INJECT_KEYCODE,
           keycode: mapToolbarMenuToDeviceKeyboard(menu),
-          timeStamp: Date.now(),
         });
       }
       if (upMenus.includes(menu)) {
@@ -228,15 +223,13 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
           action: DeviceControlAction.DEVICE_CONTROL_ACTION_AOS_KEYEVENT_ACTION_UP,
           type: DeviceControlType.DEVICE_CONTROL_TYPE_AOS_INJECT_KEYCODE,
           keycode: mapToolbarMenuToDeviceKeyboard(menu),
-          timeStamp: Date.now(),
         });
       } else {
         controls.push({
           ...input.DefaultDeviceControl(),
           action: DeviceControlAction.DEVICE_CONTROL_ACTION_AOS_KEYEVENT_ACTION_DOWN_UNSPECIFIED,
           type: DeviceControlType.DEVICE_CONTROL_TYPE_AOS_INJECT_KEYCODE,
           keycode: mapToolbarMenuToDeviceKeyboard(menu),
-          timeStamp: Date.now(),
         });
       }
 
@@ -287,7 +280,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
         action,
         type: DeviceControlType.DEVICE_CONTROL_TYPE_AOS_INJECT_KEYCODE,
         keycode,
-        timeStamp: Date.now(),
       };
 
       try {
@@ -471,7 +463,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
         const c: DeviceControl = {
           ...input.DefaultDeviceControl(),
           type: DeviceControlType.DEVICE_CONTROL_TYPE_DESKTOP_ONSCREEN_FOCUSED,
-          timeStamp: Date.now(),
         };
 
         const result = await deviceRTCCaller.call('cfGdcDaControlParam', 'cfGdcDaControlResult', {
@@ -504,7 +495,6 @@ const useDeviceInput = (deviceRTCCaller: DeviceRTCCaller | undefined, platform:
         const c: DeviceControl = {
           ...input.DefaultDeviceControl(),
           type: DeviceControlType.DEVICE_CONTROL_TYPE_DESKTOP_ONSCREEN_UNFOCUSED,
-          timeStamp: Date.now(),
         };
 
         const result = await deviceRTCCaller.call('cfGdcDaControlParam', 'cfGdcDaControlResult', {

diff --git a/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/ControlSessionListener.swift b/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/ControlSessionListener.swift
@@ -54,6 +54,25 @@ actor ControlSessionListener: IControlSessionListener {
       break
     case .dcGdcDaControlParam(let param):
       let controlResult = ControlResult(seq: abstractParam.seq, session: session)
+      let control = Inner_Types_DeviceControl.with {
+        $0.type = param.control.type
+        $0.text = param.control.text
+        $0.metaState = param.control.metaState
+        $0.action = param.control.action
+        $0.keycode = param.control.keycode
+        $0.buttons = param.control.buttons
+        $0.pointerID = param.control.pointerID
+        $0.pressure = param.control.pressure
+        $0.position = param.control.position
+        $0.hScroll = param.control.hScroll
+        $0.vScroll = param.control.vScroll
+        $0.copyKey = param.control.copyKey
+        $0.paste = param.control.paste
+        $0.repeat = param.control.repeat
+        $0.sequence = param.control.sequence
+        $0.key = param.control.key
+        $0.timeStamp = Date().unixTimeMilliseconds
+      }
       Task.catchable(
         {
           try await self.controlProcessor.push(control: param.control, result: controlResult)

diff --git a/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/ScrollControlPlayer.swift b/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/ScrollControlPlayer.swift
@@ -57,10 +57,12 @@ actor ScrollControlPlayer: IControlPlayer {
 
     let now = Date().unixTimeMilliseconds
     if control.control.timeStamp > now {
+      self.notifyTimeMismatch(controlResult: control.result)
       return
     }
     let latencyMs = now - control.control.timeStamp
-    if 900 < latencyMs {
+    if 600 < latencyMs {
+      self.notifyTimeMismatch(controlResult: control.result)
       return
     }
 
@@ -101,4 +103,12 @@ actor ScrollControlPlayer: IControlPlayer {
     lastPlayTime = control.control.timeStamp + deltaTime
     control.result.set(result: result)
   }
+
+  private func notifyTimeMismatch(controlResult: ControlResult) {
+    var result = Inner_Types_CfGdcDaControlResult()
+    result.error = Outer_ErrorResult.with {
+      $0.message = "Time mismatch."
+    }
+    controlResult.set(result: result)
+  }
 }
diff --git a/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/TouchControlPlayer.swift b/projects/ios-device-agent/IOSDeviceAgent/IOSDeviceAgentLib/TouchControlPlayer.swift
@@ -73,14 +73,17 @@ actor TouchControlPlayer: IControlPlayer {
     let endPosition = try Transform.controlSpaceToScreenSpace(controlSpacePosition: up.control.position, screenSize: screenSize)
     let now = Date().unixTimeMilliseconds
     if down.control.timeStamp > now {
+      self.notifyTimeMismatch(downUp: downUp)
       return
     }
     let latencyMs = now - down.control.timeStamp
     if 3000 < latencyMs {
+      self.notifyTimeMismatch(downUp: downUp)
       return
     }
 
     if down.control.timeStamp > up.control.timeStamp {
+      self.notifyTimeMismatch(downUp: downUp)
       return
     }
     var duration = up.control.timeStamp - down.control.timeStamp
@@ -134,4 +137,13 @@ actor TouchControlPlayer: IControlPlayer {
     downUp.down.result.set(result: result)
     downUp.up.result.set(result: result)
   }
+
+  private func notifyTimeMismatch(downUp: DownUp) {
+    var result = Inner_Types_CfGdcDaControlResult()
+    result.error = Outer_ErrorResult.with {
+      $0.message = "Time mismatch."
+    }
+    downUp.down.result.set(result: result)
+    downUp.up.result.set(result: result)
+  }
 }