Skip to content

Commit

Permalink
Merge pull request rustdesk#6097 from mcfans/master
Browse files Browse the repository at this point in the history
Physical keyboard to android support
rustdesk authored Nov 7, 2023
2 parents 82b7650 + 32a29a5 commit 5577731
Showing 17 changed files with 641 additions and 34 deletions.
28 changes: 26 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ default-net = "0.14"
wol-rs = "1.0"
flutter_rust_bridge = { version = "=1.80", features = ["uuid"], optional = true}
errno = "0.3"
rdev = { git = "https://github.com/fufesou/rdev" }
rdev = { git = "https://github.com/fufesou/rdev", branch = "master" }
url = { version = "2.3", features = ["serde"] }
crossbeam-queue = "0.3"
hex = "0.4"
29 changes: 29 additions & 0 deletions flutter/android/app/build.gradle
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import com.google.protobuf.gradle.*
plugins {
id "com.google.protobuf" version "0.9.4"
}

def keystoreProperties = new Properties()
def keystorePropertiesFile = rootProject.file('key.properties')
if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"

dependencies {
implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
}

protobuf {
protoc {
artifact = 'com.google.protobuf:protoc:3.20.1'
}

generateProtoTasks {
all().configureEach { task ->
task.builtins {
java {
option "lite"
}
}
}
}
}

android {
compileSdkVersion 33
sourceSets {
main.java.srcDirs += 'src/main/kotlin'

main.proto.srcDirs += '../../../libs/hbb_common/protos'
main.proto.includes += "message.proto"
}

compileOptions {
@@ -65,6 +93,7 @@ android {
// TODO: Add your own signing config for the release build.
// Signing with the debug keys for now, so `flutter run --release` works.
signingConfig signingConfigs.release
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules'
}
}
}
4 changes: 4 additions & 0 deletions flutter/android/app/proguard-rules
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Keep class members from protobuf generated code.
-keepclassmembers class * extends com.google.protobuf.GeneratedMessageLite {
<fields>;
}
Original file line number Diff line number Diff line change
@@ -10,12 +10,27 @@ import android.accessibilityservice.AccessibilityService
import android.accessibilityservice.GestureDescription
import android.graphics.Path
import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.widget.EditText
import android.view.accessibility.AccessibilityEvent
import android.view.ViewGroup.LayoutParams
import android.view.accessibility.AccessibilityNodeInfo
import android.graphics.Rect
import android.accessibilityservice.AccessibilityServiceInfo
import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR
import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
import android.view.inputmethod.EditorInfo
import androidx.annotation.RequiresApi
import java.util.*
import java.lang.Character
import kotlin.math.abs
import kotlin.math.max
import hbb.MessageOuterClass.KeyEvent
import hbb.MessageOuterClass.KeyboardMode
import hbb.KeyEventConverter

const val LIFT_DOWN = 9
const val LIFT_MOVE = 8
@@ -58,6 +73,8 @@ class InputService : AccessibilityService() {
private var isWheelActionsPolling = false
private var isWaitingLongPress = false

private var fakeEditTextForTextStateCalculation: EditText? = null

@RequiresApi(Build.VERSION_CODES.N)
fun onMouseInput(mask: Int, _x: Int, _y: Int) {
val x = max(0, _x)
@@ -252,9 +269,296 @@ class InputService : AccessibilityService() {
}
}

@RequiresApi(Build.VERSION_CODES.N)
fun onKeyEvent(data: ByteArray) {
val keyEvent = KeyEvent.parseFrom(data)
val keyboardMode = keyEvent.getMode()

var textToCommit: String? = null

if (keyboardMode == KeyboardMode.Legacy) {
if (keyEvent.hasChr() && keyEvent.getDown()) {
val chr = keyEvent.getChr()
if (chr != null) {
textToCommit = String(Character.toChars(chr))
}
}
} else if (keyboardMode == KeyboardMode.Translate) {
if (keyEvent.hasSeq() && keyEvent.getDown()) {
val seq = keyEvent.getSeq()
if (seq != null) {
textToCommit = seq
}
}
}

Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit")

if (Build.VERSION.SDK_INT >= 33) {
getInputMethod()?.let { inputMethod ->
inputMethod.getCurrentInputConnection()?.let { inputConnection ->
if (textToCommit != null) {
textToCommit?.let { text ->
inputConnection.commitText(text, 1, null)
}
} else {
KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
inputConnection.sendKeyEvent(event)
}
}
}
}
} else {
val handler = Handler(Looper.getMainLooper())
handler.post {
KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event ->
val possibleNodes = possibleAccessibiltyNodes()
Log.d(logTag, "possibleNodes:$possibleNodes")
for (item in possibleNodes) {
val success = trySendKeyEvent(event, item, textToCommit)
if (success) {
break
}
}
}
}
}
}

private fun insertAccessibilityNode(list: LinkedList<AccessibilityNodeInfo>, node: AccessibilityNodeInfo) {
if (node == null) {
return
}
if (list.contains(node)) {
return
}
list.add(node)
}

private fun findChildNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? {
if (node == null) {
return null
}
if (node.isEditable() && node.isFocusable()) {
return node
}
val childCount = node.getChildCount()
for (i in 0 until childCount) {
val child = node.getChild(i)
if (child != null) {
if (child.isEditable() && child.isFocusable()) {
return child
}
if (Build.VERSION.SDK_INT < 33) {
child.recycle()
}
}
}
for (i in 0 until childCount) {
val child = node.getChild(i)
if (child != null) {
val result = findChildNode(child)
if (Build.VERSION.SDK_INT < 33) {
if (child != result) {
child.recycle()
}
}
if (result != null) {
return result
}
}
}
return null
}

private fun possibleAccessibiltyNodes(): LinkedList<AccessibilityNodeInfo> {
val linkedList = LinkedList<AccessibilityNodeInfo>()
val latestList = LinkedList<AccessibilityNodeInfo>()

val focusInput = findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
var focusAccessibilityInput = findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)

val rootInActiveWindow = getRootInActiveWindow()

Log.d(logTag, "focusInput:$focusInput focusAccessibilityInput:$focusAccessibilityInput rootInActiveWindow:$rootInActiveWindow")

if (focusInput != null) {
if (focusInput.isFocusable() && focusInput.isEditable()) {
insertAccessibilityNode(linkedList, focusInput)
} else {
insertAccessibilityNode(latestList, focusInput)
}
}

if (focusAccessibilityInput != null) {
if (focusAccessibilityInput.isFocusable() && focusAccessibilityInput.isEditable()) {
insertAccessibilityNode(linkedList, focusAccessibilityInput)
} else {
insertAccessibilityNode(latestList, focusAccessibilityInput)
}
}

val childFromFocusInput = findChildNode(focusInput)
Log.d(logTag, "childFromFocusInput:$childFromFocusInput")

if (childFromFocusInput != null) {
insertAccessibilityNode(linkedList, childFromFocusInput)
}

val childFromFocusAccessibilityInput = findChildNode(focusAccessibilityInput)
if (childFromFocusAccessibilityInput != null) {
insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput)
}
Log.d(logTag, "childFromFocusAccessibilityInput:$childFromFocusAccessibilityInput")

if (rootInActiveWindow != null) {
insertAccessibilityNode(linkedList, rootInActiveWindow)
}

for (item in latestList) {
insertAccessibilityNode(linkedList, item)
}

return linkedList
}

private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean {
node.refresh()
this.fakeEditTextForTextStateCalculation?.setSelection(0,0)
this.fakeEditTextForTextStateCalculation?.setText(null)

val text = node.getText()
var isShowingHint = false
if (Build.VERSION.SDK_INT >= 26) {
isShowingHint = node.isShowingHintText()
}

var textSelectionStart = node.textSelectionStart
var textSelectionEnd = node.textSelectionEnd

if (text != null) {
if (textSelectionStart > text.length) {
textSelectionStart = text.length
}
if (textSelectionEnd > text.length) {
textSelectionEnd = text.length
}
if (textSelectionStart > textSelectionEnd) {
textSelectionStart = textSelectionEnd
}
}

var success = false

Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd")

if (textToCommit != null) {
if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
val newText = textToCommit
this.fakeEditTextForTextStateCalculation?.setText(newText)
success = updateTextForAccessibilityNode(node)
} else if (text != null) {
this.fakeEditTextForTextStateCalculation?.setText(text)
this.fakeEditTextForTextStateCalculation?.setSelection(
textSelectionStart,
textSelectionEnd
)
this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit)
success = updateTextAndSelectionForAccessibiltyNode(node)
}
} else {
if (isShowingHint) {
this.fakeEditTextForTextStateCalculation?.setText(null)
} else {
this.fakeEditTextForTextStateCalculation?.setText(text)
}
if (textSelectionStart != -1 && textSelectionEnd != -1) {
Log.d(logTag, "setting selection $textSelectionStart $textSelectionEnd")
this.fakeEditTextForTextStateCalculation?.setSelection(
textSelectionStart,
textSelectionEnd
)
}

this.fakeEditTextForTextStateCalculation?.let {
// This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created.
val rect = Rect()
node.getBoundsInScreen(rect)

it.layout(rect.left, rect.top, rect.right, rect.bottom)
it.onPreDraw()
if (event.action == android.view.KeyEvent.ACTION_DOWN) {
val succ = it.onKeyDown(event.getKeyCode(), event)
Log.d(logTag, "onKeyDown $succ")
} else if (event.action == android.view.KeyEvent.ACTION_UP) {
val success = it.onKeyUp(event.getKeyCode(), event)
Log.d(logTag, "keyup $success")
} else {}
}

success = updateTextAndSelectionForAccessibiltyNode(node)
}
return success
}

fun updateTextForAccessibilityNode(node: AccessibilityNodeInfo): Boolean {
var success = false
this.fakeEditTextForTextStateCalculation?.text?.let {
val arguments = Bundle()
arguments.putCharSequence(
AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
it.toString()
)
success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
}
return success
}

fun updateTextAndSelectionForAccessibiltyNode(node: AccessibilityNodeInfo): Boolean {
var success = updateTextForAccessibilityNode(node)

if (success) {
val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart
val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd

if (selectionStart != null && selectionEnd != null) {
val arguments = Bundle()
arguments.putInt(
AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT,
selectionStart
)
arguments.putInt(
AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT,
selectionEnd
)
success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments)
Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success")
}
}

return success
}


override fun onAccessibilityEvent(event: AccessibilityEvent) {
}

override fun onServiceConnected() {
super.onServiceConnected()
ctx = this
val info = AccessibilityServiceInfo()
if (Build.VERSION.SDK_INT >= 33) {
info.flags = FLAG_INPUT_METHOD_EDITOR or FLAG_RETRIEVE_INTERACTIVE_WINDOWS
} else {
info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS
}
setServiceInfo(info)
fakeEditTextForTextStateCalculation = EditText(this)
// Size here doesn't matter, we won't show this view.
fakeEditTextForTextStateCalculation?.layoutParams = LayoutParams(100, 100)
fakeEditTextForTextStateCalculation?.onPreDraw()
val layout = fakeEditTextForTextStateCalculation?.getLayout()
Log.d(logTag, "fakeEditTextForTextStateCalculation layout:$layout")
Log.d(logTag, "onServiceConnected!")
}

@@ -263,7 +567,5 @@ class InputService : AccessibilityService() {
super.onDestroy()
}

override fun onAccessibilityEvent(event: AccessibilityEvent?) {}

override fun onInterrupt() {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package hbb;
import android.view.KeyEvent
import android.view.KeyCharacterMap
import hbb.MessageOuterClass.KeyboardMode
import hbb.MessageOuterClass.ControlKey

object KeyEventConverter {
fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
var chrValue = 0
var modifiers = 0

val keyboardMode = keyEventProto.getMode()

if (keyEventProto.hasChr()) {
if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) {
chrValue = keyEventProto.getChr()
} else {
chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
}
} else if (keyEventProto.hasControlKey()) {
chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
}

var modifiersList = keyEventProto.getModifiersList()

if (modifiersList != null) {
for (modifier in keyEventProto.getModifiersList()) {
val modifierValue = convertModifier(modifier)
modifiers = modifiers or modifierValue
}
}

var action = 0
if (keyEventProto.getDown()) {
action = KeyEvent.ACTION_DOWN
} else {
action = KeyEvent.ACTION_UP
}

return KeyEvent(0, 0, action, chrValue, 0, modifiers)
}

private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
// Add logic to map ControlKey values to Android KeyEvent key codes.
// You'll need to provide the mapping for each key.
return when (controlKey) {
ControlKey.Alt -> KeyEvent.META_ALT_ON
ControlKey.Control -> KeyEvent.META_CTRL_ON
ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
ControlKey.Meta -> KeyEvent.META_META_ON
ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
ControlKey.Shift -> KeyEvent.META_SHIFT_ON
ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
else -> 0 // Default to unknown.
}
}

private val tag = "KeyEventConverter"

private fun convertUnicodeToKeyCode(unicode: Int): Int {
val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
android.util.Log.d(tag, "unicode: $unicode")
val events = charMap.getEvents(charArrayOf(unicode.toChar()))
if (events != null && events.size > 0) {
android.util.Log.d(tag, "keycode ${events[0].keyCode}")
return events[0].keyCode
}
return 0
}

private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
// Add logic to map ControlKey values to Android KeyEvent key codes.
// You'll need to provide the mapping for each key.
return when (controlKey) {
ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
ControlKey.PageUp -> KeyEvent.KEYCODE_PAGE_UP
ControlKey.PageDown -> KeyEvent.KEYCODE_PAGE_DOWN
ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
ControlKey.F1 -> KeyEvent.KEYCODE_F1
ControlKey.F2 -> KeyEvent.KEYCODE_F2
ControlKey.F3 -> KeyEvent.KEYCODE_F3
ControlKey.F4 -> KeyEvent.KEYCODE_F4
ControlKey.F5 -> KeyEvent.KEYCODE_F5
ControlKey.F6 -> KeyEvent.KEYCODE_F6
ControlKey.F7 -> KeyEvent.KEYCODE_F7
ControlKey.F8 -> KeyEvent.KEYCODE_F8
ControlKey.F9 -> KeyEvent.KEYCODE_F9
ControlKey.F10 -> KeyEvent.KEYCODE_F10
ControlKey.F11 -> KeyEvent.KEYCODE_F11
ControlKey.F12 -> KeyEvent.KEYCODE_F12
ControlKey.Space -> KeyEvent.KEYCODE_SPACE
ControlKey.Tab -> KeyEvent.KEYCODE_TAB
ControlKey.Return -> KeyEvent.KEYCODE_ENTER
ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
else -> 0 // Default to unknown.
}
}
}
Original file line number Diff line number Diff line change
@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
import kotlin.math.max
import kotlin.math.min


const val DEFAULT_NOTIFY_TITLE = "RustDesk"
const val DEFAULT_NOTIFY_TEXT = "Service is running"
const val DEFAULT_NOTIFY_ID = 1
@@ -94,6 +93,12 @@ class MainService : Service() {
}
}

@Keep
@RequiresApi(Build.VERSION_CODES.N)
fun rustKeyEventInput(input: ByteArray) {
InputService.ctx?.onKeyEvent(input)
}

@Keep
fun rustGetByName(name: String): String {
return when (name) {
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeWindowsChanged"
android:canRetrieveWindowContent="true"
android:accessibilityFlags="flagDefault"
android:notificationTimeout="50"
android:description="@string/accessibility_service_description"
6 changes: 3 additions & 3 deletions flutter/lib/desktop/screen/desktop_remote_screen.dart
Original file line number Diff line number Diff line change
@@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget {
final Map<String, dynamic> params;

DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) {
if (!bind.mainStartGrabKeyboard()) {
stateGlobal.grabKeyboard = true;
}
if (!bind.mainStartGrabKeyboard()) {
stateGlobal.grabKeyboard = true;
}
}

@override
4 changes: 4 additions & 0 deletions flutter/lib/mobile/pages/remote_page.dart
Original file line number Diff line number Diff line change
@@ -364,6 +364,10 @@ class _RemotePageState extends State<RemotePage> {
? []
: gFFI.ffiModel.isPeerAndroid
? [
IconButton(
color: Colors.white,
icon: Icon(Icons.keyboard),
onPressed: openKeyboard),
IconButton(
color: Colors.white,
icon: const Icon(Icons.build),
20 changes: 20 additions & 0 deletions libs/scrap/src/android/ffi.rs
Original file line number Diff line number Diff line change
@@ -173,6 +173,26 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
}
}

pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> {
if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(),
MAIN_SERVICE_CTX.read().unwrap().as_ref(),
) {
let mut env = jvm.attach_current_thread_as_daemon()?;
let data = env.byte_array_from_slice(data)?;

env.call_method(
ctx,
"rustKeyEventInput",
"([B)V",
&[JValue::Object(&JObject::from(data))],
)?;
return Ok(());
} else {
return Err(JniError::ThrowFailed(-1));
}
}

pub fn call_main_service_get_by_name(name: &str) -> JniResult<String> {
if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(),
4 changes: 2 additions & 2 deletions src/client.rs
Original file line number Diff line number Diff line change
@@ -1795,14 +1795,14 @@ impl LoginConfigHandler {
crate::flutter::push_global_event(crate::flutter::APP_TYPE_MAIN, evt);
}
if config.keyboard_mode.is_empty() {
if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version)) {
if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version), &pi.platform) {
config.keyboard_mode = KeyboardMode::Map.to_string();
} else {
config.keyboard_mode = KeyboardMode::Legacy.to_string();
}
} else {
let keyboard_modes =
crate::get_supported_keyboard_modes(get_version_number(&pi.version));
crate::get_supported_keyboard_modes(get_version_number(&pi.version), &pi.platform);
let current_mode = &KeyboardMode::from_str(&config.keyboard_mode).unwrap_or_default();
if !keyboard_modes.contains(current_mode) {
config.keyboard_mode = KeyboardMode::Legacy.to_string();
14 changes: 10 additions & 4 deletions src/common.rs
Original file line number Diff line number Diff line change
@@ -1086,18 +1086,24 @@ pub fn make_privacy_mode_msg(state: back_notification::PrivacyModeState) -> Mess
make_privacy_mode_msg_with_details(state, "".to_owned())
}

pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64) -> bool {
pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64, peer_platform: &str) -> bool {
match keyboard_mode {
KeyboardMode::Legacy => true,
KeyboardMode::Map => version_number >= hbb_common::get_version_number("1.2.0"),
KeyboardMode::Map => {
if peer_platform.to_lowercase() == crate::PLATFORM_ANDROID.to_lowercase() {
false
} else {
version_number >= hbb_common::get_version_number("1.2.0")
}
}
KeyboardMode::Translate => version_number >= hbb_common::get_version_number("1.2.0"),
KeyboardMode::Auto => version_number >= hbb_common::get_version_number("1.2.0"),
}
}

pub fn get_supported_keyboard_modes(version: i64) -> Vec<KeyboardMode> {
pub fn get_supported_keyboard_modes(version: i64, peer_platform: &str) -> Vec<KeyboardMode> {
KeyboardMode::iter()
.filter(|&mode| is_keyboard_mode_supported(mode, version))
.filter(|&mode| is_keyboard_mode_supported(mode, version, peer_platform))
.map(|&mode| mode)
.collect::<Vec<_>>()
}
1 change: 1 addition & 0 deletions src/flutter_ffi.rs
Original file line number Diff line number Diff line change
@@ -395,6 +395,7 @@ pub fn session_is_keyboard_mode_supported(session_id: SessionID, mode: String) -
SyncReturn(is_keyboard_mode_supported(
&mode,
session.get_peer_version(),
&session.peer_platform()
))
} else {
SyncReturn(false)
30 changes: 26 additions & 4 deletions src/keyboard.rs
Original file line number Diff line number Diff line change
@@ -28,6 +28,8 @@
const OS_LOWER_LINUX: &str = "linux";
#[allow(dead_code)]
const OS_LOWER_MACOS: &str = "macos";
#[allow(dead_code)]
const OS_LOWER_ANDROID: &str = "android";

#[cfg(any(target_os = "windows", target_os = "macos"))]
static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false);
@@ -163,6 +165,21 @@
}
}

pub fn map_key_to_control_key(key: &rdev::Key) -> Option<ControlKey> {

Check warning on line 168 in src/keyboard.rs

GitHub Actions / x86_64-unknown-linux-gnu (ubuntu-20.04)

function `map_key_to_control_key` is never used
match key {
Key::Alt => Some(ControlKey::Alt),
Key::ShiftLeft => Some(ControlKey::Shift),
Key::ControlLeft => Some(ControlKey::Control),
Key::MetaLeft => Some(ControlKey::Meta),
Key::AltGr => Some(ControlKey::RAlt),
Key::ShiftRight => Some(ControlKey::RShift),
Key::ControlRight => Some(ControlKey::RControl),
Key::MetaRight => Some(ControlKey::RWin),
_ => None,
}

}

pub fn event_lock_screen() -> KeyEvent {
let mut key_event = KeyEvent::new();
key_event.set_control_key(ControlKey::LockScreen);
@@ -355,7 +372,7 @@
}

#[inline]
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
pub fn is_modifier(key: &rdev::Key) -> bool {
matches!(
key,
@@ -850,12 +867,14 @@
rdev::win_scancode_to_macos_code(event.position_code)?
}
}
OS_LOWER_ANDROID => rdev::win_scancode_to_android_key_code(event.position_code)?,
_ => rdev::win_scancode_to_linux_code(event.position_code)?,
};
#[cfg(target_os = "macos")]
let keycode = match _peer {
OS_LOWER_WINDOWS => rdev::macos_code_to_win_scancode(event.platform_code as _)?,
OS_LOWER_MACOS => event.platform_code as _,
OS_LOWER_ANDROID => rdev::macos_code_to_android_key_code(event.platform_code as _)?,
_ => rdev::macos_code_to_linux_code(event.platform_code as _)?,
};
#[cfg(target_os = "linux")]
@@ -868,6 +887,7 @@
rdev::linux_code_to_macos_code(event.position_code as _)?
}
}
OS_LOWER_ANDROID => rdev::linux_code_to_android_key_code(event.position_code as _)?,
_ => event.position_code as _,
};
#[cfg(any(target_os = "android", target_os = "ios"))]
@@ -877,7 +897,7 @@
Some(key_event)
}

#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec<KeyEvent>) {
match &event.unicode {
Some(unicode_info) => {
@@ -1046,12 +1066,14 @@
events
}

#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
pub fn keycode_to_rdev_key(keycode: u32) -> Key {
#[cfg(target_os = "windows")]
return rdev::win_key_from_scancode(keycode);
#[cfg(target_os = "linux")]
#[cfg(any(target_os = "linux"))]
return rdev::linux_key_from_code(keycode);
#[cfg(any(target_os = "android"))]
return rdev::android_key_from_code(keycode);
#[cfg(target_os = "macos")]
return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default());
}
65 changes: 59 additions & 6 deletions src/server/connection.rs
Original file line number Diff line number Diff line change
@@ -38,10 +38,12 @@
sync::mpsc,
time::{self, Duration, Instant, Interval},
},
tokio_util::codec::{BytesCodec, Framed},
tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown,

Check warning on line 41 in src/server/connection.rs

GitHub Actions / x86_64-unknown-linux-gnu (ubuntu-20.04)

unused import: `protobuf::EnumOrUnknown`

Check warning on line 41 in src/server/connection.rs

GitHub Actions / x86_64-unknown-linux-gnu (ubuntu-20.04)

unused import: `protobuf::EnumOrUnknown`
};
#[cfg(any(target_os = "android", target_os = "ios"))]
use scrap::android::call_main_service_pointer_input;
use scrap::android::{call_main_service_pointer_input, call_main_service_key_event};
#[cfg(target_os = "android")]
use crate::keyboard::client::map_key_to_control_key;
use serde_derive::Serialize;
use serde_json::{json, value::Value};
use sha2::{Digest, Sha256};
@@ -57,7 +59,7 @@

#[cfg(all(windows, feature = "virtual_display_driver"))]
use crate::virtual_display_manager;
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
use std::collections::HashSet;
pub type Sender = mpsc::UnboundedSender<(Instant, Arc<Message>)>;

@@ -215,7 +217,7 @@
voice_call_request_timestamp: Option<NonZeroI64>,
audio_input_device_before_voice_call: Option<String>,
options_in_login: Option<OptionMessage>,
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
pressed_modifiers: HashSet<rdev::Key>,
#[cfg(all(target_os = "linux", feature = "linux_headless"))]
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -354,7 +356,7 @@
voice_call_request_timestamp: None,
audio_input_device_before_voice_call: None,
options_in_login: None,
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[cfg(not(any(target_os = "ios")))]
pressed_modifiers: Default::default(),
#[cfg(all(target_os = "linux", feature = "linux_headless"))]
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -1763,8 +1765,59 @@
}
self.update_auto_disconnect_timer();
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(any(target_os = "ios"))]
Some(message::Union::KeyEvent(..)) => {}
#[cfg(any(target_os = "android"))]
Some(message::Union::KeyEvent(mut me)) => {
let is_press = (me.press || me.down) && !crate::is_modifier(&me);

let key = match me.mode.enum_value() {
Ok(KeyboardMode::Map) => {
Some(crate::keyboard::keycode_to_rdev_key(me.chr()))
}
Ok(KeyboardMode::Translate) => {
if let Some(key_event::Union::Chr(code)) = me.union {
Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF))
} else {
None
}
}
_ => None,
}
.filter(crate::keyboard::is_modifier);

if let Some(key) = key {
if is_press {
self.pressed_modifiers.insert(key);
} else {
self.pressed_modifiers.remove(&key);
}
}

let mut modifiers = vec![];

for key in self.pressed_modifiers.iter() {
if let Some(control_key) = map_key_to_control_key(key) {
modifiers.push(EnumOrUnknown::new(control_key));
}
}

me.modifiers = modifiers;

let encode_result = me.write_to_bytes();

match encode_result {
Ok(data) => {
let result = call_main_service_key_event(&data);
if let Err(e) = result {
log::debug!("call_main_service_key_event fail: {}", e);
}
}
Err(e) => {
log::debug!("encode key event fail: {}", e);
}
}
}
#[cfg(not(any(target_os = "android", target_os = "ios")))]
Some(message::Union::KeyEvent(me)) => {
if self.peer_keyboard_enabled() {
36 changes: 27 additions & 9 deletions src/ui_session_interface.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL};
use crate::{input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}, common::{is_keyboard_mode_supported, get_supported_keyboard_modes}};
use async_trait::async_trait;
use bytes::Bytes;
use rdev::{Event, EventType::*, KeyCode};
@@ -213,18 +213,36 @@ impl<T: InvokeUiSession> Session<T> {
self.lc.read().unwrap().version.clone()
}

pub fn get_keyboard_mode(&self) -> String {
let mode = self.lc.read().unwrap().keyboard_mode.clone();
if ["map", "translate", "legacy"].contains(&(&mode as &str)) {
mode
pub fn fallback_keyboard_mode(&self) -> String {
let peer_version = self.get_peer_version();
let platform = self.peer_platform();

let supported_modes = get_supported_keyboard_modes(peer_version, &platform);
if let Some(mode) = supported_modes.first() {
return mode.to_string();
} else {
if self.get_peer_version() > hbb_common::get_version_number("1.2.0") {
"map"
if self.get_peer_version() >= get_version_number("1.2.0") {
return KeyboardMode::Map.to_string();
} else {
"legacy"
return KeyboardMode::Legacy.to_string();
}
}
}

pub fn get_keyboard_mode(&self) -> String {
let mode = self.lc.read().unwrap().keyboard_mode.clone();
let keyboard_mode = KeyboardMode::from_str(&mode);

let peer_version = self.get_peer_version();
let platform = self.peer_platform();

// Saved keyboard mode still exists in this version.
if let Ok(mode) = keyboard_mode {
if is_keyboard_mode_supported(&mode, peer_version, &platform) {
return mode.to_string();
}
.to_string()
}
self.fallback_keyboard_mode()
}

pub fn save_keyboard_mode(&self, value: String) {

0 comments on commit 5577731

Please sign in to comment.