Merge pull request #12556 from keymanapp/fix/mac/6578-ineffective-osk…

…-modifiers

fix(mac): make modifiers operational in OSK

mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m

@@ -46,6 +46,8 @@ - (NSString *)minimalVersionNumberString {
 
 @interface KMInputMethodAppDelegate ()
 @property (nonatomic, strong) KMPackageReader *packageReader;
+@property BOOL receivedKeyDownFromOsk;
+@property NSEventModifierFlags oskEventModifiers;
 @end
 
 @implementation KMInputMethodAppDelegate
@@ -299,7 +301,8 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef
         break;
 
       case kCGEventKeyDown:
-        os_log_debug([KMLogs eventsLog], "Event tap keydown event, keyCode: %hu", sysEvent.keyCode);
+        appDelegate.receivedKeyDownFromOsk = NO;
+        os_log_debug([KMLogs eventsLog], "Event tap keydown event, keyCode: 0x%X (%d)", sysEvent.keyCode, sysEvent.keyCode);
         // Pass back low-level backspace events to the input method event handler
         // because some non-compliant apps do not allow us to see backspace events
         // that we have generated (and we need to see them, for serialization
@@ -313,7 +316,12 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef
         if(sysEvent.keyCode == 255) {
           os_log_debug([KMLogs eventsLog], "*** kKeymanEventKeyCode = 0xFF");
         } else {
-          os_log_debug([KMLogs eventsLog], "*** other: %d(%x)", (char) sysEvent.keyCode, sysEvent.keyCode);
+          if ([OSKView isOskKeyDownEvent:event]) {
+            NSEventModifierFlags oskEventModifiers = [OSKView extractModifierFlagsFromOskEvent:event];
+            appDelegate.receivedKeyDownFromOsk = YES;
+            appDelegate.oskEventModifiers = oskEventModifiers;
+            os_log_debug([KMLogs eventsLog], "*** keydown event received from OSK, modifiers: 0x%lX",(unsigned long)oskEventModifiers);
+          }
         }
 
         switch(sysEvent.keyCode) {
@@ -1231,8 +1239,19 @@ - (void)handleKeyEvent:(NSEvent *)event {
 }
 
 - (NSEventModifierFlags) determineModifiers {
-  NSEventModifierFlags originalModifiers = self.currentModifiers;
-  return [self.modifierMapping adjustModifiers:originalModifiers];
+  NSEventModifierFlags modifierFlags = 0;
+
+  if (self.receivedKeyDownFromOsk) {
+    modifierFlags = self.oskEventModifiers;
+    os_log_debug([KMLogs eventsLog], "--- use modifiers from OSK, oskEventModifiers: 0x%lX", (unsigned long)modifierFlags);
+    self.oskEventModifiers = 0;
+  } else {
+    NSEventModifierFlags originalModifiers = self.currentModifiers;
+    modifierFlags = [self.modifierMapping adjustModifiers:originalModifiers];
+    os_log_debug([KMLogs eventsLog], "--- use adjusted modifiers from current state: 0x%lX", (unsigned long)modifierFlags);
+  }
+
+  return modifierFlags;
 }
 
 extern const CGKeyCode kProcessPendingBuffer;

mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodEventHandler.m

@@ -122,10 +122,11 @@ - (BOOL) handleEventWithKeymanEngine:(NSEvent *)event in:(id) sender {
 - (CoreKeyOutput*) processEventWithKeymanEngine:(NSEvent *)event in:(id) sender {
   CoreKeyOutput* coreKeyOutput = nil;
   if (self.appDelegate.lowLevelEventTap != nil) {
-    NSEventModifierFlags newModifiers = [self.appDelegate determineModifiers];
-    NSEvent *eventWithOriginalModifierFlags = [NSEvent keyEventWithType:event.type location:event.locationInWindow modifierFlags:newModifiers timestamp:event.timestamp windowNumber:event.windowNumber context:[NSGraphicsContext currentContext] characters:event.characters charactersIgnoringModifiers:event.charactersIgnoringModifiers isARepeat:event.isARepeat keyCode:event.keyCode];
-    coreKeyOutput = [self.kme processEvent:eventWithOriginalModifierFlags];
-    os_log_debug([KMLogs eventsLog], "processEventWithKeymanEngine, using newModifierFlag 0x%lX, instead of event.modifiers 0x%lX", newModifiers, event.modifierFlags);
+    NSEventModifierFlags modifierFlags = [self.appDelegate determineModifiers];
+    os_log_debug([KMLogs eventsLog], "processEventWithKeymanEngine, using modifierFlags 0x%lX, instead of event.modifiers 0x%lX", modifierFlags, event.modifierFlags);
+
+    NSEvent *eventWithAppropriateModifierFlags = [NSEvent keyEventWithType:event.type location:event.locationInWindow modifierFlags:modifierFlags timestamp:event.timestamp windowNumber:event.windowNumber context:[NSGraphicsContext currentContext] characters:event.characters charactersIgnoringModifiers:event.charactersIgnoringModifiers isARepeat:event.isARepeat keyCode:event.keyCode];
+    coreKeyOutput = [self.kme processEvent:eventWithAppropriateModifierFlags];
   }
   else {
     /** 

mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/OnScreenKeyboard/KeyView.m

@@ -294,6 +294,7 @@ - (void)mouseUp:(NSEvent *)theEvent {
 }
 
 -(void)processKeyClick {
+  os_log_debug([KMELogs oskLog], "KeyView processKeyClick");
   [self.target keyAction:self];
 }
 

mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/OnScreenKeyboard/OSKView.h

@@ -16,6 +16,8 @@
 
 @property (weak, nonatomic) KVKFile *kvk;
 
++ (BOOL)isOskKeyDownEvent:(CGEventRef)event;
++ (NSEventModifierFlags)extractModifierFlagsFromOskEvent:(CGEventRef)event;
 - (void)handleKeyEvent:(NSEvent *)event;
 - (void)setShiftState:(BOOL)shiftState;
 - (void)setOskShiftState:(BOOL)oskShiftState;
@@ -25,6 +27,7 @@
 - (void)setOskCtrlState:(BOOL)oskCtrlState;
 - (void)resetOSK;
 - (void)resizeOSKLayout;
+- (int64_t)createOskEventUserData;
 
 @end
 

mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/OnScreenKeyboard/OSKView.m

@@ -17,6 +17,9 @@
 #include <Carbon/Carbon.h>
 #import "KMELogs.h"
 
+const int64_t OSK_EVENT_FLAG = 0x8000000000000000;
+const int64_t OSK_EVENT_MODIFIER_MASK = 0x00000000FFFFFFFF;
+
 @interface OSKView()
 @property (nonatomic, strong) NSArray *oskLayout;
 @property (nonatomic, strong) NSArray *oskDefaultNKeys;
@@ -31,6 +34,29 @@ @interface OSKView()
 @implementation OSKView
 @synthesize tag;
 
+/**
+ * Check the kCGEventSourceUserData field of the event and determine whether the event was
+ * generated by the OSK.
+ */
++ (BOOL)isOskKeyDownEvent:(CGEventRef)event {
+  // check to see if this event was generated by the OSK
+  int64_t eventUserData = CGEventGetIntegerValueField(event, kCGEventSourceUserData);
+  return ((eventUserData & OSK_EVENT_FLAG) == OSK_EVENT_FLAG);
+}
+
+/**
+ * If the event was generated by the OSK, then return the modifier flags stored in the kCGEventSourceUserData
+ * field at the time the event was generated
+ */
++ (NSEventModifierFlags)extractModifierFlagsFromOskEvent:(CGEventRef)event {
+  NSEventModifierFlags modifierFlags = 0;
+  int64_t eventUserData = CGEventGetIntegerValueField(event, kCGEventSourceUserData);
+  if ([OSKView isOskKeyDownEvent:event]) {
+    modifierFlags = (NSEventModifierFlags) eventUserData & OSK_EVENT_MODIFIER_MASK;
+  }
+  return modifierFlags;
+}
+
 - (id)initWithFrame:(NSRect)frame {
   os_log_debug([KMELogs oskLog], "OSKView initWithFrame: %{public}@", NSStringFromRect(frame));
   self = [super initWithFrame:frame];
@@ -42,6 +68,12 @@ - (id)initWithFrame:(NSRect)frame {
   return self;
 }
 
+- (NSString *)description {
+  NSString *format = @"<shiftState:%d oskShiftState:%d altState:%d oskAltState:%d ctrlState:%d oskCtrlState:%d>";
+  NSString *str = [NSString stringWithFormat:format, _shiftState, _oskShiftState, _altState, _oskAltState, _ctrlState, _oskCtrlState];
+  return str;
+}
+
 - (void)drawRect:(NSRect)rect {
   os_log_debug([KMELogs oskLog], "OSKView drawRect: %{public}@", NSStringFromRect(rect));
 
@@ -411,30 +443,24 @@ - (void)resizeOSKLayout {
 - (void)keyAction:(id)sender {
   KeyView *keyView = (KeyView *)sender;
   NSUInteger keyCode = [keyView.key keyCode];
-  os_log_debug([KMELogs oskLog], "OSKView keyAction keyView: %{public}@", keyView);
+  os_log_debug([KMELogs oskLog], "OSKView keyAction, self: %{public}@, keyView: %{public}@", self, keyView);
   if (keyCode < 0x100) {
     NSRunningApplication *app = NSWorkspace.sharedWorkspace.frontmostApplication;
     pid_t processId = app.processIdentifier;
     CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStatePrivate);
     CGEventRef keyDownEvent = CGEventCreateKeyboardEvent(source, (CGKeyCode)keyCode, true);
     CGEventRef keyUpEvent = CGEventCreateKeyboardEvent(source, (CGKeyCode)keyCode, false);
-    if (self.shiftState || self.oskShiftState) {
-      CGEventSetFlags(keyDownEvent, CGEventGetFlags(keyDownEvent) | kCGEventFlagMaskShift);
-      CGEventSetFlags(keyUpEvent, CGEventGetFlags(keyUpEvent) | kCGEventFlagMaskShift);
-    }
-    if (self.altState || self.oskAltState) {
-      CGEventSetFlags(keyDownEvent, CGEventGetFlags(keyDownEvent) | kCGEventFlagMaskAlternate);
-      CGEventSetFlags(keyUpEvent, CGEventGetFlags(keyUpEvent) | kCGEventFlagMaskAlternate);
-    }
-    if (self.ctrlState || self.oskCtrlState) {
-      CGEventSetFlags(keyDownEvent, CGEventGetFlags(keyDownEvent) | kCGEventFlagMaskControl);
-      CGEventSetFlags(keyUpEvent, CGEventGetFlags(keyUpEvent) | kCGEventFlagMaskControl);
-    }
+
+    CGEventSetIntegerValueField(keyDownEvent, kCGEventSourceUserData, [self createOskEventUserData]);
+
+    os_log_debug([KMELogs oskLog], "OSKView keyAction, keyDownEvent: %{public}@, modifier flags: 0x%llX", keyDownEvent, CGEventGetFlags(keyDownEvent));
+
     CGEventPostToPid(processId, keyDownEvent);
     CGEventPostToPid(processId, keyUpEvent);
-    CFRelease(source);
     CFRelease(keyDownEvent);
     CFRelease(keyUpEvent);
+
+    CFRelease(source);
   }
   else {
     if (keyCode == MVK_LEFT_SHIFT || keyCode == MVK_RIGHT_SHIFT) {
@@ -449,6 +475,37 @@ - (void)keyAction:(id)sender {
   }
 }
 
+/**
+ * Create the 64-bit value to pass in the kCGEventSourceUserData field of the generated CGEvent
+ * The upper-most bit is set to distinguish it from the default value of zero, and the lower 32 bits
+ * store the modifier state of the OSK at the time that the keydown event was generated.
+ * This data is safely conveys the modifier information to the eventTap function.
+ * If the modifiers themselves were instead set by calling the function `CGEventSetFlags`
+ * directly on the generated CGEvent, then there would be no way to distinguish between the left
+ * and right option keys.
+ *
+ */
+- (int64_t) createOskEventUserData {
+  // set bit to identify this user data as originating from the OSK
+  int64_t oskEventData = OSK_EVENT_FLAG;
+
+  if (self.oskShiftState || self.shiftState) {
+    oskEventData = oskEventData | kCGEventFlagMaskShift;
+  }
+  /**
+   * Both left and right alt keys on the OSK cause oskAltState to be true without distinction of left or right.
+   * However, the generated event is a right alt so that it will trigger the right alt rules in the Keyman keyboard.
+   */
+  if (self.oskAltState || self.altState) {
+    oskEventData = oskEventData | MK_RIGHT_ALT_MASK;
+  }
+  if (self.oskCtrlState || self.oskCtrlState) {
+    oskEventData = oskEventData | kCGEventFlagMaskControl;
+  }
+
+  return oskEventData;
+}
+
 - (void)handleKeyEvent:(NSEvent *)event {
   os_log_debug([KMELogs oskLog], "OSKView handleKeyEvent event.type: %lu", event.type);
   NSView *view = [self viewWithTag:event.keyCode|0x1000];
@@ -467,7 +524,7 @@ - (void)setKeyPressedOff:(KeyView *)keyView {
 }
 
 - (void)setKeyLabels:(BOOL)shift alt:(BOOL)alt ctrl:(BOOL)ctrl {
-  os_log_debug([KMELogs keyLog], "OSKView setKeyLabels, shift: %d, alt: %d, ctrl: %d", shift, alt, ctrl);
+  os_log_debug([KMELogs oskLog], "OSKView setKeyLabels, shift: %d, alt: %d, ctrl: %d", shift, alt, ctrl);
   [self resetKeyLabels];
   NSMutableArray *mKeys = [[self keyTags] mutableCopy];
   NSArray *nkeys = [self.kvk keys];