feat(web): subkey-menu fat-finger correction 🐵

common/web/input-processor/src/corrections.ts

@@ -1,3 +1,4 @@
+import { ActiveKeyBase, KeyDistribution } from "@keymanapp/keyboard-processor";
 import { CorrectionLayout } from "./correctionLayout.js";
 
 /**
@@ -9,8 +10,8 @@ import { CorrectionLayout } from "./correctionLayout.js";
  *                          by a correction algorithm, also within <0, 0> to <1, 1>.
  * @returns A mapping of key IDs to the 'squared pseudo-distance' of the touchpoint to each key.
  */
-export function keyTouchDistances(touchCoords: {x: number, y: number}, correctiveLayout: CorrectionLayout): Map<string, number> {
-  let keyDists: Map<string, number> = new Map<string, number>();
+export function keyTouchDistances(touchCoords: {x: number, y: number}, correctiveLayout: CorrectionLayout): Map<ActiveKeyBase, number> {
+  let keyDists: Map<ActiveKeyBase, number> = new Map<ActiveKeyBase, number>();
 
   // This double-nested loop computes a pseudo-distance for the touch from each key.  Quite useful for
   // generating a probability distribution.
@@ -54,7 +55,7 @@ export function keyTouchDistances(touchCoords: {x: number, y: number}, correctiv
     distY += dy * entry.height;
 
     const distance = distX * distX + distY * distY;
-    keyDists.set(entry.keySpec.coreID, distance);
+    keyDists.set(entry.keySpec, distance);
   });
 
   return keyDists;
@@ -65,26 +66,32 @@ export function keyTouchDistances(touchCoords: {x: number, y: number}, correctiv
  * consideration.
  * @returns
  */
-export function distributionFromDistanceMap(squaredDistMap: Map<string, number>): {keyId: string, p: number}[] {
-  const keyProbs = new Map<string, number>();
+export function distributionFromDistanceMaps(squaredDistMaps: Map<ActiveKeyBase, number> | Map<ActiveKeyBase, number>[]): KeyDistribution {
+  const keyProbs = new Map<ActiveKeyBase, number>();
   let totalMass = 0;
 
-  // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place
-  // of the formula in the loop below.
-  for(let key of squaredDistMap.keys()) {
-    // We've found that in practice, dist^-4 seems to work pretty well.  (Our input has dist^2.)
-    // (Note:  our rule of thumb here has only been tested for layout-based distances.)
-    const entry = 1 / (Math.pow(squaredDistMap.get(key), 2) + 1e-6); // Prevent div-by-0 errors.
-    totalMass += entry;
+  if(!Array.isArray(squaredDistMaps)) {
+    squaredDistMaps = [squaredDistMaps];
+  }
+
+  for(let squaredDistMap of squaredDistMaps) {
+    // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place
+    // of the formula in the loop below.
+    for(let key of squaredDistMap.keys()) {
+      // We've found that in practice, dist^-4 seems to work pretty well.  (Our input has dist^2.)
+      // (Note:  our rule of thumb here has only been tested for layout-based distances.)
+      const entry = 1 / (Math.pow(squaredDistMap.get(key), 2) + 1e-6); // Prevent div-by-0 errors.
+      totalMass += entry;
 
-    // In case of duplicate key IDs.
-    keyProbs.set(key, keyProbs.get(key) ?? 0 + entry);
+      // In case of duplicate key IDs; this can occur if multiple sets are specified.
+      keyProbs.set(key, keyProbs.get(key) ?? 0 + entry);
+    }
   }
 
-  const list: {keyId: string, p: number}[] = [];
+  const list: {keySpec: ActiveKeyBase, p: number}[] = [];
 
   for(let key of keyProbs.keys()) {
-    list.push({keyId: key, p: keyProbs.get(key) / totalMass});
+    list.push({keySpec: key, p: keyProbs.get(key) / totalMass});
   }
 
   return list.sort(function(a, b) {

common/web/input-processor/src/text/inputProcessor.ts

@@ -295,7 +295,6 @@ export default class InputProcessor {
         // Sort the distribution into probability-descending order.
         keyDistribution.sort((a, b) => b.p - a.p);
 
-        let activeLayout = this.activeKeyboard.layout(keyEvent.device.formFactor);
         alternates = [];
 
         let totalMass = 0; // Tracks sum of non-error probabilities.
@@ -314,9 +313,9 @@ export default class InputProcessor {
 
           let mock = Mock.from(windowedMock, false);
 
-          let altKey = activeLayout.getLayer(keyEvent.kbdLayer).getKey(pair.keyId);
+          const altKey = pair.keySpec;
           if(!altKey) {
-            console.warn("Potential fat-finger key could not be found in layer!");
+            console.warn("Internal error:  failed to properly filter set of keys for corrections");
             continue;
           }
 

common/web/keyboard-processor/src/keyboards/keyboard.ts

@@ -1,6 +1,6 @@
 import Codes from "../text/codes.js";
 import { Layouts, type LayoutFormFactor } from "./defaultLayouts.js";
-import { ActiveKey, ActiveLayout } from "./activeLayout.js";
+import { ActiveKey, ActiveKeyBase, ActiveLayout } from "./activeLayout.js";
 import KeyEvent from "../text/keyEvent.js";
 import type OutputTarget from "../text/outputTarget.js";
 
@@ -468,7 +468,7 @@ export default class Keyboard {
     return keyEvent;
   }
 
-  constructKeyEvent(key: ActiveKey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent {
+  constructKeyEvent(key: ActiveKeyBase, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent {
     // Make a deep copy of our preconstructed key event, filling it out from there.
     const Lkc = key.baseKeyEvent;
     Lkc.device = device;

common/web/keyboard-processor/src/text/keyEvent.ts

@@ -11,10 +11,11 @@ import { type DeviceSpec } from "@keymanapp/web-utils";
 
 import Codes from './codes.js';
 import DefaultRules from './defaultRules.js';
+import { ActiveKeyBase } from "../index.js";
 
 // Represents a probability distribution over a keyboard's keys.
 // Defined here to avoid compilation issues.
-export type KeyDistribution = {keyId: string, p: number}[];
+export type KeyDistribution = {keySpec: ActiveKeyBase, p: number}[];
 
 /**
  * A simple instance of the standard 'default rules' for keystroke processing from the
@@ -45,8 +46,8 @@ export interface KeyEventSpec {
    */
   srcKeyboard?: Keyboard;
 
-  // Holds relevant event properties leading to construction of this KeyEvent.
-  source?: any; // Technically, KeyEvent|MouseEvent|Touch - but those are DOM types that must be kept out of headless mode.
+  // // Holds relevant event properties leading to construction of this KeyEvent.
+  // source?: any; // Technically, KeyEvent|MouseEvent|Touch - but those are DOM types that must be kept out of headless mode.
   // Holds a generated fat-finger distribution (when appropriate)
   keyDistribution?: KeyDistribution;
 

web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts

@@ -3,9 +3,11 @@ import { type KeyElement } from '../../../keyElement.js';
 import OSKBaseKey from '../../../keyboard-layout/oskBaseKey.js';
 import VisualKeyboard from '../../../visualKeyboard.js';
 
-import { DeviceSpec, KeyEvent, ActiveSubKey } from '@keymanapp/keyboard-processor';
+import { DeviceSpec, KeyEvent, ActiveSubKey, KeyDistribution, ActiveKeyBase } from '@keymanapp/keyboard-processor';
 import { ConfigChangeClosure, GestureRecognizerConfiguration, GestureSequence, PaddedZoneSource } from '@keymanapp/gesture-recognizer';
 import { GestureHandler } from '../gestureHandler.js';
+import { CorrectionLayout, CorrectionLayoutEntry, distributionFromDistanceMaps, keyTouchDistances } from '@keymanapp/input-processor';
+import { GestureParams } from '../specsForLayout.js';
 
 /**
  * Represents a 'realized' longpress gesture's default implementation
@@ -31,15 +33,18 @@ export default class SubkeyPopup implements GestureHandler {
   public readonly subkeys: KeyElement[];
 
   private source: GestureSequence<KeyElement>;
+  private readonly gestureParams: GestureParams;
 
   constructor(
     source: GestureSequence<KeyElement>,
     configChanger: ConfigChangeClosure<KeyElement>,
     vkbd: VisualKeyboard,
-    e: KeyElement
+    e: KeyElement,
+    gestureParams: GestureParams
   ) {
     this.baseKey = e;
     this.source = source;
+    this.gestureParams = gestureParams;
 
     source.on('complete', () => {
       this.currentSelection?.key.highlight(false);
@@ -337,6 +342,96 @@ export default class SubkeyPopup implements GestureHandler {
     return this.element.style.visibility == 'visible';
   }
 
+  buildCorrectiveLayout(): CorrectionLayout {
+    const baseBounding = this.element.getBoundingClientRect();
+    const aspectRatio = baseBounding.width / baseBounding.height;
+
+    const keys = this.subkeys.map((keyElement) => {
+      const subkeyBounds = keyElement.getBoundingClientRect();
+
+      // Ensures we have the right typing.
+      const correctiveData: CorrectionLayoutEntry = {
+        keySpec: keyElement.key.spec,
+        centerX: ((subkeyBounds.right - subkeyBounds.width / 2) - baseBounding.left) / baseBounding.width,
+        centerY: ((subkeyBounds.bottom - subkeyBounds.height / 2) - baseBounding.top) / baseBounding.height,
+        width: subkeyBounds.width / baseBounding.width,
+        height: subkeyBounds.height / baseBounding.height
+      }
+
+      return correctiveData;
+    });
+
+    return {
+      keys: keys,
+      kbdScaleRatio: aspectRatio
+    }
+  }
+
+  currentStageKeyDistances(): KeyDistribution {
+    const latestStage = this.source.stageReports[this.source.stageReports.length-1];
+    const baseStage = this.source.stageReports[0];
+    const gestureSource = latestStage.sources[0];
+    const lastCoord = gestureSource.currentSample;
+
+    const baseBounding = this.element.getBoundingClientRect();
+    const mappedCoord = {
+      x: lastCoord.targetX / baseBounding.width,
+      y: lastCoord.targetY / baseBounding.height
+    }
+
+    // Lock the coordinate within base-element bounds; corrects for the allowed 'popup roaming' zone.
+    //
+    // To consider:  add a 'clipping' feature to `keyTouchDistances`?  It could make sense for base
+    // keys, too - especially when emulating a touch OSK via the inline-OSK mode used in the
+    // Developer host page.
+    mappedCoord.x = mappedCoord.x < 0 ? 0 : (mappedCoord.x > 1 ? 1: mappedCoord.x);
+    mappedCoord.y = mappedCoord.y < 0 ? 0 : (mappedCoord.y > 1 ? 1: mappedCoord.y);
+
+    const rawSqDistances = keyTouchDistances(mappedCoord, this.buildCorrectiveLayout());
+    const currentKeyDist = rawSqDistances.get(lastCoord.item.key.spec);
+
+    /*
+     * - how long has the subkey menu been visible?
+     *   - Base key should be less likely if it's been visible a while,
+     *     but reasonably likely if it only just appeared.
+     *     - Especially if up-flicks are allowed.  Though, in that case, consider
+     *       base-layer neighbors, and particularly the one directly under the touchpoint?
+     * - raw distance traveled (since the menu appeared)
+     *   - similarly, short distance = a more likely base key?
+     */
+
+    // The concept:  how likely is it that the user MEANT to output a subkey?
+    let timeDistance = Math.min(
+      // The full path is included by the model - meaning the base wait is included here in
+      // in the stats;  we subtract it to get just the duration of the subkey menu.
+      gestureSource.path.stats.duration - baseStage.sources[0].path.stats.duration,
+      this.gestureParams.longpress.waitLength
+      ) / (2 * this.gestureParams.longpress.waitLength);  // normalize:  max time distance of 0.5
+
+    let pathDistance = Math.min(
+      gestureSource.path.stats.rawDistance,
+      this.gestureParams.longpress.noiseTolerance*4
+    ) / (this.gestureParams.longpress.noiseTolerance * 8); // normalize similarly.
+
+    // We only want to add a single distance 'dimension' - we'll choose the one that affects
+    // the interpreted distance the least.  (This matters for upflick-shortcutting in particular)
+    const layerDistance = Math.min(timeDistance * timeDistance, pathDistance * pathDistance);
+    const baseKeyDistance = currentKeyDist + layerDistance;
+
+    // Include the base key as a corrective option.
+    const baseKeyMap = new Map<ActiveKeyBase, number>();
+    const subkeyMatch = this.subkeys.find((entry) => entry.keyId == this.baseKey.keyId);
+    if(subkeyMatch) {
+      // Ensure that the base key's entry can be merged with that of its subkey.
+      // (Assuming that always makes sense.)
+      baseKeyMap.set(subkeyMatch.key.spec, baseKeyDistance);
+    } else {
+      baseKeyMap.set(this.baseKey.key.spec, baseKeyDistance);
+    }
+
+    return distributionFromDistanceMaps([rawSqDistances, baseKeyMap]);
+  }
+
   cancel() {
     this.clear();
     this.source.cancel();

web/src/engine/osk/src/input/gestures/gestureHandler.ts

@@ -1,3 +1,5 @@
+import { KeyDistribution } from "@keymanapp/keyboard-processor";
+
 export interface GestureHandler {
   /**
    * Triggers cancellation of any further processing for the gesture being handled.
@@ -9,4 +11,6 @@ export interface GestureHandler {
    * scenario in which key previews (on phones) should be disabled.
    */
   readonly hasModalVisualization: boolean;
+
+  currentStageKeyDistances(): KeyDistribution;
 }

web/src/engine/osk/src/input/gestures/heldRepeater.ts

@@ -1,4 +1,5 @@
 import { GestureSequence } from "@keymanapp/gesture-recognizer";
+import { KeyDistribution } from "@keymanapp/keyboard-processor";
 
 import { KeyElement } from "../../keyElement.js";
 import { GestureHandler } from './gestureHandler.js';
@@ -35,4 +36,8 @@ export class HeldRepeater implements GestureHandler {
 
     this.timerHandle = window.setTimeout(this.deleteRepeater, HeldRepeater.REPEAT_DELAY);
   }
+
+  currentStageKeyDistances(): KeyDistribution {
+    return null;
+  }
 }