Merge pull request #11277 from keymanapp/feat/web/gesture-engine-history

feat(web): add recent-history log to gesture engine

common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts

@@ -98,6 +98,12 @@ export interface GestureRecognizerConfiguration<HoveredItemType, StateToken = an
    * use in automated testing.
    */
   readonly recordingMode?: boolean;
+
+  /**
+   * If greater than zero, preserves this amount of previously-seen touches and gestures before
+   * permanently clearing them.
+   */
+  readonly historyLength?: number;
 }
 
 export function preprocessRecognizerConfig<HoveredItemType, StateToken = any>(
@@ -116,6 +122,7 @@ export function preprocessRecognizerConfig<HoveredItemType, StateToken = any>(
 
   processingConfig.itemIdentifier   = processingConfig.itemIdentifier   ?? (() => null);
   processingConfig.recordingMode = !!processingConfig.recordingMode;
+  processingConfig.historyLength = (processingConfig.historyLength ?? 0) > 0 ? processingConfig.historyLength : 0;
 
   if(!config.paddedSafeBounds) {
     let paddingArray = config.safeBoundPadding;

common/web/gesture-recognizer/src/engine/gestureRecognizer.ts

@@ -19,7 +19,7 @@ export class GestureRecognizer<HoveredItemType, StateToken = any> extends Touchp
     // overhead.
     gestureModelDefinitions = gestureModelDefinitions || EMPTY_GESTURE_DEFS;
 
-    super(gestureModelDefinitions);
+    super(gestureModelDefinitions, null, preprocessedConfig.historyLength);
     this.config = preprocessedConfig;
 
     this.mouseEngine = new MouseEventEngine<HoveredItemType>(this.config);

common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts

@@ -435,7 +435,7 @@ export class CumulativePathStats<Type = any> {
    * Provides a JSON.stringify()-friendly object with the properties most useful for
    * debugger-based inspection and/or console-logging statements.
    */
-  public get summaryObject() {
+  public toJSON() {
     return {
       angle: this.angle,
       cardinal: this.cardinalDirection,

common/web/gesture-recognizer/src/engine/headless/gestureDebugPath.ts

@@ -9,6 +9,7 @@ import { GesturePath } from "./gesturePath.js";
 export type SerializedGesturePath<Type, StateToken> = {
   coords: Mutable<InputSample<Type, StateToken>>[]; // ensures type match with public class property.
   wasCancelled?: boolean;
+  stats?: CumulativePathStats
 }
 
 interface EventMap<Type, StateToken> {
@@ -121,7 +122,8 @@ export class GestureDebugPath<Type, StateToken = any> extends GesturePath<Type,
         t:       obj.t,
         item:    obj.item
       }))),
-      wasCancelled: this.wasCancelled
+      wasCancelled: this.wasCancelled,
+      stats: this.stats
     }
 
     // Removes components of each sample that we don't want serialized.

common/web/gesture-recognizer/src/engine/headless/gestureDebugSource.ts

@@ -3,17 +3,7 @@ import { GestureRecognizerConfiguration, preprocessRecognizerConfig } from "../c
 import { Nonoptional } from "../nonoptional.js";
 import { MatcherSelector } from "./gestures/matchers/matcherSelector.js";
 import { SerializedGesturePath, GestureDebugPath } from "./gestureDebugPath.js";
-import { GestureSource } from "./gestureSource.js";
-
-/**
- * Documents the expected typing of serialized versions of the `GestureSource` class.
- */
-export type SerializedGestureSource<HoveredItemType = any, StateToken = any> = {
-  isFromTouch: boolean;
-  path: SerializedGesturePath<HoveredItemType, StateToken>;
-  // identifier is not included b/c it's only needed during live processing.
-}
-
+import { GestureSource, SerializedGestureSource } from "./gestureSource.js";
 /**
  * Represents all metadata needed internally for tracking a single "touch contact point" / "touchpoint"
  * involved in a potential / recognized gesture as tracked over time.
@@ -35,8 +25,6 @@ export type SerializedGestureSource<HoveredItemType = any, StateToken = any> = {
  */
 export class GestureDebugSource<HoveredItemType, StateToken=any> extends GestureSource<HoveredItemType, StateToken, GestureDebugPath<HoveredItemType, StateToken>> {
   // Assertion:  must always contain an index 0 - the base recognizer config.
-  protected recognizerConfigStack: Nonoptional<GestureRecognizerConfiguration<HoveredItemType, StateToken>>[];
-
   private static _jsonIdSeed: -1;
 
   /**
@@ -78,22 +66,4 @@ export class GestureDebugSource<HoveredItemType, StateToken=any> extends Gesture
     instance._path = path;
     return instance;
   }
-
-  /**
-   * Creates a serialization-friendly version of this instance for use by
-   * `JSON.stringify`.
-   */
-  /* c8 ignore start */
-  toJSON(): SerializedGestureSource {
-    const path = this.path as GestureDebugPath<HoveredItemType, StateToken>;
-    let jsonClone: SerializedGestureSource = {
-      isFromTouch: this.isFromTouch,
-      path: path.toJSON()
-    };
-
-    return jsonClone;
-    /* c8 ignore stop */
-    /* c8 ignore next 2 */
-    // esbuild or tsc seems to mangle the 'ignore stop' if put outside the ending brace.
-  }
 }

common/web/gesture-recognizer/src/engine/headless/gesturePath.ts

@@ -138,4 +138,13 @@ export class GesturePath<Type, StateToken = any> extends EventEmitter<EventMap<T
 
     this.removeAllListeners();
   }
+
+  public toJSON(): any {
+    return {
+      // Replicate array and its entries, but with certain fields of each entry missing.
+      // No .clientX, no .clientY.
+      stats: this.stats,
+      wasCancelled: this.wasCancelled
+    }
+  }
 }

common/web/gesture-recognizer/src/engine/headless/gestureSource.ts

@@ -3,13 +3,26 @@ import { GesturePath } from "./gesturePath.js";
 import { GestureRecognizerConfiguration, preprocessRecognizerConfig } from "../configuration/gestureRecognizerConfiguration.js";
 import { Nonoptional } from "../nonoptional.js";
 import { MatcherSelector } from "./gestures/matchers/matcherSelector.js";
+import { SerializedGesturePath } from "./gestureDebugPath.js";
 
 export function buildGestureMatchInspector<Type, StateToken>(selector: MatcherSelector<Type, StateToken>) {
   return (source: GestureSource<Type, StateToken>) => {
     return selector.potentialMatchersForSource(source).map((matcher) => matcher.model.id);
   };
 }
 
+/**
+ * Documents the expected typing of serialized versions of the `GestureSource` class.
+ */
+export type SerializedGestureSource<HoveredItemType = any, StateToken = any> = {
+  isFromTouch: boolean;
+  path: SerializedGesturePath<HoveredItemType, StateToken>;
+  stateToken?: StateToken;
+  identifier?: string;
+  // identifier is not included b/c it's only needed during live processing.
+}
+
+
 /**
  * Represents all metadata needed internally for tracking a single "touch contact point" / "touchpoint"
  * involved in a potential / recognized gesture as tracked over time.
@@ -215,6 +228,25 @@ export class GestureSource<
   public get currentRecognizerConfig() {
     return this.recognizerConfigStack[this.recognizerConfigStack.length-1];
   }
+
+ /**
+  * Creates a serialization-friendly version of this instance for use by
+  * `JSON.stringify`.
+  */
+ /* c8 ignore start */
+ toJSON(): SerializedGestureSource {
+   let jsonClone: SerializedGestureSource = {
+     identifier: this.identifier,
+     isFromTouch: this.isFromTouch,
+     path: this.path.toJSON(),
+     stateToken: this.stateToken
+   };
+
+   return jsonClone;
+   /* c8 ignore stop */
+   /* c8 ignore next 2 */
+   // esbuild or tsc seems to mangle the 'ignore stop' if put outside the ending brace.
+ }
 }
 
 export class GestureSourceSubview<

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts

@@ -15,6 +15,12 @@ export class GestureStageReport<Type, StateToken = any> {
    */
   public readonly matchedId: string;
 
+  /**
+   * The set id of gesture models that were allowed for this stage of the
+   * GestureSequence.
+   */
+  public readonly gestureSetId: string;
+
   public readonly linkType: MatchResult<Type>['action']['type'];
   /**
    * The `item`, if any, specified for selection by the matched gesture model.
@@ -29,8 +35,9 @@ export class GestureStageReport<Type, StateToken = any> {
 
   public readonly allSourceIds: string[];
 
-  constructor(selection: MatcherSelection<Type, StateToken>) {
+  constructor(selection: MatcherSelection<Type, StateToken>, gestureSetId: string) {
     const { matcher, result } = selection;
+    this.gestureSetId = gestureSetId;
     this.matchedId = matcher?.model.id;
     this.linkType = result.action.type;
     this.item = result.action.item;
@@ -194,7 +201,8 @@ export class GestureSequence<Type, StateToken = any> extends EventEmitter<EventM
     }
 
   private readonly selectionHandler = async (selection: MatcherSelection<Type, StateToken>) => {
-    const matchReport = new GestureStageReport<Type, StateToken>(selection);
+    const gestureSet = this.pushedSelector?.baseGestureSetId || this.selector?.baseGestureSetId;
+    const matchReport = new GestureStageReport<Type, StateToken>(selection, gestureSet);
     if(selection.matcher) {
       this.stageReports.push(matchReport);
     }
@@ -381,6 +389,10 @@ export class GestureSequence<Type, StateToken = any> extends EventEmitter<EventM
       this.emit('complete');
     }
   }
+
+  public toJSON(): any {
+    return this.stageReports;
+  }
 }
 
 export function modelSetForAction<Type, StateToken>(

common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts

@@ -39,9 +39,14 @@ export class TouchpointCoordinator<HoveredItemType, StateToken=any> extends Even
 
   private _stateToken: StateToken;
 
-  public constructor(gestureModelDefinitions: GestureModelDefs<HoveredItemType, StateToken>, inputEngines?: InputEngineBase<HoveredItemType, StateToken>[]) {
+  private _history: (GestureSource<HoveredItemType> | GestureSequence<HoveredItemType, StateToken>)[] = [];
+  private historyMax: number;
+
+  public constructor(gestureModelDefinitions: GestureModelDefs<HoveredItemType, StateToken>, inputEngines?: InputEngineBase<HoveredItemType, StateToken>[], historyLength?: number) {
     super();
 
+    this.historyMax = historyLength > 0 ? historyLength : 0;
+
     this.gestureModelDefinitions = gestureModelDefinitions;
     this.inputEngines = [];
     if(inputEngines) {
@@ -165,6 +170,16 @@ export class TouchpointCoordinator<HoveredItemType, StateToken=any> extends Even
     this.inputEngines.push(engine);
   }
 
+  private recordHistory(gesture: typeof this._history[0]) {
+    const histMax = this.historyMax;
+    if(histMax > 0) {
+      if(this._history.length == histMax) {
+        this._history.shift();
+      }
+      this._history.push(gesture);
+    }
+  }
+
   private readonly onNewTrackedPath = async (touchpoint: GestureSource<HoveredItemType>) => {
     this.addSimpleSourceHooks(touchpoint);
     const modelDefs = this.gestureModelDefinitions;
@@ -224,12 +239,17 @@ export class TouchpointCoordinator<HoveredItemType, StateToken=any> extends Even
 
     touchpoint.setGestureMatchInspector(this.buildGestureMatchInspector(selector));
 
+    const preGestureScribe = () => {
+      this.recordHistory(touchpoint);
+    }
+
     /*
       If there's an error in code receiving this event, we must not let that break the flow of
       event input processing - we may still have a locking Promise corresponding to our active
       GestureSource.  (See: next comment)
     */
     try {
+      touchpoint.path.on('invalidated', preGestureScribe);
       this.emit('inputstart', touchpoint);
     } catch (err) {
       reportError("Error from 'inputstart' event listener", err);
@@ -283,6 +303,10 @@ export class TouchpointCoordinator<HoveredItemType, StateToken=any> extends Even
     // Could track sequences easily enough; the question is how to tell when to 'let go'.
 
     // No try-catch because only there's no critical code after it.
+    if(!touchpoint.path.wasCancelled) {
+      touchpoint.path.off('invalidated', preGestureScribe);
+      gestureSequence.on('complete', () => this.recordHistory(gestureSequence));
+    }
     this.emit('recognizedgesture', gestureSequence);
   }
 
@@ -294,6 +318,24 @@ export class TouchpointCoordinator<HoveredItemType, StateToken=any> extends Even
     return [].concat(this.inputEngines.map((engine) => engine.activeSources).reduce((merged, arr) => merged.concat(arr), []));
   }
 
+  public get history() {
+    return this._history;
+  }
+
+  public get historyJSON() {
+    const sanitizingReplacer = function (key: string, value) {
+      if(key == 'item') {
+        // KMW 'key' elements involve circular refs.
+        // Just return the key ID.  (Assumes use in KMW)
+        return value?.id;
+      } else {
+        return value;
+      }
+    }
+
+    return JSON.stringify(this.history, sanitizingReplacer, 2);
+  }
+
   /**
    * The current 'state token' to be set for newly-starting gestures for use by gesture-recognizer
    * consumers, their item-identifier lookup functions, and their gesture model definitions.

common/web/gesture-recognizer/src/engine/index.ts

@@ -10,8 +10,8 @@ export { InputSample } from "./headless/inputSample.js";
 export { GesturePath } from "./headless/gesturePath.js";
 export { GestureDebugPath } from "./headless/gestureDebugPath.js"
 export { ConfigChangeClosure, GestureStageReport, GestureSequence } from "./headless/gestures/matchers/gestureSequence.js";
-export { GestureSource, GestureSourceSubview, buildGestureMatchInspector } from "./headless/gestureSource.js";
-export { SerializedGestureSource, GestureDebugSource } from "./headless/gestureDebugSource.js";
+export { GestureSource, GestureSourceSubview, buildGestureMatchInspector, SerializedGestureSource } from "./headless/gestureSource.js";
+export { GestureDebugSource } from "./headless/gestureDebugSource.js";
 export { MouseEventEngine } from "./mouseEventEngine.js";
 export { PaddedZoneSource } from './configuration/paddedZoneSource.js';
 export { RecognitionZoneSource } from './configuration/recognitionZoneSource.js';

common/web/gesture-recognizer/src/test/auto/headless/gesturePath.js

@@ -31,8 +31,10 @@ describe("GesturePath", function() {
       assert.isFalse(reconstructedPath.wasCancelled);
 
       assert.sameDeepOrderedMembers(reconstructedPath.coords, rawPathObject.coords);
-      assert.notEqual(reconstructedPath.toJSON(), rawPathObject);
-      assert.deepEqual(reconstructedPath.toJSON(), rawPathObject);
+      const reconstructedPathJSON = reconstructedPath.toJSON();
+      delete reconstructedPathJSON.stats;
+      assert.notEqual(reconstructedPathJSON, rawPathObject);
+      assert.deepEqual(reconstructedPathJSON, rawPathObject);
     });
 
     it('synthetic', () => {
@@ -92,8 +94,7 @@ describe("GesturePath", function() {
   "wasCancelled": true
 }
       `.trim();
-
-      assert.equal(JSON.stringify(serializationObj, null, 2), SERIALIZATION_TO_MATCH);
+      assert.equal(JSON.stringify(serializationObj, (key, value) => key == 'stats' ? undefined : value, 2), SERIALIZATION_TO_MATCH);
     });
   });
 

web/src/engine/osk/src/visualKeyboard.ts

@@ -61,6 +61,21 @@ import Flick, { buildFlickScroller } from './input/gestures/browser/flick.js';
 import { GesturePreviewHost } from './keyboard-layout/gesturePreviewHost.js';
 import OSKBaseKey from './keyboard-layout/oskBaseKey.js';
 import { OSKResourcePathConfiguration } from './index.js';
+import KEYMAN_VERSION from '@keymanapp/keyman-version';
+
+/**
+ * Gesture history data will include each touchpath sample observed during its
+ * lifetime in addition to its lifetime stats.
+ */
+// @ts-ignore
+const DEBUG_GESTURES: boolean = KEYMAN_VERSION.TIER != 'stable' || KEYMAN_VERSION.VERSION_ENVIRONMENT != '';
+
+/**
+ * If greater than zero, `this.gestureEngine.history` & `this.gestureEngine.historyJSON`
+ * will contain report-data this many of the most-recently completed gesture inputs in
+ * order of their time of completion.
+ */
+const DEBUG_HISTORY_COUNT: number = DEBUG_GESTURES ? 10 : 0;
 
 interface KeyRuleEffects {
   contextToken?: number,
@@ -403,7 +418,14 @@ export default class VisualKeyboard extends EventEmitter<EventMap> implements Ke
          */
 
         return this.layerGroup.findNearestKey(sample);
-      }
+      },
+      /* When enabled, facilitates investigation of perceived odd behaviors observed on Android devices
+        in association with issues like #11221 and #11183.  "Recordings" are only accessible within
+        the mobile apps via WebView inspection and outside the apps via Developer mode in the browser;
+        they are not transmitted or uploaded automatically.
+      */
+      recordingMode: DEBUG_GESTURES,
+      historyLength: DEBUG_HISTORY_COUNT
     };
 
     this.gestureParams.longpress.permitsFlick = (key) => {