diff --git a/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts b/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts index 7568b6df530..bacbcbd8d97 100644 --- a/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts +++ b/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts @@ -98,6 +98,12 @@ export interface GestureRecognizerConfiguration( @@ -116,6 +122,7 @@ export function preprocessRecognizerConfig( processingConfig.itemIdentifier = processingConfig.itemIdentifier ?? (() => null); processingConfig.recordingMode = !!processingConfig.recordingMode; + processingConfig.historyLength = (processingConfig.historyLength ?? 0) > 0 ? processingConfig.historyLength : 0; if(!config.paddedSafeBounds) { let paddingArray = config.safeBoundPadding; diff --git a/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts b/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts index d1722551228..bdae72f3f2e 100644 --- a/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts +++ b/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts @@ -19,7 +19,7 @@ export class GestureRecognizer extends Touchp // overhead. gestureModelDefinitions = gestureModelDefinitions || EMPTY_GESTURE_DEFS; - super(gestureModelDefinitions); + super(gestureModelDefinitions, null, preprocessedConfig.historyLength); this.config = preprocessedConfig; this.mouseEngine = new MouseEventEngine(this.config); diff --git a/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts b/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts index 58e3975da3b..b0653691c4c 100644 --- a/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts @@ -435,7 +435,7 @@ export class CumulativePathStats { * Provides a JSON.stringify()-friendly object with the properties most useful for * debugger-based inspection and/or console-logging statements. */ - public get summaryObject() { + public toJSON() { return { angle: this.angle, cardinal: this.cardinalDirection, diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureDebugPath.ts b/common/web/gesture-recognizer/src/engine/headless/gestureDebugPath.ts index 18bfc4cc591..2a685f34494 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureDebugPath.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureDebugPath.ts @@ -9,6 +9,7 @@ import { GesturePath } from "./gesturePath.js"; export type SerializedGesturePath = { coords: Mutable>[]; // ensures type match with public class property. wasCancelled?: boolean; + stats?: CumulativePathStats } interface EventMap { @@ -121,7 +122,8 @@ export class GestureDebugPath extends GesturePath = { - isFromTouch: boolean; - path: SerializedGesturePath; - // identifier is not included b/c it's only needed during live processing. -} - +import { GestureSource, SerializedGestureSource } from "./gestureSource.js"; /** * Represents all metadata needed internally for tracking a single "touch contact point" / "touchpoint" * involved in a potential / recognized gesture as tracked over time. @@ -35,8 +25,6 @@ export type SerializedGestureSource = { */ export class GestureDebugSource extends GestureSource> { // Assertion: must always contain an index 0 - the base recognizer config. - protected recognizerConfigStack: Nonoptional>[]; - private static _jsonIdSeed: -1; /** @@ -78,22 +66,4 @@ export class GestureDebugSource extends Gesture instance._path = path; return instance; } - - /** - * Creates a serialization-friendly version of this instance for use by - * `JSON.stringify`. - */ - /* c8 ignore start */ - toJSON(): SerializedGestureSource { - const path = this.path as GestureDebugPath; - let jsonClone: SerializedGestureSource = { - isFromTouch: this.isFromTouch, - path: path.toJSON() - }; - - return jsonClone; - /* c8 ignore stop */ - /* c8 ignore next 2 */ - // esbuild or tsc seems to mangle the 'ignore stop' if put outside the ending brace. - } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/gesturePath.ts b/common/web/gesture-recognizer/src/engine/headless/gesturePath.ts index b16234cc0e9..02104d75ab0 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gesturePath.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gesturePath.ts @@ -138,4 +138,13 @@ export class GesturePath extends EventEmitter(selector: MatcherSelector) { return (source: GestureSource) => { @@ -10,6 +11,18 @@ export function buildGestureMatchInspector(selector: MatcherSe }; } +/** + * Documents the expected typing of serialized versions of the `GestureSource` class. + */ +export type SerializedGestureSource = { + isFromTouch: boolean; + path: SerializedGesturePath; + stateToken?: StateToken; + identifier?: string; + // identifier is not included b/c it's only needed during live processing. +} + + /** * Represents all metadata needed internally for tracking a single "touch contact point" / "touchpoint" * involved in a potential / recognized gesture as tracked over time. @@ -215,6 +228,25 @@ export class GestureSource< public get currentRecognizerConfig() { return this.recognizerConfigStack[this.recognizerConfigStack.length-1]; } + + /** + * Creates a serialization-friendly version of this instance for use by + * `JSON.stringify`. + */ + /* c8 ignore start */ + toJSON(): SerializedGestureSource { + let jsonClone: SerializedGestureSource = { + identifier: this.identifier, + isFromTouch: this.isFromTouch, + path: this.path.toJSON(), + stateToken: this.stateToken + }; + + return jsonClone; + /* c8 ignore stop */ + /* c8 ignore next 2 */ + // esbuild or tsc seems to mangle the 'ignore stop' if put outside the ending brace. + } } export class GestureSourceSubview< diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts index 51055eefc47..2ca854a0ba2 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts @@ -15,6 +15,12 @@ export class GestureStageReport { */ public readonly matchedId: string; + /** + * The set id of gesture models that were allowed for this stage of the + * GestureSequence. + */ + public readonly gestureSetId: string; + public readonly linkType: MatchResult['action']['type']; /** * The `item`, if any, specified for selection by the matched gesture model. @@ -29,8 +35,9 @@ export class GestureStageReport { public readonly allSourceIds: string[]; - constructor(selection: MatcherSelection) { + constructor(selection: MatcherSelection, gestureSetId: string) { const { matcher, result } = selection; + this.gestureSetId = gestureSetId; this.matchedId = matcher?.model.id; this.linkType = result.action.type; this.item = result.action.item; @@ -194,7 +201,8 @@ export class GestureSequence extends EventEmitter) => { - const matchReport = new GestureStageReport(selection); + const gestureSet = this.pushedSelector?.baseGestureSetId || this.selector?.baseGestureSetId; + const matchReport = new GestureStageReport(selection, gestureSet); if(selection.matcher) { this.stageReports.push(matchReport); } @@ -381,6 +389,10 @@ export class GestureSequence extends EventEmitter( diff --git a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts index 157da4b99b4..8c681b7ea79 100644 --- a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts +++ b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts @@ -39,9 +39,14 @@ export class TouchpointCoordinator extends Even private _stateToken: StateToken; - public constructor(gestureModelDefinitions: GestureModelDefs, inputEngines?: InputEngineBase[]) { + private _history: (GestureSource | GestureSequence)[] = []; + private historyMax: number; + + public constructor(gestureModelDefinitions: GestureModelDefs, inputEngines?: InputEngineBase[], historyLength?: number) { super(); + this.historyMax = historyLength > 0 ? historyLength : 0; + this.gestureModelDefinitions = gestureModelDefinitions; this.inputEngines = []; if(inputEngines) { @@ -165,6 +170,16 @@ export class TouchpointCoordinator extends Even this.inputEngines.push(engine); } + private recordHistory(gesture: typeof this._history[0]) { + const histMax = this.historyMax; + if(histMax > 0) { + if(this._history.length == histMax) { + this._history.shift(); + } + this._history.push(gesture); + } + } + private readonly onNewTrackedPath = async (touchpoint: GestureSource) => { this.addSimpleSourceHooks(touchpoint); const modelDefs = this.gestureModelDefinitions; @@ -224,12 +239,17 @@ export class TouchpointCoordinator extends Even touchpoint.setGestureMatchInspector(this.buildGestureMatchInspector(selector)); + const preGestureScribe = () => { + this.recordHistory(touchpoint); + } + /* If there's an error in code receiving this event, we must not let that break the flow of event input processing - we may still have a locking Promise corresponding to our active GestureSource. (See: next comment) */ try { + touchpoint.path.on('invalidated', preGestureScribe); this.emit('inputstart', touchpoint); } catch (err) { reportError("Error from 'inputstart' event listener", err); @@ -283,6 +303,10 @@ export class TouchpointCoordinator extends Even // Could track sequences easily enough; the question is how to tell when to 'let go'. // No try-catch because only there's no critical code after it. + if(!touchpoint.path.wasCancelled) { + touchpoint.path.off('invalidated', preGestureScribe); + gestureSequence.on('complete', () => this.recordHistory(gestureSequence)); + } this.emit('recognizedgesture', gestureSequence); } @@ -294,6 +318,24 @@ export class TouchpointCoordinator extends Even return [].concat(this.inputEngines.map((engine) => engine.activeSources).reduce((merged, arr) => merged.concat(arr), [])); } + public get history() { + return this._history; + } + + public get historyJSON() { + const sanitizingReplacer = function (key: string, value) { + if(key == 'item') { + // KMW 'key' elements involve circular refs. + // Just return the key ID. (Assumes use in KMW) + return value?.id; + } else { + return value; + } + } + + return JSON.stringify(this.history, sanitizingReplacer, 2); + } + /** * The current 'state token' to be set for newly-starting gestures for use by gesture-recognizer * consumers, their item-identifier lookup functions, and their gesture model definitions. diff --git a/common/web/gesture-recognizer/src/engine/index.ts b/common/web/gesture-recognizer/src/engine/index.ts index 3da7cc16dac..e6570362d1f 100644 --- a/common/web/gesture-recognizer/src/engine/index.ts +++ b/common/web/gesture-recognizer/src/engine/index.ts @@ -10,8 +10,8 @@ export { InputSample } from "./headless/inputSample.js"; export { GesturePath } from "./headless/gesturePath.js"; export { GestureDebugPath } from "./headless/gestureDebugPath.js" export { ConfigChangeClosure, GestureStageReport, GestureSequence } from "./headless/gestures/matchers/gestureSequence.js"; -export { GestureSource, GestureSourceSubview, buildGestureMatchInspector } from "./headless/gestureSource.js"; -export { SerializedGestureSource, GestureDebugSource } from "./headless/gestureDebugSource.js"; +export { GestureSource, GestureSourceSubview, buildGestureMatchInspector, SerializedGestureSource } from "./headless/gestureSource.js"; +export { GestureDebugSource } from "./headless/gestureDebugSource.js"; export { MouseEventEngine } from "./mouseEventEngine.js"; export { PaddedZoneSource } from './configuration/paddedZoneSource.js'; export { RecognitionZoneSource } from './configuration/recognitionZoneSource.js'; diff --git a/common/web/gesture-recognizer/src/test/auto/headless/gesturePath.js b/common/web/gesture-recognizer/src/test/auto/headless/gesturePath.js index dcb1a231fd7..24ae343a6ff 100644 --- a/common/web/gesture-recognizer/src/test/auto/headless/gesturePath.js +++ b/common/web/gesture-recognizer/src/test/auto/headless/gesturePath.js @@ -31,8 +31,10 @@ describe("GesturePath", function() { assert.isFalse(reconstructedPath.wasCancelled); assert.sameDeepOrderedMembers(reconstructedPath.coords, rawPathObject.coords); - assert.notEqual(reconstructedPath.toJSON(), rawPathObject); - assert.deepEqual(reconstructedPath.toJSON(), rawPathObject); + const reconstructedPathJSON = reconstructedPath.toJSON(); + delete reconstructedPathJSON.stats; + assert.notEqual(reconstructedPathJSON, rawPathObject); + assert.deepEqual(reconstructedPathJSON, rawPathObject); }); it('synthetic', () => { @@ -92,8 +94,7 @@ describe("GesturePath", function() { "wasCancelled": true } `.trim(); - - assert.equal(JSON.stringify(serializationObj, null, 2), SERIALIZATION_TO_MATCH); + assert.equal(JSON.stringify(serializationObj, (key, value) => key == 'stats' ? undefined : value, 2), SERIALIZATION_TO_MATCH); }); }); diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 4a07df109ba..6b8af2e0c17 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -61,6 +61,21 @@ import Flick, { buildFlickScroller } from './input/gestures/browser/flick.js'; import { GesturePreviewHost } from './keyboard-layout/gesturePreviewHost.js'; import OSKBaseKey from './keyboard-layout/oskBaseKey.js'; import { OSKResourcePathConfiguration } from './index.js'; +import KEYMAN_VERSION from '@keymanapp/keyman-version'; + +/** + * Gesture history data will include each touchpath sample observed during its + * lifetime in addition to its lifetime stats. + */ +// @ts-ignore +const DEBUG_GESTURES: boolean = KEYMAN_VERSION.TIER != 'stable' || KEYMAN_VERSION.VERSION_ENVIRONMENT != ''; + +/** + * If greater than zero, `this.gestureEngine.history` & `this.gestureEngine.historyJSON` + * will contain report-data this many of the most-recently completed gesture inputs in + * order of their time of completion. + */ +const DEBUG_HISTORY_COUNT: number = DEBUG_GESTURES ? 10 : 0; interface KeyRuleEffects { contextToken?: number, @@ -403,7 +418,14 @@ export default class VisualKeyboard extends EventEmitter implements Ke */ return this.layerGroup.findNearestKey(sample); - } + }, + /* When enabled, facilitates investigation of perceived odd behaviors observed on Android devices + in association with issues like #11221 and #11183. "Recordings" are only accessible within + the mobile apps via WebView inspection and outside the apps via Developer mode in the browser; + they are not transmitted or uploaded automatically. + */ + recordingMode: DEBUG_GESTURES, + historyLength: DEBUG_HISTORY_COUNT }; this.gestureParams.longpress.permitsFlick = (key) => {