diff --git a/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts b/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts index 98516b2dde5..99ad3334a7a 100644 --- a/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts +++ b/common/web/gesture-recognizer/src/engine/configuration/gestureRecognizerConfiguration.ts @@ -8,6 +8,8 @@ import { Nonoptional } from "../nonoptional.js"; import { PaddedZoneSource } from "./paddedZoneSource.js"; import { RecognitionZoneSource } from "./recognitionZoneSource.js"; +export type ItemIdentifier = (coord: Omit, 'item'>, target: EventTarget) => ItemType; + // For example, customization of a longpress timer's length need not be readonly. export interface GestureRecognizerConfiguration { /** @@ -84,10 +86,11 @@ export interface GestureRecognizerConfiguration, 'item'>, target: EventTarget) => HoveredItemType; + readonly itemIdentifier?: ItemIdentifier; } export function preprocessRecognizerConfig( diff --git a/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts b/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts index 32620c45b16..86fd645d0a7 100644 --- a/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts +++ b/common/web/gesture-recognizer/src/engine/gestureRecognizer.ts @@ -11,7 +11,7 @@ export class GestureRecognizer extends Touchp private readonly mouseEngine: MouseEventEngine; private readonly touchEngine: TouchEventEngine; - public constructor(gestureModelDefinitions: GestureModelDefs, config: GestureRecognizerConfiguration) { + public constructor(gestureModelDefinitions: GestureModelDefs, config: GestureRecognizerConfiguration) { const preprocessedConfig = preprocessRecognizerConfig(config); // Possibly just a stop-gap measure... but this provides an empty gesture-spec set definition diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts index b1dc3fb7a2e..064a429b347 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts @@ -4,8 +4,8 @@ import { GestureRecognizerConfiguration, preprocessRecognizerConfig } from "../c import { Nonoptional } from "../nonoptional.js"; import { MatcherSelector } from "./gestures/matchers/matcherSelector.js"; -export function buildGestureMatchInspector(selector: MatcherSelector) { - return (source: GestureSource) => { +export function buildGestureMatchInspector(selector: MatcherSelector) { + return (source: GestureSource) => { return selector.potentialMatchersForSource(source).map((matcher) => matcher.model.id); }; } @@ -277,6 +277,7 @@ export class GestureSourceSubview extends Ges super(source.rawIdentifier, configStack, source.isFromTouch); const baseSource = this._baseSource = source instanceof GestureSourceSubview ? source._baseSource : source; + this.stateToken = source.stateToken; /** * Provides a coordinate-system translation for source subviews. @@ -286,7 +287,23 @@ export class GestureSourceSubview extends Ges const translation = this.recognizerTranslation; // Provide a coordinate-system translation for source subviews. // The base version still needs to use the original coord system, though. - return {...sample, targetX: sample.targetX - translation.x, targetY: sample.targetY - translation.y}; + const transformedSample = {...sample, targetX: sample.targetX - translation.x, targetY: sample.targetY - translation.y}; + + // If the subview is operating from the perspective of a different state token than its base source, + // its samples' item fields will need correction. + // + // This can arise during multitap-like scenarios. + if(source.stateToken != baseSource.stateToken) { + transformedSample.item = this.currentRecognizerConfig.itemIdentifier( + { + ...sample, + stateToken: this.stateToken + }, + null + ); + } + + return transformedSample; } // Note: we don't particularly need subviews to track the actual coords aside from diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index 795147068d4..e8880c395ce 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -4,6 +4,7 @@ import { GestureModel, GestureResolution, GestureResolutionSpec, RejectionDefaul import { ManagedPromise, TimeoutPromise } from "@keymanapp/web-utils"; import { FulfillmentCause, PathMatcher } from "./pathMatcher.js"; +import { ItemIdentifier } from "../../../configuration/gestureRecognizerConfiguration.js"; /** * This interface specifies the minimal data necessary for setting up gesture-selection @@ -18,12 +19,12 @@ import { FulfillmentCause, PathMatcher } from "./pathMatcher.js"; * of this interface, they can be integrated into the gesture-sequence staging system - * even if not matched directly by the recognizer itself. */ -export interface PredecessorMatch { +export interface PredecessorMatch { readonly sources: GestureSource[]; readonly allSourceIds: string[]; - readonly primaryPath: GestureSource; + readonly primaryPath: GestureSource; readonly result: MatchResult; - readonly model?: GestureModel; + readonly model?: GestureModel; readonly baseItem: Type; } @@ -37,9 +38,9 @@ export interface MatchResultSpec { readonly action: GestureResolutionSpec } -export class GestureMatcher implements PredecessorMatch { +export class GestureMatcher implements PredecessorMatch { private sustainTimerPromise?: TimeoutPromise; - public readonly model: GestureModel; + public readonly model: GestureModel; private readonly pathMatchers: PathMatcher[]; @@ -55,7 +56,7 @@ export class GestureMatcher implements PredecessorMatch { private _isCancelled: boolean = false; - private readonly predecessor?: PredecessorMatch; + private readonly predecessor?: PredecessorMatch; private readonly publishedPromise: ManagedPromise>; // unsure on the actual typing at the moment. private _result: MatchResult; @@ -64,7 +65,10 @@ export class GestureMatcher implements PredecessorMatch { return this.publishedPromise.corePromise; } - constructor(model: GestureModel, sourceObj: GestureSource | PredecessorMatch) { + constructor( + model: GestureModel, + sourceObj: GestureSource | PredecessorMatch + ) { /* c8 ignore next 5 */ if(!model || !sourceObj) { throw new Error("Construction of GestureMatcher requires a gesture-model spec and a source for related contact points."); @@ -244,6 +248,15 @@ export class GestureMatcher implements PredecessorMatch { const matcher = this.pathMatchers[i]; const contactSpec = this.model.contacts[i]; + /* Future TODO: + * This should probably include "committing" the state token and items used by the subview, + * should they differ from the base source's original values. + * + * That said, this is only a 'polish' task, as we aren't actually relying on the base source + * once we've started identifying gestures. It'll likely only matter if external users + * desire to utilize the recognizer. + */ + // If the path already terminated, no need to evaluate further for this contact point. if(matcher.source.isPathComplete) { continue; @@ -348,11 +361,14 @@ export class GestureMatcher implements PredecessorMatch { // Add it early, as we need it to be accessible for reference via .primaryPath stuff below. this.pathMatchers.push(contactModel); + let ancestorSource: GestureSource = null; let baseItem: Type = null; // If there were no existing contacts but a predecessor exists and a sustain timer // has been specified, it needs special base-item handling. if(!existingContacts && this.predecessor && this.model.sustainTimer) { + ancestorSource = this.predecessor.primaryPath; const baseItemMode = this.model.sustainTimer.baseItem ?? 'result'; + let baseStateToken: StateToken; switch(baseItemMode) { case 'none': @@ -360,9 +376,11 @@ export class GestureMatcher implements PredecessorMatch { break; case 'base': baseItem = this.predecessor.primaryPath.baseItem; + baseStateToken = this.predecessor.primaryPath.stateToken; break; case 'result': baseItem = this.predecessor.result.action.item; + baseStateToken = this.predecessor.primaryPath.currentSample.stateToken; break; } @@ -370,14 +388,31 @@ export class GestureMatcher implements PredecessorMatch { // continuation and successor to `predecessor.primaryPath`. Its base `item` // should reflect this. simpleSource.baseItem = baseItem ?? simpleSource.baseItem; + simpleSource.stateToken = baseStateToken; + + // May be missing during unit tests. + if(simpleSource.currentRecognizerConfig) { + simpleSource.currentSample.item = simpleSource.currentRecognizerConfig.itemIdentifier({ + ...simpleSource.currentSample, + stateToken: baseStateToken + }, + null + ); + } } else { // just use the highest-priority item source's base item and call it a day. // There's no need to refer to some previously-existing source for comparison. baseItem = this.primaryPath.baseItem; + ancestorSource = this.primaryPath; } if(contactSpec.model.allowsInitialState) { - const initialStateCheck = contactSpec.model.allowsInitialState(simpleSource.currentSample, this.primaryPath?.currentSample, baseItem); + const initialStateCheck = contactSpec.model.allowsInitialState( + simpleSource.currentSample, + ancestorSource.currentSample, + baseItem, + ancestorSource.stateToken + ); if(!initialStateCheck) { this.finalize(false, 'path'); @@ -385,7 +420,10 @@ export class GestureMatcher implements PredecessorMatch { } // Now that we've done the initial-state check, we can check for instantly-matching path models. - contactModel.update(); + const result = contactModel.update(); + if(result.type == 'reject' && this.model.id == 'modipress-multitap-end') { + console.log('temp'); + } contactModel.promise.then((resolution) => { this.finalize(resolution.type == 'resolve', resolution.cause); diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts index 9577b40b197..8b989ad354e 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureSequence.ts @@ -7,7 +7,7 @@ import { GestureModel, GestureResolution } from "../specs/gestureModel.js"; import { MatcherSelection, MatcherSelector } from "./matcherSelector.js"; import { GestureRecognizerConfiguration, TouchpointCoordinator } from "../../../index.js"; -export class GestureStageReport { +export class GestureStageReport { /** * The id of the GestureModel spec that was matched at this stage of the * GestureSequence. @@ -28,7 +28,7 @@ export class GestureStageReport { public readonly allSourceIds: string[]; - constructor(selection: MatcherSelection) { + constructor(selection: MatcherSelection) { const { matcher, result } = selection; this.matchedId = matcher?.model.id; this.linkType = result.action.type; @@ -74,34 +74,34 @@ interface PopConfig { export type ConfigChangeClosure = (configStackCommand: PushConfig | PopConfig) => void; -interface EventMap { +interface EventMap { stage: ( - stageReport: GestureStageReport, + stageReport: GestureStageReport, changeConfiguration: ConfigChangeClosure ) => void; complete: () => void; } -export class GestureSequence extends EventEmitter> { - public stageReports: GestureStageReport[]; +export class GestureSequence extends EventEmitter> { + public stageReports: GestureStageReport[]; // It's not specific to just this sequence... but it does have access to // the potential next stages. - private selector: MatcherSelector; + private selector: MatcherSelector; // We need this reference in order to properly handle 'setchange' resolution actions when staging. private touchpointCoordinator: TouchpointCoordinator; // Selectors have locked-in 'base gesture sets'; this is only non-null if // in a 'setchange' action. - private pushedSelector?: MatcherSelector; + private pushedSelector?: MatcherSelector; - private gestureConfig: GestureModelDefs; + private gestureConfig: GestureModelDefs; // Note: the first stage will be available under `stageReports` after awaiting a simple Promise.resolve(). constructor( - firstSelectionMatch: MatcherSelection, - gestureModelDefinitions: GestureModelDefs, - selector: MatcherSelector, + firstSelectionMatch: MatcherSelection, + gestureModelDefinitions: GestureModelDefs, + selector: MatcherSelector, touchpointCoordinator: TouchpointCoordinator ) { super(); @@ -174,8 +174,8 @@ export class GestureSequence extends EventEmitter> { return potentialMatches; } - private readonly selectionHandler = (selection: MatcherSelection) => { - const matchReport = new GestureStageReport(selection); + private readonly selectionHandler = (selection: MatcherSelection) => { + const matchReport = new GestureStageReport(selection); if(selection.matcher) { this.stageReports.push(matchReport); } @@ -267,10 +267,12 @@ export class GestureSequence extends EventEmitter> { if(selection.result.action.type == 'chain') { const targetSet = selection.result.action.selectionMode; - // push the new one. - const changedSetSelector = new MatcherSelector(targetSet); - this.pushedSelector = changedSetSelector; - this.touchpointCoordinator?.pushSelector(changedSetSelector); + if(targetSet) { + // push the new one. + const changedSetSelector = new MatcherSelector(targetSet); + this.pushedSelector = changedSetSelector; + this.touchpointCoordinator?.pushSelector(changedSetSelector); + } } } } else { @@ -284,7 +286,10 @@ export class GestureSequence extends EventEmitter> { } } - private readonly modelResetHandler = (selection: MatcherSelection, replaceModelWith: (model: GestureModel) => void) => { + private readonly modelResetHandler = ( + selection: MatcherSelection, + replaceModelWith: (model: GestureModel) => void + ) => { const sourceIds = selection.matcher.allSourceIds; // If none of the sources involved match a source already included in the sequence, bypass @@ -304,11 +309,11 @@ export class GestureSequence extends EventEmitter> { }; } -export function modelSetForAction( +export function modelSetForAction( action: GestureResolution, - gestureModelDefinitions: GestureModelDefs, + gestureModelDefinitions: GestureModelDefs, activeSetId: string -): GestureModel[] { +): GestureModel[] { switch(action.type) { case 'none': case 'complete': diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts index 3a4c0d1a5f1..ef5679522bf 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts @@ -6,24 +6,27 @@ import { GestureSource, GestureSourceSubview } from "../../gestureSource.js"; import { GestureMatcher, MatchResult, PredecessorMatch } from "./gestureMatcher.js"; import { GestureModel } from "../specs/gestureModel.js"; import { GestureSequence } from "./index.js"; +import { ItemIdentifier } from "../../../configuration/gestureRecognizerConfiguration.js"; -interface GestureSourceTracker { +interface GestureSourceTracker { /** * Should be the actual GestureSource instance, not a subview thereof. * Each `GestureMatcher` will construct its own 'subview' into the GestureSource * based on its model's needs. */ source: GestureSource; - matchPromise: ManagedPromise>; + matchPromise: ManagedPromise>; } -export interface MatcherSelection { - matcher: PredecessorMatch, +export interface MatcherSelection { + matcher: PredecessorMatch, result: MatchResult } -interface EventMap { - 'rejectionwithaction': (selection: MatcherSelection, replaceModelWith: (replacementModel: GestureModel) => void) => void; +interface EventMap { + 'rejectionwithaction': ( + selection: MatcherSelection, + replaceModelWith: (replacementModel: GestureModel) => void) => void; } /** @@ -38,9 +41,9 @@ interface EventMap { * GestureSource, the Promise will resolve when the last potential model is rejected, * providing values indicating match failure and the action to be taken. */ -export class MatcherSelector extends EventEmitter> { - private _sourceSelector: GestureSourceTracker[] = []; - private potentialMatchers: GestureMatcher[] = []; +export class MatcherSelector extends EventEmitter> { + private _sourceSelector: GestureSourceTracker[] = []; + private potentialMatchers: GestureMatcher[] = []; public readonly baseGestureSetId: string; @@ -120,9 +123,9 @@ export class MatcherSelector extends EventEmitter> { * @param gestureModelSet */ public matchGesture( - source: GestureSource, - gestureModelSet: GestureModel[] - ): Promise>; + source: GestureSource, + gestureModelSet: GestureModel[] + ): Promise>; /** * Facilitates matching a new stage in an ongoing gesture-stage sequence based on a previously- @@ -131,14 +134,14 @@ export class MatcherSelector extends EventEmitter> { * @param gestureModelSet */ public matchGesture( - priorStageMatcher: PredecessorMatch, - gestureModelSet: GestureModel[] - ): Promise>; + priorStageMatcher: PredecessorMatch, + gestureModelSet: GestureModel[] + ): Promise>; public matchGesture( - source: GestureSource | PredecessorMatch, - gestureModelSet: GestureModel[] - ): Promise> { + source: GestureSource | PredecessorMatch, + gestureModelSet: GestureModel[] + ): Promise> { /* * To be clear, this _starts_ the source-tracking process. It's an async process, though. * @@ -158,7 +161,7 @@ export class MatcherSelector extends EventEmitter> { }) } - const matchPromise = new ManagedPromise>(); + const matchPromise = new ManagedPromise>(); /* * First... @@ -174,7 +177,7 @@ export class MatcherSelector extends EventEmitter> { // Sets up source selectors - the object that matches a source against its Promise. // Promises only resolve once, after all - once called, a "selection" has been made. - const sourceSelectors: GestureSourceTracker = { + const sourceSelectors: GestureSourceTracker = { source: src, matchPromise: matchPromise }; @@ -319,7 +322,7 @@ export class MatcherSelector extends EventEmitter> { }); } - private matcherSelectionFilter(matcher: GestureMatcher, matchSynchronizers: ManagedPromise[]) { + private matcherSelectionFilter(matcher: GestureMatcher, matchSynchronizers: ManagedPromise[]) { // Returns a closure-captured Promise-resolution handler used by individual GestureMatchers managed // by this class instance. return async (result: MatchResult) => { @@ -422,7 +425,7 @@ export class MatcherSelector extends EventEmitter> { if(!result.matched) { // There is an action to be resolved... // But we didn't actually MATCH a gesture. - const replacer = (replacementModel: GestureModel) => { + const replacer = (replacementModel: GestureModel) => { const replacementMatcher = new GestureMatcher(replacementModel, matcher); /* IMPORTANT: verify that the replacement model is initially valid. diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/pathMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/pathMatcher.ts index f1aeec7e2e9..8b288afaebb 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/pathMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/pathMatcher.ts @@ -22,9 +22,9 @@ export interface PathNotFulfilled { type PathMatchResult = PathMatchRejection | PathMatchResolution; type PathUpdateResult = PathMatchResult | PathNotFulfilled; -export class PathMatcher { +export class PathMatcher { private timerPromise?: TimeoutPromise; - public readonly model: ContactModel; + public readonly model: ContactModel; // During execution, source.path is fine... but once this matcher's role is done, // `source` will continue to receive edits and may even change the instance @@ -38,7 +38,7 @@ export class PathMatcher { return this.publishedPromise.corePromise; } - constructor(model: ContactModel, source: GestureSource) { + constructor(model: ContactModel, source: GestureSource) { /* c8 ignore next 3 */ if(!model || !source) { throw new Error("A gesture-path source and contact-path model must be specified."); diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts index 2fe1b589c8d..6c7492f21fd 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/contactModel.ts @@ -6,7 +6,7 @@ type SimpleStringResult = 'resolve' | 'reject'; export type PointModelResolution = SimpleStringResult; -export interface ContactModel { +export interface ContactModel { pathModel: PathModel, pathResolutionAction: PointModelResolution, @@ -72,6 +72,7 @@ export interface ContactModel { readonly allowsInitialState?: ( incomingSample: InputSample, comparisonSample?: InputSample, - baseItem?: Type + baseItem?: Type, + stateToken?: StateToken ) => boolean; } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModel.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModel.ts index 622a85ff967..f72438382b6 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModel.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModel.ts @@ -83,7 +83,7 @@ type ResolutionStruct = ResolutionChain | ResolutionComplete; export type GestureResolutionSpec = ResolutionStruct & ResolutionItemSpec; export type GestureResolution = (ResolutionStruct | RejectionDefault | RejectionReplace) & ResolutionItem; -export interface GestureModel { +export interface GestureModel { // Gestures may want to say "build gesture of type `id`" for a followup-gesture. readonly id: string; @@ -97,7 +97,7 @@ export interface GestureModel { // One or more "touchpath models" - how a touchpath matching this gesture would look, based on its // ordinal position. (Same order as in the TrackedInput) readonly contacts: { - model: ContactModel, + model: ContactModel, /** * Indicates that the corresponding GestureSource should not be considered part of the * Gesture sequence being matched, acting more as a separate gesture that 'triggers' a state diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModelDefs.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModelDefs.ts index 002b3cce074..9ba8ecd5a2f 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModelDefs.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/specs/gestureModelDefs.ts @@ -3,15 +3,15 @@ import * as gestures from "../index.js"; // Prototype spec for the main gesture & gesture-set definitions. // A work in-progress. Should probably land somewhere within headless/gestures/specs/. // ... with the following two functions, as well. -export interface GestureModelDefs { - gestures: gestures.specs.GestureModel[], +export interface GestureModelDefs { + gestures: gestures.specs.GestureModel[], sets: { default: string[], } & Record; } -export function getGestureModel(defs: GestureModelDefs, id: string): gestures.specs.GestureModel { +export function getGestureModel(defs: GestureModelDefs, id: string): gestures.specs.GestureModel { const result = defs.gestures.find((spec) => spec.id == id); if(!result) { throw new Error(`Could not find spec for gesture with id '${id}'`); @@ -20,7 +20,7 @@ export function getGestureModel(defs: GestureModelDefs, id: string): return result; } -export function getGestureModelSet(defs: GestureModelDefs, id: string): gestures.specs.GestureModel[] { +export function getGestureModelSet(defs: GestureModelDefs, id: string): gestures.specs.GestureModel[] { let idSet = defs.sets[id]; if(!idSet) { throw new Error(`Could not find a defined gesture-set with id '${id}'`); @@ -42,4 +42,4 @@ export const EMPTY_GESTURE_DEFS = { sets: { default: [] } -} as GestureModelDefs \ No newline at end of file +} as GestureModelDefs \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts index 312378f8fb6..1ce7d161cc7 100644 --- a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts +++ b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts @@ -14,7 +14,7 @@ interface EventMap { */ 'inputstart': (input: GestureSource) => void; - 'recognizedgesture': (sequence: GestureSequence) => void; + 'recognizedgesture': (sequence: GestureSequence) => void; } /** @@ -27,16 +27,16 @@ interface EventMap { */ export class TouchpointCoordinator extends EventEmitter> { private inputEngines: InputEngineBase[]; - private selectorStack: MatcherSelector[] = [new MatcherSelector()]; + private selectorStack: MatcherSelector[] = [new MatcherSelector()]; - private gestureModelDefinitions: GestureModelDefs; + private gestureModelDefinitions: GestureModelDefs; private _activeSources: GestureSource[] = []; - private _activeGestures: GestureSequence[] = []; + private _activeGestures: GestureSequence[] = []; private _stateToken: StateToken; - public constructor(gestureModelDefinitions: GestureModelDefs, inputEngines?: InputEngineBase[]) { + public constructor(gestureModelDefinitions: GestureModelDefs, inputEngines?: InputEngineBase[]) { super(); this.gestureModelDefinitions = gestureModelDefinitions; @@ -50,7 +50,10 @@ export class TouchpointCoordinator extends Even this.selectorStack[0].on('rejectionwithaction', this.modelResetHandler) } - private readonly modelResetHandler = (selection: MatcherSelection, replaceModelWith: (model: GestureModel) => void) => { + private readonly modelResetHandler = ( + selection: MatcherSelection, + replaceModelWith: (model: GestureModel) => void + ) => { const sourceIds = selection.matcher.allSourceIds; // If there's an active gesture that uses a source noted in the selection, it's the responsibility @@ -68,12 +71,12 @@ export class TouchpointCoordinator extends Even } }; - public pushSelector(selector: MatcherSelector) { + public pushSelector(selector: MatcherSelector) { this.selectorStack.push(selector); selector.on('rejectionwithaction', this.modelResetHandler); } - public popSelector(selector: MatcherSelector) { + public popSelector(selector: MatcherSelector) { /* c8 ignore start */ if(this.selectorStack.length <= 1) { throw new Error("May not pop the original, base gesture selector."); @@ -144,7 +147,7 @@ export class TouchpointCoordinator extends Even this.emit('inputstart', touchpoint); } - public get activeGestures(): GestureSequence[] { + public get activeGestures(): GestureSequence[] { return [].concat(this._activeGestures); } diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 9eeddffd54b..c8515bd8755 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -3,6 +3,7 @@ import KeyEvent, { KeyEventSpec } from "../text/keyEvent.js"; import KeyMapping from "../text/keyMapping.js"; import type { KeyDistribution } from "../text/keyEvent.js"; import type { LayoutKey, LayoutSubKey, LayoutRow, LayoutLayer, LayoutFormFactor, ButtonClass } from "./defaultLayouts.js"; +import { Layouts } from "./defaultLayouts.js"; import type Keyboard from "./keyboard.js"; import { TouchLayout } from "@keymanapp/common-types"; @@ -225,6 +226,27 @@ class ActiveKeyBase { hasMultitaps: false } + // The default-layer shift key on mobile platforms should have a default multitap under + // select conditions. + // + // Note: whether or not any other key has multitaps doesn't matter here. Just THIS one. + if(key.id == 'K_SHIFT' && displayLayer == 'default' && layout.formFactor != 'desktop') { + /* Extra requirements: + * + * 1. The SHIFT key must not specify subkeys or have already-specified multitaps. + * + * Note: touch layouts specified on desktop layouts often do specify subkeys; + * utilized modifiers aside from 'shift' become subkeys of K_SHIFT) + * + * 2. There exists a specified 'caps' layer. Otherwise, there's no destination for + * the default multitap. + * + */ + if(!key.sk && !key.multitap && !!layout.layer.find((entry) => entry.id == 'caps')) { + key.multitap = [Layouts.dfltShiftMultitap]; + } + } + // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. let dummy = new ActiveKeyBase(); let proto = Object.getPrototypeOf(dummy); @@ -788,9 +810,11 @@ export class ActiveLayout implements LayoutFormFactor{ * @param formFactor */ static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout { + /* c8 ignore start */ if(layout == null) { throw new Error("Cannot build an ActiveLayout for a null specification."); } + /* c8 ignore end */ const analysisMetadata: AnalysisMetadata = { hasFlicks: false, diff --git a/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts b/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts index ef1cc4c4b83..be65bbc68c7 100644 --- a/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts +++ b/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts @@ -535,6 +535,15 @@ export class Layouts { return KLS; } + static dfltShiftMultitap: LayoutSubKey = { + // Needs to be something special and unique. Typing restricts us from + // using a reserved key-id prefix, though. + id: "T_MT_SHIFT_TO_CAPS", + text: '*ShiftLock*', + sp: 1, + nextlayer: 'caps' + } + // Defines the default visual layout for a keyboard. /* c8 ignore start */ static dfltLayout: LayoutSpec = { diff --git a/common/web/keyboard-processor/src/keyboards/keyboard.ts b/common/web/keyboard-processor/src/keyboards/keyboard.ts index f70f30fcb90..dff4bdaabfd 100644 --- a/common/web/keyboard-processor/src/keyboards/keyboard.ts +++ b/common/web/keyboard-processor/src/keyboards/keyboard.ts @@ -1,6 +1,6 @@ import Codes from "../text/codes.js"; import { Layouts, type LayoutFormFactor } from "./defaultLayouts.js"; -import { ActiveKey, ActiveLayout } from "./activeLayout.js"; +import { ActiveKey, ActiveLayout, ActiveSubkey } from "./activeLayout.js"; import KeyEvent from "../text/keyEvent.js"; import type OutputTarget from "../text/outputTarget.js"; @@ -468,7 +468,7 @@ export default class Keyboard { return keyEvent; } - constructKeyEvent(key: ActiveKey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent { + constructKeyEvent(key: ActiveKey | ActiveSubkey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent { // Make a deep copy of our preconstructed key event, filling it out from there. const Lkc = key.baseKeyEvent; Lkc.device = device; diff --git a/web/src/engine/osk/src/input/gestures/browser/multitap.ts b/web/src/engine/osk/src/input/gestures/browser/multitap.ts new file mode 100644 index 00000000000..0db724789c2 --- /dev/null +++ b/web/src/engine/osk/src/input/gestures/browser/multitap.ts @@ -0,0 +1,78 @@ +import OSKSubKey from './oskSubKey.js'; +import { type KeyElement } from '../../../keyElement.js'; +import OSKBaseKey from '../../../keyboard-layout/oskBaseKey.js'; +import VisualKeyboard from '../../../visualKeyboard.js'; + +import { DeviceSpec, KeyEvent, ActiveSubkey, ActiveKey } from '@keymanapp/keyboard-processor'; +import { ConfigChangeClosure, GestureRecognizerConfiguration, GestureSequence, PaddedZoneSource } from '@keymanapp/gesture-recognizer'; + +/** + * Represents a potential multitap gesture's implementation within KeymanWeb. + * Once a simple-tap gesture occurs on a key with specified multitap subkeys, + * this class is designed to take over further processing of said gesture. + * This includes providing: + * * UI feedback regarding the state of the ongoing multitap, as appropriate + * * Proper selection of the appropriate multitap key for subsequent taps. + */ +export default class Multitap { + public readonly baseKey: KeyElement; + private readonly multitaps: ActiveSubkey[]; + private tapIndex = 0; + + private sequence: GestureSequence; + + constructor( + source: GestureSequence, + vkbd: VisualKeyboard, + e: KeyElement + ) { + this.baseKey = e; + this.multitaps = e.key.spec.multitap; + + // // For multitaps, keeping the key highlighted makes sense. I think. + // this.baseKey.key.highlight(true); + + source.on('complete', () => { + if(source.stageReports.length > 1) { + } + // this.currentSelection?.key.highlight(false); + this.clear(); + }); + + source.on('stage', (tap) => { + switch(tap.matchedId) { + case 'modipress-multitap-start': + return; + case 'multitap': + case 'modipress-multitap-end': + break; + default: + throw new Error("Unsupported gesture state encountered during multitap sequence"); + } + + // For rota-style behavior + this.tapIndex = (this.tapIndex + 1) % (this.baseKey.key.spec.multitap.length+1); + + const selection = this.tapIndex == 0 + ? this.baseKey.key.spec + : this.multitaps[this.tapIndex-1]; + + const keyEvent = vkbd.keyEventFromSpec(selection); + // TODO: special fat-finger alternates stuff? + vkbd.raiseKeyEvent(keyEvent, null); + }); + + /* In theory, setting up a specialized recognizer config limited to the base key's surface area + * would be pretty ideal - it'd provide automatic cancellation if anywhere else were touched. + * + * However, because multitap keys can swap layers, and because an invisible layer doesn't provide + * the expected bounding-box that it would were it visible, it's anything but straightforward to + * do for certain supported cases. It's simpler to handle this problem by leveraging the + * key-finding operation specified on the gesture model and ensuring the base key remains in place. + */ + } + + clear() { + // TODO: for hint stuff. + } +} \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts b/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts index 449fd384ee6..e599b235716 100644 --- a/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts +++ b/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts @@ -30,7 +30,7 @@ export default class SubkeyPopup { public readonly subkeys: KeyElement[]; constructor( - source: GestureSequence, + source: GestureSequence, configChanger: ConfigChangeClosure, vkbd: VisualKeyboard, e: KeyElement diff --git a/web/src/engine/osk/src/input/gestures/heldRepeater.ts b/web/src/engine/osk/src/input/gestures/heldRepeater.ts index 93b80be99f2..f67ae2851d2 100644 --- a/web/src/engine/osk/src/input/gestures/heldRepeater.ts +++ b/web/src/engine/osk/src/input/gestures/heldRepeater.ts @@ -6,12 +6,12 @@ export class HeldRepeater { static readonly INITIAL_DELAY = 500; static readonly REPEAT_DELAY = 100; - readonly source: GestureSequence; + readonly source: GestureSequence; readonly repeatClosure: () => void; timerHandle: number; - constructor(source: GestureSequence, closureToRepeat: () => void) { + constructor(source: GestureSequence, closureToRepeat: () => void) { this.source = source; this.repeatClosure = closureToRepeat; diff --git a/web/src/engine/osk/src/input/gestures/specsForLayout.ts b/web/src/engine/osk/src/input/gestures/specsForLayout.ts index ff9ae757183..f8263cadbec 100644 --- a/web/src/engine/osk/src/input/gestures/specsForLayout.ts +++ b/web/src/engine/osk/src/input/gestures/specsForLayout.ts @@ -65,7 +65,7 @@ export const DEFAULT_GESTURE_PARAMS: GestureParams = { * immediate effect during gesture processing. * @returns */ -export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GestureParams): GestureModelDefs { +export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GestureParams): GestureModelDefs { const layout = layerGroup.spec; // To be used among the `allowsInitialState` contact-model specifications as needed. @@ -77,15 +77,11 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa case 'longpress': return !!keySpec.sk; case 'multitap': + case 'modipress-multitap-start': if(layout.hasMultitaps) { return !!keySpec.multitap; - } else if(layout.formFactor != 'desktop') { - // maintain our special caps-shifting? - // if(keySpec.baseKeyID == 'K_SHIFT') { - - // } else { + } else { return false; - // } } case 'flick': // This is a gesture-start check; there won't yet be any directional info available. @@ -124,59 +120,19 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa return model; } - - function withLayerChangeItemFix(model: GestureModel, contactIndices: number | number[]) { - // Creates deep copies of the model specifications that are safe to customize to the - // keyboard layout. - model = deepCopy(model); - - if(typeof contactIndices == 'number') { - contactIndices = [contactIndices]; - } - - model.contacts.forEach((contact, index) => { - if((contactIndices as number[]).indexOf(index) != -1) { - const baseInitialStateCheck = contact.model.allowsInitialState ?? (() => true); - - contact.model = { - ...contact.model, - // And now for the true purpose of the method. - allowsInitialState: (sample, ancestorSample, baseKey) => { - // By default, the state token is set to whatever the current layer is for a source. - // - // So, if the first tap of a key swaps layers, the second tap will be on the wrong layer and - // thus have a different state token. This is the perfect place to detect and correct that. - if(ancestorSample.stateToken != sample.stateToken) { - sample.stateToken = ancestorSample.stateToken; - - // Specialized item lookup is required here for proper 'correction' - we want the key - // corresponding to our original layer, not the new layer here. Now that we've identified - // the original OSK layer (state) for the gesture, we can find the best matching key - // from said layer instead of the current layer. - // - // Matters significantly for multitaps if and when they include layer-switching specs. - sample.item = layerGroup.findNearestKey(sample); - } - - return baseInitialStateCheck(sample, ancestorSample, baseKey); - } - }; - } - }); - - return model; - } // #endregion const gestureModels = [ withKeySpecFiltering(longpressModel, 0), - withLayerChangeItemFix(withKeySpecFiltering(MultitapModel, 0), 0), + withKeySpecFiltering(MultitapModel, 0), simpleTapModel, withKeySpecFiltering(SpecialKeyStartModel, 0), SpecialKeyEndModel, SubkeySelectModel, withKeySpecFiltering(ModipressStartModel, 0), - ModipressEndModel + ModipressEndModel, + withKeySpecFiltering(ModipressMultitapStartModel, 0), + ModipressMultitapEndModel ]; const defaultSet = [ @@ -206,7 +162,7 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa // #region Definition of models for paths comprising gesture-stage models -type ContactModel = specs.ContactModel; +type ContactModel = specs.ContactModel; export const InstantContactRejectionModel: ContactModel = { itemPriority: 0, @@ -337,7 +293,7 @@ export const SubkeySelectContactModel: ContactModel = { // #endregion // #region Gesture-stage model definitions -type GestureModel = specs.GestureModel; +type GestureModel = specs.GestureModel; // TODO: customization of the gesture models depending upon properties of the keyboard. // - has flicks? no longpress shortcut, also no longpress reset(?) @@ -608,6 +564,7 @@ export const SubkeySelectModel: GestureModel = { sustainWhenNested: true } +/* NOTE: modipress stuff is still incomplete at this time! */ export const ModipressStartModel: GestureModel = { id: 'modipress-start', resolutionPriority: 5, @@ -648,13 +605,93 @@ export const ModipressEndModel: GestureModel = { { model: { ...ModipressContactEndModel, - itemChangeAction: 'reject' + itemChangeAction: 'reject', + pathInheritance: 'full' } } ], resolutionAction: { - type: 'complete', + type: 'chain', + // Because SHIFT -> CAPS multitap is a thing. Shift gets handled as a modipress first. + // Modipresses resolve before multitaps... unless there's a model designed to handle & disambiguate both. + next: 'modipress-multitap-start', + // Key was already emitted from the 'modipress-start' stage. item: 'none' } } + +export const ModipressMultitapStartModel: GestureModel = { + id: 'modipress-multitap-start', + resolutionPriority: 6, + contacts: [ + { + model: { + ...ModipressContactStartModel, + pathInheritance: 'reject', + allowsInitialState(incomingSample, comparisonSample, baseItem) { + if(incomingSample.item != baseItem) { + return false; + } + // TODO: needs better abstraction, probably. + + // But, to get started... we can just use a simple hardcoded approach. + const modifierKeyIds = ['K_SHIFT', 'K_ALT', 'K_CTRL']; + for(const modKeyId of modifierKeyIds) { + if(baseItem.key.spec.id == modKeyId) { + return true; + } + } + + return false; + }, + itemChangeAction: 'reject', + itemPriority: 1 + } + } + ], + sustainTimer: { + duration: 500, + expectedResult: false, + baseItem: 'base' + }, + resolutionAction: { + type: 'chain', + next: 'modipress-multitap-end', + selectionMode: 'modipress', + item: 'current' // return the modifier key ID so that we know to shift to it! + } +} + +export const ModipressMultitapEndModel: GestureModel = { + id: 'modipress-multitap-end', + resolutionPriority: 5, + contacts: [ + { + model: { + ...ModipressContactEndModel, + itemChangeAction: 'reject', + pathInheritance: 'full', + timer: { + // will need something similar for base modipress. + duration: 500, + expectedResult: false + } + } + } + ], + resolutionAction: { + type: 'chain', + // Because SHIFT -> CAPS multitap is a thing. Shift gets handled as a modipress first. + // TODO: maybe be selective about it: if the tap occurs within a set amount of time? + next: 'modipress-multitap-start', + // Key was already emitted from the 'modipress-start' stage. + item: 'none' + }, + rejectionActions: { + timer: { + type: 'replace', + replace: 'modipress-end' + } + } +} // #endregion \ No newline at end of file diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 87c1c4f9cee..1ac75540fc8 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -12,7 +12,8 @@ import { KeyEvent, Layouts, StateKeyMap, - LayoutKey + LayoutKey, + ActiveSubkey } from '@keymanapp/keyboard-processor'; import { @@ -47,6 +48,7 @@ import { DEFAULT_GESTURE_PARAMS, GestureParams, gestureSetForLayout } from './in import { getViewportScale } from './screenUtils.js'; import { HeldRepeater } from './input/gestures/heldRepeater.js'; import SubkeyPopup from './input/gestures/browser/subkeyPopup.js'; +import Multitap from './input/gestures/browser/multitap.js'; export interface VisualKeyboardConfiguration extends CommonConfiguration { /** @@ -473,26 +475,26 @@ export default class VisualKeyboard extends EventEmitter implements Ke } if(gestureKey) { - - if(gestureStage.matchedId == 'multitap') { - // TODO: examine sequence, determine rota-style index to apply; select THAT item instead. - } - if(gestureStage.matchedId == 'subkey-select') { // TODO: examine subkey menu, determine proper set of fat-finger alternates. } - // Once the best coord to use for fat-finger calculations has been determined: - this.modelKeyClick(gestureStage.item, coord); + // Multitaps do special key-mapping stuff internally and produce + raise their + // key events directly. + if(gestureStage.matchedId != 'multitap') { + // Once the best coord to use for fat-finger calculations has been determined: + this.modelKeyClick(gestureStage.item, coord); + } } // Outside of passing keys along... the handling of later stages is delegated // to gesture-specific handling classes. - if(gestureSequence.stageReports.length > 1) { + if(gestureSequence.stageReports.length > 1 && gestureStage.matchedId != 'modipress-end') { return; } // So, if this is the first stage, this is where we need to perform that delegation. + const baseItem = gestureSequence.stageReports[0].item; // -- Scratch-space as gestures start becoming integrated -- // Reordering may follow at some point. @@ -510,7 +512,13 @@ export default class VisualKeyboard extends EventEmitter implements Ke } else if(gestureStage.matchedId.indexOf('longpress') > -1) { // Matches: 'longpress', 'longpress-reset'. // Likewise. - new SubkeyPopup(gestureSequence, configChanger, this, gestureSequence.stageReports[0].sources[0].baseItem); + + // But... a longpress doesn't generate a keystroke; we'll need to properly lookup the base key. + const baseKey = gestureSequence.stageReports[0].sources[0].baseItem + new SubkeyPopup(gestureSequence, configChanger, this, baseKey); + } else if(baseItem.key.spec.multitap && (gestureStage.matchedId == 'simple-tap' || gestureStage.matchedId == 'multitap' || gestureStage.matchedId == 'modipress-end')) { + // Likewise - mere construction is enough. + new Multitap(gestureSequence, this, baseItem); } // TODO: depending upon the gesture type, what sort of UI shifts should happen to @@ -717,7 +725,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke * @param keySpec The spec of the key directly triggered by the input event. May be for a subkey. * @returns */ - getTouchProbabilities(input: InputSample, keySpec?: ActiveKey): KeyDistribution { + getTouchProbabilities(input: InputSample, keySpec?: ActiveKey | ActiveSubkey): KeyDistribution { // TODO: It'd be nice to optimize by keeping these off when unused, but the wiring // necessary would get in the way of modularization at the moment. // let keyman = com.keyman.singleton; @@ -1057,6 +1065,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke modelKeyClick(e: KeyElement, input?: InputSample) { let keyEvent = this.initKeyEvent(e, input); + // Override alternates here as appropriate. Will likely need extra param. this.raiseKeyEvent(keyEvent, e); } @@ -1080,7 +1089,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke return this.keyEventFromSpec(keySpec, input); } - keyEventFromSpec(keySpec: ActiveKey, input?: InputSample) { + keyEventFromSpec(keySpec: ActiveKey | ActiveSubkey, input?: InputSample) { //let core = com.keyman.singleton.core; // only singleton-based ref currently needed here. // Start: mirrors _GetKeyEventProperties @@ -1131,6 +1140,8 @@ export default class VisualKeyboard extends EventEmitter implements Ke return; } + this.gestureEngine.stateToken = layerId; + // So... through KMW 14, we actually never tracked the capsKey, numKey, and scrollKey // properly for keyboard-defined layouts - only _default_, desktop-style layouts. //