Skip to content

Commit

Permalink
Merge pull request #11265 from keymanapp/feat/web/layer-processing-de…
Browse files Browse the repository at this point in the history 
…ferral

feat(web): optimization via lazy preprocessing of keyboard touch-layout info ⏩
  • Loading branch information
@jahorton
jahorton authored Jun 24, 2024
2 parents 8b4e6dc + 7166428 commit 7f55d74
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 51 deletions.
88 changes: 50 additions & 38 deletions common/web/keyboard-processor/src/keyboards/activeLayout.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import type Keyboard from "./keyboard.js";
import { TouchLayout } from "@keymanapp/common-types";
import TouchLayoutDefaultHint = TouchLayout.TouchLayoutDefaultHint;
import TouchLayoutFlick = TouchLayout.TouchLayoutFlick;
import TouchLayoutSpec = TouchLayout.TouchLayoutPlatform;
import TouchLayerSpec = TouchLayout.TouchLayoutLayer;
import TouchLayoutKeySp = TouchLayout.TouchLayoutKeySp;
import { type DeviceSpec } from "@keymanapp/web-utils";

Expand Down Expand Up @@ -558,12 +560,10 @@ export class ActiveRow implements LayoutRow {

static polyfill(
row: LayoutRow,
keyboard: Keyboard,
layout: ActiveLayout,
displayLayer: string,
totalWidth: number,
proportionalY: number,
analysisFlagObj: AnalysisMetadata
proportionalY: number
) {
// Apply defaults, setting the width and other undefined properties for each key
let keys=row['key'];
Expand Down Expand Up @@ -598,16 +598,6 @@ export class ActiveRow implements LayoutRow {

const processedKey = new ActiveKey(key, layout, displayLayer);
keys[j] = processedKey;

if(processedKey.sk) {
analysisFlagObj.hasLongpresses = true;
}
if(processedKey.multitap) {
analysisFlagObj.hasMultitaps = true;
}
if(processedKey.flick) {
analysisFlagObj.hasFlicks = true;
}
}

/* The calculations here are effectively 'virtualized'. When used with the OSK, the VisualKeyboard
Expand Down Expand Up @@ -705,7 +695,7 @@ export class ActiveLayer implements LayoutLayer {
}
}

static polyfill(layer: LayoutLayer, keyboard: Keyboard, layout: ActiveLayout, analysisFlagObj: AnalysisMetadata) {
static polyfill(layer: LayoutLayer, layout: ActiveLayout) {
layer.aligned=false;

// Create a DIV for each row of the group
Expand Down Expand Up @@ -738,7 +728,7 @@ export class ActiveLayer implements LayoutLayer {
for(let i=0; i<rowCount; i++) {
// Calculate proportional y-coord of row. 0 is at top with highest y-coord.
let rowProportionalY = (i + 0.5) / rowCount;
ActiveRow.polyfill(layer.row[i], keyboard, layout, layer.id, totalWidth, rowProportionalY, analysisFlagObj);
ActiveRow.polyfill(layer.row[i], layout, layer.id, totalWidth, rowProportionalY);
}

assignDefaultsWithPropDefs(layer, new ActiveLayer());
Expand Down Expand Up @@ -778,7 +768,11 @@ export class ActiveLayer implements LayoutLayer {
}

export class ActiveLayout implements LayoutFormFactor{
layer: ActiveLayer[];
/**
* Holds all layer specifications for the layout. There is no guarantee that they
* have been fully preprocessed.
*/
layer: TouchLayerSpec[];
font: string;
keyLabels: boolean;
isDefault?: boolean;
Expand All @@ -801,8 +795,25 @@ export class ActiveLayout implements LayoutFormFactor{

}

/**
* Returns a fully preprocessed version of the specified layer spec.
* @param layerId
* @returns
*/
@Enumerable
getLayer(layerId: string): ActiveLayer {
if(!this.layerMap[layerId]) {
const spec = this.layer.find((layerSpec) => layerSpec.id == layerId);
if(!spec) {
return null;
}

// Prepare the layer-spec for actual use.
ActiveLayer.sanitize(spec);
ActiveLayer.polyfill(spec, this);
this.layerMap[layerId] = spec as ActiveLayer;
}

return this.layerMap[layerId];
}

Expand All @@ -821,30 +832,26 @@ export class ActiveLayout implements LayoutFormFactor{
static correctLayerEmptyRowBug(layers: LayoutLayer[]) {
for(let n=0; n<layers.length; n++) {
let layer=layers[n];
let rows=layer['row'];
let rows=layer.row;
let i: number;
for(i=rows.length-1; i>=0; i--) {
if(!Array.isArray(rows[i]['key']) || rows[i]['key'].length == 0) {
if(!Array.isArray(rows[i].key) || rows[i].key.length == 0) {
rows.splice(i, 1)
}
}
}
}

static sanitize(rawLayout: LayoutFormFactor) {
static sanitize(rawLayout: TouchLayoutSpec) {
ActiveLayout.correctLayerEmptyRowBug(rawLayout.layer);

for(const layer of rawLayout.layer) {
ActiveLayer.sanitize(layer);
}
}

/**
*
* @param layout
* @param formFactor
*/
static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout {
static polyfill(layout: TouchLayoutSpec, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout {
/* c8 ignore start */
if(layout == null) {
throw new Error("Cannot build an ActiveLayout for a null specification.");
Expand All @@ -865,33 +872,38 @@ export class ActiveLayout implements LayoutFormFactor{
*/
this.sanitize(layout);

// This bit of preprocessing is a must; we need to know what gestures are available
// across all layers, "out of the gate".
for(let layer of layout.layer) {
for(let row of layer.row) {
for(let key of row.key) {
analysisMetadata.hasLongpresses ||= !!key.sk;
analysisMetadata.hasFlicks ||= !!key.flick;
analysisMetadata.hasMultitaps ||= !!key.multitap;
}
}
}

// Create a separate OSK div for each OSK layer, only one of which will ever be visible
var n: number;
let layerMap: {[layerId: string]: ActiveLayer} = {};

let layers=layout.layer;

// Add class functions to the existing layout object, allowing it to act as an ActiveLayout.
assignDefaultsWithPropDefs(layout, new ActiveLayout());

let aLayout = layout as ActiveLayout;
let aLayout = layout as unknown as ActiveLayout;
aLayout.keyboard = keyboard;
aLayout.formFactor = formFactor;

for(n=0; n<layers.length; n++) {
ActiveLayer.polyfill(layers[n], keyboard, aLayout, analysisMetadata);
layerMap[layers[n].id] = layers[n] as ActiveLayer;
}

// After all layers are preprocessed...
aLayout.layerMap = layerMap;

// The default-layer shift key & shift-layer shift key on mobile platforms should have a
// default multitap re: a 'caps' layer under select conditions.
//
// Note: whether or not any other keys have multitaps doesn't matter here. Just THESE.
if(formFactor != 'desktop' && !!layout.layer.find((entry) => entry.id == 'caps')) {
const defaultLayer = layout.layer.find((entry) => entry.id == 'default') as ActiveLayer;
const shiftLayer = layout.layer.find((entry) => entry.id == 'shift') as ActiveLayer;
// Triggers preprocessing for both default and shift layers. They're the
// most-frequently referenced, at least.
const defaultLayer = aLayout.getLayer('default') as ActiveLayer;
const shiftLayer = aLayout.getLayer('shift') as ActiveLayer;

const defaultShift = defaultLayer.getKey('K_SHIFT');
const shiftShift = shiftLayer ?.getKey('K_SHIFT');
Expand All @@ -916,7 +928,7 @@ export class ActiveLayout implements LayoutFormFactor{
aLayout.hasLongpresses = analysisMetadata.hasLongpresses;
aLayout.hasMultitaps = analysisMetadata.hasMultitaps;

aLayout.layerMap = layerMap;
// All layers are lazy-processed, with the usual processing applied when first referenced.

return aLayout;
}
Expand Down
16 changes: 8 additions & 8 deletions common/web/keyboard-processor/src/keyboards/defaultLayouts.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import { Version, deepCopy } from "@keymanapp/web-utils";
import { TouchLayout } from "@keymanapp/common-types";

import LayoutFormFactorBase = TouchLayout.TouchLayoutPlatform;
import LayoutFormFactorSpec = TouchLayout.TouchLayoutPlatform;
import LayoutLayerBase = TouchLayout.TouchLayoutLayer;
export type LayoutRow = TouchLayout.TouchLayoutRow;
export type LayoutKey = TouchLayout.TouchLayoutKey;
Expand Down Expand Up @@ -52,15 +52,15 @@ export interface LayoutLayer extends LayoutLayerBase {
aligned?: boolean,
nextlayer?: string
};
export interface LayoutFormFactor extends LayoutFormFactorBase {
export interface LayoutFormFactor extends LayoutFormFactorSpec {
// To facilitate those post-processing elements.
layer: LayoutLayer[]
};

export type LayoutSpec = {
"desktop"?: LayoutFormFactor,
"phone"?: LayoutFormFactor,
"tablet"?: LayoutFormFactor
"desktop"?: LayoutFormFactorSpec,
"phone"?: LayoutFormFactorSpec,
"tablet"?: LayoutFormFactorSpec
}

const KEY_102_WIDTH = 200;
Expand Down Expand Up @@ -136,9 +136,9 @@ export class Layouts {
}

// Clone the default layout object for this device
var layout: LayoutFormFactor = deepCopy(Layouts.dfltLayout[layoutType]);
var layout: LayoutFormFactorSpec = deepCopy(Layouts.dfltLayout[layoutType]);

var n,layers=layout['layer'], keyLabels: EncodedVisualKeyboard['KLS'] = PVK['KLS'], key102=PVK['K102'];
var n,layers=layout['layer'] as LayoutLayer[], keyLabels: EncodedVisualKeyboard['KLS'] = PVK['KLS'], key102=PVK['K102'];
var i, j, k, rows: LayoutRow[], key: LayoutKey, keys: LayoutKey[];
var chiral: boolean = (kbdBitmask & Codes.modifierBitmasks.IS_CHIRAL) != 0;

Expand Down Expand Up @@ -255,7 +255,7 @@ export class Layouts {

// *** Step 2: Layer objects now exist; time to fill them with the appropriate key labels and key styles ***
for(n=0; n<layers.length; n++) {
var layer=layers[n], kx, shiftKey: LayoutKey = null;
var layer=layers[n] as LayoutLayer, kx, shiftKey: LayoutKey = null;
var capsKey: LayoutKey = null, numKey: LayoutKey = null, scrollKey: LayoutKey = null; // null if not in the OSK layout.
var layerSpec = keyLabels[layer['id']];
var isShift = layer['id'] == 'shift' ? 1 : 0;
Expand Down
14 changes: 9 additions & 5 deletions common/web/keyboard-processor/src/keyboards/keyboard.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import Codes from "../text/codes.js";
import { EncodedVisualKeyboard, LayoutSpec, Layouts, type LayoutFormFactor } from "./defaultLayouts.js";
import { EncodedVisualKeyboard, LayoutSpec, Layouts } from "./defaultLayouts.js";
import { ActiveKey, ActiveLayout, ActiveSubKey } from "./activeLayout.js";
import KeyEvent from "../text/keyEvent.js";
import type OutputTarget from "../text/outputTarget.js";
import { TouchLayout } from "@keymanapp/common-types";
type TouchLayoutSpec = TouchLayout.TouchLayoutPlatform & { isDefault?: boolean};

import type { ComplexKeyboardStore } from "../text/kbdInterface.js";

Expand Down Expand Up @@ -506,7 +508,7 @@ export default class Keyboard {
}
}

private findOrConstructLayout(formFactor: DeviceSpec.FormFactor): LayoutFormFactor {
private findOrConstructLayout(formFactor: DeviceSpec.FormFactor): TouchLayoutSpec {
if(this._layouts) {
// Search for viable layouts. `null` is allowed for desktop form factors when help text is available,
// so we check explicitly against `undefined`.
Expand Down Expand Up @@ -550,7 +552,7 @@ export default class Keyboard {
// Final check - do we construct a layout, or is this a case where helpText / insertHelpHTML should take over?
if(rawSpecifications) {
// Now to generate a layout from our raw specifications.
let layout = this._layouts[formFactor] = Layouts.buildDefaultLayout(rawSpecifications, this, formFactor) as ActiveLayout;
let layout: TouchLayoutSpec = this._layouts[formFactor] = Layouts.buildDefaultLayout(rawSpecifications, this, formFactor);
layout.isDefault = true;
return layout;
} else {
Expand All @@ -572,11 +574,13 @@ export default class Keyboard {
if(rawLayout) {
// Prevents accidentally reprocessing layouts; it's a simple enough check.
if(this.layoutStates[formFactor] == LayoutState.NOT_LOADED) {
rawLayout = ActiveLayout.polyfill(rawLayout, this, formFactor);
const layout = ActiveLayout.polyfill(rawLayout, this, formFactor);
this.layoutStates[formFactor] = LayoutState.POLYFILLED;
return layout;
} else {
return rawLayout as unknown as ActiveLayout;
}

return rawLayout as ActiveLayout;
} else {
return null;
}
Expand Down
6 changes: 6 additions & 0 deletions web/src/engine/osk/src/visualKeyboard.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,12 @@ export default class VisualKeyboard extends EventEmitter<EventMap> implements Ke
return allottedHeight;
}

/*
Note: these may not be fully preprocessed yet!
However, any "empty row bug" preprocessing has been applied, and that's
what we care about here.
*/
const layers = this.layerGroup.spec.layer;
let oskHeight = 0;

Expand Down

0 comments on commit 7f55d74

Please sign in to comment.