Merge pull request #12713 from keymanapp/refactor/common/LdmlKeyboard…

…Types

refactor(common): move remaining LDML keyboard types into `LdmlKeyboardTypes`

common/web/types/src/main-ldml-keyboard.ts

@@ -0,0 +1,8 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * Just a wrapper to make `LdmlKeyboardTypes` export work.
+ */
+export { UnicodeSetParser, UnicodeSet } from './ldml-keyboard/unicodeset-parser-api.js';
+export { VariableParser, MarkerParser } from './ldml-keyboard/pattern-parser.js';
+export { ElementString } from './kmx/kmx-plus/element-string.js';

common/web/types/src/main.ts

@@ -25,10 +25,7 @@ export * as Schemas from './schemas.js';
 export * as SchemaValidators from './schema-validators.js';
 
 export * as KMXPlus from './kmx/kmx-plus/kmx-plus.js';
-// TODO: these exports are really not well named
-export { UnicodeSetParser, UnicodeSet } from './ldml-keyboard/unicodeset-parser-api.js';
-export { VariableParser, MarkerParser } from './ldml-keyboard/pattern-parser.js';
-export { ElementString } from './kmx/kmx-plus/element-string.js';
+export * as LdmlKeyboardTypes from './main-ldml-keyboard.js';
 
 export * as LexicalModelTypes from './lexical-model-types.js';
 

common/web/types/test/lexical-model-types.tests.ts

@@ -5,7 +5,7 @@
  * imported and compiled without any compiler errors.
  */
 
-import { KMXPlus, ElementString, LexicalModelTypes } from "@keymanapp/common-types";
+import { KMXPlus, LdmlKeyboardTypes, LexicalModelTypes } from "@keymanapp/common-types";
 
 export let u: LexicalModelTypes.USVString;
 export let l: LexicalModelTypes.Transform;
@@ -21,5 +21,5 @@ export let lmp: LexicalModelTypes.LexicalModelPunctuation;
 
 
 // try some of the other types - that should still work
-export let elemString: ElementString;
+export let elemString: LdmlKeyboardTypes.ElementString;
 export let section: KMXPlus.Section;

developer/src/common/web/utils/src/types/kmx/kmx-plus-builder/build-elem.ts

@@ -1,5 +1,5 @@
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, ElementString } from "@keymanapp/common-types";
+import { KMXPlus, LdmlKeyboardTypes } from "@keymanapp/common-types";
 import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js";
 import { BUILDER_SECTION, BUILDER_U32CHAR } from "./builder-section.js";
 import { build_uset_index, BUILDER_USET, BUILDER_USET_REF } from "./build-uset.js";
@@ -23,7 +23,7 @@ interface BUILDER_ELEM_STRING {
   offset: number;
   length: number;
   items: BUILDER_ELEM_ELEMENT[];
-  _value: ElementString;
+  _value: LdmlKeyboardTypes.ElementString;
 };
 
 /**
@@ -102,8 +102,8 @@ export function build_elem(source_elem: Elem, sect_strs: BUILDER_STRS, sect_uset
   return result;
 }
 
-export function build_elem_index(sect_elem: BUILDER_ELEM, value: ElementString) : BUILDER_ELEM_REF{
-  if(!(value instanceof ElementString)) {
+export function build_elem_index(sect_elem: BUILDER_ELEM, value: LdmlKeyboardTypes.ElementString) : BUILDER_ELEM_REF{
+  if(!(value instanceof LdmlKeyboardTypes.ElementString)) {
     throw new Error('unexpected value '+value);
   }
 

developer/src/kmc-kmn/src/compiler/compiler.ts

@@ -6,7 +6,7 @@ TODO: implement additional interfaces:
 */
 
 // TODO: rename wasm-host?
-import { VisualKeyboard, KvkFileReader, UnicodeSetParser, UnicodeSet, KeymanFileTypes, KvkFileWriter } from '@keymanapp/common-types';
+import { VisualKeyboard, KvkFileReader, LdmlKeyboardTypes, KeymanFileTypes, KvkFileWriter } from '@keymanapp/common-types';
 import {
   CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanCompiler, KeymanCompilerArtifacts,
   KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact, KvksFileReader
@@ -140,7 +140,7 @@ let
  * or write from filesystem or network directly, but relies on callbacks for all
  * external IO.
  */
-export class KmnCompiler implements KeymanCompiler, UnicodeSetParser {
+export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSetParser {
   private callbacks: CompilerCallbacks;
   private wasmExports: MallocAndFree;
   private options: KmnCompilerOptions;
@@ -529,7 +529,7 @@ export class KmnCompiler implements KeymanCompiler, UnicodeSetParser {
    * @param rangeCount - number of ranges to allocate
    * @returns            UnicodeSet accessor object, or null on failure
    */
-  public parseUnicodeSet(pattern: string, rangeCount: number) : UnicodeSet | null {
+  public parseUnicodeSet(pattern: string, rangeCount: number) : LdmlKeyboardTypes.UnicodeSet | null {
     if(!this.verifyInitialized()) {
       /* c8 ignore next 2 */
       // verifyInitialized will set a callback if needed
@@ -557,7 +557,7 @@ export class KmnCompiler implements KeymanCompiler, UnicodeSetParser {
         ranges.push([start, end]);
       }
       this.wasmExports.free(buf);
-      return new UnicodeSet(pattern, ranges);
+      return new LdmlKeyboardTypes.UnicodeSet(pattern, ranges);
     } else {
       // rc is negative: it's an error code.
       this.wasmExports.free(buf);

developer/src/kmc-ldml/src/compiler/compiler.ts

@@ -3,7 +3,7 @@
  *
  * Compiles a LDML XML keyboard file into a Keyman KMXPlus file
  */
-import { KMXPlus, UnicodeSetParser, KvkFileWriter, KMX } from '@keymanapp/common-types';
+import { KMXPlus, LdmlKeyboardTypes, KvkFileWriter, KMX } from '@keymanapp/common-types';
 import {
   CompilerCallbacks, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts,
   defaultCompilerOptions, LDMLKeyboardXMLSourceFileReader, LDMLKeyboard,
@@ -93,7 +93,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
   private options: LdmlCompilerOptions;
 
   // uset parser
-  private usetparser?: UnicodeSetParser = undefined;
+  private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined;
 
   /**
    * Initialize the compiler, including loading the WASM host for uset parsing.
@@ -211,7 +211,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
    * Construct or return a UnicodeSetParser, aka KmnCompiler
    * @returns the held UnicodeSetParser
    */
-  async getUsetParser(): Promise<UnicodeSetParser> {
+  async getUsetParser(): Promise<LdmlKeyboardTypes.UnicodeSetParser> {
     if (this.usetparser === undefined) {
       // initialize
       const compiler = new KmnCompiler();

developer/src/kmc-ldml/src/compiler/empty-compiler.ts

@@ -1,6 +1,6 @@
 import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants';
 import { SectionCompiler } from "./section-compiler.js";
-import { util, KMXPlus, MarkerParser } from "@keymanapp/common-types";
+import { util, KMXPlus, LdmlKeyboardTypes } from "@keymanapp/common-types";
 import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils";
 import { VarsCompiler } from './vars.js';
 import { LdmlCompilerMessages } from './ldml-compiler-messages.js';
@@ -35,7 +35,7 @@ export class StrsCompiler extends EmptyCompiler {
 
     if (strs) {
       const badStringAnalyzer = new util.BadStringAnalyzer();
-      const CONTAINS_MARKER_REGEX = new RegExp(MarkerParser.ANY_MARKER_MATCH);
+      const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH);
       for (let s of strs.allProcessedStrings.values()) {
         // replace all \\uXXXX with the actual code point.
         // this lets us analyze whether there are PUA, unassigned, etc.
@@ -47,7 +47,7 @@ export class StrsCompiler extends EmptyCompiler {
         if (CONTAINS_MARKER_REGEX.test(s)) {
           // it had a marker, take out all marker strings, as the sentinel is illegal
           // need a new regex to match
-          const REPLACE_MARKER_REGEX = new RegExp(MarkerParser.ANY_MARKER_MATCH, 'g');
+          const REPLACE_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH, 'g');
           s = s.replaceAll(REPLACE_MARKER_REGEX, ''); // remove markers.
         }
         badStringAnalyzer.add(s);

developer/src/kmc-ldml/src/compiler/linter-keycaps.ts

@@ -1,4 +1,4 @@
-import { MarkerParser } from "@keymanapp/common-types";
+import { LdmlKeyboardTypes } from "@keymanapp/common-types";
 import { Linter } from "./linter.js";
 import { LdmlCompilerMessages } from "./ldml-compiler-messages.js";
 
@@ -20,7 +20,7 @@ export class LinterKeycaps extends Linter {
     }
 
     public async lint(): Promise<boolean> {
-        const ANY_MARKER_REGEX = new RegExp(MarkerParser.ANY_MARKER_MATCH,'g');
+        const ANY_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH,'g');
         // Are there any keys which:
         // 0. Are present in the layout, flicks or gestures, AND
         // 1. Consist entirely of marker output, AND

developer/src/kmc-ldml/src/compiler/substitution-tracker.ts

@@ -1,4 +1,4 @@
-import { MarkerParser, VariableParser } from "@keymanapp/common-types";
+import { LdmlKeyboardTypes } from "@keymanapp/common-types";
 
 /**
  * Verb for SubstitutionTracker.add()
@@ -76,7 +76,7 @@ export class SubstitutionTracker {
 /** rollup of several substitution types */
 export class Substitutions {
   addSetAndStringSubtitution(verb: SubstitutionUse, str?: string) {
-    this.set.add(verb, VariableParser.allSetReferences(str));
+    this.set.add(verb, LdmlKeyboardTypes.VariableParser.allSetReferences(str));
     this.addStringAndMarkerSubstitution(verb, str);
   }
   /** add a string that can have string var substitutions or markers */
@@ -85,12 +85,12 @@ export class Substitutions {
     this.addStringSubstitution(verb, str);
   }
   addStringSubstitution(verb: SubstitutionUse, str?: string) {
-    this.string.add(verb, VariableParser.allStringReferences(str));
+    this.string.add(verb, LdmlKeyboardTypes.VariableParser.allStringReferences(str));
   }
   /** add a string that's just markers */
   addMarkers(verb: SubstitutionUse, str?: string) {
-    this.markers.add(verb, MarkerParser.allReferences(str));
-    MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.add(m));
+    this.markers.add(verb, LdmlKeyboardTypes.MarkerParser.allReferences(str));
+    LdmlKeyboardTypes.MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.add(m));
   }
   // all valid markers
   markers: SubstitutionTracker;

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -1,5 +1,5 @@
 import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, VariableParser, MarkerParser, util } from '@keymanapp/common-types';
+import { KMXPlus, LdmlKeyboardTypes, util } from '@keymanapp/common-types';
 import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils";
 import { SectionCompiler } from "./section-compiler.js";
 
@@ -28,8 +28,8 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
         transformGroup.transform?.forEach(({ to, from }) => {
           st.addSetAndStringSubtitution(SubstitutionUse.consume, from);
           st.addSetAndStringSubtitution(SubstitutionUse.emit, to);
-          const mapFrom = VariableParser.CAPTURE_SET_REFERENCE.exec(from);
-          const mapTo = VariableParser.MAPPED_SET_REFERENCE.exec(to || '');
+          const mapFrom = LdmlKeyboardTypes.VariableParser.CAPTURE_SET_REFERENCE.exec(from);
+          const mapTo = LdmlKeyboardTypes.VariableParser.MAPPED_SET_REFERENCE.exec(to || '');
           if (mapFrom) {
             // add the 'from' as a match
             st.set.add(SubstitutionUse.consume, [mapFrom[1]]);
@@ -144,8 +144,8 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
     cookedFrom = this.checkEscapes(cookedFrom); // check for \uXXXX escapes before normalizing
 
     cookedFrom = sections.vars.substituteStrings(cookedFrom, sections, true);
-    const mapFrom = VariableParser.CAPTURE_SET_REFERENCE.exec(cookedFrom);
-    const mapTo = VariableParser.MAPPED_SET_REFERENCE.exec(transform.to || '');
+    const mapFrom = LdmlKeyboardTypes.VariableParser.CAPTURE_SET_REFERENCE.exec(cookedFrom);
+    const mapTo = LdmlKeyboardTypes.VariableParser.MAPPED_SET_REFERENCE.exec(transform.to || '');
     if (mapFrom && mapTo) { // TODO-LDML: error cases
       result.mapFrom = sections.strs.allocString(mapFrom[1]); // var name
       result.mapTo = sections.strs.allocString(mapTo[1]); // var name
@@ -175,7 +175,7 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
 
     if (!sections?.meta?.normalizationDisabled) {
       // nfd here.
-      cookedFrom = MarkerParser.nfd_markers(cookedFrom, true);
+      cookedFrom = LdmlKeyboardTypes.MarkerParser.nfd_markers(cookedFrom, true);
     }
 
     // perform regex validation

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -1,5 +1,5 @@
 import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, VariableParser, MarkerParser } from '@keymanapp/common-types';
+import { KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types';
 import { LDMLKeyboard, CompilerCallbacks } from '@keymanapp/developer-utils';
 import { SectionCompiler } from "./section-compiler.js";
 import Vars = KMXPlus.Vars;
@@ -41,7 +41,7 @@ export class VarsCompiler extends SectionCompiler {
   }
 
   private validateIdentifier(id: string) {
-    if(!id.match(VariableParser.ID)) { // From <string> DTD
+    if(!id.match(LdmlKeyboardTypes.VariableParser.ID)) { // From <string> DTD
       this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidVariableIdentifier({id}));
       return false;
     }
@@ -80,7 +80,7 @@ export class VarsCompiler extends SectionCompiler {
           continue;
         }
         addId(id);
-        const stringrefs = VariableParser.allStringReferences(value);
+        const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value);
         for(const ref of stringrefs) {
           if(!allStrings.has(ref)) {
             valid = false;
@@ -101,13 +101,13 @@ export class VarsCompiler extends SectionCompiler {
         addId(id);
         allSets.add(id);
         // check for illegal references, here.
-        const stringrefs = VariableParser.allStringReferences(value);
+        const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value);
         st.string.add(SubstitutionUse.variable, stringrefs);
 
         // Now split into spaces.
-        const items: string[] = VariableParser.setSplitter(value);
+        const items: string[] = LdmlKeyboardTypes.VariableParser.setSplitter(value);
         for (const item of items) {
-          const setrefs = VariableParser.allSetReferences(item);
+          const setrefs = LdmlKeyboardTypes.VariableParser.allSetReferences(item);
           if (setrefs.length > 1) {
             // this is the form $[seta]$[setb]
             valid = false;
@@ -126,9 +126,9 @@ export class VarsCompiler extends SectionCompiler {
         }
         addId(id);
         allUnicodeSets.add(id);
-        const stringrefs = VariableParser.allStringReferences(value);
+        const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value);
         st.string.add(SubstitutionUse.variable, stringrefs);
-        const setrefs = VariableParser.allSetReferences(value);
+        const setrefs = LdmlKeyboardTypes.VariableParser.allSetReferences(value);
         for (const id2 of setrefs) {
           if (!allUnicodeSets.has(id2)) {
             valid = false;
@@ -193,13 +193,13 @@ export class VarsCompiler extends SectionCompiler {
     const matchedNotEmitted : Set<string> = new Set<string>();
     const mt = st.markers;
     for (const m of mt.matched.values()) {
-      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker
       if (!mt.emitted.has(m)) {
         matchedNotEmitted.add(m);
       }
     }
     for (const m of mt.consumed.values()) {
-      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker
       if (!mt.emitted.has(m)) {
         matchedNotEmitted.add(m);
       }
@@ -225,10 +225,10 @@ export class VarsCompiler extends SectionCompiler {
 
   validateSubstitutions(keyboard: LDMLKeyboard.LKKeyboard, st : Substitutions) : boolean {
     keyboard?.variables?.string?.forEach(({value}) =>
-          st.markers.add(SubstitutionUse.variable, MarkerParser.allReferences(value)));
+          st.markers.add(SubstitutionUse.variable, LdmlKeyboardTypes.MarkerParser.allReferences(value)));
     // get markers mentioned in a set
     keyboard?.variables?.set?.forEach(({ value }) =>
-      VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, MarkerParser.allReferences(v))));
+      LdmlKeyboardTypes.VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, LdmlKeyboardTypes.MarkerParser.allReferences(v))));
     return true;
   }
 
@@ -254,7 +254,7 @@ export class VarsCompiler extends SectionCompiler {
     const mt = st.markers;
 
     // collect all markers, excluding the match-all
-    const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort();
+    const allMarkers : string[] = Array.from(mt.all).filter(m => m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort();
     result.markers = sections.list.allocList(allMarkers, {}, sections);
 
     // sets need to be added late, because they can refer to markers
@@ -280,7 +280,7 @@ export class VarsCompiler extends SectionCompiler {
     // OK to do this as a substitute, because we've already validated the set above.
     value = result.substituteSets(value, sections);
     // raw items - without marker substitution
-    const rawItems: string[] = VariableParser.setSplitter(value);
+    const rawItems: string[] = LdmlKeyboardTypes.VariableParser.setSplitter(value);
     // cooked items - has substutition of markers
     // this is not 'forMatch', all variables are to be assumed as string literals, not regex
     // content.

developer/src/kmc-ldml/test/helpers/index.ts

@@ -7,7 +7,7 @@ import 'mocha';
 import * as path from 'path';
 import { fileURLToPath } from 'url';
 import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js';
-import { util, KMXPlus, UnicodeSetParser } from '@keymanapp/common-types';
+import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types';
 import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, } from "@keymanapp/developer-utils";
 import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles
 import { assert } from 'chai';
@@ -297,7 +297,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi
     });
   }
 }
-async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise<UnicodeSetParser> {
+async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise<LdmlKeyboardTypes.UnicodeSetParser> {
   // for tests, just create a new one
   // see LdmlKeyboardCompiler.getUsetParser()
   const compiler = new KmnCompiler();