fix(developer): rewrite ldml visual keyboard compiler 🍒 🏠

common/web/types/src/consts/modifier-key-constants.ts

@@ -0,0 +1,33 @@
+/*
+ * Keyman is copyright (C) SIL International. MIT License.
+ *
+ * Modifier key bit-flags
+ */
+
+export const ModifierKeyConstants = {
+  // Define Keyman Developer modifier bit-flags (exposed for use by other modules)
+  // Compare against /common/include/kmx_file.h.  CTRL+F "#define LCTRLFLAG" to find the secton.
+  LCTRLFLAG:      0x0001, // Left Control flag
+  RCTRLFLAG:      0x0002, // Right Control flag
+  LALTFLAG:       0x0004, // Left Alt flag
+  RALTFLAG:       0x0008, // Right Alt flag
+  K_SHIFTFLAG:    0x0010, // Either shift flag
+  K_CTRLFLAG:     0x0020, // Either ctrl flag
+  K_ALTFLAG:      0x0040, // Either alt flag
+  K_METAFLAG:     0x0080, // Either Meta-key flag (tentative).  Not usable in keyboard rules;
+                          // Used internally (currently, only by KMW) to ensure Meta-key
+                          // shortcuts safely bypass rules
+                          // Meta key = Command key on macOS, Windows key on Windows/Linux
+  CAPITALFLAG:    0x0100, // Caps lock on
+  NOTCAPITALFLAG: 0x0200, // Caps lock NOT on
+  NUMLOCKFLAG:    0x0400, // Num lock on
+  NOTNUMLOCKFLAG: 0x0800, // Num lock NOT on
+  SCROLLFLAG:     0x1000, // Scroll lock on
+  NOTSCROLLFLAG:  0x2000, // Scroll lock NOT on
+  ISVIRTUALKEY:   0x4000, // It is a Virtual Key Sequence
+  VIRTUALCHARKEY: 0x8000, // Keyman 6.0: Virtual Key Cap Sequence NOT YET
+
+  // Note: OTHER_MODIFIER = 0x10000, used by KMX+ for the
+  // other modifier flag in layers, > 16 bit so not available here.
+  // See keys_mod_other in keyman_core_ldml.ts
+}

common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -113,6 +113,7 @@ export class LDMLKeyboardXMLSourceFileReader {
       for (const sub of obj) {
         // retain the same subtag
         if (!this.boxImportsAndSpecials(sub, subtag)) {
+          // resolveImports has already logged a message
           return false;
         }
       }
@@ -125,13 +126,15 @@ export class LDMLKeyboardXMLSourceFileReader {
           boxXmlArray(obj, key);
           // Now, resolve the import
           if (!this.resolveImports(obj, subtag)) {
+            // resolveImports has already logged a message
             return false;
           }
           // now delete the import array we so carefully constructed, the caller does not
           // want to see it.
           delete obj['import'];
         } else {
           if (!this.boxImportsAndSpecials(obj[key], key)) {
+            // resolveImports has already logged a message
             return false;
           }
         }
@@ -151,6 +154,7 @@ export class LDMLKeyboardXMLSourceFileReader {
     // first, the explicit imports
     for (const asImport of ([...obj['import'] as LKImport[]].reverse())) {
       if (!this.resolveOneImport(obj, subtag, asImport)) {
+        // resolveOneImport has already logged a message
         return false;
       }
     }
@@ -161,6 +165,7 @@ export class LDMLKeyboardXMLSourceFileReader {
         base: constants.cldr_import_base,
         path: constants.cldr_implied_keys_import
       }, true)) {
+        // resolveOneImport has already logged a message
         return false;
       }
     } else if (subtag === 'forms') {
@@ -169,6 +174,7 @@ export class LDMLKeyboardXMLSourceFileReader {
         base: constants.cldr_import_base,
         path: constants.cldr_implied_forms_import
       }, true)) {
+        // resolveOneImport has already logged a message
         return false;
       }
     }
@@ -266,7 +272,8 @@ export class LDMLKeyboardXMLSourceFileReader {
         // An alternative fix would be to pull xml2js directly from github
         // rather than using the version tagged on npmjs.com.
       });
-      parser.parseString(file, (e: unknown, r: unknown) => { a = r as LDMLKeyboardXMLSourceFile }); // TODO-LDML: isn't 'e' the error?
+      const data = new TextDecoder().decode(file);
+      parser.parseString(data, (e: unknown, r: unknown) => { if(e) throw e; a = r as LDMLKeyboardXMLSourceFile }); // TODO-LDML: isn't 'e' the error?
       return a;
     })();
     return source;
@@ -278,14 +285,23 @@ export class LDMLKeyboardXMLSourceFileReader {
    */
   public load(file: Uint8Array): LDMLKeyboardXMLSourceFile | null {
     if (!file) {
+      throw new Error(`file parameter must not be null`);
+    }
+
+    let source: LDMLKeyboardXMLSourceFile = null;
+    try {
+      source = this.loadUnboxed(file);
+    } catch(e) {
+      this.callbacks.reportMessage(CommonTypesMessages.Error_InvalidXml({e}));
       return null;
     }
-    const source = this.loadUnboxed(file);
-    if(this.boxArrays(source)) {
+
+    if (this.boxArrays(source)) {
       return source;
-    } else {
-      return null;
     }
+
+    // boxArrays ... resolveImports has already logged a message
+    return null;
   }
 
   loadTestDataUnboxed(file: Uint8Array): any {

common/web/types/src/main.ts

@@ -20,6 +20,7 @@ export { VariableParser, MarkerParser } from './ldml-keyboard/pattern-parser.js'
 export { LDMLKeyboardXMLSourceFileReader, LDMLKeyboardXMLSourceFileReaderOptions } from './ldml-keyboard/ldml-keyboard-xml-reader.js';
 
 export * as Constants from './consts/virtual-key-constants.js';
+export { ModifierKeyConstants } from './consts/modifier-key-constants.js';
 
 export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, CompilerOptions, CompilerEvent, CompilerErrorNamespace,
          CompilerErrorSeverity, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerCallbackOptions,

common/web/types/src/util/common-events.ts

@@ -1,3 +1,6 @@
+/*
+ * Keyman is copyright (C) SIL International. MIT License.
+ */
 import { CompilerErrorNamespace, CompilerErrorSeverity,  CompilerMessageDef as def, CompilerMessageSpec as m } from './compiler-interfaces.js';
 import { constants } from '@keymanapp/ldml-keyboard-constants';
 
@@ -43,4 +46,8 @@ export class CommonTypesMessages {
   static Error_TestDataUnexpectedArray = (o: {subtag: string}) =>
     m(this.ERROR_TestDataUnexpectedArray,
       `Problem reading test data: expected single ${def(o.subtag)} element, found multiple`);
+
+  static ERROR_InvalidXml = SevError | 0x0008;
+  static Error_InvalidXml = (o:{e: any}) =>
+  m(this.ERROR_InvalidXml, `The XML file could not be read: ${(o.e ?? '').toString()}`);
 };

developer/src/kmc-ldml/package.json

@@ -35,12 +35,14 @@
     "@keymanapp/developer-test-helpers": "*",
     "@keymanapp/resources-gosh": "*",
     "@types/chai": "^4.1.7",
+    "@types/common-tags": "^1.8.4",
     "@types/mocha": "^5.2.7",
     "@types/node": "^20.4.1",
     "@types/semver": "^7.3.12",
     "c8": "^7.12.0",
     "chai": "^4.3.4",
     "chalk": "^2.4.2",
+    "common-tags": "^1.8.2",
     "mocha": "^8.4.0",
     "ts-node": "^9.1.1",
     "typescript": "^4.9.5"

developer/src/kmc-ldml/src/compiler/compiler.ts

@@ -125,6 +125,8 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
       return null;
     }
 
+    outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx');
+
     // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful
     // to duplicate some of the metadata
     KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions);
@@ -134,7 +136,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
     const kmx_binary = builder.compile();
 
     const vkcompiler = new LdmlKeyboardVisualKeyboardCompiler(this.callbacks);
-    const vk = vkcompiler.compile(source);
+    const vk = vkcompiler.compile(kmx.kmxplus, this.callbacks.path.basename(outputFilename, '.kmx'));
     const writer = new KvkFileWriter();
     const kvk_binary = writer.write(vk);
 
@@ -149,7 +151,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
     //KMW17.0: const encoder = new TextEncoder();
     //KMW17.0: const kmw_binary = encoder.encode(kmw_string);
 
-    outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx');
 
     return {
       artifacts: {