feat(developer): support local imports

- added a new reader callback option, localImportsPaths - due to the CLDR issue #12749 use base="" - add tests - some bugfixes in import messages - add an ImportStatus section to determine if something is a local import Fixes: #10649
keymanapp · Nov 29, 2024 · 01a47c3 · 01a47c3
1 parent add502e
commit 01a47c3
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 16 deletions.
diff --git a/developer/src/common/web/utils/src/common-messages.ts b/developer/src/common/web/utils/src/common-messages.ts
@@ -20,17 +20,19 @@ export class CommonTypesMessages {
   static ERROR_ImportInvalidBase = SevError | 0x0002;
   static Error_ImportInvalidBase = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportInvalidBase,
-      `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} is supported.`);
+      `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) is supported.`);
 
   static ERROR_ImportInvalidPath = SevError | 0x0003;
   static Error_ImportInvalidPath = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportInvalidPath,
-      `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml`);
+      `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`);
 
   static ERROR_ImportReadFail = SevError | 0x0004;
   static Error_ImportReadFail = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportReadFail,
-      `Import could not read data with path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`);
+      `Import could not read data with path ${def(o.path)}`,
+      // for CLDR, give guidance on the suggested path
+      (o.base === constants.cldr_import_base) ? `expected the form '${constants.cldr_version_latest}/*.xml' for ${o.base}` : undefined);
 
   static ERROR_ImportWrongRoot = SevError | 0x0005;
   static Error_ImportWrongRoot = (o: { base: string, path: string, subtag: string }) =>

diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts
@@ -19,7 +19,10 @@ interface NameAndProps  {
 };
 
 export class LDMLKeyboardXMLSourceFileReaderOptions {
+  /** path to the CLDR imports */
   importsPath: string;
+  /** ordered list of paths for local imports */
+  localImportsPaths: string[];
 };
 
 export class LDMLKeyboardXMLSourceFileReader {
@@ -35,6 +38,16 @@ export class LDMLKeyboardXMLSourceFileReader {
     return this.callbacks.loadFile(importPath);
   }
 
+  readLocalImportFile(path: string): Uint8Array {
+    // try each of the local imports paths
+    for (const localPath of this.options.localImportsPaths) {
+      const importPath = this.callbacks.resolveFilename(localPath, path);
+      const data = this.callbacks.loadFile(importPath);
+      if (data) return data;
+    }
+    return null; // was not able to load from any of the paths
+  }
+
   /**
    * xml2js will not place single-entry objects into arrays.
    * Easiest way to fix this is to box them ourselves as needed
@@ -203,16 +216,24 @@ export class LDMLKeyboardXMLSourceFileReader {
    */
   private resolveOneImport(obj: any, subtag: string, asImport: LKImport, implied? : boolean) : boolean {
     const { base, path } = asImport;
-    if (base !== constants.cldr_import_base) {
+    if (base && base !== constants.cldr_import_base) {
       this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidBase({base, path, subtag}));
       return false;
     }
-    const paths = path.split('/');
-    if (paths[0] == '' || paths[1] == '' || paths.length !== 2) {
-      this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag}));
-      return false;
+    let importData: Uint8Array;
+
+    if (base === constants.cldr_import_base) {
+      // CLDR import
+      const paths = path.split('/');
+      if (paths[0] == '' || paths[1] == '' || paths.length !== 2) {
+        this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag}));
+        return false;
+      }
+      importData = this.readImportFile(paths[0], paths[1]);
+    } else {
+      // local import
+      importData = this.readLocalImportFile(path);
     }
-    const importData: Uint8Array = this.readImportFile(paths[0], paths[1]);
     if (!importData || !importData.length) {
       this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag}));
       return false;
@@ -241,6 +262,9 @@ export class LDMLKeyboardXMLSourceFileReader {
         // mark all children as an implied import
         subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath);
       }
+      if (!base) {
+        subsubval.forEach(o => o[ImportStatus.localImport] = path);
+      }
 
       if (!obj[subsubtag]) {
         obj[subsubtag] = []; // start with empty array

diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts
@@ -38,7 +38,7 @@ export interface LKImport {
   /**
    * import base, currently `cldr` is supported
    */
-  base: string;
+  base?: string;
   /**
    * path to imported resource, of the form `45/*.xml`
    */
@@ -199,6 +199,8 @@ export class ImportStatus {
   static impliedImport = Symbol('LDML implied import');
   /** item came in via import */
   static import = Symbol('LDML import');
+  /** item came in via local (not CLDR) import */
+  static localImport = Symbol('LDML local import');
 
   /** @returns true if the object was loaded through an implied import */
   static isImpliedImport(o : any) : boolean {
@@ -208,5 +210,9 @@ export class ImportStatus {
   static isImport(o : any) : boolean {
     return o && !!o[ImportStatus.import];
   }
+  /** @returns true if the object was loaded through an explicit import */
+  static isLocalImport(o : any) : boolean {
+    return o && !!o[ImportStatus.localImport];
+  }
 };
 
diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="und" conformsTo="45">
+  <info author="srl295" indicator="🙀" layout="qwerty"  name="Test Minimal Keyboard"/>
+  <keys>
+    <import base="" path="keys-Zyyy-morepunctuation.xml"/>
+  </keys>
+</keyboard3>
diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="und" conformsTo="45">
+  <info author="srl295" indicator="🙀" layout="qwerty"  name="Test Minimal Keyboard"/>
+  <keys>
+  <import base="" path="keys-Zyyy-DOESNOTEXIST.xml"/>
+  </keys>
+</keyboard3>
diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keys>
+    <!-- Special Symbols -->
+    <key id="interrobang" output="‽" />
+    <key id="snail" output="@" />
+</keys>
diff --git a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts
@@ -7,9 +7,11 @@ import { LDMLKeyboardXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-ke
 import { LDMLKeyboardTestDataXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-keyboard-testdata-xml.js';
 import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
 import { fileURLToPath } from 'url';
+import { dirname, sep } from 'node:path';
 
 const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = {
-  importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
+  importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)),
+  localImportsPaths: [],
 };
 
 export interface CompilationCase {
@@ -76,18 +78,29 @@ export interface TestDataCase {
 export function testReaderCases(cases : CompilationCase[]) {
   // we need our own callbacks rather than using the global so messages don't get mixed
   const callbacks = new TestCompilerCallbacks();
-  const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks);
   for (const testcase of cases) {
     const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail
     const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`:
                                         `should load: ${testcase.subpath}`;
     it(testHeading, function () {
       callbacks.clear();
-
-      const data = loadFile(makePathToFixture('ldml-keyboard', testcase.subpath));
+      const path = makePathToFixture('ldml-keyboard', testcase.subpath);
+      // update readerOptions to point to the source dir. Need a trailing separator.
+      readerOptions.localImportsPaths = [ dirname(path) + sep ];
+      const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks);
+      const data = loadFile(path);
       assert.ok(data, `reading ${testcase.subpath}`);
       const source = reader.load(data);
       if (!testcase.loadfail) {
+        if (!source) {
+          // print any loading errs here
+          if (testcase.warnings) {
+            assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included');
+          } else if (!expectFailure) {
+            // no warnings, so expect zero messages
+            assert.deepEqual(callbacks.messages, [], 'expected zero messages');
+          }
+        }
         assert.ok(source, `loading ${testcase.subpath}`);
       } else {
         assert.notOk(source, `loading ${testcase.subpath} (expected failure)`);

diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts
@@ -119,11 +119,33 @@ describe('ldml keyboard xml reader tests', function () {
         // 'hash' is an import but not implied
         assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'hash')));
         assert.isTrue(ImportStatus.isImport(k.find(({id}) => id === 'hash')));
+        assert.isFalse(ImportStatus.isLocalImport(k.find(({id}) => id === 'hash')));
         // 'zz' is not imported
         assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'zz')));
         assert.isFalse(ImportStatus.isImport(k.find(({id}) => id === 'zz')));
       },
     },
+    {
+      subpath: 'import-local.xml',
+      callback: (data, source, subpath, callbacks) => {
+        assert.ok(source?.keyboard3?.keys);
+        const k = pluckKeysFromKeybag(source?.keyboard3?.keys.key, ['interrobang','snail']);
+        assert.sameDeepOrderedMembers(k.map((entry) => {
+          // Drop the Symbol members from the returned keys; assertions may expect their presence.
+          return {
+            id: entry.id,
+            output: entry.output
+          };
+        }), [
+          { id: 'interrobang', output: '‽' },
+          { id: 'snail', output: '@' },
+        ]);
+        // all of the keys are implied imports here
+        assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+        assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+        assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+      },
+    },
     {
       subpath: 'invalid-import-base.xml',
       loadfail: true,
@@ -135,12 +157,23 @@ describe('ldml keyboard xml reader tests', function () {
         }),
       ],
     },
+    {
+      subpath: 'invalid-import-local.xml',
+      loadfail: true,
+      errors: [
+        CommonTypesMessages.Error_ImportReadFail({
+          base: undefined,
+          path: 'keys-Zyyy-DOESNOTEXIST.xml',
+          subtag: 'keys'
+        }),
+      ],
+    },
     {
       subpath: 'invalid-import-path.xml',
       loadfail: true,
       errors: [
         CommonTypesMessages.Error_ImportInvalidPath({
-          base: null,
+          base: 'cldr',
           path: '45/too/many/slashes/leading/to/nothing-Zxxx-does-not-exist.xml',
           subtag: null,
         }),
@@ -151,7 +184,7 @@ describe('ldml keyboard xml reader tests', function () {
       loadfail: true,
       errors: [
         CommonTypesMessages.Error_ImportReadFail({
-          base: null,
+          base: 'cldr',
           path: '45/none-Zxxx-does-not-exist.xml',
           subtag: null,
         }),