Merge pull request #12750 from keymanapp/feat/developer/10649-local-i…

…mports

feat(developer,core): local imports 🙀

developer/src/common/web/utils/src/common-messages.ts

@@ -20,17 +20,19 @@ export class CommonTypesMessages {
   static ERROR_ImportInvalidBase = SevError | 0x0002;
   static Error_ImportInvalidBase = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportInvalidBase,
-      `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} is supported.`);
+      `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) are supported.`);
 
   static ERROR_ImportInvalidPath = SevError | 0x0003;
   static Error_ImportInvalidPath = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportInvalidPath,
-      `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml`);
+      `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`);
 
   static ERROR_ImportReadFail = SevError | 0x0004;
   static Error_ImportReadFail = (o: { base: string, path: string, subtag: string }) =>
     m(this.ERROR_ImportReadFail,
-      `Import could not read data with path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`);
+      `Import could not read data with path ${def(o.path)}`,
+      // for CLDR, give guidance on the suggested path
+      (o.base === constants.cldr_import_base) ? `expected the form '${constants.cldr_version_latest}/*.xml' for ${o.base}` : undefined);
 
   static ERROR_ImportWrongRoot = SevError | 0x0005;
   static Error_ImportWrongRoot = (o: { base: string, path: string, subtag: string }) =>

developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -19,7 +19,10 @@ interface NameAndProps  {
 };
 
 export class LDMLKeyboardXMLSourceFileReaderOptions {
-  importsPath: string;
+  /** path to the CLDR imports */
+  cldrImportsPath: string;
+  /** ordered list of paths for local imports */
+  localImportsPaths: string[];
 };
 
 export class LDMLKeyboardXMLSourceFileReader {
@@ -31,10 +34,21 @@ export class LDMLKeyboardXMLSourceFileReader {
   }
 
   readImportFile(version: string, subpath: string): Uint8Array {
-    const importPath = this.callbacks.resolveFilename(this.options.importsPath, `${version}/${subpath}`);
+    const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`);
     return this.callbacks.loadFile(importPath);
   }
 
+  readLocalImportFile(path: string): Uint8Array {
+    // try each of the local imports paths
+    for (const localPath of this.options.localImportsPaths) {
+      const importPath = this.callbacks.path.join(localPath, path);
+      if(this.callbacks.fs.existsSync(importPath)) {
+        return this.callbacks.loadFile(importPath);
+      }
+    }
+    return null; // was not able to load from any of the paths
+  }
+
   /**
    * xml2js will not place single-entry objects into arrays.
    * Easiest way to fix this is to box them ourselves as needed
@@ -203,16 +217,25 @@ export class LDMLKeyboardXMLSourceFileReader {
    */
   private resolveOneImport(obj: any, subtag: string, asImport: LKImport, implied? : boolean) : boolean {
     const { base, path } = asImport;
-    if (base !== constants.cldr_import_base) {
+    // If base is not an empty string (or null/undefined), then it must be 'cldr'
+    if (base && base !== constants.cldr_import_base) {
       this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidBase({base, path, subtag}));
       return false;
     }
-    const paths = path.split('/');
-    if (paths[0] == '' || paths[1] == '' || paths.length !== 2) {
-      this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag}));
-      return false;
+    let importData: Uint8Array;
+
+    if (base === constants.cldr_import_base) {
+      // CLDR import
+      const paths = path.split('/');
+      if (paths[0] == '' || paths[1] == '' || paths.length !== 2) {
+        this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag}));
+        return false;
+      }
+      importData = this.readImportFile(paths[0], paths[1]);
+    } else {
+      // local import
+      importData = this.readLocalImportFile(path);
     }
-    const importData: Uint8Array = this.readImportFile(paths[0], paths[1]);
     if (!importData || !importData.length) {
       this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag}));
       return false;
@@ -241,6 +264,9 @@ export class LDMLKeyboardXMLSourceFileReader {
         // mark all children as an implied import
         subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath);
       }
+      if (base !== constants.cldr_import_base) {
+        subsubval.forEach(o => o[ImportStatus.localImport] = path);
+      }
 
       if (!obj[subsubtag]) {
         obj[subsubtag] = []; // start with empty array

developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts

@@ -38,7 +38,7 @@ export interface LKImport {
   /**
    * import base, currently `cldr` is supported
    */
-  base: string;
+  base?: 'cldr' | '';
   /**
    * path to imported resource, of the form `45/*.xml`
    */
@@ -199,6 +199,8 @@ export class ImportStatus {
   static impliedImport = Symbol('LDML implied import');
   /** item came in via import */
   static import = Symbol('LDML import');
+  /** item came in via local (not CLDR) import */
+  static localImport = Symbol('LDML local import');
 
   /** @returns true if the object was loaded through an implied import */
   static isImpliedImport(o : any) : boolean {
@@ -208,5 +210,9 @@ export class ImportStatus {
   static isImport(o : any) : boolean {
     return o && !!o[ImportStatus.import];
   }
+  /** @returns true if the object was loaded through an explicit import */
+  static isLocalImport(o : any) : boolean {
+    return o && !!o[ImportStatus.localImport];
+  }
 };
 

developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="und" conformsTo="45">
+  <info author="srl295" indicator="🙀" layout="qwerty"  name="Test Minimal Keyboard"/>
+  <keys>
+    <import base="" path="keys-Zyyy-morepunctuation.xml"/>
+  </keys>
+</keyboard3>

developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="und" conformsTo="45">
+  <info author="srl295" indicator="🙀" layout="qwerty"  name="Test Minimal Keyboard"/>
+  <keys>
+  <import base="" path="keys-Zyyy-DOESNOTEXIST.xml"/>
+  </keys>
+</keyboard3>

developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keys>
+    <!-- Special Symbols -->
+    <key id="interrobang" output="‽" />
+    <key id="snail" output="@" />
+</keys>

developer/src/common/web/utils/test/helpers/reader-callback-test.ts

@@ -7,9 +7,11 @@ import { LDMLKeyboardXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-ke
 import { LDMLKeyboardTestDataXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-keyboard-testdata-xml.js';
 import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
 import { fileURLToPath } from 'url';
+import { dirname } from 'node:path';
 
 const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = {
-  importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
+  cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)),
+  localImportsPaths: [],
 };
 
 export interface CompilationCase {
@@ -76,18 +78,29 @@ export interface TestDataCase {
 export function testReaderCases(cases : CompilationCase[]) {
   // we need our own callbacks rather than using the global so messages don't get mixed
   const callbacks = new TestCompilerCallbacks();
-  const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks);
   for (const testcase of cases) {
     const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail
     const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`:
                                         `should load: ${testcase.subpath}`;
     it(testHeading, function () {
       callbacks.clear();
-
-      const data = loadFile(makePathToFixture('ldml-keyboard', testcase.subpath));
+      const path = makePathToFixture('ldml-keyboard', testcase.subpath);
+      // update readerOptions to point to the source dir.
+      readerOptions.localImportsPaths = [ dirname(path) ];
+      const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks);
+      const data = loadFile(path);
       assert.ok(data, `reading ${testcase.subpath}`);
       const source = reader.load(data);
       if (!testcase.loadfail) {
+        if (!source) {
+          // print any loading errs here
+          if (testcase.warnings) {
+            assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included');
+          } else if (!expectFailure) {
+            // no warnings, so expect zero messages
+            assert.deepEqual(callbacks.messages, [], 'expected zero messages');
+          }
+        }
         assert.ok(source, `loading ${testcase.subpath}`);
       } else {
         assert.notOk(source, `loading ${testcase.subpath} (expected failure)`);

developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts

@@ -119,11 +119,33 @@ describe('ldml keyboard xml reader tests', function () {
         // 'hash' is an import but not implied
         assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'hash')));
         assert.isTrue(ImportStatus.isImport(k.find(({id}) => id === 'hash')));
+        assert.isFalse(ImportStatus.isLocalImport(k.find(({id}) => id === 'hash')));
         // 'zz' is not imported
         assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'zz')));
         assert.isFalse(ImportStatus.isImport(k.find(({id}) => id === 'zz')));
       },
     },
+    {
+      subpath: 'import-local.xml',
+      callback: (data, source, subpath, callbacks) => {
+        assert.ok(source?.keyboard3?.keys);
+        const k = pluckKeysFromKeybag(source?.keyboard3?.keys.key, ['interrobang','snail']);
+        assert.sameDeepOrderedMembers(k.map((entry) => {
+          // Drop the Symbol members from the returned keys; assertions may expect their presence.
+          return {
+            id: entry.id,
+            output: entry.output
+          };
+        }), [
+          { id: 'interrobang', output: '‽' },
+          { id: 'snail', output: '@' },
+        ]);
+        // all of the keys are implied imports here
+        assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+        assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+        assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
+      },
+    },
     {
       subpath: 'invalid-import-base.xml',
       loadfail: true,
@@ -135,12 +157,23 @@ describe('ldml keyboard xml reader tests', function () {
         }),
       ],
     },
+    {
+      subpath: 'invalid-import-local.xml',
+      loadfail: true,
+      errors: [
+        CommonTypesMessages.Error_ImportReadFail({
+          base: undefined,
+          path: 'keys-Zyyy-DOESNOTEXIST.xml',
+          subtag: 'keys'
+        }),
+      ],
+    },
     {
       subpath: 'invalid-import-path.xml',
       loadfail: true,
       errors: [
         CommonTypesMessages.Error_ImportInvalidPath({
-          base: null,
+          base: 'cldr',
           path: '45/too/many/slashes/leading/to/nothing-Zxxx-does-not-exist.xml',
           subtag: null,
         }),
@@ -151,7 +184,7 @@ describe('ldml keyboard xml reader tests', function () {
       loadfail: true,
       errors: [
         CommonTypesMessages.Error_ImportReadFail({
-          base: null,
+          base: 'cldr',
           path: '45/none-Zxxx-does-not-exist.xml',
           subtag: null,
         }),

developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <info name="keys-minimal"/>
+
+  <keys>
+    <import base="" path="keys-Zyyy-morepunctuation.xml"/>
+  </keys>
+
+  <layers formId="us">
+    <layer id="base">
+      <row keys="snail interrobang" />
+    </layer>
+  </layers>
+
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<keys>
+    <!-- Special Symbols -->
+    <key id="interrobang" output="‽" />
+    <key id="snail" output="@" />
+</keys>

developer/src/kmc-ldml/test/helpers/index.ts

@@ -38,7 +38,8 @@ export const compilerTestCallbacks = new TestCompilerCallbacks();
 
 export const compilerTestOptions: LdmlCompilerOptions = {
   readerOptions: {
-    importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
+    cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)),
+    localImportsPaths: [], // will be fixed up in loadSectionFixture
   }
 };
 
@@ -59,8 +60,14 @@ export async function loadSectionFixture(compilerClass: SectionCompilerNew, file
   const data = callbacks.loadFile(inputFilename);
   assert.isNotNull(data, `Failed to read file ${inputFilename}`);
 
+  compilerTestOptions.readerOptions.localImportsPaths =  [ path.dirname(inputFilename) ];
+
   const reader = new LDMLKeyboardXMLSourceFileReader(compilerTestOptions.readerOptions, callbacks);
   const source = reader.load(data);
+  if (!source) {
+    // print any callbacks here
+    assert.sameDeepMembers(callbacks.messages, [], `Errors loading ${inputFilename}`);
+  }
   assert.isNotNull(source, `Failed to load XML from ${inputFilename}`);
 
   if (!reader.validate(source)) {

developer/src/kmc-ldml/test/keys.tests.ts

@@ -199,6 +199,19 @@ describe('keys', function () {
         assert.equal(flickw.flicks[0].keyId.value, 'dd');
       },
     },
+    {
+      subpath: 'sections/keys/import-local.xml',
+      callback: (keys, subpath, callbacks) => {
+        assert.isNotNull(keys);
+        assert.equal((<Keys>keys).keys.length, 2 + KeysCompiler.reserved_count);
+        const [snail] = (<Keys>keys).keys.filter(({ id }) => id.value === 'snail');
+        assert.ok(snail,`Missing the snail`);
+        assert.equal(snail.to.value, `@`, `Snail's value`);
+        const [interrobang] = (<Keys>keys).keys.filter(({ id }) => id.value === 'interrobang');
+        assert.ok(interrobang,`Missing the interrobang`);
+        assert.equal(interrobang.to.value, `‽`, `Interrobang's value`);
+      },
+    },
   ], keysDependencies);
 });
 

developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts

@@ -4,6 +4,7 @@ import { CompilerOptions, CompilerCallbacks } from '@keymanapp/developer-utils';
 import { LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils';
 import { BuildActivity } from './BuildActivity.js';
 import { fileURLToPath } from 'url';
+import { dirname } from 'node:path';
 
 export class BuildLdmlKeyboard extends BuildActivity {
   public get name(): string { return 'LDML keyboard'; }
@@ -13,7 +14,8 @@ export class BuildLdmlKeyboard extends BuildActivity {
   public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean> {
     // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk
     const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: {
-      importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
+      cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)),
+      localImportsPaths: [ dirname(infile) ], // local dir
     }};
     const compiler = new kmcLdml.LdmlKeyboardCompiler();
     return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions);

developer/src/kmc/src/commands/buildTestData/index.ts

@@ -7,6 +7,7 @@ import { fileURLToPath } from 'url';
 import { CommandLineBaseOptions } from 'src/util/baseOptions.js';
 import { exitProcess } from '../../util/sysexits.js';
 import { InfrastructureMessages } from '../../messages/infrastructureMessages.js';
+import { dirname } from 'node:path';
 
 export async function buildTestData(infile: string, _options: any, commander: any): Promise<void> {
   const options: CommandLineBaseOptions = commander.optsWithGlobals();
@@ -17,7 +18,8 @@ export async function buildTestData(infile: string, _options: any, commander: an
     saveDebug: false,
     shouldAddCompilerVersion: false,
     readerOptions: {
-      importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL))
+      cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)),
+      localImportsPaths: [ dirname(infile) ], // local dir
     }
   };