diff --git a/developer/src/common/web/utils/src/common-messages.ts b/developer/src/common/web/utils/src/common-messages.ts index 724dadf9cd8..6eca08d37c4 100644 --- a/developer/src/common/web/utils/src/common-messages.ts +++ b/developer/src/common/web/utils/src/common-messages.ts @@ -20,17 +20,19 @@ export class CommonTypesMessages { static ERROR_ImportInvalidBase = SevError | 0x0002; static Error_ImportInvalidBase = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportInvalidBase, - `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} is supported.`); + `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) are supported.`); static ERROR_ImportInvalidPath = SevError | 0x0003; static Error_ImportInvalidPath = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportInvalidPath, - `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml`); + `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`); static ERROR_ImportReadFail = SevError | 0x0004; static Error_ImportReadFail = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportReadFail, - `Import could not read data with path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`); + `Import could not read data with path ${def(o.path)}`, + // for CLDR, give guidance on the suggested path + (o.base === constants.cldr_import_base) ? `expected the form '${constants.cldr_version_latest}/*.xml' for ${o.base}` : undefined); static ERROR_ImportWrongRoot = SevError | 0x0005; static Error_ImportWrongRoot = (o: { base: string, path: string, subtag: string }) => diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 41d4c699f40..928bb94bef7 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -19,7 +19,10 @@ interface NameAndProps { }; export class LDMLKeyboardXMLSourceFileReaderOptions { - importsPath: string; + /** path to the CLDR imports */ + cldrImportsPath: string; + /** ordered list of paths for local imports */ + localImportsPaths: string[]; }; export class LDMLKeyboardXMLSourceFileReader { @@ -31,10 +34,21 @@ export class LDMLKeyboardXMLSourceFileReader { } readImportFile(version: string, subpath: string): Uint8Array { - const importPath = this.callbacks.resolveFilename(this.options.importsPath, `${version}/${subpath}`); + const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); return this.callbacks.loadFile(importPath); } + readLocalImportFile(path: string): Uint8Array { + // try each of the local imports paths + for (const localPath of this.options.localImportsPaths) { + const importPath = this.callbacks.path.join(localPath, path); + if(this.callbacks.fs.existsSync(importPath)) { + return this.callbacks.loadFile(importPath); + } + } + return null; // was not able to load from any of the paths + } + /** * xml2js will not place single-entry objects into arrays. * Easiest way to fix this is to box them ourselves as needed @@ -203,16 +217,25 @@ export class LDMLKeyboardXMLSourceFileReader { */ private resolveOneImport(obj: any, subtag: string, asImport: LKImport, implied? : boolean) : boolean { const { base, path } = asImport; - if (base !== constants.cldr_import_base) { + // If base is not an empty string (or null/undefined), then it must be 'cldr' + if (base && base !== constants.cldr_import_base) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidBase({base, path, subtag})); return false; } - const paths = path.split('/'); - if (paths[0] == '' || paths[1] == '' || paths.length !== 2) { - this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag})); - return false; + let importData: Uint8Array; + + if (base === constants.cldr_import_base) { + // CLDR import + const paths = path.split('/'); + if (paths[0] == '' || paths[1] == '' || paths.length !== 2) { + this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag})); + return false; + } + importData = this.readImportFile(paths[0], paths[1]); + } else { + // local import + importData = this.readLocalImportFile(path); } - const importData: Uint8Array = this.readImportFile(paths[0], paths[1]); if (!importData || !importData.length) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag})); return false; @@ -241,6 +264,9 @@ export class LDMLKeyboardXMLSourceFileReader { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); } + if (base !== constants.cldr_import_base) { + subsubval.forEach(o => o[ImportStatus.localImport] = path); + } if (!obj[subsubtag]) { obj[subsubtag] = []; // start with empty array diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts index 696194573f8..587b02dd719 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts @@ -38,7 +38,7 @@ export interface LKImport { /** * import base, currently `cldr` is supported */ - base: string; + base?: 'cldr' | ''; /** * path to imported resource, of the form `45/*.xml` */ @@ -199,6 +199,8 @@ export class ImportStatus { static impliedImport = Symbol('LDML implied import'); /** item came in via import */ static import = Symbol('LDML import'); + /** item came in via local (not CLDR) import */ + static localImport = Symbol('LDML local import'); /** @returns true if the object was loaded through an implied import */ static isImpliedImport(o : any) : boolean { @@ -208,5 +210,9 @@ export class ImportStatus { static isImport(o : any) : boolean { return o && !!o[ImportStatus.import]; } + /** @returns true if the object was loaded through an explicit import */ + static isLocalImport(o : any) : boolean { + return o && !!o[ImportStatus.localImport]; + } }; diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml new file mode 100644 index 00000000000..b3cc7cabdff --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml new file mode 100644 index 00000000000..e8ac9dc9640 --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml new file mode 100644 index 00000000000..768e81e7cd2 --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts index c67d49dfdb4..d865ab2f46a 100644 --- a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts +++ b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts @@ -7,9 +7,11 @@ import { LDMLKeyboardXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-ke import { LDMLKeyboardTestDataXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-keyboard-testdata-xml.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { fileURLToPath } from 'url'; +import { dirname } from 'node:path'; const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [], }; export interface CompilationCase { @@ -76,18 +78,29 @@ export interface TestDataCase { export function testReaderCases(cases : CompilationCase[]) { // we need our own callbacks rather than using the global so messages don't get mixed const callbacks = new TestCompilerCallbacks(); - const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); for (const testcase of cases) { const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`: `should load: ${testcase.subpath}`; it(testHeading, function () { callbacks.clear(); - - const data = loadFile(makePathToFixture('ldml-keyboard', testcase.subpath)); + const path = makePathToFixture('ldml-keyboard', testcase.subpath); + // update readerOptions to point to the source dir. + readerOptions.localImportsPaths = [ dirname(path) ]; + const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); + const data = loadFile(path); assert.ok(data, `reading ${testcase.subpath}`); const source = reader.load(data); if (!testcase.loadfail) { + if (!source) { + // print any loading errs here + if (testcase.warnings) { + assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included'); + } else if (!expectFailure) { + // no warnings, so expect zero messages + assert.deepEqual(callbacks.messages, [], 'expected zero messages'); + } + } assert.ok(source, `loading ${testcase.subpath}`); } else { assert.notOk(source, `loading ${testcase.subpath} (expected failure)`); diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index 17c35660245..e3e6fc1e9fb 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -119,11 +119,33 @@ describe('ldml keyboard xml reader tests', function () { // 'hash' is an import but not implied assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'hash'))); assert.isTrue(ImportStatus.isImport(k.find(({id}) => id === 'hash'))); + assert.isFalse(ImportStatus.isLocalImport(k.find(({id}) => id === 'hash'))); // 'zz' is not imported assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'zz'))); assert.isFalse(ImportStatus.isImport(k.find(({id}) => id === 'zz'))); }, }, + { + subpath: 'import-local.xml', + callback: (data, source, subpath, callbacks) => { + assert.ok(source?.keyboard3?.keys); + const k = pluckKeysFromKeybag(source?.keyboard3?.keys.key, ['interrobang','snail']); + assert.sameDeepOrderedMembers(k.map((entry) => { + // Drop the Symbol members from the returned keys; assertions may expect their presence. + return { + id: entry.id, + output: entry.output + }; + }), [ + { id: 'interrobang', output: '‽' }, + { id: 'snail', output: '@' }, + ]); + // all of the keys are implied imports here + assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + }, + }, { subpath: 'invalid-import-base.xml', loadfail: true, @@ -135,12 +157,23 @@ describe('ldml keyboard xml reader tests', function () { }), ], }, + { + subpath: 'invalid-import-local.xml', + loadfail: true, + errors: [ + CommonTypesMessages.Error_ImportReadFail({ + base: undefined, + path: 'keys-Zyyy-DOESNOTEXIST.xml', + subtag: 'keys' + }), + ], + }, { subpath: 'invalid-import-path.xml', loadfail: true, errors: [ CommonTypesMessages.Error_ImportInvalidPath({ - base: null, + base: 'cldr', path: '45/too/many/slashes/leading/to/nothing-Zxxx-does-not-exist.xml', subtag: null, }), @@ -151,7 +184,7 @@ describe('ldml keyboard xml reader tests', function () { loadfail: true, errors: [ CommonTypesMessages.Error_ImportReadFail({ - base: null, + base: 'cldr', path: '45/none-Zxxx-does-not-exist.xml', subtag: null, }), diff --git a/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml b/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml new file mode 100644 index 00000000000..d97d711301d --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml b/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml new file mode 100644 index 00000000000..768e81e7cd2 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 3bb67278fbe..bd438c4ca11 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -38,7 +38,8 @@ export const compilerTestCallbacks = new TestCompilerCallbacks(); export const compilerTestOptions: LdmlCompilerOptions = { readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [], // will be fixed up in loadSectionFixture } }; @@ -59,8 +60,14 @@ export async function loadSectionFixture(compilerClass: SectionCompilerNew, file const data = callbacks.loadFile(inputFilename); assert.isNotNull(data, `Failed to read file ${inputFilename}`); + compilerTestOptions.readerOptions.localImportsPaths = [ path.dirname(inputFilename) ]; + const reader = new LDMLKeyboardXMLSourceFileReader(compilerTestOptions.readerOptions, callbacks); const source = reader.load(data); + if (!source) { + // print any callbacks here + assert.sameDeepMembers(callbacks.messages, [], `Errors loading ${inputFilename}`); + } assert.isNotNull(source, `Failed to load XML from ${inputFilename}`); if (!reader.validate(source)) { diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index b639c194cc3..cfb37e6baf5 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -199,6 +199,19 @@ describe('keys', function () { assert.equal(flickw.flicks[0].keyId.value, 'dd'); }, }, + { + subpath: 'sections/keys/import-local.xml', + callback: (keys, subpath, callbacks) => { + assert.isNotNull(keys); + assert.equal((keys).keys.length, 2 + KeysCompiler.reserved_count); + const [snail] = (keys).keys.filter(({ id }) => id.value === 'snail'); + assert.ok(snail,`Missing the snail`); + assert.equal(snail.to.value, `@`, `Snail's value`); + const [interrobang] = (keys).keys.filter(({ id }) => id.value === 'interrobang'); + assert.ok(interrobang,`Missing the interrobang`); + assert.equal(interrobang.to.value, `‽`, `Interrobang's value`); + }, + }, ], keysDependencies); }); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 6ac98c9b92e..6a6300b646b 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -4,6 +4,7 @@ import { CompilerOptions, CompilerCallbacks } from '@keymanapp/developer-utils'; import { LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; +import { dirname } from 'node:path'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -13,7 +14,8 @@ export class BuildLdmlKeyboard extends BuildActivity { public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) ], // local dir }}; const compiler = new kmcLdml.LdmlKeyboardCompiler(); return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index c958673c2fa..5343c81702d 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -7,6 +7,7 @@ import { fileURLToPath } from 'url'; import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; import { exitProcess } from '../../util/sysexits.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; +import { dirname } from 'node:path'; export async function buildTestData(infile: string, _options: any, commander: any): Promise { const options: CommandLineBaseOptions = commander.optsWithGlobals(); @@ -17,7 +18,8 @@ export async function buildTestData(infile: string, _options: any, commander: an saveDebug: false, shouldAddCompilerVersion: false, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) ], // local dir } };