From 01a47c3ea967bda8f4cefa2373d47e099d6dc8a7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 13:48:21 -0600 Subject: [PATCH 1/7] feat(developer): support local imports - added a new reader callback option, localImportsPaths - due to the CLDR issue #12749 use base="" - add tests - some bugfixes in import messages - add an ImportStatus section to determine if something is a local import Fixes: #10649 --- .../common/web/utils/src/common-messages.ts | 8 ++-- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 36 +++++++++++++++--- .../types/ldml-keyboard/ldml-keyboard-xml.ts | 8 +++- .../fixtures/ldml-keyboard/import-local.xml | 7 ++++ .../ldml-keyboard/invalid-import-local.xml | 7 ++++ .../keys-Zyyy-morepunctuation.xml | 6 +++ .../test/helpers/reader-callback-test.ts | 21 +++++++++-- .../kmx/ldml-keyboard-xml-reader.tests.ts | 37 ++++++++++++++++++- 8 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml create mode 100644 developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml create mode 100644 developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml diff --git a/developer/src/common/web/utils/src/common-messages.ts b/developer/src/common/web/utils/src/common-messages.ts index 724dadf9cd8..c35ae136fde 100644 --- a/developer/src/common/web/utils/src/common-messages.ts +++ b/developer/src/common/web/utils/src/common-messages.ts @@ -20,17 +20,19 @@ export class CommonTypesMessages { static ERROR_ImportInvalidBase = SevError | 0x0002; static Error_ImportInvalidBase = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportInvalidBase, - `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} is supported.`); + `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) is supported.`); static ERROR_ImportInvalidPath = SevError | 0x0003; static Error_ImportInvalidPath = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportInvalidPath, - `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml`); + `Import element with invalid path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`); static ERROR_ImportReadFail = SevError | 0x0004; static Error_ImportReadFail = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportReadFail, - `Import could not read data with path ${def(o.path)}: expected the form '${constants.cldr_version_latest}/*.xml'`); + `Import could not read data with path ${def(o.path)}`, + // for CLDR, give guidance on the suggested path + (o.base === constants.cldr_import_base) ? `expected the form '${constants.cldr_version_latest}/*.xml' for ${o.base}` : undefined); static ERROR_ImportWrongRoot = SevError | 0x0005; static Error_ImportWrongRoot = (o: { base: string, path: string, subtag: string }) => diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 41d4c699f40..559a7c421ff 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -19,7 +19,10 @@ interface NameAndProps { }; export class LDMLKeyboardXMLSourceFileReaderOptions { + /** path to the CLDR imports */ importsPath: string; + /** ordered list of paths for local imports */ + localImportsPaths: string[]; }; export class LDMLKeyboardXMLSourceFileReader { @@ -35,6 +38,16 @@ export class LDMLKeyboardXMLSourceFileReader { return this.callbacks.loadFile(importPath); } + readLocalImportFile(path: string): Uint8Array { + // try each of the local imports paths + for (const localPath of this.options.localImportsPaths) { + const importPath = this.callbacks.resolveFilename(localPath, path); + const data = this.callbacks.loadFile(importPath); + if (data) return data; + } + return null; // was not able to load from any of the paths + } + /** * xml2js will not place single-entry objects into arrays. * Easiest way to fix this is to box them ourselves as needed @@ -203,16 +216,24 @@ export class LDMLKeyboardXMLSourceFileReader { */ private resolveOneImport(obj: any, subtag: string, asImport: LKImport, implied? : boolean) : boolean { const { base, path } = asImport; - if (base !== constants.cldr_import_base) { + if (base && base !== constants.cldr_import_base) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidBase({base, path, subtag})); return false; } - const paths = path.split('/'); - if (paths[0] == '' || paths[1] == '' || paths.length !== 2) { - this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag})); - return false; + let importData: Uint8Array; + + if (base === constants.cldr_import_base) { + // CLDR import + const paths = path.split('/'); + if (paths[0] == '' || paths[1] == '' || paths.length !== 2) { + this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidPath({base, path, subtag})); + return false; + } + importData = this.readImportFile(paths[0], paths[1]); + } else { + // local import + importData = this.readLocalImportFile(path); } - const importData: Uint8Array = this.readImportFile(paths[0], paths[1]); if (!importData || !importData.length) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag})); return false; @@ -241,6 +262,9 @@ export class LDMLKeyboardXMLSourceFileReader { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); } + if (!base) { + subsubval.forEach(o => o[ImportStatus.localImport] = path); + } if (!obj[subsubtag]) { obj[subsubtag] = []; // start with empty array diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts index 696194573f8..62322ae25e2 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts @@ -38,7 +38,7 @@ export interface LKImport { /** * import base, currently `cldr` is supported */ - base: string; + base?: string; /** * path to imported resource, of the form `45/*.xml` */ @@ -199,6 +199,8 @@ export class ImportStatus { static impliedImport = Symbol('LDML implied import'); /** item came in via import */ static import = Symbol('LDML import'); + /** item came in via local (not CLDR) import */ + static localImport = Symbol('LDML local import'); /** @returns true if the object was loaded through an implied import */ static isImpliedImport(o : any) : boolean { @@ -208,5 +210,9 @@ export class ImportStatus { static isImport(o : any) : boolean { return o && !!o[ImportStatus.import]; } + /** @returns true if the object was loaded through an explicit import */ + static isLocalImport(o : any) : boolean { + return o && !!o[ImportStatus.localImport]; + } }; diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml new file mode 100644 index 00000000000..b3cc7cabdff --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/import-local.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml new file mode 100644 index 00000000000..e8ac9dc9640 --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/invalid-import-local.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml new file mode 100644 index 00000000000..768e81e7cd2 --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/ldml-keyboard/keys-Zyyy-morepunctuation.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts index c67d49dfdb4..8c2e4295209 100644 --- a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts +++ b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts @@ -7,9 +7,11 @@ import { LDMLKeyboardXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-ke import { LDMLKeyboardTestDataXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-keyboard-testdata-xml.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { fileURLToPath } from 'url'; +import { dirname, sep } from 'node:path'; const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [], }; export interface CompilationCase { @@ -76,18 +78,29 @@ export interface TestDataCase { export function testReaderCases(cases : CompilationCase[]) { // we need our own callbacks rather than using the global so messages don't get mixed const callbacks = new TestCompilerCallbacks(); - const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); for (const testcase of cases) { const expectFailure = testcase.throws || !!(testcase.errors); // if true, we expect this to fail const testHeading = expectFailure ? `should fail to load: ${testcase.subpath}`: `should load: ${testcase.subpath}`; it(testHeading, function () { callbacks.clear(); - - const data = loadFile(makePathToFixture('ldml-keyboard', testcase.subpath)); + const path = makePathToFixture('ldml-keyboard', testcase.subpath); + // update readerOptions to point to the source dir. Need a trailing separator. + readerOptions.localImportsPaths = [ dirname(path) + sep ]; + const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); + const data = loadFile(path); assert.ok(data, `reading ${testcase.subpath}`); const source = reader.load(data); if (!testcase.loadfail) { + if (!source) { + // print any loading errs here + if (testcase.warnings) { + assert.includeDeepMembers(callbacks.messages, testcase.warnings, 'expected warnings to be included'); + } else if (!expectFailure) { + // no warnings, so expect zero messages + assert.deepEqual(callbacks.messages, [], 'expected zero messages'); + } + } assert.ok(source, `loading ${testcase.subpath}`); } else { assert.notOk(source, `loading ${testcase.subpath} (expected failure)`); diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index 17c35660245..e3e6fc1e9fb 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -119,11 +119,33 @@ describe('ldml keyboard xml reader tests', function () { // 'hash' is an import but not implied assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'hash'))); assert.isTrue(ImportStatus.isImport(k.find(({id}) => id === 'hash'))); + assert.isFalse(ImportStatus.isLocalImport(k.find(({id}) => id === 'hash'))); // 'zz' is not imported assert.isFalse(ImportStatus.isImpliedImport(k.find(({id}) => id === 'zz'))); assert.isFalse(ImportStatus.isImport(k.find(({id}) => id === 'zz'))); }, }, + { + subpath: 'import-local.xml', + callback: (data, source, subpath, callbacks) => { + assert.ok(source?.keyboard3?.keys); + const k = pluckKeysFromKeybag(source?.keyboard3?.keys.key, ['interrobang','snail']); + assert.sameDeepOrderedMembers(k.map((entry) => { + // Drop the Symbol members from the returned keys; assertions may expect their presence. + return { + id: entry.id, + output: entry.output + }; + }), [ + { id: 'interrobang', output: '‽' }, + { id: 'snail', output: '@' }, + ]); + // all of the keys are implied imports here + assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + }, + }, { subpath: 'invalid-import-base.xml', loadfail: true, @@ -135,12 +157,23 @@ describe('ldml keyboard xml reader tests', function () { }), ], }, + { + subpath: 'invalid-import-local.xml', + loadfail: true, + errors: [ + CommonTypesMessages.Error_ImportReadFail({ + base: undefined, + path: 'keys-Zyyy-DOESNOTEXIST.xml', + subtag: 'keys' + }), + ], + }, { subpath: 'invalid-import-path.xml', loadfail: true, errors: [ CommonTypesMessages.Error_ImportInvalidPath({ - base: null, + base: 'cldr', path: '45/too/many/slashes/leading/to/nothing-Zxxx-does-not-exist.xml', subtag: null, }), @@ -151,7 +184,7 @@ describe('ldml keyboard xml reader tests', function () { loadfail: true, errors: [ CommonTypesMessages.Error_ImportReadFail({ - base: null, + base: 'cldr', path: '45/none-Zxxx-does-not-exist.xml', subtag: null, }), From e99f6ac46a5d2d4b983fd0319e68b1c641f50a49 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 13:56:41 -0600 Subject: [PATCH 2/7] feat(developer): support local imports in kmc - basic changes to support local imports Fixes: #10649 --- .../src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts | 4 +++- developer/src/kmc/src/commands/buildTestData/index.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 6ac98c9b92e..643f31e9596 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -4,6 +4,7 @@ import { CompilerOptions, CompilerCallbacks } from '@keymanapp/developer-utils'; import { LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; +import { dirname, sep } from 'node:path'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -13,7 +14,8 @@ export class BuildLdmlKeyboard extends BuildActivity { public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) + sep ], // local dir }}; const compiler = new kmcLdml.LdmlKeyboardCompiler(); return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index c958673c2fa..39b149765c5 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -7,6 +7,7 @@ import { fileURLToPath } from 'url'; import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; import { exitProcess } from '../../util/sysexits.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; +import { dirname, sep } from 'node:path'; export async function buildTestData(infile: string, _options: any, commander: any): Promise { const options: CommandLineBaseOptions = commander.optsWithGlobals(); @@ -17,7 +18,8 @@ export async function buildTestData(infile: string, _options: any, commander: an saveDebug: false, shouldAddCompilerVersion: false, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) + sep ], // local dir } }; From 01fed4b2abb8b68e9842fe6119d69da0289febab Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 14:11:29 -0600 Subject: [PATCH 3/7] feat(developer): test local imports in kmc-ldml - also improve callback visibility when XML fails to load Fixes: #10649 --- .../test/fixtures/sections/keys/import-local.xml | 16 ++++++++++++++++ .../sections/keys/keys-Zyyy-morepunctuation.xml | 6 ++++++ developer/src/kmc-ldml/test/helpers/index.ts | 9 ++++++++- developer/src/kmc-ldml/test/keys.tests.ts | 13 +++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml diff --git a/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml b/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml new file mode 100644 index 00000000000..d97d711301d --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/keys/import-local.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml b/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml new file mode 100644 index 00000000000..768e81e7cd2 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/keys/keys-Zyyy-morepunctuation.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 3bb67278fbe..283459688fe 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -38,7 +38,8 @@ export const compilerTestCallbacks = new TestCompilerCallbacks(); export const compilerTestOptions: LdmlCompilerOptions = { readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [], // will be fixed up in loadSectionFixture } }; @@ -59,8 +60,14 @@ export async function loadSectionFixture(compilerClass: SectionCompilerNew, file const data = callbacks.loadFile(inputFilename); assert.isNotNull(data, `Failed to read file ${inputFilename}`); + compilerTestOptions.readerOptions.localImportsPaths = [ path.dirname(inputFilename) + path.sep ]; + const reader = new LDMLKeyboardXMLSourceFileReader(compilerTestOptions.readerOptions, callbacks); const source = reader.load(data); + if (!source) { + // print any callbacks here + assert.sameDeepMembers(callbacks.messages, [], `Errors loading ${inputFilename}`); + } assert.isNotNull(source, `Failed to load XML from ${inputFilename}`); if (!reader.validate(source)) { diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index b639c194cc3..cfb37e6baf5 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -199,6 +199,19 @@ describe('keys', function () { assert.equal(flickw.flicks[0].keyId.value, 'dd'); }, }, + { + subpath: 'sections/keys/import-local.xml', + callback: (keys, subpath, callbacks) => { + assert.isNotNull(keys); + assert.equal((keys).keys.length, 2 + KeysCompiler.reserved_count); + const [snail] = (keys).keys.filter(({ id }) => id.value === 'snail'); + assert.ok(snail,`Missing the snail`); + assert.equal(snail.to.value, `@`, `Snail's value`); + const [interrobang] = (keys).keys.filter(({ id }) => id.value === 'interrobang'); + assert.ok(interrobang,`Missing the interrobang`); + assert.equal(interrobang.to.value, `‽`, `Interrobang's value`); + }, + }, ], keysDependencies); }); From b1b6587a7994810e19fd3d66ed01a9e351638e75 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 14:19:35 -0600 Subject: [PATCH 4/7] feat(core): test local imports in kmc Fixes: #10649 --- .../unit/ldml/keyboards/k_015_importlocal.xml | 22 +++++++++++++++++++ .../keyboards/keys-Zyyy-morepunctuation.xml | 6 +++++ core/tests/unit/ldml/keyboards/meson.build | 1 + 3 files changed, 29 insertions(+) create mode 100644 core/tests/unit/ldml/keyboards/k_015_importlocal.xml create mode 100644 core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml diff --git a/core/tests/unit/ldml/keyboards/k_015_importlocal.xml b/core/tests/unit/ldml/keyboards/k_015_importlocal.xml new file mode 100644 index 00000000000..36bd5c6942f --- /dev/null +++ b/core/tests/unit/ldml/keyboards/k_015_importlocal.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + diff --git a/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml b/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml new file mode 100644 index 00000000000..768e81e7cd2 --- /dev/null +++ b/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index 15f458c587f..949acb78996 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -26,6 +26,7 @@ tests_without_testdata = [ 'k_010_mt', 'k_011_mt_iso', 'k_012_other', + 'k_015_importlocal', 'k_030_transform_plus', 'k_100_keytest', 'k_101_keytest', From 48da8fd613272148f9acb7ba70b0396dee39d755 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Dec 2024 11:55:13 -0600 Subject: [PATCH 5/7] Apply suggestions from code review Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/common-messages.ts | 2 +- .../src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 8 +++++--- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/developer/src/common/web/utils/src/common-messages.ts b/developer/src/common/web/utils/src/common-messages.ts index c35ae136fde..6eca08d37c4 100644 --- a/developer/src/common/web/utils/src/common-messages.ts +++ b/developer/src/common/web/utils/src/common-messages.ts @@ -20,7 +20,7 @@ export class CommonTypesMessages { static ERROR_ImportInvalidBase = SevError | 0x0002; static Error_ImportInvalidBase = (o: { base: string, path: string, subtag: string }) => m(this.ERROR_ImportInvalidBase, - `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) is supported.`); + `Import element with base ${def(o.base)} is unsupported. Only ${constants.cldr_import_base} or empty (for local) are supported.`); static ERROR_ImportInvalidPath = SevError | 0x0003; static Error_ImportInvalidPath = (o: { base: string, path: string, subtag: string }) => diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 559a7c421ff..2a5754b45b2 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -42,8 +42,9 @@ export class LDMLKeyboardXMLSourceFileReader { // try each of the local imports paths for (const localPath of this.options.localImportsPaths) { const importPath = this.callbacks.resolveFilename(localPath, path); - const data = this.callbacks.loadFile(importPath); - if (data) return data; + if(this.callbacks.fs.existsSync(importPath)) { + return this.callbacks.loadFile(importPath); + } } return null; // was not able to load from any of the paths } @@ -216,6 +217,7 @@ export class LDMLKeyboardXMLSourceFileReader { */ private resolveOneImport(obj: any, subtag: string, asImport: LKImport, implied? : boolean) : boolean { const { base, path } = asImport; + // If base is not an empty string (or null/undefined), then it must be 'cldr' if (base && base !== constants.cldr_import_base) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportInvalidBase({base, path, subtag})); return false; @@ -262,7 +264,7 @@ export class LDMLKeyboardXMLSourceFileReader { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); } - if (!base) { + if (base !== constants.cldr_import_base) { subsubval.forEach(o => o[ImportStatus.localImport] = path); } diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts index 62322ae25e2..cfff0fd433b 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts @@ -38,7 +38,7 @@ export interface LKImport { /** * import base, currently `cldr` is supported */ - base?: string; + base?: 'cldr'; /** * path to imported resource, of the form `45/*.xml` */ From f4f03551096c384d2ea1be5cf591953f8ce56c64 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Dec 2024 15:40:54 -0600 Subject: [PATCH 6/7] feat(developer): local imports directory fixes - use path.join instead of path.resolve - rename importsPath to cldrImportsPath - remove k_015_importlocal from core, as it duplicated tests in common and developer Fixes: #10649 --- .../unit/ldml/keyboards/k_015_importlocal.xml | 22 ------------------- .../keyboards/keys-Zyyy-morepunctuation.xml | 6 ----- core/tests/unit/ldml/keyboards/meson.build | 1 - .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 6 ++--- .../test/helpers/reader-callback-test.ts | 8 +++---- developer/src/kmc-ldml/test/helpers/index.ts | 4 ++-- .../buildClasses/BuildLdmlKeyboard.ts | 6 ++--- .../kmc/src/commands/buildTestData/index.ts | 6 ++--- 8 files changed, 15 insertions(+), 44 deletions(-) delete mode 100644 core/tests/unit/ldml/keyboards/k_015_importlocal.xml delete mode 100644 core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml diff --git a/core/tests/unit/ldml/keyboards/k_015_importlocal.xml b/core/tests/unit/ldml/keyboards/k_015_importlocal.xml deleted file mode 100644 index 36bd5c6942f..00000000000 --- a/core/tests/unit/ldml/keyboards/k_015_importlocal.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml b/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml deleted file mode 100644 index 768e81e7cd2..00000000000 --- a/core/tests/unit/ldml/keyboards/keys-Zyyy-morepunctuation.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index 949acb78996..15f458c587f 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -26,7 +26,6 @@ tests_without_testdata = [ 'k_010_mt', 'k_011_mt_iso', 'k_012_other', - 'k_015_importlocal', 'k_030_transform_plus', 'k_100_keytest', 'k_101_keytest', diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 2a5754b45b2..928bb94bef7 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -20,7 +20,7 @@ interface NameAndProps { export class LDMLKeyboardXMLSourceFileReaderOptions { /** path to the CLDR imports */ - importsPath: string; + cldrImportsPath: string; /** ordered list of paths for local imports */ localImportsPaths: string[]; }; @@ -34,14 +34,14 @@ export class LDMLKeyboardXMLSourceFileReader { } readImportFile(version: string, subpath: string): Uint8Array { - const importPath = this.callbacks.resolveFilename(this.options.importsPath, `${version}/${subpath}`); + const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); return this.callbacks.loadFile(importPath); } readLocalImportFile(path: string): Uint8Array { // try each of the local imports paths for (const localPath of this.options.localImportsPaths) { - const importPath = this.callbacks.resolveFilename(localPath, path); + const importPath = this.callbacks.path.join(localPath, path); if(this.callbacks.fs.existsSync(importPath)) { return this.callbacks.loadFile(importPath); } diff --git a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts index 8c2e4295209..d865ab2f46a 100644 --- a/developer/src/common/web/utils/test/helpers/reader-callback-test.ts +++ b/developer/src/common/web/utils/test/helpers/reader-callback-test.ts @@ -7,10 +7,10 @@ import { LDMLKeyboardXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-ke import { LDMLKeyboardTestDataXMLSourceFile } from '../../src/types/ldml-keyboard/ldml-keyboard-testdata-xml.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { fileURLToPath } from 'url'; -import { dirname, sep } from 'node:path'; +import { dirname } from 'node:path'; const readerOptions: LDMLKeyboardXMLSourceFileReaderOptions = { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), localImportsPaths: [], }; @@ -85,8 +85,8 @@ export function testReaderCases(cases : CompilationCase[]) { it(testHeading, function () { callbacks.clear(); const path = makePathToFixture('ldml-keyboard', testcase.subpath); - // update readerOptions to point to the source dir. Need a trailing separator. - readerOptions.localImportsPaths = [ dirname(path) + sep ]; + // update readerOptions to point to the source dir. + readerOptions.localImportsPaths = [ dirname(path) ]; const reader = new LDMLKeyboardXMLSourceFileReader(readerOptions, callbacks); const data = loadFile(path); assert.ok(data, `reading ${testcase.subpath}`); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 283459688fe..bd438c4ca11 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -38,7 +38,7 @@ export const compilerTestCallbacks = new TestCompilerCallbacks(); export const compilerTestOptions: LdmlCompilerOptions = { readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), localImportsPaths: [], // will be fixed up in loadSectionFixture } }; @@ -60,7 +60,7 @@ export async function loadSectionFixture(compilerClass: SectionCompilerNew, file const data = callbacks.loadFile(inputFilename); assert.isNotNull(data, `Failed to read file ${inputFilename}`); - compilerTestOptions.readerOptions.localImportsPaths = [ path.dirname(inputFilename) + path.sep ]; + compilerTestOptions.readerOptions.localImportsPaths = [ path.dirname(inputFilename) ]; const reader = new LDMLKeyboardXMLSourceFileReader(compilerTestOptions.readerOptions, callbacks); const source = reader.load(data); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 643f31e9596..6a6300b646b 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -4,7 +4,7 @@ import { CompilerOptions, CompilerCallbacks } from '@keymanapp/developer-utils'; import { LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; -import { dirname, sep } from 'node:path'; +import { dirname } from 'node:path'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -14,8 +14,8 @@ export class BuildLdmlKeyboard extends BuildActivity { public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), - localImportsPaths: [ dirname(infile) + sep ], // local dir + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) ], // local dir }}; const compiler = new kmcLdml.LdmlKeyboardCompiler(); return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 39b149765c5..5343c81702d 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'url'; import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; import { exitProcess } from '../../util/sysexits.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; -import { dirname, sep } from 'node:path'; +import { dirname } from 'node:path'; export async function buildTestData(infile: string, _options: any, commander: any): Promise { const options: CommandLineBaseOptions = commander.optsWithGlobals(); @@ -18,8 +18,8 @@ export async function buildTestData(infile: string, _options: any, commander: an saveDebug: false, shouldAddCompilerVersion: false, readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), - localImportsPaths: [ dirname(infile) + sep ], // local dir + cldrImportsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)), + localImportsPaths: [ dirname(infile) ], // local dir } }; From 9a338abf914934833219974c5d884a37f5687cf2 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 4 Dec 2024 08:34:36 -0600 Subject: [PATCH 7/7] chore(developer): update per code review - support empty string as import base --- .../web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts index cfff0fd433b..587b02dd719 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml.ts @@ -38,7 +38,7 @@ export interface LKImport { /** * import base, currently `cldr` is supported */ - base?: 'cldr'; + base?: 'cldr' | ''; /** * path to imported resource, of the form `45/*.xml` */