From c27ac9e4be8d5f5c07fd106960694f5c3526b30c Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 26 Nov 2024 19:14:16 +0100 Subject: [PATCH 01/13] chore(common): rename test files This change rename the test files in Common according to the discussion at the Keyman conference in November 2024. --- common/web/types/package.json | 2 +- .../kmx/{test-keyman-targets.ts => keyman-targets.tests.ts} | 0 .../web/types/test/kmx/{test-kmx-file.ts => kmx-file.tests.ts} | 0 .../web/types/test/kvk/{test-kvk-file.ts => kvk-file.tests.ts} | 2 +- .../types/test/kvk/{test-kvk-utils.ts => kvk-utils.tests.ts} | 0 .../{test-pattern-parser.ts => pattern-parser.tests.ts} | 0 .../ldml-keyboard/{test-string-list.ts => string-list.tests.ts} | 0 common/web/types/test/tsconfig.json | 1 - .../web/types/test/util/{test-unescape.ts => unescape.tests.ts} | 0 9 files changed, 2 insertions(+), 3 deletions(-) rename common/web/types/test/kmx/{test-keyman-targets.ts => keyman-targets.tests.ts} (100%) rename common/web/types/test/kmx/{test-kmx-file.ts => kmx-file.tests.ts} (100%) rename common/web/types/test/kvk/{test-kvk-file.ts => kvk-file.tests.ts} (90%) rename common/web/types/test/kvk/{test-kvk-utils.ts => kvk-utils.tests.ts} (100%) rename common/web/types/test/ldml-keyboard/{test-pattern-parser.ts => pattern-parser.tests.ts} (100%) rename common/web/types/test/ldml-keyboard/{test-string-list.ts => string-list.tests.ts} (100%) rename common/web/types/test/util/{test-unescape.ts => unescape.tests.ts} (100%) diff --git a/common/web/types/package.json b/common/web/types/package.json index e9f7c412947..f78557cb524 100644 --- a/common/web/types/package.json +++ b/common/web/types/package.json @@ -48,7 +48,7 @@ "typescript": "^5.4.5" }, "mocha": { - "spec": "build/test/**/test-*.js", + "spec": "build/test/**/*.tests.js", "require": [ "source-map-support/register" ] diff --git a/common/web/types/test/kmx/test-keyman-targets.ts b/common/web/types/test/kmx/keyman-targets.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-keyman-targets.ts rename to common/web/types/test/kmx/keyman-targets.tests.ts diff --git a/common/web/types/test/kmx/test-kmx-file.ts b/common/web/types/test/kmx/kmx-file.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-kmx-file.ts rename to common/web/types/test/kmx/kmx-file.tests.ts diff --git a/common/web/types/test/kvk/test-kvk-file.ts b/common/web/types/test/kvk/kvk-file.tests.ts similarity index 90% rename from common/web/types/test/kvk/test-kvk-file.ts rename to common/web/types/test/kvk/kvk-file.tests.ts index 8a1179b67b2..8e0387fefe3 100644 --- a/common/web/types/test/kvk/test-kvk-file.ts +++ b/common/web/types/test/kvk/kvk-file.tests.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import 'mocha'; import { makePathToFixture } from '../helpers/index.js'; import KvkFileReader from "../../src/kvk/kvk-file-reader.js"; -import { verify_balochi_inpage, verify_khmer_angkor } from './test-kvk-utils.js'; +import { verify_balochi_inpage, verify_khmer_angkor } from './kvk-utils.tests.js'; describe('kvk-file-reader', function () { it('kvk-file-reader should read a valid file', function() { diff --git a/common/web/types/test/kvk/test-kvk-utils.ts b/common/web/types/test/kvk/kvk-utils.tests.ts similarity index 100% rename from common/web/types/test/kvk/test-kvk-utils.ts rename to common/web/types/test/kvk/kvk-utils.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/pattern-parser.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/test-pattern-parser.ts rename to common/web/types/test/ldml-keyboard/pattern-parser.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-string-list.ts b/common/web/types/test/ldml-keyboard/string-list.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/test-string-list.ts rename to common/web/types/test/ldml-keyboard/string-list.tests.ts diff --git a/common/web/types/test/tsconfig.json b/common/web/types/test/tsconfig.json index 9678c49945f..21b4145e594 100644 --- a/common/web/types/test/tsconfig.json +++ b/common/web/types/test/tsconfig.json @@ -10,7 +10,6 @@ "allowSyntheticDefaultImports": true }, "include": [ - "**/test-*.ts", "**/*.tests.ts", "./helpers/*.ts", ], diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/test/util/unescape.tests.ts similarity index 100% rename from common/web/types/test/util/test-unescape.ts rename to common/web/types/test/util/unescape.tests.ts From 8e19508a8185c673cf59e7bc541e83232eebbd63 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 11:06:53 -0600 Subject: [PATCH 02/13] feat(common): isDenormalized() function for checking strings that are neither NFC nor NFD Fixes: #7394 --- common/web/types/src/util/util.ts | 9 +++++++++ common/web/types/test/util/test-unescape.ts | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 0d13505cfc9..7395c27ba0f 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -305,6 +305,15 @@ export function isPUA(ch: number) { (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END)); } +/** @returns true if s is NEITHER NFC nor NFD */ +export function isDenormalized(s: string) : boolean { + if(!s) return false; // empty or null + const nfc = s.normalize("NFC"); + const nfd = s.normalize("NFD"); + if (s !== nfc && s !== nfd) return true; + return false; +} + class BadStringMap extends Map> { public toString() : string { if (!this.size) { diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/test/util/test-unescape.ts index 8c75b491824..853fb6a09b0 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/test/util/test-unescape.ts @@ -1,6 +1,6 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isDenormalized} from '../../src/util/util.js'; describe('test UTF32 functions()', function() { it('should properly categorize strings', () => { @@ -186,6 +186,24 @@ describe('test bad char functions', () => { assert.isTrue(isPUA(ch), describeCodepoint(ch)); } }); + describe('test isDenormalized()', () => { + it('should correctly categorize strings', () => { + [ + undefined, + null, + '', + 'ABC', + 'fa\u1E69cinating', // NFC + 'fas\u0323\u0307cinating', // NFD + 'd\u0323\u0307', // NFD + '\u1e0d\u0307', // NFC + ].map(s => assert.isFalse(isDenormalized(s), `for string ${s}`)); + [ + 'd\u0307\u0323', // NFD but reversed marks + 'fas\u0307\u0323cinating', // not-NFD + ].map(s => assert.isTrue(isDenormalized(s), `for string ${s}`)); + }); + }); }); describe('test BadStringAnalyzer', () => { From add502e1a9faba0651210bc1881bb1ccd233cc95 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 12:16:11 -0600 Subject: [PATCH 03/13] feat(common,developer): tests and warning on denormalized content - common: shift the 'strs' processing slightly, because otherwise we normalize to NFD before even tracking the strings - if any string is neither NFC nor NFD, give a warning - add tests for the same Fixes: #7394 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 15 ++++++----- .../kmc-ldml/src/compiler/empty-compiler.ts | 7 +++++ .../src/compiler/ldml-compiler-messages.ts | 6 ++++- .../fixtures/sections/strs/warn-denorm-1.xml | 18 +++++++++++++ .../fixtures/sections/strs/warn-denorm-2.xml | 22 ++++++++++++++++ .../fixtures/sections/strs/warn-denorm-3.xml | 26 +++++++++++++++++++ .../fixtures/sections/strs/warn-denorm-4.xml | 26 +++++++++++++++++++ developer/src/kmc-ldml/test/strs.tests.ts | 20 ++++++++++++++ 8 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 0cb49e527c7..f78e2d6d2eb 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -174,12 +174,7 @@ export class Strs extends Section { */ allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem { // Run the string processing pipeline - s = Strs.processString(s, opts, sections); - - // add to the set, for testing - if (s) { - this.allProcessedStrings.add(s); - } + s = this.processString(s, opts, sections); // if it's a single char, don't push it into the strs table if (opts?.singleOk && isOneChar(s)) { @@ -197,7 +192,7 @@ export class Strs extends Section { } /** process everything according to opts */ - static processString(s: string, opts: StrsOptions, sections: DependencySections) { + processString(s: string, opts: StrsOptions, sections: DependencySections) { s = s ?? ''; // type check everything else if (typeof s !== 'string') { @@ -215,6 +210,12 @@ export class Strs extends Section { if (opts?.unescape) { s = unescapeString(s); } + + if (s) { + // add all processed strings here, so that we catch denormalized strings in the input + this.allProcessedStrings.add(s); + } + // nfd if (opts?.nfd) { if (!sections?.meta?.normalizationDisabled) { diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index aaba0950260..12c39d9d620 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -34,9 +34,12 @@ export class StrsCompiler extends EmptyCompiler { const strs = section; if (strs) { + let hadDenormalized = false; const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { + // stop at the first denormalized string + hadDenormalized = hadDenormalized || util.isDenormalized(s); // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. // the results might not be valid regex of course. @@ -72,6 +75,10 @@ export class StrsCompiler extends EmptyCompiler { return false; } } + if (hadDenormalized) { + // warn once + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm()); + } } return true; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1445b40036a..f947b7ca189 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -199,7 +199,11 @@ export class LdmlCompilerMessages { `Invalid marker identifier "\m{${def(o.id)}}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); - // Available: 0x02B-0x2F + static WARN_StringDenorm = SevWarn | 0x002B; + static Warn_StringDenorm = () => + m(this.WARN_StringDenorm, `File contains text that is neither NFC nor NFD.`); + + // Available: 0x02C-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; static Error_InvalidQuadEscape = (o: { cp: number }) => diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml new file mode 100644 index 00000000000..49cd61d358a --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml new file mode 100644 index 00000000000..ec001db7684 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml new file mode 100644 index 00000000000..a40c119099c --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml new file mode 100644 index 00000000000..cf443c61fe5 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/strs.tests.ts b/developer/src/kmc-ldml/test/strs.tests.ts index a5f7e89e35c..14250d993ae 100644 --- a/developer/src/kmc-ldml/test/strs.tests.ts +++ b/developer/src/kmc-ldml/test/strs.tests.ts @@ -71,4 +71,24 @@ describe('strs', function () { ]); assert.isNull(kmx); // should fail post-validate }); + describe('should warn on denorm strings', async function () { + for (const num of [1, 2, 3, 4]) { + const path = `sections/strs/warn-denorm-${num}.xml`; + it(path, async function () { + const inputFilename = makePathToFixture(path); + // Compile the keyboard + const kmx = await compileKeyboard(inputFilename, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }, + [ + // validation messages + LdmlCompilerMessages.Warn_StringDenorm(), + ], + false, // validation should pass + [ + // same messages + LdmlCompilerMessages.Warn_StringDenorm(), + ]); + assert.isNotNull(kmx); // not failing + }); + } + }); }); From f4b7e0aea37af956b44d29d2921debf03da0543f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 29 Nov 2024 22:13:25 -0600 Subject: [PATCH 04/13] Apply suggestions from code review Co-authored-by: Marc Durdin --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index f78e2d6d2eb..a88688f7ab7 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -191,8 +191,8 @@ export class Strs extends Section { return result; } - /** process everything according to opts */ - processString(s: string, opts: StrsOptions, sections: DependencySections) { + /** process everything according to opts, and add the string to this.allProcessedStrings */ + private processString(s: string, opts: StrsOptions, sections: DependencySections) { s = s ?? ''; // type check everything else if (typeof s !== 'string') { From 42ba0a4aabff07831b8087915c81a7550a303724 Mon Sep 17 00:00:00 2001 From: sgschantz Date: Mon, 2 Dec 2024 17:08:46 +0700 Subject: [PATCH 05/13] refactor(mac): pass kmx data blob to keyman core instead of file path Fixes: #12499 --- .../CoreWrapper/CoreWrapper.m | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index c1cd9b81f85..adcf8c14b60 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -110,11 +110,22 @@ -(void) dealloc{ -(void)loadKeyboardUsingCore:(NSString*) path { km_core_path_name keyboardPath = [path UTF8String]; - km_core_status result = km_core_keyboard_load(keyboardPath, &_coreKeyboard); + NSError* dataError = nil; + NSData *data = [NSData dataWithContentsOfFile:path options:0 error:&dataError]; - if (result != KM_CORE_STATUS_OK) { - NSString *message = [NSString stringWithFormat:@"Unexpected Keyman Core result: %u", result]; - [NSException raise:@"LoadKeyboardException" format:@"%@", message]; + if (dataError != nil) { + os_log_error([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataError: %{public}@", path, dataError); + } else { + NSUInteger dataLength = data.length; + os_log_info([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataLength: %lu", path, dataLength); + + km_core_status result = km_core_keyboard_load_from_blob(keyboardPath, + data.bytes, dataLength, &_coreKeyboard); + if (result != KM_CORE_STATUS_OK) { + NSString *message = [NSString stringWithFormat:@"Unexpected Keyman Core result: %u", result]; + os_log_error([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n core result: %{public}@", path, message); + [NSException raise:@"LoadKeyboardException" format:@"%@", message]; + } } } @@ -126,7 +137,7 @@ -(void)readKeyboardAttributesUsingCore { if (result==KM_CORE_STATUS_OK) { _keyboardVersion = [self.coreHelper createNSStringFromUnicharString:keyboardAttributes->version_string]; _keyboardId = [self.coreHelper createNSStringFromUnicharString:keyboardAttributes->id]; - os_log_debug([KMELogs coreLog], "readKeyboardAttributesUsingCore, keyboardVersion: %{public}@\n, keyboardId: %{public}@\n", _keyboardVersion, _keyboardId); + os_log_debug([KMELogs coreLog], "readKeyboardAttributesUsingCore, keyboardVersion: %{public}@, keyboardId: %{public}@\n", _keyboardVersion, _keyboardId); } else { os_log_error([KMELogs coreLog], "km_core_keyboard_get_attrs() failed with result = %u\n", result); } From 6f2a6b2ebfd4c1664a3372b362b1828e61029bda Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Dec 2024 11:35:06 -0600 Subject: [PATCH 06/13] feat(common,developer): tests and warning on denormalized content per review comments - report a warning on each separate instance of a denormalized string (once per string). Fixes: #7394 --- common/web/types/src/util/util.ts | 10 +++++----- common/web/types/test/util/test-unescape.ts | 6 +++--- developer/src/kmc-ldml/src/compiler/empty-compiler.ts | 9 +++------ .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 4 ++-- developer/src/kmc-ldml/test/strs.tests.ts | 5 +++-- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 7395c27ba0f..70e1077cc5a 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -305,13 +305,13 @@ export function isPUA(ch: number) { (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END)); } -/** @returns true if s is NEITHER NFC nor NFD */ -export function isDenormalized(s: string) : boolean { - if(!s) return false; // empty or null +/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */ +export function isNormalized(s: string) : boolean { + if(!s) return true; // empty or null const nfc = s.normalize("NFC"); const nfd = s.normalize("NFD"); - if (s !== nfc && s !== nfd) return true; - return false; + if (s !== nfc && s !== nfd) return false; + return true; } class BadStringMap extends Map> { diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/test/util/test-unescape.ts index 853fb6a09b0..3eaafd7150f 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/test/util/test-unescape.ts @@ -1,6 +1,6 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isDenormalized} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js'; describe('test UTF32 functions()', function() { it('should properly categorize strings', () => { @@ -197,11 +197,11 @@ describe('test bad char functions', () => { 'fas\u0323\u0307cinating', // NFD 'd\u0323\u0307', // NFD '\u1e0d\u0307', // NFC - ].map(s => assert.isFalse(isDenormalized(s), `for string ${s}`)); + ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`)); [ 'd\u0307\u0323', // NFD but reversed marks 'fas\u0307\u0323cinating', // not-NFD - ].map(s => assert.isTrue(isDenormalized(s), `for string ${s}`)); + ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`)); }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index 12c39d9d620..4b9aec76133 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -34,12 +34,13 @@ export class StrsCompiler extends EmptyCompiler { const strs = section; if (strs) { - let hadDenormalized = false; const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { // stop at the first denormalized string - hadDenormalized = hadDenormalized || util.isDenormalized(s); + if (!util.isNormalized(s)) { + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s})); + } // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. // the results might not be valid regex of course. @@ -75,10 +76,6 @@ export class StrsCompiler extends EmptyCompiler { return false; } } - if (hadDenormalized) { - // warn once - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm()); - } } return true; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index f947b7ca189..5df0da4b73a 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -200,8 +200,8 @@ export class LdmlCompilerMessages { ); static WARN_StringDenorm = SevWarn | 0x002B; - static Warn_StringDenorm = () => - m(this.WARN_StringDenorm, `File contains text that is neither NFC nor NFD.`); + static Warn_StringDenorm = (o: { s: string }) => + m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); // Available: 0x02C-0x2F diff --git a/developer/src/kmc-ldml/test/strs.tests.ts b/developer/src/kmc-ldml/test/strs.tests.ts index 14250d993ae..24e8ff4c1ed 100644 --- a/developer/src/kmc-ldml/test/strs.tests.ts +++ b/developer/src/kmc-ldml/test/strs.tests.ts @@ -76,16 +76,17 @@ describe('strs', function () { const path = `sections/strs/warn-denorm-${num}.xml`; it(path, async function () { const inputFilename = makePathToFixture(path); + const s = 's\u0307\u0323'; // sĖ‡ĖĢ // Compile the keyboard const kmx = await compileKeyboard(inputFilename, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }, [ // validation messages - LdmlCompilerMessages.Warn_StringDenorm(), + LdmlCompilerMessages.Warn_StringDenorm({s}), ], false, // validation should pass [ // same messages - LdmlCompilerMessages.Warn_StringDenorm(), + LdmlCompilerMessages.Warn_StringDenorm({s}), ]); assert.isNotNull(kmx); // not failing }); From 8bc144b6d1d2affb295466ead2350ce745dc1acc Mon Sep 17 00:00:00 2001 From: sgschantz Date: Tue, 3 Dec 2024 10:48:57 +0700 Subject: [PATCH 07/13] refactor(mac): raise exception when unable to read kmx data --- mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m | 1 + 1 file changed, 1 insertion(+) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index adcf8c14b60..ed2227f318e 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -115,6 +115,7 @@ -(void)loadKeyboardUsingCore:(NSString*) path { if (dataError != nil) { os_log_error([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataError: %{public}@", path, dataError); + [NSException raise:@"LoadKeyboardException" format:@"%@", dataError]; } else { NSUInteger dataLength = data.length; os_log_info([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataLength: %lu", path, dataLength); From cbbb643ed02a4070c7138705c08f347cf9911f38 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 3 Dec 2024 13:53:13 +0100 Subject: [PATCH 08/13] chore(common): rename `common/web/types/test` to `.../tests` Addresses code review comment. --- common/web/types/.eslintrc.cjs | 4 ++-- common/web/types/build.sh | 2 +- common/web/types/package.json | 6 +++--- .../{test => tests}/fixtures/kmx/khmer_angkor.kmx | Bin .../{test => tests}/fixtures/kvk/balochi_inpage.kvk | Bin .../{test => tests}/fixtures/kvk/khmer_angkor.kvk | Bin common/web/types/{test => tests}/helpers/index.ts | 2 +- .../{test => tests}/kmx/keyman-targets.tests.ts | 0 .../web/types/{test => tests}/kmx/kmx-file.tests.ts | 0 .../web/types/{test => tests}/kvk/kvk-file.tests.ts | 0 .../types/{test => tests}/kvk/kvk-utils.tests.ts | 0 .../ldml-keyboard/pattern-parser.tests.ts | 0 .../ldml-keyboard/string-list.tests.ts | 0 .../{test => tests}/lexical-model-types.tests.ts | 0 common/web/types/{test => tests}/tsconfig.json | 2 +- .../types/{test => tests}/util/unescape.tests.ts | 0 16 files changed, 8 insertions(+), 8 deletions(-) rename common/web/types/{test => tests}/fixtures/kmx/khmer_angkor.kmx (100%) rename common/web/types/{test => tests}/fixtures/kvk/balochi_inpage.kvk (100%) rename common/web/types/{test => tests}/fixtures/kvk/khmer_angkor.kvk (100%) rename common/web/types/{test => tests}/helpers/index.ts (87%) rename common/web/types/{test => tests}/kmx/keyman-targets.tests.ts (100%) rename common/web/types/{test => tests}/kmx/kmx-file.tests.ts (100%) rename common/web/types/{test => tests}/kvk/kvk-file.tests.ts (100%) rename common/web/types/{test => tests}/kvk/kvk-utils.tests.ts (100%) rename common/web/types/{test => tests}/ldml-keyboard/pattern-parser.tests.ts (100%) rename common/web/types/{test => tests}/ldml-keyboard/string-list.tests.ts (100%) rename common/web/types/{test => tests}/lexical-model-types.tests.ts (100%) rename common/web/types/{test => tests}/tsconfig.json (93%) rename common/web/types/{test => tests}/util/unescape.tests.ts (100%) diff --git a/common/web/types/.eslintrc.cjs b/common/web/types/.eslintrc.cjs index b28ba0941b1..4fd5954e495 100644 --- a/common/web/types/.eslintrc.cjs +++ b/common/web/types/.eslintrc.cjs @@ -1,13 +1,13 @@ module.exports = { parserOptions: { - project: ["./tsconfig.json", "./test/tsconfig.json"], + project: ["./tsconfig.json", "./tests/tsconfig.json"], }, ignorePatterns: [ ".*/*", "build/*", "coverage/*", "node_modules/*", - "test/fixtures/*", + "tests/fixtures/*", "tools/*", "src/schemas/*" ], diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 75273aaa2c9..418840d1a85 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -88,7 +88,7 @@ function do_test() { fi eslint . - tsc --build test + tsc --build tests readonly C8_THRESHOLD=60 diff --git a/common/web/types/package.json b/common/web/types/package.json index f78557cb524..d6efc635039 100644 --- a/common/web/types/package.json +++ b/common/web/types/package.json @@ -23,7 +23,7 @@ "build": "tsc -b", "build:schema": "ajv compile", "lint": "eslint .", - "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" + "test": "npm run lint && cd tests && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" }, "author": "Marc Durdin (https://github.com/mcdurdin)", "license": "MIT", @@ -48,7 +48,7 @@ "typescript": "^5.4.5" }, "mocha": { - "spec": "build/test/**/*.tests.js", + "spec": "build/tests/**/*.tests.js", "require": [ "source-map-support/register" ] @@ -76,7 +76,7 @@ "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts", "src/osk/osk.ts", "src/schemas/*", - "test/" + "tests/" ] }, "sideEffects": false diff --git a/common/web/types/test/fixtures/kmx/khmer_angkor.kmx b/common/web/types/tests/fixtures/kmx/khmer_angkor.kmx similarity index 100% rename from common/web/types/test/fixtures/kmx/khmer_angkor.kmx rename to common/web/types/tests/fixtures/kmx/khmer_angkor.kmx diff --git a/common/web/types/test/fixtures/kvk/balochi_inpage.kvk b/common/web/types/tests/fixtures/kvk/balochi_inpage.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/balochi_inpage.kvk rename to common/web/types/tests/fixtures/kvk/balochi_inpage.kvk diff --git a/common/web/types/test/fixtures/kvk/khmer_angkor.kvk b/common/web/types/tests/fixtures/kvk/khmer_angkor.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/khmer_angkor.kvk rename to common/web/types/tests/fixtures/kvk/khmer_angkor.kvk diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/tests/helpers/index.ts similarity index 87% rename from common/web/types/test/helpers/index.ts rename to common/web/types/tests/helpers/index.ts index 3e305eeca8a..717b2e75191 100644 --- a/common/web/types/test/helpers/index.ts +++ b/common/web/types/tests/helpers/index.ts @@ -9,5 +9,5 @@ import { fileURLToPath } from "url"; * @param components One or more path components. */ export function makePathToFixture(...components: string[]): string { - return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); + return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url)); } diff --git a/common/web/types/test/kmx/keyman-targets.tests.ts b/common/web/types/tests/kmx/keyman-targets.tests.ts similarity index 100% rename from common/web/types/test/kmx/keyman-targets.tests.ts rename to common/web/types/tests/kmx/keyman-targets.tests.ts diff --git a/common/web/types/test/kmx/kmx-file.tests.ts b/common/web/types/tests/kmx/kmx-file.tests.ts similarity index 100% rename from common/web/types/test/kmx/kmx-file.tests.ts rename to common/web/types/tests/kmx/kmx-file.tests.ts diff --git a/common/web/types/test/kvk/kvk-file.tests.ts b/common/web/types/tests/kvk/kvk-file.tests.ts similarity index 100% rename from common/web/types/test/kvk/kvk-file.tests.ts rename to common/web/types/tests/kvk/kvk-file.tests.ts diff --git a/common/web/types/test/kvk/kvk-utils.tests.ts b/common/web/types/tests/kvk/kvk-utils.tests.ts similarity index 100% rename from common/web/types/test/kvk/kvk-utils.tests.ts rename to common/web/types/tests/kvk/kvk-utils.tests.ts diff --git a/common/web/types/test/ldml-keyboard/pattern-parser.tests.ts b/common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/pattern-parser.tests.ts rename to common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts diff --git a/common/web/types/test/ldml-keyboard/string-list.tests.ts b/common/web/types/tests/ldml-keyboard/string-list.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/string-list.tests.ts rename to common/web/types/tests/ldml-keyboard/string-list.tests.ts diff --git a/common/web/types/test/lexical-model-types.tests.ts b/common/web/types/tests/lexical-model-types.tests.ts similarity index 100% rename from common/web/types/test/lexical-model-types.tests.ts rename to common/web/types/tests/lexical-model-types.tests.ts diff --git a/common/web/types/test/tsconfig.json b/common/web/types/tests/tsconfig.json similarity index 93% rename from common/web/types/test/tsconfig.json rename to common/web/types/tests/tsconfig.json index 21b4145e594..45e5311fdc0 100644 --- a/common/web/types/test/tsconfig.json +++ b/common/web/types/tests/tsconfig.json @@ -4,7 +4,7 @@ "compilerOptions": { "rootDir": ".", "rootDirs": ["./", "../src/"], - "outDir": "../build/test", + "outDir": "../build/tests", "baseUrl": ".", "strictNullChecks": false, // TODO: get rid of this as some point "allowSyntheticDefaultImports": true diff --git a/common/web/types/test/util/unescape.tests.ts b/common/web/types/tests/util/unescape.tests.ts similarity index 100% rename from common/web/types/test/util/unescape.tests.ts rename to common/web/types/tests/util/unescape.tests.ts From f1f026029d82f2af12cfaebdf051828f547ece57 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 3 Dec 2024 13:01:20 -0500 Subject: [PATCH 09/13] auto: increment master version to 18.0.152 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 5800446af11..d92ae3e0f4e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 18.0.151 alpha 2024-12-03 + +* feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745) +* feat(developer): report on mismatching lang tag scripts when building keyboard-info (#12753) + ## 18.0.150 alpha 2024-12-02 * fix(core,developer): use `NDEBUG` flag to disable assertions in release build (#12715) diff --git a/VERSION.md b/VERSION.md index de0d1bb346a..a08197ede11 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.151 \ No newline at end of file +18.0.152 \ No newline at end of file From 148938eb1de0fb04681a8ba9635dc56a19be11ac Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 4 Dec 2024 12:51:00 +0700 Subject: [PATCH 10/13] fix(developer): honour provided script when checking for matching scripts Fixes: #12765 --- .../src/kmc-keyboard-info/src/keyboard-info-compiler.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts index d91f9db0ba3..74c06d5d129 100644 --- a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts @@ -575,11 +575,12 @@ export class KeyboardInfoCompiler implements KeymanCompiler { '' ); + const resolvedScript = locale.script ?? langtagsByTag[bcp47]?.script ?? langtagsByTag[locale.language]?.script ?? undefined; if(commonScript === null) { - commonScript = tag?.script ?? undefined; + commonScript = resolvedScript; } else { - if(tag?.script !== commonScript) { - this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: tag?.script})) + if(resolvedScript !== commonScript) { + this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: resolvedScript})) } } } From ed22cc00a6df5314f1650e05b00b5a6cabdeb0be Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 4 Dec 2024 10:44:05 +0100 Subject: [PATCH 11/13] fix(common): rename test file This change renames a new test file and moves it to the new location. This file was overlooked in #12709. --- .../ldml-keyboard/unicodeset-parser-api.tests.ts} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename common/web/types/{test/ldml-keyboard/test-unicodeset-parser-api.ts => tests/ldml-keyboard/unicodeset-parser-api.tests.ts} (98%) diff --git a/common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts similarity index 98% rename from common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts rename to common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts index cc36e78ae28..763bc6d5bb9 100644 --- a/common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts +++ b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts @@ -1,8 +1,8 @@ /* * Keyman is copyright (C) SIL Global. MIT License. - * + * * Created by Dr Mark C. Sinclair on 2024-11-29 - * + * * Test code for unicodeset-parser-api.ts */ From 917cae39f0a913f2061c8a45e95bf7516c6272d5 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 4 Dec 2024 13:01:45 -0500 Subject: [PATCH 12/13] auto: increment master version to 18.0.153 --- HISTORY.md | 7 +++++++ VERSION.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d92ae3e0f4e..b831dadee8a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # Keyman Version History +## 18.0.152 alpha 2024-12-04 + +* refactor(mac): pass kmx data blob to keyman core instead of file path (#12760) +* fix(developer): honour provided script when checking for matching scripts (#12768) +* chore(common): rename test files (#12709) +* fix(common): rename test file (#12770) + ## 18.0.151 alpha 2024-12-03 * feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745) diff --git a/VERSION.md b/VERSION.md index a08197ede11..6269a6496fb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.152 \ No newline at end of file +18.0.153 \ No newline at end of file From b7990d1b46bdf1395affef49b25ebf830a4b6a36 Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 5 Dec 2024 06:27:22 +0100 Subject: [PATCH 13/13] chore(core): Add link to Keyman Glossary --- core/docs/api/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/core/docs/api/index.md b/core/docs/api/index.md index 8112d9c8793..c33246f7f44 100644 --- a/core/docs/api/index.md +++ b/core/docs/api/index.md @@ -75,6 +75,7 @@ modifiers such as Windows key are excluded from this set. Some modifiers are transient, such as Control, and others have long-lasting state, such as Caps Lock. +- __See more in__ [Keyman Glossary](https://github.com/keymanapp/keyman/wiki/Keyman-glossary) [km_core_cp]: background#km_core_cp "km_core_cp type" [km_core_usv]: background#km_core_usv "km_core_usv type"