diff --git a/HISTORY.md b/HISTORY.md index 5800446af11..b831dadee8a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,17 @@ # Keyman Version History +## 18.0.152 alpha 2024-12-04 + +* refactor(mac): pass kmx data blob to keyman core instead of file path (#12760) +* fix(developer): honour provided script when checking for matching scripts (#12768) +* chore(common): rename test files (#12709) +* fix(common): rename test file (#12770) + +## 18.0.151 alpha 2024-12-03 + +* feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745) +* feat(developer): report on mismatching lang tag scripts when building keyboard-info (#12753) + ## 18.0.150 alpha 2024-12-02 * fix(core,developer): use `NDEBUG` flag to disable assertions in release build (#12715) diff --git a/VERSION.md b/VERSION.md index de0d1bb346a..6269a6496fb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.151 \ No newline at end of file +18.0.153 \ No newline at end of file diff --git a/common/web/types/.eslintrc.cjs b/common/web/types/.eslintrc.cjs index b28ba0941b1..4fd5954e495 100644 --- a/common/web/types/.eslintrc.cjs +++ b/common/web/types/.eslintrc.cjs @@ -1,13 +1,13 @@ module.exports = { parserOptions: { - project: ["./tsconfig.json", "./test/tsconfig.json"], + project: ["./tsconfig.json", "./tests/tsconfig.json"], }, ignorePatterns: [ ".*/*", "build/*", "coverage/*", "node_modules/*", - "test/fixtures/*", + "tests/fixtures/*", "tools/*", "src/schemas/*" ], diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 75273aaa2c9..418840d1a85 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -88,7 +88,7 @@ function do_test() { fi eslint . - tsc --build test + tsc --build tests readonly C8_THRESHOLD=60 diff --git a/common/web/types/package.json b/common/web/types/package.json index e9f7c412947..d6efc635039 100644 --- a/common/web/types/package.json +++ b/common/web/types/package.json @@ -23,7 +23,7 @@ "build": "tsc -b", "build:schema": "ajv compile", "lint": "eslint .", - "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" + "test": "npm run lint && cd tests && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" }, "author": "Marc Durdin (https://github.com/mcdurdin)", "license": "MIT", @@ -48,7 +48,7 @@ "typescript": "^5.4.5" }, "mocha": { - "spec": "build/test/**/test-*.js", + "spec": "build/tests/**/*.tests.js", "require": [ "source-map-support/register" ] @@ -76,7 +76,7 @@ "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts", "src/osk/osk.ts", "src/schemas/*", - "test/" + "tests/" ] }, "sideEffects": false diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 0cb49e527c7..a88688f7ab7 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -174,12 +174,7 @@ export class Strs extends Section { */ allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem { // Run the string processing pipeline - s = Strs.processString(s, opts, sections); - - // add to the set, for testing - if (s) { - this.allProcessedStrings.add(s); - } + s = this.processString(s, opts, sections); // if it's a single char, don't push it into the strs table if (opts?.singleOk && isOneChar(s)) { @@ -196,8 +191,8 @@ export class Strs extends Section { return result; } - /** process everything according to opts */ - static processString(s: string, opts: StrsOptions, sections: DependencySections) { + /** process everything according to opts, and add the string to this.allProcessedStrings */ + private processString(s: string, opts: StrsOptions, sections: DependencySections) { s = s ?? ''; // type check everything else if (typeof s !== 'string') { @@ -215,6 +210,12 @@ export class Strs extends Section { if (opts?.unescape) { s = unescapeString(s); } + + if (s) { + // add all processed strings here, so that we catch denormalized strings in the input + this.allProcessedStrings.add(s); + } + // nfd if (opts?.nfd) { if (!sections?.meta?.normalizationDisabled) { diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 0d13505cfc9..70e1077cc5a 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -305,6 +305,15 @@ export function isPUA(ch: number) { (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END)); } +/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */ +export function isNormalized(s: string) : boolean { + if(!s) return true; // empty or null + const nfc = s.normalize("NFC"); + const nfd = s.normalize("NFD"); + if (s !== nfc && s !== nfd) return false; + return true; +} + class BadStringMap extends Map> { public toString() : string { if (!this.size) { diff --git a/common/web/types/test/fixtures/kmx/khmer_angkor.kmx b/common/web/types/tests/fixtures/kmx/khmer_angkor.kmx similarity index 100% rename from common/web/types/test/fixtures/kmx/khmer_angkor.kmx rename to common/web/types/tests/fixtures/kmx/khmer_angkor.kmx diff --git a/common/web/types/test/fixtures/kvk/balochi_inpage.kvk b/common/web/types/tests/fixtures/kvk/balochi_inpage.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/balochi_inpage.kvk rename to common/web/types/tests/fixtures/kvk/balochi_inpage.kvk diff --git a/common/web/types/test/fixtures/kvk/khmer_angkor.kvk b/common/web/types/tests/fixtures/kvk/khmer_angkor.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/khmer_angkor.kvk rename to common/web/types/tests/fixtures/kvk/khmer_angkor.kvk diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/tests/helpers/index.ts similarity index 87% rename from common/web/types/test/helpers/index.ts rename to common/web/types/tests/helpers/index.ts index 3e305eeca8a..717b2e75191 100644 --- a/common/web/types/test/helpers/index.ts +++ b/common/web/types/tests/helpers/index.ts @@ -9,5 +9,5 @@ import { fileURLToPath } from "url"; * @param components One or more path components. */ export function makePathToFixture(...components: string[]): string { - return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); + return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url)); } diff --git a/common/web/types/test/kmx/test-keyman-targets.ts b/common/web/types/tests/kmx/keyman-targets.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-keyman-targets.ts rename to common/web/types/tests/kmx/keyman-targets.tests.ts diff --git a/common/web/types/test/kmx/test-kmx-file.ts b/common/web/types/tests/kmx/kmx-file.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-kmx-file.ts rename to common/web/types/tests/kmx/kmx-file.tests.ts diff --git a/common/web/types/test/kvk/test-kvk-file-writer.ts b/common/web/types/tests/kvk/kvk-file-writer.tests.ts similarity index 100% rename from common/web/types/test/kvk/test-kvk-file-writer.ts rename to common/web/types/tests/kvk/kvk-file-writer.tests.ts diff --git a/common/web/types/test/kvk/test-kvk-file.ts b/common/web/types/tests/kvk/kvk-file.tests.ts similarity index 90% rename from common/web/types/test/kvk/test-kvk-file.ts rename to common/web/types/tests/kvk/kvk-file.tests.ts index 8a1179b67b2..8e0387fefe3 100644 --- a/common/web/types/test/kvk/test-kvk-file.ts +++ b/common/web/types/tests/kvk/kvk-file.tests.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import 'mocha'; import { makePathToFixture } from '../helpers/index.js'; import KvkFileReader from "../../src/kvk/kvk-file-reader.js"; -import { verify_balochi_inpage, verify_khmer_angkor } from './test-kvk-utils.js'; +import { verify_balochi_inpage, verify_khmer_angkor } from './kvk-utils.tests.js'; describe('kvk-file-reader', function () { it('kvk-file-reader should read a valid file', function() { diff --git a/common/web/types/test/kvk/test-kvk-utils.ts b/common/web/types/tests/kvk/kvk-utils.tests.ts similarity index 100% rename from common/web/types/test/kvk/test-kvk-utils.ts rename to common/web/types/tests/kvk/kvk-utils.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/test-pattern-parser.ts rename to common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-string-list.ts b/common/web/types/tests/ldml-keyboard/string-list.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/test-string-list.ts rename to common/web/types/tests/ldml-keyboard/string-list.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts similarity index 98% rename from common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts rename to common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts index cc36e78ae28..763bc6d5bb9 100644 --- a/common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts +++ b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts @@ -1,8 +1,8 @@ /* * Keyman is copyright (C) SIL Global. MIT License. - * + * * Created by Dr Mark C. Sinclair on 2024-11-29 - * + * * Test code for unicodeset-parser-api.ts */ diff --git a/common/web/types/test/lexical-model-types.tests.ts b/common/web/types/tests/lexical-model-types.tests.ts similarity index 100% rename from common/web/types/test/lexical-model-types.tests.ts rename to common/web/types/tests/lexical-model-types.tests.ts diff --git a/common/web/types/test/tsconfig.json b/common/web/types/tests/tsconfig.json similarity index 90% rename from common/web/types/test/tsconfig.json rename to common/web/types/tests/tsconfig.json index 9678c49945f..45e5311fdc0 100644 --- a/common/web/types/test/tsconfig.json +++ b/common/web/types/tests/tsconfig.json @@ -4,13 +4,12 @@ "compilerOptions": { "rootDir": ".", "rootDirs": ["./", "../src/"], - "outDir": "../build/test", + "outDir": "../build/tests", "baseUrl": ".", "strictNullChecks": false, // TODO: get rid of this as some point "allowSyntheticDefaultImports": true }, "include": [ - "**/test-*.ts", "**/*.tests.ts", "./helpers/*.ts", ], diff --git a/common/web/types/test/util/test-file-types.ts b/common/web/types/tests/util/file-types.tests.ts similarity index 100% rename from common/web/types/test/util/test-file-types.ts rename to common/web/types/tests/util/file-types.tests.ts diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/tests/util/unescape.tests.ts similarity index 94% rename from common/web/types/test/util/test-unescape.ts rename to common/web/types/tests/util/unescape.tests.ts index 8c75b491824..3eaafd7150f 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/tests/util/unescape.tests.ts @@ -1,6 +1,6 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js'; describe('test UTF32 functions()', function() { it('should properly categorize strings', () => { @@ -186,6 +186,24 @@ describe('test bad char functions', () => { assert.isTrue(isPUA(ch), describeCodepoint(ch)); } }); + describe('test isDenormalized()', () => { + it('should correctly categorize strings', () => { + [ + undefined, + null, + '', + 'ABC', + 'fa\u1E69cinating', // NFC + 'fas\u0323\u0307cinating', // NFD + 'd\u0323\u0307', // NFD + '\u1e0d\u0307', // NFC + ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`)); + [ + 'd\u0307\u0323', // NFD but reversed marks + 'fas\u0307\u0323cinating', // not-NFD + ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`)); + }); + }); }); describe('test BadStringAnalyzer', () => { diff --git a/core/docs/api/index.md b/core/docs/api/index.md index 8112d9c8793..c33246f7f44 100644 --- a/core/docs/api/index.md +++ b/core/docs/api/index.md @@ -75,6 +75,7 @@ modifiers such as Windows key are excluded from this set. Some modifiers are transient, such as Control, and others have long-lasting state, such as Caps Lock. +- __See more in__ [Keyman Glossary](https://github.com/keymanapp/keyman/wiki/Keyman-glossary) [km_core_cp]: background#km_core_cp "km_core_cp type" [km_core_usv]: background#km_core_usv "km_core_usv type" diff --git a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts index d91f9db0ba3..74c06d5d129 100644 --- a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts @@ -575,11 +575,12 @@ export class KeyboardInfoCompiler implements KeymanCompiler { '' ); + const resolvedScript = locale.script ?? langtagsByTag[bcp47]?.script ?? langtagsByTag[locale.language]?.script ?? undefined; if(commonScript === null) { - commonScript = tag?.script ?? undefined; + commonScript = resolvedScript; } else { - if(tag?.script !== commonScript) { - this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: tag?.script})) + if(resolvedScript !== commonScript) { + this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: resolvedScript})) } } } diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index aaba0950260..4b9aec76133 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -37,6 +37,10 @@ export class StrsCompiler extends EmptyCompiler { const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { + // stop at the first denormalized string + if (!util.isNormalized(s)) { + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s})); + } // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. // the results might not be valid regex of course. diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1445b40036a..5df0da4b73a 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -199,7 +199,11 @@ export class LdmlCompilerMessages { `Invalid marker identifier "\m{${def(o.id)}}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); - // Available: 0x02B-0x2F + static WARN_StringDenorm = SevWarn | 0x002B; + static Warn_StringDenorm = (o: { s: string }) => + m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); + + // Available: 0x02C-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; static Error_InvalidQuadEscape = (o: { cp: number }) => diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml new file mode 100644 index 00000000000..49cd61d358a --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml new file mode 100644 index 00000000000..ec001db7684 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml new file mode 100644 index 00000000000..a40c119099c --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml new file mode 100644 index 00000000000..cf443c61fe5 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/strs.tests.ts b/developer/src/kmc-ldml/test/strs.tests.ts index a5f7e89e35c..24e8ff4c1ed 100644 --- a/developer/src/kmc-ldml/test/strs.tests.ts +++ b/developer/src/kmc-ldml/test/strs.tests.ts @@ -71,4 +71,25 @@ describe('strs', function () { ]); assert.isNull(kmx); // should fail post-validate }); + describe('should warn on denorm strings', async function () { + for (const num of [1, 2, 3, 4]) { + const path = `sections/strs/warn-denorm-${num}.xml`; + it(path, async function () { + const inputFilename = makePathToFixture(path); + const s = 's\u0307\u0323'; // sĖ‡ĖĢ + // Compile the keyboard + const kmx = await compileKeyboard(inputFilename, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }, + [ + // validation messages + LdmlCompilerMessages.Warn_StringDenorm({s}), + ], + false, // validation should pass + [ + // same messages + LdmlCompilerMessages.Warn_StringDenorm({s}), + ]); + assert.isNotNull(kmx); // not failing + }); + } + }); }); diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m index c1cd9b81f85..ed2227f318e 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/CoreWrapper/CoreWrapper.m @@ -110,11 +110,23 @@ -(void) dealloc{ -(void)loadKeyboardUsingCore:(NSString*) path { km_core_path_name keyboardPath = [path UTF8String]; - km_core_status result = km_core_keyboard_load(keyboardPath, &_coreKeyboard); + NSError* dataError = nil; + NSData *data = [NSData dataWithContentsOfFile:path options:0 error:&dataError]; - if (result != KM_CORE_STATUS_OK) { - NSString *message = [NSString stringWithFormat:@"Unexpected Keyman Core result: %u", result]; - [NSException raise:@"LoadKeyboardException" format:@"%@", message]; + if (dataError != nil) { + os_log_error([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataError: %{public}@", path, dataError); + [NSException raise:@"LoadKeyboardException" format:@"%@", dataError]; + } else { + NSUInteger dataLength = data.length; + os_log_info([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n dataLength: %lu", path, dataLength); + + km_core_status result = km_core_keyboard_load_from_blob(keyboardPath, + data.bytes, dataLength, &_coreKeyboard); + if (result != KM_CORE_STATUS_OK) { + NSString *message = [NSString stringWithFormat:@"Unexpected Keyman Core result: %u", result]; + os_log_error([KMELogs coreLog], "loadKeyboardUsingCore, path: %{public}@\n core result: %{public}@", path, message); + [NSException raise:@"LoadKeyboardException" format:@"%@", message]; + } } } @@ -126,7 +138,7 @@ -(void)readKeyboardAttributesUsingCore { if (result==KM_CORE_STATUS_OK) { _keyboardVersion = [self.coreHelper createNSStringFromUnicharString:keyboardAttributes->version_string]; _keyboardId = [self.coreHelper createNSStringFromUnicharString:keyboardAttributes->id]; - os_log_debug([KMELogs coreLog], "readKeyboardAttributesUsingCore, keyboardVersion: %{public}@\n, keyboardId: %{public}@\n", _keyboardVersion, _keyboardId); + os_log_debug([KMELogs coreLog], "readKeyboardAttributesUsingCore, keyboardVersion: %{public}@, keyboardId: %{public}@\n", _keyboardVersion, _keyboardId); } else { os_log_error([KMELogs coreLog], "km_core_keyboard_get_attrs() failed with result = %u\n", result); }