chore(common/web): Merge branch 'master' into test/common/web/types/9…

…052-unit-tests-kvk-file-writer

HISTORY.md

@@ -1,5 +1,17 @@
 # Keyman Version History
 
+## 18.0.152 alpha 2024-12-04
+
+* refactor(mac): pass kmx data blob to keyman core instead of file path (#12760)
+* fix(developer): honour provided script when checking for matching scripts (#12768)
+* chore(common): rename test files (#12709)
+* fix(common): rename test file (#12770)
+
+## 18.0.151 alpha 2024-12-03
+
+* feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745)
+* feat(developer): report on mismatching lang tag scripts when building keyboard-info (#12753)
+
 ## 18.0.150 alpha 2024-12-02
 
 * fix(core,developer): use `NDEBUG` flag to disable assertions in release build (#12715)

VERSION.md

@@ -1 +1 @@
-18.0.151
+18.0.153

common/web/types/.eslintrc.cjs

@@ -1,13 +1,13 @@
 module.exports = {
   parserOptions: {
-    project: ["./tsconfig.json", "./test/tsconfig.json"],
+    project: ["./tsconfig.json", "./tests/tsconfig.json"],
   },
   ignorePatterns: [
     ".*/*",
     "build/*",
     "coverage/*",
     "node_modules/*",
-    "test/fixtures/*",
+    "tests/fixtures/*",
     "tools/*",
     "src/schemas/*"
   ],

common/web/types/build.sh

@@ -88,7 +88,7 @@ function do_test() {
   fi
 
   eslint .
-  tsc --build test
+  tsc --build tests
 
   readonly C8_THRESHOLD=60
 

common/web/types/package.json

@@ -23,7 +23,7 @@
     "build": "tsc -b",
     "build:schema": "ajv compile",
     "lint": "eslint .",
-    "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha"
+    "test": "npm run lint && cd tests && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha"
   },
   "author": "Marc Durdin <[email protected]> (https://github.com/mcdurdin)",
   "license": "MIT",
@@ -48,7 +48,7 @@
     "typescript": "^5.4.5"
   },
   "mocha": {
-    "spec": "build/test/**/test-*.js",
+    "spec": "build/tests/**/*.tests.js",
     "require": [
       "source-map-support/register"
     ]
@@ -76,7 +76,7 @@
       "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
       "src/osk/osk.ts",
       "src/schemas/*",
-      "test/"
+      "tests/"
     ]
   },
   "sideEffects": false

common/web/types/src/kmx/kmx-plus/kmx-plus.ts

@@ -174,12 +174,7 @@ export class Strs extends Section {
    */
   allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem {
     // Run the string processing pipeline
-    s = Strs.processString(s, opts, sections);
-
-    // add to the set, for testing
-    if (s) {
-      this.allProcessedStrings.add(s);
-    }
+    s = this.processString(s, opts, sections);
 
     // if it's a single char, don't push it into the strs table
     if (opts?.singleOk && isOneChar(s)) {
@@ -196,8 +191,8 @@ export class Strs extends Section {
     return result;
   }
 
-  /** process everything according to opts */
-  static processString(s: string, opts: StrsOptions, sections: DependencySections) {
+  /** process everything according to opts, and add the string to this.allProcessedStrings */
+  private processString(s: string, opts: StrsOptions, sections: DependencySections) {
     s = s ?? '';
     // type check everything else
     if (typeof s !== 'string') {
@@ -215,6 +210,12 @@ export class Strs extends Section {
     if (opts?.unescape) {
       s = unescapeString(s);
     }
+
+    if (s) {
+      // add all processed strings here, so that we catch denormalized strings in the input
+      this.allProcessedStrings.add(s);
+    }
+
     // nfd
     if (opts?.nfd) {
       if (!sections?.meta?.normalizationDisabled) {

common/web/types/src/util/util.ts

@@ -305,6 +305,15 @@ export function isPUA(ch: number) {
     (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END));
 }
 
+/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */
+export function isNormalized(s: string) : boolean {
+  if(!s) return true; // empty or null
+  const nfc = s.normalize("NFC");
+  const nfd = s.normalize("NFD");
+  if (s !== nfc && s !== nfd) return false;
+  return true;
+}
+
 class BadStringMap extends Map<BadStringType, Set<number>> {
   public toString() : string {
     if (!this.size) {

common/web/types/test/helpers/index.ts → common/web/types/tests/helpers/index.ts

@@ -9,5 +9,5 @@ import { fileURLToPath } from "url";
  * @param components One or more path components.
  */
 export function makePathToFixture(...components: string[]): string {
-  return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url));
+  return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url));
 }

common/web/types/test/kvk/test-kvk-file.ts → common/web/types/tests/kvk/kvk-file.tests.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import 'mocha';
 import { makePathToFixture } from '../helpers/index.js';
 import KvkFileReader from "../../src/kvk/kvk-file-reader.js";
-import { verify_balochi_inpage, verify_khmer_angkor } from './test-kvk-utils.js';
+import { verify_balochi_inpage, verify_khmer_angkor } from './kvk-utils.tests.js';
 
 describe('kvk-file-reader', function () {
   it('kvk-file-reader should read a valid file', function() {

common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts → common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts

@@ -1,8 +1,8 @@
 /*
  * Keyman is copyright (C) SIL Global. MIT License.
- * 
+ *
  * Created by Dr Mark C. Sinclair on 2024-11-29
- * 
+ *
  * Test code for unicodeset-parser-api.ts
  */
 

common/web/types/test/tsconfig.json → common/web/types/tests/tsconfig.json

@@ -4,13 +4,12 @@
   "compilerOptions": {
     "rootDir": ".",
     "rootDirs": ["./", "../src/"],
-    "outDir": "../build/test",
+    "outDir": "../build/tests",
     "baseUrl": ".",
     "strictNullChecks": false, // TODO: get rid of this as some point
     "allowSyntheticDefaultImports": true
   },
   "include": [
-    "**/test-*.ts",
     "**/*.tests.ts",
     "./helpers/*.ts",
   ],

common/web/types/test/util/test-unescape.ts → common/web/types/tests/util/unescape.tests.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import {assert} from 'chai';
-import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer} from '../../src/util/util.js';
+import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js';
 
 describe('test UTF32 functions()', function() {
   it('should properly categorize strings', () => {
@@ -186,6 +186,24 @@ describe('test bad char functions', () => {
       assert.isTrue(isPUA(ch), describeCodepoint(ch));
     }
   });
+  describe('test isDenormalized()', () => {
+    it('should correctly categorize strings', () => {
+      [
+        undefined,
+        null,
+        '',
+        'ABC',
+        'fa\u1E69cinating', // NFC
+        'fas\u0323\u0307cinating', // NFD
+        'd\u0323\u0307', // NFD
+        '\u1e0d\u0307', // NFC
+      ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`));
+      [
+        'd\u0307\u0323', // NFD but reversed marks
+        'fas\u0307\u0323cinating', // not-NFD
+      ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`));
+    });
+  });
 });
 
 describe('test BadStringAnalyzer', () => {

core/docs/api/index.md

@@ -75,6 +75,7 @@ modifiers such as Windows key are excluded from this set. Some modifiers are
 transient, such as Control, and others have long-lasting state, such as
 Caps Lock.
 
+- __See more in__   [Keyman Glossary](https://github.com/keymanapp/keyman/wiki/Keyman-glossary)
 
 [km_core_cp]: background#km_core_cp "km_core_cp type"
 [km_core_usv]: background#km_core_usv "km_core_usv type"

developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts

@@ -575,11 +575,12 @@ export class KeyboardInfoCompiler implements KeymanCompiler {
         ''
       );
 
+      const resolvedScript = locale.script ?? langtagsByTag[bcp47]?.script ?? langtagsByTag[locale.language]?.script ?? undefined;
       if(commonScript === null) {
-        commonScript = tag?.script ?? undefined;
+        commonScript = resolvedScript;
       } else {
-        if(tag?.script !== commonScript) {
-          this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: tag?.script}))
+        if(resolvedScript !== commonScript) {
+          this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: resolvedScript}))
         }
       }
     }

developer/src/kmc-ldml/src/compiler/empty-compiler.ts

@@ -37,6 +37,10 @@ export class StrsCompiler extends EmptyCompiler {
       const badStringAnalyzer = new util.BadStringAnalyzer();
       const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH);
       for (let s of strs.allProcessedStrings.values()) {
+        // stop at the first denormalized string
+        if (!util.isNormalized(s)) {
+          this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s}));
+        }
         // replace all \\uXXXX with the actual code point.
         // this lets us analyze whether there are PUA, unassigned, etc.
         // the results might not be valid regex of course.

developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts

@@ -199,7 +199,11 @@ export class LdmlCompilerMessages {
     `Invalid marker identifier "\m{${def(o.id)}}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`,
   );
 
-  // Available: 0x02B-0x2F
+  static WARN_StringDenorm = SevWarn | 0x002B;
+  static Warn_StringDenorm = (o: { s: string }) =>
+  m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`);
+
+  // Available: 0x02C-0x2F
 
   static ERROR_InvalidQuadEscape = SevError | 0x0030;
   static Error_InvalidQuadEscape = (o: { cp: number }) =>

developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-1.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0" />
+
+  <info author="srl295" indicator="🙀" layout="qwerty" name="TestKbd" />
+
+  <keys>
+    <key id="ess" output="s\u{0307}\u{0323}" /> <!-- not NFD-->
+  </keys>
+
+  <layers formId="us" minDeviceWidth="123">
+    <layer id="zz">
+      <row keys="ess" />
+    </layer>
+  </layers>
+
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-2.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0" />
+
+  <info author="srl295" indicator="🙀" layout="qwerty" name="TestKbd" />
+
+  <keys>
+    <key id="ess" output="${ess}" />
+  </keys>
+
+  <variables>
+    <string id="ess" value="s\u{0307}\u{0323}" /> <!-- Not NFD -->
+  </variables>
+
+  <layers formId="us" minDeviceWidth="123">
+    <layer id="zz">
+      <row keys="ess" />
+    </layer>
+  </layers>
+
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-3.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0" />
+
+  <info author="srl295" indicator="🙀" layout="qwerty" name="TestKbd" />
+
+  <displays>
+    <display keyId="ess" display="s\u{0307}\u{0323}" /> <!-- not NFD-->
+  </displays>
+
+  <keys>
+    <key id="ess" output="${ess}" />
+  </keys>
+
+  <variables>
+    <string id="ess" value="S" />
+  </variables>
+
+  <layers formId="us" minDeviceWidth="123">
+    <layer id="zz">
+      <row keys="ess" />
+    </layer>
+  </layers>
+
+</keyboard3>

developer/src/kmc-ldml/test/fixtures/sections/strs/warn-denorm-4.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0" />
+
+  <info author="srl295" indicator="🙀" layout="qwerty" name="TestKbd" />
+
+
+  <keys>
+    <key id="ess" output="S" />
+  </keys>
+
+
+  <layers formId="us" minDeviceWidth="123">
+    <layer id="zz">
+      <row keys="ess" />
+    </layer>
+  </layers>
+
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="S" to="s\u{0307}\u{0323}" /> <!-- Not NFD-->
+    </transformGroup>
+  </transforms>
+
+</keyboard3>

developer/src/kmc-ldml/test/strs.tests.ts

@@ -71,4 +71,25 @@ describe('strs', function () {
             ]);
         assert.isNull(kmx); // should fail post-validate
     });
+    describe('should warn on denorm strings', async function () {
+      for (const num of [1, 2, 3, 4]) {
+        const path = `sections/strs/warn-denorm-${num}.xml`;
+        it(path, async function () {
+          const inputFilename = makePathToFixture(path);
+          const s = 's\u0307\u0323'; // ṩ
+          // Compile the keyboard
+          const kmx = await compileKeyboard(inputFilename, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false },
+            [
+              // validation messages
+              LdmlCompilerMessages.Warn_StringDenorm({s}),
+            ],
+            false, // validation should pass
+            [
+              // same messages
+              LdmlCompilerMessages.Warn_StringDenorm({s}),
+            ]);
+          assert.isNotNull(kmx); // not failing
+        });
+      }
+    });
 });