Merge pull request #12622 from keymanapp/feat/developer/9767-kmc-anal…

…yze-extend-mapping

feat(developer): analyze osk-char-use merge with existing mapping file

developer/docs/help/reference/kmc/cli/reference.md

@@ -280,6 +280,14 @@ Note: paths shown above may vary.
 
 : Result file to write to (.json, .md, or .txt)
 
+`-i, --input-mapping-file <filename>`
+
+: Merge result file with existing mapping file. If supplied, existing
+  codepoint mappings will be kept, to ensure that updated fonts are
+  backwardly compatible with deployed keyboards. The
+  `--include-counts` flag will be set according to the format of
+  the input mapping file.
+
 For more information on the purpose of `analyze osk-char-use` and
 `analyze rewrite-osk-from-char-use`, see
 [`&displayMap`](/developer/language/reference/displaymap).

developer/src/kmc-analyze/build.sh

@@ -27,10 +27,11 @@ builder_parse "$@"
 
 function do_test() {
   eslint .
-  # TODO: enable tests
-  #     cd test && tsc --build && cd .. && mocha
-  # TODO: enable c8 (disabled because no coverage at present)
-  #     c8 --reporter=lcov --reporter=text --exclude-after-remap mocha
+  cd test
+  tsc --build
+  cd ..
+  readonly C8_THRESHOLD=70
+  c8 --reporter=lcov --reporter=text --lines $C8_THRESHOLD --statements $C8_THRESHOLD --branches $C8_THRESHOLD --functions $C8_THRESHOLD mocha
 }
 
 builder_run_action clean      rm -rf ./build/

developer/src/kmc-analyze/package.json

@@ -36,6 +36,12 @@
     "mocha": "^8.4.0",
     "typescript": "^5.4.5"
   },
+  "mocha": {
+    "spec": "build/test/**/test-*.js",
+    "require": [
+      "source-map-support/register"
+    ]
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/keymanapp/keyman.git"

developer/src/kmc-analyze/src/analyzer-messages.ts

@@ -1,9 +1,12 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
 import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, CompilerMessageSpecWithException, KeymanUrls } from "@keymanapp/developer-utils";
 
 const Namespace = CompilerErrorNamespace.Analyzer;
 const SevInfo = CompilerErrorSeverity.Info | Namespace;
 // const SevHint = CompilerErrorSeverity.Hint | Namespace;
-// const SevWarn = CompilerErrorSeverity.Warn | Namespace;
+const SevWarn = CompilerErrorSeverity.Warn | Namespace;
 // const SevError = CompilerErrorSeverity.Error | Namespace;
 const SevFatal = CompilerErrorSeverity.Fatal | Namespace;
 
@@ -28,4 +31,29 @@ export class AnalyzerMessages {
     `Scanning ${def(o.type)} file ${def(o.name)}`,
     `Informative message reporting on the current file being scanned`
   );
+
+  static readonly WARN_PreviousMapFileCouldNotBeLoaded = SevWarn | 0x0003;
+  static readonly Warn_PreviousMapFileCouldNotBeLoaded = (o:{filename: string}) => m(
+    this.WARN_PreviousMapFileCouldNotBeLoaded,
+    `The map file ${def(o.filename)} is missing or not a valid JSON map file`,
+  );
+
+  static readonly WARN_PreviousMapFileCouldNotBeLoadedDueToError = SevWarn | 0x0004;
+  static readonly Warn_PreviousMapFileCouldNotBeLoadedDueToError = (o:{filename: string, e: any}) => m(
+    this.WARN_PreviousMapFileCouldNotBeLoadedDueToError,
+    `The map file ${def(o.filename)} could not be loaded due to ${def(o.e ?? 'unknown error')}`,
+  );
+
+  static readonly WARN_PreviousMapDidNotIncludeCounts = SevWarn | 0x0005;
+  static readonly Warn_PreviousMapDidNotIncludeCounts = (o:{filename: string}) => m(
+    this.WARN_PreviousMapDidNotIncludeCounts,
+    `The map file ${def(o.filename)} did not include counts. Changing includeCounts option to 'false' to match`,
+  );
+
+  static readonly WARN_PreviousMapDidIncludeCounts = SevWarn | 0x0006;
+  static readonly Warn_PreviousMapDidIncludeCounts = (o:{filename: string}) => m(
+    this.WARN_PreviousMapDidIncludeCounts,
+    `The map file ${def(o.filename)} did include counts. Changing includeCounts option to 'true' to match`,
+  );
+
 };

developer/src/kmc-analyze/src/osk-character-use/index.ts

@@ -1,3 +1,6 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ */
 import { KeymanFileTypes, TouchLayout } from "@keymanapp/common-types";
 import { KmnCompilerMessages, Osk } from '@keymanapp/kmc-kmn';
 import { CompilerCallbacks, escapeMarkdownChar, KvksFile, KvksFileReader, TouchLayoutFileReader } from '@keymanapp/developer-utils';
@@ -26,6 +29,10 @@ export interface AnalyzeOskCharacterUseOptions {
    * source file
    */
   includeCounts?: boolean;
+  /**
+   * Filename of an existing mapping file to merge the results into
+   */
+  mergeMapFile?: string;
 }
 
 const defaultOptions: AnalyzeOskCharacterUseOptions = {
@@ -233,17 +240,45 @@ export class AnalyzeOskCharacterUse {
   // Results reporting
   //
 
-  private prepareResults(strings: StringRefUsageMap): Osk.StringResult[] {
-    let result: Osk.StringResult[] = [];
-    let pua = this.options.puaBase;
+  private prepareResults(previousMap: Osk.StringResult[], strings: StringRefUsageMap): Osk.StringResult[] {
+
+    // https://stackoverflow.com/a/1584377/1836776 - because we need to compare
+    // objects, we can't use Set
+    const mergeArrays = (a: any, b: any, predicate = (a:any, b:any) => a === b) => {
+      const c = [...a]; // copy to avoid side effects
+      // add all items from B to copy C if they're not already present
+      b.forEach((bItem: any) => (c.some((cItem) => predicate(bItem, cItem)) ? null : c.push(bItem)))
+      return c;
+    }
+
+    if(!previousMap) {
+      previousMap = [];
+    }
+
+    let result: Osk.StringResult[] = [...previousMap];
+
+    // Note: we are assuming same base as previous runs
+    let pua = Math.max(this.options.puaBase, ...previousMap.map(item => parseInt(item.pua,16) + 1));
+
     for(let str of Object.keys(strings)) {
-      result.push({
-        pua: pua.toString(16).toUpperCase(),
-        str,
-        unicode: AnalyzeOskCharacterUse.stringToUnicodeSequence(str, false),
-        usages: this.options.includeCounts ? strings[str] : strings[str].map(item => item.filename)
-      });
-      pua++;
+      const r = result.find(item => item.str == str);
+      if(!r) {
+        result.push({
+          pua: pua.toString(16).toUpperCase(),
+          str,
+          unicode: AnalyzeOskCharacterUse.stringToUnicodeSequence(str, false),
+          usages: this.options.includeCounts ? strings[str] : strings[str].map(item => item.filename)
+        });
+        pua++;
+      } else {
+        if(this.options.includeCounts) {
+          // merge StringUsageRefs
+          r.usages = mergeArrays(r.usages, strings[str], (a: Osk.StringRefUsage, b: Osk.StringRefUsage) => a.filename === b.filename);
+        } else {
+          // merge strings
+          r.usages = mergeArrays(r.usages, strings[str].map(item => item.filename));
+        }
+      }
     }
     return result;
   }
@@ -273,7 +308,8 @@ export class AnalyzeOskCharacterUse {
    *           parameter.
    */
   public getStrings(format?: '.txt'|'.md'|'.json'): string[] {
-    const final = this.prepareResults(this._strings);
+    const previousMap = this.loadPreviousMap(this.options.mergeMapFile);
+    const final = this.prepareResults(previousMap, this._strings);
     switch(format) {
       case '.md':
         return AnalyzeOskCharacterUse.getStringsAsMarkdown(final);
@@ -283,6 +319,49 @@ export class AnalyzeOskCharacterUse {
     return AnalyzeOskCharacterUse.getStringsAsText(final);
   }
 
+  /**
+   * Load a JSON-format result file to merge from
+   * @param filename
+   * @returns
+   */
+  private loadPreviousMap(filename: string): Osk.StringResult[] {
+    if(!filename) {
+      return null;
+    }
+
+    const data = this.callbacks.loadFile(filename);
+    if(!data) {
+      this.callbacks.reportMessage(AnalyzerMessages.Warn_PreviousMapFileCouldNotBeLoaded({filename}));
+      return null;
+    }
+    let json: any;
+    try {
+      json = JSON.parse(new TextDecoder().decode(data));
+      if(!json || typeof json != 'object' || !Array.isArray(json.map)) {
+        this.callbacks.reportMessage(AnalyzerMessages.Warn_PreviousMapFileCouldNotBeLoaded({filename}));
+        return null;
+      }
+    } catch(e) {
+      this.callbacks.reportMessage(AnalyzerMessages.Warn_PreviousMapFileCouldNotBeLoadedDueToError({filename, e}));
+      return null;
+    }
+
+    const map: Osk.StringResult[] = json.map;
+    const usages = map.find(item => item?.usages?.length).usages;
+    if(usages) {
+      if(typeof usages[0] == 'string' && this.options.includeCounts) {
+        this.callbacks.reportMessage(AnalyzerMessages.Warn_PreviousMapDidNotIncludeCounts({filename}));
+        this.options.includeCounts = false;
+      } else if(typeof usages[0] != 'string' && !this.options.includeCounts) {
+        this.callbacks.reportMessage(AnalyzerMessages.Warn_PreviousMapDidIncludeCounts({filename}));
+        this.options.includeCounts = true;
+      }
+    }
+
+    return map;
+  }
+
+
   // Following functions are static so that we can keep them pure
   // and potentially refactor into separate reporting class later