Merge branch 'chore/core/ldml-test-robust-epic-ldml' into feat/develo…

…per/9121-reject-bad-regex-epic-ldml

HISTORY.md

@@ -1,5 +1,10 @@
 # Keyman Version History
 
+## 17.0.237 alpha 2024-01-02
+
+* feat(developer): Consolidate public APIs for kmc modules (#10208)
+* chore(developer): honor prompt to upgrade in kmc (#10218)
+
 ## 17.0.236 alpha 2023-12-22
 
 * fix(oem/fv): Add fv_kwadacha_tsekene (#10292)

VERSION.md

@@ -1 +1 @@
-17.0.237
+17.0.238

common/web/input-processor/tests/cases/languageProcessor.js

@@ -57,7 +57,13 @@ describe('LanguageProcessor', function() {
   });
 
   describe('.predict', function() {
-    let compiler = new LexicalModelCompiler(callbacks);
+    let compiler = null;
+
+    this.beforeAll(async function() {
+      compiler = new LexicalModelCompiler();
+      assert.isTrue(await compiler.init(callbacks, {}));
+    });
+
     const MODEL_ID = 'example.qaa.trivial';
 
     // ES-module mode leaves out `__dirname`, so we rebuild it using other components.
@@ -67,20 +73,23 @@ describe('LanguageProcessor', function() {
     const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID);
 
     describe('using angle brackets for quotes', function() {
-      let modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
-        format: 'trie-1.0',
-        sources: ['wordlist.tsv'],
-        punctuation: {
-          quotesForKeepSuggestion: { open: `«`, close: `»`},
-          insertAfterWord: " " , // OGHAM SPACE MARK
-        }
-      }, PATH);
-
-      let modelSpec = {
-        id: MODEL_ID,
-        languages: ['en'],
-        code: modelCode
-      };
+      let modelCode = null, modelSpec = null;
+      this.beforeAll(function() {
+        modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
+          format: 'trie-1.0',
+          sources: ['wordlist.tsv'],
+          punctuation: {
+            quotesForKeepSuggestion: { open: `«`, close: `»`},
+            insertAfterWord: " " , // OGHAM SPACE MARK
+          }
+        }, PATH);
+
+        modelSpec = {
+          id: MODEL_ID,
+          languages: ['en'],
+          code: modelCode
+        };
+      });
 
       it("successfully loads the model", function(done) {
         let languageProcessor = new LanguageProcessor(worker, new TranscriptionCache());
@@ -115,18 +124,21 @@ describe('LanguageProcessor', function() {
       });
 
       describe('properly cases generated suggestions', function() {
-        let modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
-          format: 'trie-1.0',
-          sources: ['wordlist.tsv'],
-          languageUsesCasing: true,
-          //applyCasing // we rely on the compiler's default implementation here.
-        }, PATH);
-
-        let modelSpec = {
-          id: MODEL_ID,
-          languages: ['en'],
-          code: modelCode
-        };
+        let modelCode = null, modelSpec = null;
+        this.beforeAll(function () {
+          modelCode = compiler.generateLexicalModelCode(MODEL_ID, {
+            format: 'trie-1.0',
+            sources: ['wordlist.tsv'],
+            languageUsesCasing: true,
+            //applyCasing // we rely on the compiler's default implementation here.
+          }, PATH);
+
+          modelSpec = {
+            id: MODEL_ID,
+            languages: ['en'],
+            code: modelCode
+          };
+        });
 
         describe("does not alter casing when input is lowercased", function() {
           it("when input is fully lowercased", function(done) {

common/web/types/src/kpj/keyman-developer-project.ts

@@ -37,8 +37,16 @@ export class KeymanDeveloperProject {
     for(let filename of files) {
       let fullPath = this.callbacks.path.join(sourcePath, filename);
       if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Source.LdmlKeyboard)) {
-        if(!this.callbacks.fs.readFileSync(fullPath, 'utf-8').match(/ldmlKeyboard3\.dtd/)) {
-          // Skip this .xml because we assume it isn't really a keyboard .xml
+        try {
+          const data = this.callbacks.loadFile(fullPath);
+          const text = new TextDecoder().decode(data);
+          if(!text?.match(/ldmlKeyboard3\.dtd/)) {
+            // Skip this .xml because we assume it isn't really a keyboard .xml
+            continue;
+          }
+        } catch(e) {
+          // We'll just silently skip this file because we were not able to load it,
+          // so let's hope it wasn't a real LDML keyboard XML :-)
           continue;
         }
       }

common/web/types/src/main.ts

@@ -27,6 +27,13 @@ export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, Compile
          compilerExceptionToString, compilerErrorFormatCode,
          compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS,
          ALL_COMPILER_LOG_FORMATS, CompilerLogFormat,
+
+         KeymanCompilerArtifact,
+         KeymanCompilerArtifactOptional,
+         KeymanCompilerArtifacts,
+         KeymanCompilerResult,
+         KeymanCompiler
+
          } from './util/compiler-interfaces.js';
 export { CommonTypesMessages } from './util/common-events.js';
 

common/web/types/src/util/compiler-interfaces.ts

@@ -253,6 +253,40 @@ export interface CompilerCallbackOptions {
   compilerWarningsAsErrors?: boolean;
 };
 
+export interface KeymanCompilerArtifact {
+  data: Uint8Array;
+  filename: string;
+};
+
+export type KeymanCompilerArtifactOptional = KeymanCompilerArtifact | undefined;
+
+export interface KeymanCompilerArtifacts {
+  readonly [type:string]: KeymanCompilerArtifactOptional;
+};
+
+export interface KeymanCompilerResult {
+  artifacts: KeymanCompilerArtifacts;
+};
+
+export interface KeymanCompiler {
+  init(callbacks: CompilerCallbacks, options: CompilerOptions): Promise<boolean>;
+  /**
+   * Run the compiler, and save the result in memory arrays. Note that while
+   * `outputFilename` is provided here, the output file is not written to in
+   * this function.
+   * @param inputFilename
+   * @param outputFilename The intended output filename, optional, if missing,
+   *                       calculated from inputFilename
+   * @param data
+   */
+  run(inputFilename:string, outputFilename?:string /*, data?: any*/): Promise<KeymanCompilerResult>;
+  /**
+   * Writes the compiled output files to disk
+   * @param artifacts
+   */
+  write(artifacts: KeymanCompilerArtifacts): Promise<boolean>;
+};
+
 /**
  * Abstract interface for callbacks, to abstract out file i/o
  */
@@ -369,10 +403,6 @@ export interface CompilerBaseOptions {
    * Format of output for log to console
    */
   logFormat?: CompilerLogFormat;
-  /**
-   * Optional output file for activities that generate output
-   */
-  outFile?: string;
   /**
    * Colorize log output, default is detected from console
    */

core/tests/unit/ldml/test_transforms.cpp

@@ -790,8 +790,8 @@ int test_normalize() {
     // from tests
     marker_map map;
     std::cout << __FILE__ << ":" << __LINE__ << "   - complex test 9c" << std::endl;
-    const std::u32string src    = U"9ce\u0300\uFFFF\b\u0002\u0320\uFFFF\b\u0001";
-    const std::u32string expect = U"9ce\uFFFF\b\u0002\u0320\u0300\uFFFF\b\u0001";
+    const std::u32string src    = U"9ce\u0300\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001";
+    const std::u32string expect = U"9ce\uFFFF\u0008\u0002\u0320\u0300\uFFFF\u0008\u0001";
     std::u32string dst = src;
     assert(normalize_nfd_markers(dst, map));
     if (dst != expect) {

developer/src/common/web/utils/index.ts

@@ -1,3 +1,3 @@
 export { validateMITLicense } from './src/validate-mit-license.js';
 export { KeymanSentry } from './src/KeymanSentry.js';
-export { getOption, loadOptions } from './src/options.js';
+export { getOption, loadOptions, clearOptions } from './src/options.js';

developer/src/common/web/utils/src/options.ts

@@ -35,12 +35,16 @@ export interface KeymanDeveloperOptions {
   "automatically report usage"?: boolean;
   "toolbar visible"?: boolean;
   "active project"?: string;
+  "prompt to upgrade projects"?: boolean;
 };
 
 type KeymanDeveloperOption = keyof KeymanDeveloperOptions;
 
+// Default has no options set, and unit tests will use the defaults (won't call
+// `loadOptions()`)
+let options: KeymanDeveloperOptions = {};
+
 // We only load the options from disk once on first use
-let options: KeymanDeveloperOptions = null;
 let optionsLoaded = false;
 
 export async function loadOptions(): Promise<KeymanDeveloperOptions> {
@@ -74,9 +78,18 @@ export async function loadOptions(): Promise<KeymanDeveloperOptions> {
     // low level.
     options = {};
   }
+  optionsLoaded = true;
   return options;
 }
 
 export function getOption<T extends KeymanDeveloperOption>(valueName: T, defaultValue: KeymanDeveloperOptions[T]): KeymanDeveloperOptions[T] {
   return options[valueName] ?? defaultValue;
 }
+
+/**
+ * unit tests will clear options before running, for consistency
+ */
+export function clearOptions() {
+  options = {};
+  optionsLoaded = true;
+}

developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts

@@ -9,15 +9,15 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename:
   let touchLayoutFilename: string;
 
   const kmnCompiler = new KmnCompiler();
-  if(!await kmnCompiler.init(callbacks)) {
+  if(!await kmnCompiler.init(callbacks, {
+    shouldAddCompilerVersion: false,
+    saveDebug: false,
+  })) {
     // kmnCompiler will report errors
     return null;
   }
 
-  let result = kmnCompiler.runCompiler(filename, {
-    shouldAddCompilerVersion: false,
-    saveDebug: false,
-  });
+  let result = await kmnCompiler.run(filename, null);
 
   if(!result) {
     // kmnCompiler will report any errors
@@ -29,7 +29,7 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename:
   }
 
   const reader = new KmxFileReader();
-  const keyboard: KMX.KEYBOARD = reader.read(result.kmx.data);
+  const keyboard: KMX.KEYBOARD = reader.read(result.artifacts.kmx.data);
   const touchLayoutStore = keyboard.stores.find(store => store.dwSystemID == KMX.KMXFile.TSS_LAYOUTFILE);
 
   if(touchLayoutStore) {

developer/src/kmc-keyboard-info/src/index.ts

@@ -5,7 +5,7 @@
 
 import { minKeymanVersion } from "./min-keyman-version.js";
 import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFileLanguageFont, KeyboardInfoFilePlatform } from "./keyboard-info-file.js";
-import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types";
+import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from "@keymanapp/common-types";
 import { KeyboardInfoCompilerMessages } from "./messages.js";
 import langtags from "./imports/langtags.js";
 import { validateMITLicense } from "@keymanapp/developer-utils";
@@ -24,7 +24,7 @@ const HelpRoot = 'https://help.keyman.com/keyboard/';
  * Build a dictionary of language tags from langtags.json
  */
 
-function init(): void {
+function preinit(): void {
   if(langtagsByTag['en']) {
     // Already initialized, we can reasonably assume that 'en' will always be in
     // langtags.json.
@@ -62,9 +62,30 @@ export interface KeyboardInfoSources {
   forPublishing: boolean;
 };
 
-export class KeyboardInfoCompiler {
-  constructor(private callbacks: CompilerCallbacks) {
-    init();
+export interface KeyboardInfoCompilerOptions extends CompilerOptions {
+  sources: KeyboardInfoSources;
+};
+
+export interface KeyboardInfoCompilerArtifacts extends KeymanCompilerArtifacts {
+  keyboard_info: KeymanCompilerArtifact;
+};
+
+export interface KeyboardInfoCompilerResult extends KeymanCompilerResult {
+  artifacts: KeyboardInfoCompilerArtifacts;
+};
+
+export class KeyboardInfoCompiler implements KeymanCompiler {
+  private callbacks: CompilerCallbacks;
+  private options: KeyboardInfoCompilerOptions;
+
+  constructor() {
+    preinit();
+  }
+
+  public async init(callbacks: CompilerCallbacks, options: KeyboardInfoCompilerOptions): Promise<boolean> {
+    this.callbacks = callbacks;
+    this.options = {...options};
+    return true;
   }
 
   /**
@@ -77,15 +98,18 @@ export class KeyboardInfoCompiler {
    *
    * @param sources                     Details on files from which to extract metadata
    */
-  public async writeKeyboardInfoFile(
-    sources: KeyboardInfoSources
-  ): Promise<Uint8Array> {
+  public async run(inputFilename: string, outputFilename?: string): Promise<KeyboardInfoCompilerResult> {
+    const sources = this.options.sources;
 
     // TODO(lowpri): work from .kpj and nothing else as input. Blocked because
     // .kpj work is largely in kmc at present, so that would need to move to
     // a separate module.
 
-    const kmpCompiler = new KmpCompiler(this.callbacks);
+    const kmpCompiler = new KmpCompiler();
+    if(!await kmpCompiler.init(this.callbacks, {})) {
+      // Errors will have been emitted by KmpCompiler
+      return null;
+    }
     const kmpJsonData = kmpCompiler.transformKpsToKmpObject(sources.kpsFilename);
     if(!kmpJsonData) {
       // Errors will have been emitted by KmpCompiler
@@ -307,7 +331,22 @@ export class KeyboardInfoCompiler {
       }, null, 2));
     }
 
-    return new TextEncoder().encode(jsonOutput);
+    const data = new TextEncoder().encode(jsonOutput);
+    const result: KeyboardInfoCompilerResult = {
+      artifacts: {
+        keyboard_info: {
+          data,
+          filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.keyboard_info')
+        }
+      }
+    };
+
+    return result;
+  }
+
+  public async write(artifacts: KeyboardInfoCompilerArtifacts): Promise<boolean> {
+    this.callbacks.fs.writeFileSync(artifacts.keyboard_info.filename, artifacts.keyboard_info.data);
+    return true;
   }
 
   private mapKeymanTargetToPlatform(target: KeymanTargets.KeymanTarget): KeyboardInfoFilePlatform[] {