From 6bebefda10bf3fd1a22b42f301a45d4036cdc172 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:05:31 +0700 Subject: [PATCH 01/18] feat(developer): move kmc-kmn to KeymanCompiler interface Part of #9473. KmnCompiler now implements KeymanCompiler, including the returned artifacts. Also establishes the common types for the compiler interfaces and consolidates and renames various API surfaces for kmc-kmn. --- common/web/types/src/main.ts | 7 ++ .../web/types/src/util/compiler-interfaces.ts | 39 +++++++- .../src/util/get-osk-from-kmn-file.ts | 12 +-- .../src/kmc-kmn/src/compiler/compiler.ts | 90 ++++++++++--------- .../src/kmw-compiler/compiler-globals.ts | 6 +- .../kmc-kmn/src/kmw-compiler/kmw-compiler.ts | 4 +- .../src/kmc-kmn/test/kmw/test-kmw-compiler.ts | 17 ++-- developer/src/kmc-kmn/test/test-compiler.ts | 45 +++++++--- developer/src/kmc-kmn/test/test-features.ts | 8 +- developer/src/kmc-kmn/test/test-messages.ts | 4 +- developer/src/kmc-kmn/test/test-wasm-uset.ts | 10 +-- .../src/kmc-ldml/src/compiler/compiler.ts | 2 +- .../commands/buildClasses/BuildKmnKeyboard.ts | 27 +++--- .../src/kmc/src/util/TestKeymanSentry.ts | 2 +- 14 files changed, 172 insertions(+), 101 deletions(-) diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index 06671a13d75..4a1afc1bf96 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -27,6 +27,13 @@ export { defaultCompilerOptions, CompilerBaseOptions, CompilerCallbacks, Compile compilerExceptionToString, compilerErrorFormatCode, compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS, ALL_COMPILER_LOG_FORMATS, CompilerLogFormat, + + KeymanCompilerArtifact, + KeymanCompilerArtifactOptional, + KeymanCompilerArtifacts, + KeymanCompilerResult, + KeymanCompiler + } from './util/compiler-interfaces.js'; export { CommonTypesMessages } from './util/common-events.js'; diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index b27a170ab34..fc6a33fa1a9 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -253,6 +253,40 @@ export interface CompilerCallbackOptions { compilerWarningsAsErrors?: boolean; }; +export interface KeymanCompilerArtifact { + data: Uint8Array; + filename: string; +}; + +export type KeymanCompilerArtifactOptional = KeymanCompilerArtifact | undefined; + +export interface KeymanCompilerArtifacts { + readonly [type:string]: KeymanCompilerArtifactOptional; +}; + +export interface KeymanCompilerResult { + artifacts: KeymanCompilerArtifacts; +}; + +export interface KeymanCompiler { + init(callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + /** + * Run the compiler, and save the result in memory arrays. Note that while + * `outputFilename` is provided here, the output file is not written to in + * this function. + * @param inputFilename + * @param outputFilename The intended output filename, optional, if missing, + * calculated from inputFilename + * @param data + */ + run(inputFilename:string, outputFilename?:string /*, data?: any*/): Promise; + /** + * Writes the compiled output files to disk + * @param artifacts + */ + write(artifacts: KeymanCompilerArtifacts): Promise; +}; + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -369,10 +403,7 @@ export interface CompilerBaseOptions { * Format of output for log to console */ logFormat?: CompilerLogFormat; - /** - * Optional output file for activities that generate output - */ - outFile?: string; + outFile?: string; //TODO:REMOVE /** * Colorize log output, default is detected from console */ diff --git a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts index 2df34e9dae0..0e2ec289956 100644 --- a/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts +++ b/developer/src/kmc-analyze/src/util/get-osk-from-kmn-file.ts @@ -9,15 +9,15 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: let touchLayoutFilename: string; const kmnCompiler = new KmnCompiler(); - if(!await kmnCompiler.init(callbacks)) { + if(!await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: false, + })) { // kmnCompiler will report errors return null; } - let result = kmnCompiler.runCompiler(filename, { - shouldAddCompilerVersion: false, - saveDebug: false, - }); + let result = await kmnCompiler.run(filename, null); if(!result) { // kmnCompiler will report any errors @@ -29,7 +29,7 @@ export async function getOskFromKmnFile(callbacks: CompilerCallbacks, filename: } const reader = new KmxFileReader(); - const keyboard: KMX.KEYBOARD = reader.read(result.kmx.data); + const keyboard: KMX.KEYBOARD = reader.read(result.artifacts.kmx.data); const touchLayoutStore = keyboard.stores.find(store => store.dwSystemID == KMX.KMXFile.TSS_LAYOUTFILE); if(touchLayoutStore) { diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 317cfc757c3..a4033ad89c5 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -6,17 +6,12 @@ TODO: implement additional interfaces: */ // TODO: rename wasm-host? -import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader } from '@keymanapp/common-types'; +import { UnicodeSetParser, UnicodeSet, Osk, VisualKeyboard, KvkFileReader, KeymanCompiler, KeymanCompilerArtifacts, KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanFileTypes, KvkFileWriter, KvksFileReader } from '@keymanapp/common-types'; import loadWasmHost from '../import/kmcmplib/wasm-host.js'; import { CompilerMessages, mapErrorFromKmcmplib } from './kmn-compiler-messages.js'; import { WriteCompiledKeyboard } from '../kmw-compiler/kmw-compiler.js'; -export interface CompilerResultFile { - filename: string; - data: Uint8Array; -}; - // // Matches kmcmplibapi.h definitions // @@ -46,7 +41,7 @@ export const COMPILETARGETS__MASK = 0x03; /** * Data in CompilerResultExtra comes from kmcmplib */ -export interface CompilerResultExtra { +export interface KmnCompilerResultExtra { /** * A bitmask, consisting of COMPILETARGETS_KMX and/or COMPILETARGETS_JS */ @@ -61,11 +56,15 @@ export interface CompilerResultExtra { // Internal in-memory result from a successful compilation // -export interface CompilerResult { - kmx?: CompilerResultFile; - kvk?: CompilerResultFile; - js?: CompilerResultFile; - extra: CompilerResultExtra; +export interface KmnCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface KmnCompilerResult extends KeymanCompilerResult { + artifacts: KmnCompilerArtifacts; + extra: KmnCompilerResultExtra; displayMap?: Osk.PuaMap; }; @@ -97,18 +96,20 @@ interface MallocAndFree { let Module: any; -export class KmnCompiler implements UnicodeSetParser { +export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; wasmExports: MallocAndFree; + options: KmnCompilerOptions; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); callbackProcIdentifier++; } - public async init(callbacks: CompilerCallbacks): Promise { + public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise { this.callbacks = callbacks; + this.options = options; if(!Module) { try { Module = await loadWasmHost(); @@ -140,20 +141,22 @@ export class KmnCompiler implements UnicodeSetParser { return true; } - public run(infile: string, options?: KmnCompilerOptions): boolean { - let result = this.runCompiler(infile, options); - if(result) { - if(result.kmx) { - this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); - } - if(result.kvk) { - this.callbacks.fs.writeFileSync(result.kvk.filename, result.kvk.data); - } - if(result.js) { - this.callbacks.fs.writeFileSync(result.js.filename, result.js.data); - } + public async write(artifacts: KmnCompilerArtifacts): Promise { + if(!artifacts) { + throw Error('artifacts must be defined'); } - return !!result; + + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } private compilerMessageCallback = (line: number, code: number, msg: string): number => { @@ -196,9 +199,10 @@ export class KmnCompiler implements UnicodeSetParser { return 1; } - private copyWasmResult(wasm_result: any): CompilerResult { - let result: CompilerResult = { + private copyWasmResult(wasm_result: any): KmnCompilerResult { + let result: KmnCompilerResult = { // We cannot Object.assign or {...} on a wasm-defined object, so... + artifacts: {}, extra: { targets: wasm_result.extra.targets, displayMapFilename: wasm_result.extra.displayMapFilename, @@ -234,15 +238,15 @@ export class KmnCompiler implements UnicodeSetParser { return new Uint8Array(new Uint8Array(Module.HEAP8.buffer, offset, size)); } - public runCompiler(infile: string, options: KmnCompilerOptions): CompilerResult { + public async run(infile: string, outfile: string): Promise { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ return null; } - options = {...baseOptions, ...options}; + const options = {...baseOptions, ...this.options}; - options.outFile = options.outFile ?? infile.replace(/\.kmn$/i, '.kmx'); + outfile = outfile ?? infile.replace(/\.kmn$/i, '.kmx'); (globalThis as any)[this.callbackID] = { message: this.compilerMessageCallback, @@ -264,11 +268,11 @@ export class KmnCompiler implements UnicodeSetParser { return null; } - const result: CompilerResult = this.copyWasmResult(wasm_result); + const result: KmnCompilerResult = this.copyWasmResult(wasm_result); if(result.extra.targets & COMPILETARGETS_KMX) { - result.kmx = { - filename: options.outFile, + result.artifacts.kmx = { + filename: outfile, data: this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize) }; } @@ -286,8 +290,8 @@ export class KmnCompiler implements UnicodeSetParser { } if(result.extra.kvksFilename) { - result.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, options.outFile, result.displayMap); - if(!result.kvk) { + result.artifacts.kvk = this.runKvkCompiler(result.extra.kvksFilename, infile, outfile, result.displayMap); + if(!result.artifacts.kvk) { return null; } } @@ -308,12 +312,12 @@ export class KmnCompiler implements UnicodeSetParser { if(!wasm_result.result) { return null; } - const kmw_result: CompilerResult = this.copyWasmResult(wasm_result); + const kmw_result: KmnCompilerResult = this.copyWasmResult(wasm_result); kmw_result.displayMap = result.displayMap; // we can safely re-use the kmx compile displayMap const web_kmx = this.copyWasmBuffer(wasm_result.kmx, wasm_result.kmxSize); - result.js = this.runWebCompiler(infile, options.outFile, web_kmx, result.kvk?.data, kmw_result, options); - if(!result.js) { + result.artifacts.js = this.runWebCompiler(infile, outfile, web_kmx, result.artifacts.kvk?.data, kmw_result, options); + if(!result.artifacts.js) { return null; } } @@ -338,9 +342,9 @@ export class KmnCompiler implements UnicodeSetParser { kmxFilename: string, web_kmx: Uint8Array, kvk: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, options: CompilerOptions - ): CompilerResultFile { + ): KeymanCompilerArtifact { const data = WriteCompiledKeyboard(this.callbacks, kmnFilename, web_kmx, kvk, kmxResult, options.saveDebug); if(!data) { return null; @@ -348,7 +352,7 @@ export class KmnCompiler implements UnicodeSetParser { return { filename: this.callbacks.path.join(this.callbacks.path.dirname(kmxFilename), - this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), + this.keyboardIdFromKmnFilename(kmnFilename) + KeymanFileTypes.Binary.WebKeyboard), data: new TextEncoder().encode(data) }; } diff --git a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts index 92cba914c5e..3b09a18f479 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/compiler-globals.ts @@ -1,10 +1,10 @@ import { KMX, CompilerCallbacks, CompilerOptions } from "@keymanapp/common-types"; -import { CompilerResult } from "../compiler/compiler.js"; +import { KmnCompilerResult } from "../compiler/compiler.js"; export let FTabStop: string; export let nl: string; export let FCompilerWarningsAsErrors = false; -export let kmxResult: CompilerResult; +export let kmxResult: KmnCompilerResult; export let fk: KMX.KEYBOARD; export let FMnemonic: boolean; export let options: CompilerOptions; @@ -19,7 +19,7 @@ export function setupGlobals( _options: CompilerOptions, _tab: string, _nl: string, - _kmxResult: CompilerResult, + _kmxResult: KmnCompilerResult, _keyboard: KMX.KEYBOARD, _kmnfile: string ) { diff --git a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts index dd891cd7cb1..974e4fd485d 100644 --- a/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts +++ b/developer/src/kmc-kmn/src/kmw-compiler/kmw-compiler.ts @@ -5,7 +5,7 @@ import { JavaScript_ContextMatch, JavaScript_KeyAsString, JavaScript_Name, JavaS import { KmwCompilerMessages } from "./kmw-compiler-messages.js"; import { ValidateLayoutFile } from "./validate-layout-file.js"; import { VisualKeyboardFromFile } from "./visual-keyboard-compiler.js"; -import { CompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; +import { KmnCompilerResult, STORETYPE_DEBUG, STORETYPE_OPTION, STORETYPE_RESERVED } from "../compiler/compiler.js"; function requote(s: string): string { return "'" + s.replaceAll(/(['\\])/g, "\\$1") + "'"; @@ -41,7 +41,7 @@ export function WriteCompiledKeyboard( kmnfile: string, keyboardData: Uint8Array, kvkData: Uint8Array, - kmxResult: CompilerResult, + kmxResult: KmnCompilerResult, FDebug: boolean = false ): string { let opts: CompilerOptions = { diff --git a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts index a7eecda019e..67323515767 100644 --- a/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts +++ b/developer/src/kmc-kmn/test/kmw/test-kmw-compiler.ts @@ -5,7 +5,7 @@ import { dirname } from 'path'; import { fileURLToPath } from 'url'; import fs from 'fs'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; -import { CompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; +import { KmnCompilerResult, KmnCompiler } from '../../src/compiler/compiler.js'; import { ETLResult, extractTouchLayout as parseWebTestResult } from './util.js'; import { KeymanFileTypes } from '@keymanapp/common-types'; @@ -27,7 +27,10 @@ describe('KeymanWeb Compiler', function() { const kmnCompiler: KmnCompiler = new KmnCompiler(); this.beforeAll(async function() { - assert.isTrue(await kmnCompiler.init(callbacks)); + assert.isTrue(await kmnCompiler.init(callbacks, { + shouldAddCompilerVersion: false, + saveDebug: true, + })); }); this.afterEach(function() { @@ -79,18 +82,16 @@ describe('KeymanWeb Compiler', function() { }); -function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): { result: CompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult } { +async function run_test_keyboard(kmnCompiler: KmnCompiler, id: string): + Promise<{ result: KmnCompilerResult, actualCode: string, actual: ETLResult, expectedCode: string, expected: ETLResult }> { const filenames = generateTestFilenames(id); - let result = kmnCompiler.runCompiler(filenames.source, { - shouldAddCompilerVersion: false, - saveDebug: true, - }); + let result = await kmnCompiler.run(filenames.source, null); assert.isNotNull(result); let value = { result, - actualCode: new TextDecoder().decode(result.js.data), + actualCode: new TextDecoder().decode(result.artifacts.js.data), expectedCode: fs.readFileSync(filenames.fixture, 'utf8'), expected: null, actual: null, diff --git a/developer/src/kmc-kmn/test/test-compiler.ts b/developer/src/kmc-kmn/test/test-compiler.ts index 0b776266ce8..d103625e9ba 100644 --- a/developer/src/kmc-kmn/test/test-compiler.ts +++ b/developer/src/kmc-kmn/test/test-compiler.ts @@ -15,7 +15,7 @@ describe('Compiler class', function() { const compiler = new KmnCompiler(); const callbacks : any = null; // ERROR try { - await compiler.init(callbacks) + await compiler.init(callbacks, null) assert.fail('Expected exception'); } catch(e) { assert.ok(e); @@ -26,21 +26,27 @@ describe('Compiler class', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const fixtureName = baselineDir + 'k_000___null_keyboard.kmx'; const infile = baselineDir + 'k_000___null_keyboard.kmn'; const outFile = __dirname + '/k_000___null_keyboard.kmx'; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -54,7 +60,7 @@ describe('Compiler class', function() { it('should build all baseline fixtures', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const files = fs.readdirSync(baselineDir); @@ -64,7 +70,13 @@ describe('Compiler class', function() { const infile = baselineDir + file.replace(/x$/, 'n'); const outFile = __dirname + '/' + file; - assert(compiler.run(infile, {saveDebug: true, outFile, shouldAddCompilerVersion: false})); + if(fs.existsSync(outFile)) { + fs.rmSync(outFile); + } + + const result = await compiler.run(infile, outFile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert(fs.existsSync(outFile)); const outfileData = fs.readFileSync(outFile); @@ -78,7 +90,10 @@ describe('Compiler class', function() { it('should compile a keyboard with visual keyboard', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert.isTrue(await compiler.init(callbacks)); + assert.isTrue(await compiler.init(callbacks, { + saveDebug: true, + shouldAddCompilerVersion: false, + })); assert.isTrue(compiler.verifyInitialized()); const fixtureDir = keyboardsDir + 'caps_lock_layer_3620/' @@ -89,11 +104,17 @@ describe('Compiler class', function() { const resultingKmxfile = __dirname + '/caps_lock_layer_3620.kmx'; const resultingKvkfile = __dirname + '/caps_lock_layer_3620.kvk'; - assert.isTrue(compiler.run(infile, { - saveDebug: true, - shouldAddCompilerVersion: false, - outFile: resultingKmxfile, - })); + if(fs.existsSync(resultingKmxfile)) { + fs.rmSync(resultingKmxfile); + } + + if(fs.existsSync(resultingKvkfile)) { + fs.rmSync(resultingKvkfile); + } + + const result = await compiler.run(infile, resultingKmxfile); + assert.isNotNull(result); + assert.isTrue(await compiler.write(result.artifacts)); assert.isTrue(fs.existsSync(resultingKmxfile)); assert.isTrue(fs.existsSync(resultingKvkfile)); diff --git a/developer/src/kmc-kmn/test/test-features.ts b/developer/src/kmc-kmn/test/test-features.ts index 81c9aa24ea6..7f4be109516 100644 --- a/developer/src/kmc-kmn/test/test-features.ts +++ b/developer/src/kmc-kmn/test/test-features.ts @@ -12,7 +12,7 @@ describe('Keyboard compiler features', async function() { this.beforeAll(async function() { compiler = new KmnCompiler(); callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true})); assert(compiler.verifyInitialized()); }); @@ -29,15 +29,15 @@ describe('Keyboard compiler features', async function() { ]; for(const v of versions) { - it(`should build a version ${v[0]} keyboard`, function() { + it(`should build a version ${v[0]} keyboard`, async function() { const fixtureName = makePathToFixture('features', `version_${v[1]}.kmn`); - const result = compiler.runCompiler(fixtureName, {outFile: `version_${v[1]}.kmx`, saveDebug: true}); + const result = await compiler.run(fixtureName, `version_${v[1]}.kmx`); if(result === null) callbacks.printMessages(); assert.isNotNull(result); const reader = new KmxFileReader(); - const keyboard = reader.read(result.kmx.data); + const keyboard = reader.read(result.artifacts.kmx.data); assert.equal(keyboard.fileVersion, v[2]); }); } diff --git a/developer/src/kmc-kmn/test/test-messages.ts b/developer/src/kmc-kmn/test/test-messages.ts index 51e6894daa7..92f8936f327 100644 --- a/developer/src/kmc-kmn/test/test-messages.ts +++ b/developer/src/kmc-kmn/test/test-messages.ts @@ -23,13 +23,13 @@ describe('CompilerMessages', function () { callbacks.clear(); const compiler = new KmnCompiler(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, {saveDebug: true, shouldAddCompilerVersion: false})); assert(compiler.verifyInitialized()); const kmnPath = makePathToFixture(...fixture); // Note: throwing away compile results (just to memory) - compiler.runCompiler(kmnPath, {saveDebug: true, shouldAddCompilerVersion: false}); + await compiler.run(kmnPath, null); if(messageId) { assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts index e2bf5fcf1d6..13288e10ca8 100644 --- a/developer/src/kmc-kmn/test/test-wasm-uset.ts +++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts @@ -25,14 +25,14 @@ describe('Compiler UnicodeSet function', function() { it('should start', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); }); it('should compile a basic uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[abc]"; @@ -50,7 +50,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile a more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[[🙀A-C]-[CB]]"; @@ -70,7 +70,7 @@ describe('Compiler UnicodeSet function', function() { it('should compile an even more complex uset', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); const pat = "[\\u{10FFFD}\\u{2019}\\u{22}\\u{a}\\u{ead}\\u{1F640}]"; @@ -97,7 +97,7 @@ describe('Compiler UnicodeSet function', function() { it('should fail in various ways', async function() { const compiler = new KmnCompiler(); const callbacks = new TestCompilerCallbacks(); - assert(await compiler.init(callbacks)); + assert(await compiler.init(callbacks, null)); assert(compiler.verifyInitialized()); // map from string to failing error const failures = { diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b6f51f553b2..b7dcbf9dfb3 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -59,7 +59,7 @@ export class LdmlKeyboardCompiler { if (this.usetparser === undefined) { // initialize const compiler = new KmnCompiler(); - const ok = await compiler.init(this.callbacks); + const ok = await compiler.init(this.callbacks, null); if (ok) { this.usetparser = compiler; } else { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 61dd2f97a69..519d251b61d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -11,16 +11,12 @@ export class BuildKmnKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a Keyman keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let compiler = new KmnCompiler(); - if(!await compiler.init(callbacks)) { - return false; - } - + public async build(infile: string, /*TODO: outfile?: string,*/ callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn - if(options.outFile) { - options.outFile = getPosixAbsolutePath(options.outFile); - const folderName = path.dirname(options.outFile); + let outfile = options.outFile;//TODO: remove here + if(outfile) { + outfile = getPosixAbsolutePath(outfile); + const folderName = path.dirname(outfile); try { fs.mkdirSync(folderName, {recursive: true}); } catch(e) { @@ -29,7 +25,18 @@ export class BuildKmnKeyboard extends BuildActivity { } } infile = getPosixAbsolutePath(infile); - return compiler.run(infile, options); + + const compiler = new KmnCompiler(); + if(!await compiler.init(callbacks, options)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + return await compiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/util/TestKeymanSentry.ts b/developer/src/kmc/src/util/TestKeymanSentry.ts index 75da80e53de..b1c4aa40ee7 100644 --- a/developer/src/kmc/src/util/TestKeymanSentry.ts +++ b/developer/src/kmc/src/util/TestKeymanSentry.ts @@ -26,7 +26,7 @@ export class TestKeymanSentry { if(cli.includes('kmcmplib')) { const compiler = new KmnCompiler(); const callbacks = new NodeCompilerCallbacks({}); - if(!await compiler.init(callbacks)) { + if(!await compiler.init(callbacks, null)) { throw new Error('Failed to instantiate WASM compiler'); } try { From d18d4833a96d7237db3f81f3a3361617f89aff04 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:41:27 +0700 Subject: [PATCH 02/18] chore(developer): tidyup of KmnCompiler members --- developer/src/kmc-kmn/src/compiler/compiler.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index a4033ad89c5..6878fc1b47c 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -97,10 +97,10 @@ let Module: any; export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { - callbackID: string; // a unique numeric id added to globals with prefixed names - callbacks: CompilerCallbacks; - wasmExports: MallocAndFree; - options: KmnCompilerOptions; + private readonly callbackID: string; // a unique numeric id added to globals with prefixed names + private callbacks: CompilerCallbacks; + private wasmExports: MallocAndFree; + private options: KmnCompilerOptions; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); @@ -109,7 +109,7 @@ export class KmnCompiler implements KeymanCompiler, UnicodeSetParser { public async init(callbacks: CompilerCallbacks, options: KmnCompilerOptions): Promise { this.callbacks = callbacks; - this.options = options; + this.options = {...options}; if(!Module) { try { Module = await loadWasmHost(); From d3ee8c98020056085e8552c5e1b942a22ef611c4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 7 Dec 2023 11:42:10 +0700 Subject: [PATCH 03/18] feat(developer): LdmlKeyboardCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of LdmlKeyboardCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities out of kmc and into kmc-ldml. --- .../src/kmc-ldml/src/compiler/compiler.ts | 94 +++++++++++++++++-- developer/src/kmc-ldml/test/helpers/index.ts | 13 ++- .../src/kmc-ldml/test/test-compiler-e2e.ts | 27 +++--- .../kmc-ldml/test/test-keymanweb-compiler.ts | 3 +- .../src/kmc-ldml/test/test-testdata-e2e.ts | 4 +- .../buildClasses/BuildLdmlKeyboard.ts | 92 ++++-------------- .../kmc/src/commands/buildTestData/index.ts | 13 ++- 7 files changed, 138 insertions(+), 108 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b7dcbf9dfb3..9cd19bf19eb 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -1,4 +1,4 @@ -import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser } from '@keymanapp/common-types'; +import { LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, KMXPlus, CompilerCallbacks, LDMLKeyboardTestDataXMLSourceFile, UnicodeSetParser, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, KMXBuilder, KvkFileWriter, KeymanCompilerArtifactOptional } from '@keymanapp/common-types'; import { LdmlCompilerOptions } from './ldml-compiler-options.js'; import { CompilerMessages } from './messages.js'; import { BkspCompiler, TranCompiler } from './tran.js'; @@ -16,6 +16,9 @@ import KMXPlusFile = KMXPlus.KMXPlusFile; import DependencySections = KMXPlus.DependencySections; import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; +import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; +import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; +import { LdmlKeyboardKeymanWebCompiler } from './keymanweb-compiler.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -37,18 +40,93 @@ export const SECTION_COMPILERS = [ TranCompiler, ]; -export class LdmlKeyboardCompiler { - private readonly callbacks: CompilerCallbacks; - private readonly options: LdmlCompilerOptions; +export interface LdmlKeyboardCompilerArtifacts extends KeymanCompilerArtifacts { + kmx?: KeymanCompilerArtifactOptional; + kvk?: KeymanCompilerArtifactOptional; + js?: KeymanCompilerArtifactOptional; +}; + +export interface LdmlKeyboardCompilerResult extends KeymanCompilerResult { + artifacts: LdmlKeyboardCompilerArtifacts; +}; + +export class LdmlKeyboardCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: LdmlCompilerOptions; // uset parser private usetparser?: UnicodeSetParser = undefined; - constructor (callbacks: CompilerCallbacks, options: LdmlCompilerOptions) { - this.options = { - ...options - }; + async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { + this.options = {...options}; this.callbacks = callbacks; + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + + let compilerOptions: LdmlCompilerOptions = { + ...defaultCompilerOptions, + ...this.options, + }; + + let source = this.load(inputFilename); + if (!source) { + return null; + } + let kmx = await this.compile(source); + if (!kmx) { + return null; + } + + // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful + // to duplicate some of the metadata + KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); + + // Use the builder to generate the binary output file + const builder = new KMXBuilder(kmx, compilerOptions.saveDebug); + const kmx_binary = builder.compile(); + + const vkcompiler = new LdmlKeyboardVisualKeyboardCompiler(this.callbacks); + const vk = vkcompiler.compile(source); + const writer = new KvkFileWriter(); + const kvk_binary = writer.write(vk); + + // Note: we could have a step of generating source files here + // KvksFileWriter()... + // const tlcompiler = new kmc.TouchLayoutCompiler(); + // const tl = tlcompiler.compile(source); + // const tlwriter = new TouchLayoutFileWriter(); + const kmwcompiler = new LdmlKeyboardKeymanWebCompiler(this.callbacks, compilerOptions); + const kmw_string = kmwcompiler.compile(inputFilename, source); + const encoder = new TextEncoder(); + const kmw_binary = encoder.encode(kmw_string); + + outputFilename = outputFilename ?? inputFilename.replace(/\.xml$/, '.kmx'); + + return { + artifacts: { + kmx: { data: kmx_binary, filename: outputFilename }, + kvk: { data: kvk_binary, filename: outputFilename.replace(/\.kmx$/, '.kvk') }, + js: { data: kmw_binary, filename: outputFilename.replace(/\.kmx$/, '.js') }, + } + }; + } + + async write(artifacts: LdmlKeyboardCompilerArtifacts): Promise { + if(artifacts.kmx) { + this.callbacks.fs.writeFileSync(artifacts.kmx.filename, artifacts.kmx.data); + } + + if(artifacts.kvk) { + this.callbacks.fs.writeFileSync(artifacts.kvk.filename, artifacts.kvk.data); + } + + if(artifacts.js) { + this.callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + } + + return true; } /** diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 8f6d73ef439..53406873a42 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -112,14 +112,16 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section } } -export function loadTestdata(inputFilename: string, options: LdmlCompilerOptions) : LDMLKeyboardTestDataXMLSourceFile { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); +export async function loadTestData(inputFilename: string, options: LdmlCompilerOptions) : Promise { + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.loadTestData(inputFilename); return source; } export async function compileKeyboard(inputFilename: string, options: LdmlCompilerOptions, validateMessages?: CompilerEvent[], expectFailValidate?: boolean, compileMessages?: CompilerEvent[]): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -151,7 +153,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil } export async function compileVisualKeyboard(inputFilename: string, options: LdmlCompilerOptions): Promise { - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, options); + const k = new LdmlKeyboardCompiler(); + assert.isTrue(await k.init(compilerTestCallbacks, options)); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); @@ -254,7 +257,7 @@ async function getTestUnicodeSetParser(callbacks: CompilerCallbacks): Promise(code, expected); }); - it('should handle non existent files', () => { + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle unparseable files', () => { + it('should handle unparseable files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle not-valid files', () => { + it('should handle not-valid files', async () => { const filename = makePathToFixture('test-fr.xml'); // not a keyboard .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to load(${filename})`); }); - it('should handle non existent test files', () => { + it('should handle non existent test files', async () => { const filename = 'DOES_NOT_EXIST.xml'; - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.loadTestData(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); - it('should handle unparseable test files', () => { + it('should handle unparseable test files', async () => { const filename = makePathToFixture('basic-kvk.txt'); // not an .xml file - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); const source = k.load(filename); assert.notOk(source, `Trying to loadTestData(${filename})`); }); diff --git a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts index e1a570ea128..9a14b38c33d 100644 --- a/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts +++ b/developer/src/kmc-ldml/test/test-keymanweb-compiler.ts @@ -16,7 +16,8 @@ describe('LdmlKeyboardKeymanWebCompiler', function() { // Load input data; we'll use the LDML keyboard compiler loader to save us // effort here - const k = new LdmlKeyboardCompiler(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); const source = k.load(inputFilename); checkMessages(); assert.isNotNull(source, 'k.load should not have returned null'); diff --git a/developer/src/kmc-ldml/test/test-testdata-e2e.ts b/developer/src/kmc-ldml/test/test-testdata-e2e.ts index eea3631759d..cf233523282 100644 --- a/developer/src/kmc-ldml/test/test-testdata-e2e.ts +++ b/developer/src/kmc-ldml/test/test-testdata-e2e.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import 'mocha'; import {assert} from 'chai'; -import {compilerTestOptions, loadTestdata, makePathToFixture} from './helpers/index.js'; +import {compilerTestOptions, loadTestData, makePathToFixture} from './helpers/index.js'; describe('testdata-tests', function() { this.slow(500); // 0.5 sec -- json schema validation takes a while @@ -15,7 +15,7 @@ describe('testdata-tests', function() { const jsonFilename = makePathToFixture('test-fr.json'); // Compile the keyboard - const testData = loadTestdata(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + const testData = await loadTestData(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); assert.isNotNull(testData); const jsonData = JSON.parse(readFileSync(jsonFilename, 'utf-8')); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index 551ce66fa4b..fbb69f921c5 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -1,7 +1,6 @@ -import * as path from 'path'; import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { KvkFileWriter, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, defaultCompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; @@ -14,88 +13,33 @@ export class BuildLdmlKeyboard extends BuildActivity { public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk // Compile: - let [kmx,kvk,kmw] = await buildLdmlKeyboardToMemory(infile, callbacks, options); - // Output: + const outfile = options.outFile; //TODO - const fileBaseName = options.outFile ?? infile; - const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); - const outFileDir = path.dirname(fileBaseName); + const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + }}; - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); + const compiler = new kmcLdml.LdmlKeyboardCompiler(); + if(!await compiler.init(callbacks, ldmlCompilerOptions)) { return false; } - if(kmx && kvk) { - const outFileKmx = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.Keyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled keyboard to ${outFileKmx}`); - fs.writeFileSync(outFileKmx, kmx); - - const outFileKvk = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.VisualKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled visual keyboard to ${outFileKvk}`); - fs.writeFileSync(outFileKvk, kvk); - } else { - // TODO: console needs to be replaced with InfrastructureMessages - console.error(`An error occurred compiling ${infile}`); + const result = await compiler.run(infile, outfile); + if(!result) { return false; } - if(kmw) { - const outFileKmw = path.join(outFileDir, outFileBase + KeymanFileTypes.Binary.WebKeyboard); - // TODO: console needs to be replaced with InfrastructureMessages - console.log(`Writing compiled js keyboard to ${outFileKmw}`); - fs.writeFileSync(outFileKmw, kmw); - } + // TODO: Consider if this mkdir should be in write() + // TODO: This pattern may be needed for kmc-kmn as well? + const outFileDir = callbacks.path.dirname(outfile ?? infile); - return true; - } -} - -async function buildLdmlKeyboardToMemory(inputFilename: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise<[Uint8Array, Uint8Array, Uint8Array]> { - let compilerOptions: kmcLdml.LdmlCompilerOptions = { - ...defaultCompilerOptions, - ...options, - readerOptions: { - importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + try { + fs.mkdirSync(outFileDir, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); + return false; } - }; - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, compilerOptions); - let source = k.load(inputFilename); - if (!source) { - return [null, null, null]; - } - let kmx = await k.compile(source); - if (!kmx) { - return [null, null, null]; + return await compiler.write(result.artifacts); } - - // In order for the KMX file to be loaded by non-KMXPlus components, it is helpful - // to duplicate some of the metadata - kmcLdml.KMXPlusMetadataCompiler.addKmxMetadata(kmx.kmxplus, kmx.keyboard, compilerOptions); - - // Use the builder to generate the binary output file - const builder = new kmcLdml.KMXBuilder(kmx, options.saveDebug); - const kmx_binary = builder.compile(); - - const vkcompiler = new kmcLdml.LdmlKeyboardVisualKeyboardCompiler(callbacks); - const vk = vkcompiler.compile(source); - const writer = new KvkFileWriter(); - const kvk_binary = writer.write(vk); - - // Note: we could have a step of generating source files here - // KvksFileWriter()... - // const tlcompiler = new kmc.TouchLayoutCompiler(); - // const tl = tlcompiler.compile(source); - // const tlwriter = new TouchLayoutFileWriter(); - const kmwcompiler = new kmcLdml.LdmlKeyboardKeymanWebCompiler(callbacks, compilerOptions); - const kmw_string = kmwcompiler.compile(inputFilename, source); - const encoder = new TextEncoder(); - const kmw_binary = encoder.encode(kmw_string); - - return [kmx_binary, kvk_binary, kmw_binary]; } diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 834112bc130..5d0f6a2380f 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -5,7 +5,7 @@ import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKey import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; -export function buildTestData(infile: string, _options: any, commander: any) { +export async function buildTestData(infile: string, _options: any, commander: any) { const options: CompilerBaseOptions = commander.optsWithGlobals(); let compilerOptions: kmcLdml.LdmlCompilerOptions = { @@ -18,7 +18,7 @@ export function buildTestData(infile: string, _options: any, commander: any) { } }; - let testData = loadTestData(infile, compilerOptions); + let testData = await loadTestData(infile, compilerOptions); if (!testData) { return; } @@ -31,12 +31,11 @@ export function buildTestData(infile: string, _options: any, commander: any) { fs.writeFileSync(outFileJson, JSON.stringify(testData, null, ' ')); } -function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): LDMLKeyboardTestDataXMLSourceFile { +async function loadTestData(inputFilename: string, options: kmcLdml.LdmlCompilerOptions): Promise { const callbacks: CompilerCallbacks = new NodeCompilerCallbacks(options); - const k = new kmcLdml.LdmlKeyboardCompiler(callbacks, options); - let source = k.loadTestData(inputFilename); - if (!source) { + const k = new kmcLdml.LdmlKeyboardCompiler(); + if(!await k.init(callbacks, options)) { return null; } - return source; + return await k.loadTestData(inputFilename); } From 4c53509df9420f512d49a45b2eeb09e5ebe2a840 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 8 Dec 2023 11:45:53 +0700 Subject: [PATCH 04/18] feat(developer): LexicalModelCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of LexicalModelCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities into the class itself. Removes top-level `compileModel()` and `loadFromFilename()` functions, as you should now use `LexicalModelCompiler` API. Most of the test cases needed only minor patching, but introducing the `async init()` function has caused them to all be async, despite the model compiler itself having no async requirements. I think this is fine, because it makes any future additional async initialization tasks much less painful to implement. --- .../kmc-model/src/lexical-model-compiler.ts | 92 ++++++++++++++++++- developer/src/kmc-model/src/main.ts | 80 +--------------- .../test-compile-model-with-pseudoclosure.ts | 32 ++++--- .../src/kmc-model/test/test-compile-model.ts | 13 ++- .../src/kmc-model/test/test-compile-trie.ts | 27 +++--- .../src/kmc-model/test/test-punctuation.ts | 7 +- .../src/commands/buildClasses/BuildModel.ts | 22 ++--- developer/src/kmc/src/kmlmc.ts | 18 +++- 8 files changed, 156 insertions(+), 135 deletions(-) diff --git a/developer/src/kmc-model/src/lexical-model-compiler.ts b/developer/src/kmc-model/src/lexical-model-compiler.ts index 0edd818968d..8785ed32c49 100644 --- a/developer/src/kmc-model/src/lexical-model-compiler.ts +++ b/developer/src/kmc-model/src/lexical-model-compiler.ts @@ -2,20 +2,102 @@ lexical-model-compiler.ts: base file for lexical model compiler. */ -import * as ts from "typescript"; +import ts from "typescript"; import { createTrieDataStructure } from "./build-trie.js"; import { ModelDefinitions } from "./model-definitions.js"; import {decorateWithJoin} from "./join-word-breaker-decorator.js"; import {decorateWithScriptOverrides} from "./script-overrides-decorator.js"; import { LexicalModelSource, WordBreakerSpec, SimpleWordBreakerSpec } from "./lexical-model.js"; -import { ModelCompilerError, ModelCompilerMessages } from "./model-compiler-errors.js"; +import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from "./model-compiler-errors.js"; import { callbacks, setCompilerCallbacks } from "./compiler-callbacks.js"; -import { CompilerCallbacks } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult } from "@keymanapp/common-types"; -export default class LexicalModelCompiler { +/** + * An ECMAScript module as emitted by the TypeScript compiler. + */ +interface ES2015Module { + /** This is always true. */ + __esModule: boolean; + 'default'?: unknown; +}; + +export interface LexicalModelCompilerArtifacts extends KeymanCompilerArtifacts { + js: KeymanCompilerArtifact; +}; + +export interface LexicalModelCompilerResult extends KeymanCompilerResult { + artifacts: LexicalModelCompilerArtifacts; +}; + +export class LexicalModelCompiler implements KeymanCompiler { - constructor(callbacks: CompilerCallbacks) { + async init(callbacks: CompilerCallbacks, _options: CompilerOptions): Promise { setCompilerCallbacks(callbacks); + return true; + } + + async run(inputFilename: string, outputFilename?: string): Promise { + try { + let modelSource = this.loadFromFilename(inputFilename); + let containingDirectory = callbacks.path.dirname(inputFilename); + let code = this.generateLexicalModelCode('', modelSource, containingDirectory); + const result: LexicalModelCompilerResult = { + artifacts: { + js: { + data: new TextEncoder().encode(code), + filename: outputFilename ?? inputFilename.replace(/\.model\.ts$/, '.model.js') + } + } + } + return result; + } catch(e) { + callbacks.reportMessage( + e instanceof ModelCompilerError + ? e.event + : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) + ); + return null; + } + } + + async write(artifacts: LexicalModelCompilerArtifacts): Promise { + callbacks.fs.writeFileSync(artifacts.js.filename, artifacts.js.data); + return true; + } + + /** + * Loads a lexical model's source module from the given filename. + * + * @param filename path to the model source file. + */ + public loadFromFilename(filename: string): LexicalModelSource { + + let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); + // Compile the module to JavaScript code. + // NOTE: transpile module does a very simple TS to JS compilation. + // It DOES NOT check for types! + let compilationOutput = ts.transpile(sourceCode, { + // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. + // When we drop Android 5.0 support, we can update this to a `ScriptTarget` + // matrix against target version of Keyman, here and in + // lexical-model-compiler.ts. + target: ts.ScriptTarget.ES3, + module: ts.ModuleKind.CommonJS, + }); + // Turn the module into a function in which we can inject a global. + let moduleCode = '(function(exports){' + compilationOutput + '})'; + + // Run the module; its exports will be assigned to `moduleExports`. + let moduleExports: Partial = {}; + let module = eval(moduleCode); + module(moduleExports); + + if (!moduleExports['__esModule'] || !moduleExports['default']) { + ModelCompilerMessageContext.filename = filename; + throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); + } + + return moduleExports['default'] as LexicalModelSource; } /** diff --git a/developer/src/kmc-model/src/main.ts b/developer/src/kmc-model/src/main.ts index 667253353d9..68fae091e3a 100644 --- a/developer/src/kmc-model/src/main.ts +++ b/developer/src/kmc-model/src/main.ts @@ -1,79 +1 @@ -import { CompilerCallbacks } from '@keymanapp/common-types'; -import ts from 'typescript'; -import { setCompilerCallbacks } from './compiler-callbacks.js'; - -import LexicalModelCompiler from './lexical-model-compiler.js'; -import { LexicalModelSource } from './lexical-model.js'; -import { ModelCompilerError, ModelCompilerMessageContext, ModelCompilerMessages } from './model-compiler-errors.js'; - -export { default as LexicalModelCompiler } from './lexical-model-compiler.js'; - -/** - * Compiles a model.ts file, using paths relative to its location. - * - * @param filename path to model.ts source. - * @return model source code, or null on error - */ -export function compileModel(filename: string, callbacks: CompilerCallbacks): string { - setCompilerCallbacks(callbacks); - - try { - let modelSource = loadFromFilename(filename, callbacks); - let containingDirectory = callbacks.path.dirname(filename); - - return (new LexicalModelCompiler(callbacks)) - .generateLexicalModelCode('', modelSource, containingDirectory); - } catch(e) { - callbacks.reportMessage( - e instanceof ModelCompilerError - ? e.event - : ModelCompilerMessages.Fatal_UnexpectedException({e:e}) - ); - } - return null; -} - -/** - * An ECMAScript module as emitted by the TypeScript compiler. - */ -interface ES2015Module { - /** This is always true. */ - __esModule: boolean; - 'default'?: unknown; -} - -/** - * Loads a lexical model's source module from the given filename. - * - * @param filename path to the model source file. - */ -export function loadFromFilename(filename: string, callbacks: CompilerCallbacks): LexicalModelSource { - setCompilerCallbacks(callbacks); - - let sourceCode = new TextDecoder().decode(callbacks.loadFile(filename)); - // Compile the module to JavaScript code. - // NOTE: transpile module does a very simple TS to JS compilation. - // It DOES NOT check for types! - let compilationOutput = ts.transpile(sourceCode, { - // Our runtime only supports ES3 with Node/CommonJS modules on Android 5.0. - // When we drop Android 5.0 support, we can update this to a `ScriptTarget` - // matrix against target version of Keyman, here and in - // lexical-model-compiler.ts. - target: ts.ScriptTarget.ES3, - module: ts.ModuleKind.CommonJS, - }); - // Turn the module into a function in which we can inject a global. - let moduleCode = '(function(exports){' + compilationOutput + '})'; - - // Run the module; its exports will be assigned to `moduleExports`. - let moduleExports: Partial = {}; - let module = eval(moduleCode); - module(moduleExports); - - if (!moduleExports['__esModule'] || !moduleExports['default']) { - ModelCompilerMessageContext.filename = filename; - throw new ModelCompilerError(ModelCompilerMessages.Error_NoDefaultExport()); - } - - return moduleExports['default'] as LexicalModelSource; -} +export { LexicalModelCompiler } from './lexical-model-compiler.js'; diff --git a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts index d6fcec39599..08247e17c1d 100644 --- a/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts +++ b/developer/src/kmc-model/test/test-compile-model-with-pseudoclosure.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -26,8 +26,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { } }; - it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 1: applyCasing prepends symbols, searchTermToKey removes them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -80,8 +81,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 2: applyCasing prepends symbols, searchTermToKey keeps them', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -124,8 +126,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('variant 3: applyCasing prepends symbols, default searchTermToKey', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('variant 3: applyCasing prepends symbols, default searchTermToKey', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -164,8 +167,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { }); describe('relying on default applyCasing + searchTermToKey', function() { - it('languageUsesCasing: true', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: true', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -188,8 +192,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: false', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: false', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -213,8 +218,9 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { assert.isNotNull(compilation.exportedModel); }); - it('languageUsesCasing: undefined', function() { - let compiler = new LexicalModelCompiler(callbacks); + it('languageUsesCasing: undefined', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-compile-model.ts b/developer/src/kmc-model/test/test-compile-model.ts index 2db28b6e09b..e3074d697ac 100644 --- a/developer/src/kmc-model/test/test-compile-model.ts +++ b/developer/src/kmc-model/test/test-compile-model.ts @@ -1,12 +1,12 @@ import 'mocha'; import {assert} from 'chai'; -import {compileModel} from '../src/main.js'; +import { LexicalModelCompiler } from '../src/main.js'; import {makePathToFixture, compileModelSourceCode, CompilationResult} from './helpers/index.js'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KeymanFileTypes } from '@keymanapp/common-types'; -describe('compileModel', function () { +describe('LexicalModelCompiler', function () { let callbacks = new TestCompilerCallbacks(); // Try to compile ALL of the correct models. @@ -22,9 +22,14 @@ describe('compileModel', function () { for (let modelID of MODELS) { let modelPath = makePathToFixture(modelID, modelID + KeymanFileTypes.Source.Model); - it(`should compile ${modelID}`, function () { - let code = compileModel(modelPath, callbacks); + it(`should compile ${modelID}`, async function () { + const compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); + const result = await compiler.run(modelPath, null); callbacks.printMessages(); + assert.isNotNull(result); + const decoder = new TextDecoder(); + const code = decoder.decode(result.artifacts.js.data); let r = compileModelSourceCode(code); let compilation = r as CompilationResult; diff --git a/developer/src/kmc-model/test/test-compile-trie.ts b/developer/src/kmc-model/test/test-compile-trie.ts index 544924da3eb..5d80b72ac17 100644 --- a/developer/src/kmc-model/test/test-compile-trie.ts +++ b/developer/src/kmc-model/test/test-compile-trie.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -14,11 +14,12 @@ describe('LexicalModelCompiler', function () { }); describe('#generateLexicalModelCode', function () { - it('should compile a trivial word list', function () { + it('should compile a trivial word list', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -37,11 +38,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s*wordBreakers\b/); }); - it('should compile a word list exported by Microsoft Excel', function () { + it('should compile a word list exported by Microsoft Excel', async function () { const MODEL_ID = 'example.qaa.utf16le'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.txt'] @@ -58,11 +60,12 @@ describe('LexicalModelCompiler', function () { }); }); - it('should compile a word list with a custom word breaking function', function () { + it('should compile a word list with a custom word breaking function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], @@ -81,11 +84,12 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\bwordBreaker\b["']?:\s+function\b/); }); - it('should not generate unpaired surrogate code units', function () { + it('should not generate unpaired surrogate code units', async function () { const MODEL_ID = 'example.qaa.smp'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] @@ -115,10 +119,11 @@ describe('LexicalModelCompiler', function () { assert.match(code, /\btotalWeight\b["']?:\s*27596\b/); }); - it('should include the source code of its search term to key function', function () { + it('should include the source code of its search term to key function', async function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'] diff --git a/developer/src/kmc-model/test/test-punctuation.ts b/developer/src/kmc-model/test/test-punctuation.ts index 133eed5fedd..f3ad64433ec 100644 --- a/developer/src/kmc-model/test/test-punctuation.ts +++ b/developer/src/kmc-model/test/test-punctuation.ts @@ -1,4 +1,4 @@ -import LexicalModelCompiler from '../src/lexical-model-compiler.js'; +import { LexicalModelCompiler } from '../src/lexical-model-compiler.js'; import {assert} from 'chai'; import 'mocha'; @@ -10,10 +10,11 @@ describe('LexicalModelCompiler', function () { const MODEL_ID = 'example.qaa.trivial'; const PATH = makePathToFixture(MODEL_ID); - it('should compile punctuation into the generated code', function () { + it('should compile punctuation into the generated code', async function () { const callbacks = new TestCompilerCallbacks(); - let compiler = new LexicalModelCompiler(callbacks); + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, null)); let code = compiler.generateLexicalModelCode(MODEL_ID, { format: 'trie-1.0', sources: ['wordlist.tsv'], diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index e69aeca7801..00614c27310 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -1,6 +1,5 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; export class BuildModel extends BuildActivity { @@ -9,25 +8,18 @@ export class BuildModel extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - let outputFilename: string = this.getOutputFilename(infile, options); - let code = null; + const outputFilename: string = this.getOutputFilename(infile, options); - // Compile: - try { - code = compileModel(infile, callbacks); - } catch(e) { - console.error(e); + const compiler = new LexicalModelCompiler(); + if(!await compiler.init(callbacks, options)) { return false; } - if(!code) { - console.error('Compilation failed.') + const result = await compiler.run(infile, outputFilename); + if(!result) { return false; } - // Output: - fs.writeFileSync(outputFilename, code, 'utf8'); - - return true; + return await compiler.write(result.artifacts); } } \ No newline at end of file diff --git a/developer/src/kmc/src/kmlmc.ts b/developer/src/kmc/src/kmlmc.ts index 1ce171fd15e..ca2fd4d6b0b 100644 --- a/developer/src/kmc/src/kmlmc.ts +++ b/developer/src/kmc/src/kmlmc.ts @@ -3,9 +3,8 @@ * kmlmc - Keyman Lexical Model Compiler */ -import * as fs from 'fs'; import { Command } from 'commander'; -import { compileModel } from '@keymanapp/kmc-model'; +import { LexicalModelCompiler } from '@keymanapp/kmc-model'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -30,10 +29,16 @@ if (!inputFilename) { const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); +const compiler = new LexicalModelCompiler(); +if(!await compiler.init(callbacks, null)) { + console.error('Initialization failed.'); + process.exit(SysExits.EX_DATAERR); +} + let code = null; // Compile: try { - code = compileModel(inputFilename, callbacks); + code = await compiler.run(inputFilename, program.opts().outFile); } catch(e) { console.error(e); process.exit(SysExits.EX_DATAERR); @@ -46,9 +51,12 @@ if(!code) { // Output: if (program.opts().outFile) { - fs.writeFileSync(program.opts().outFile, code, 'utf8'); + compiler.write(code.artifacts); } else { - console.log(code); + // TODO(lowpri): if writing to console then log messages should all be to stderr? + const decoder = new TextDecoder(); + const text = decoder.decode(code.artifacts.js.data); + console.log(text); } function exitDueToUsageError(message: string): never { From 69a2448176db50f251707370ffbeb38950b7e1a5 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 06:48:53 +0700 Subject: [PATCH 05/18] feat(developer): PackageCompiler and WindowPackageInstallerCompiler now implement KeymanCompiler Relates to #9473. Refactors the public API of PackageCompiler and WindowsPackageInstallerCompiler to meet KeymanCompiler, including asyncing a bunch of functions, and moving file write responsibilities into the classes themselves. The classes were updated together in a single commit because WindowsPackageInstallerCompiler depends on PackageCompiler, and needed refactoring for the updated API access there anyway. Most of the test cases needed only minor patching, but introducing the `async init()` function has caused them to all be async. The test-messages module has a slight functional change with the compile process running completely rather than stopping on first message, which meant we needed to add extra dummy fixtures to avoid other errors that arose later in the compile process. The keyman.exe fixture is a text file, not a Windows executable. --- developer/src/kmc-keyboard-info/src/index.ts | 6 +- .../kmc-package/src/compiler/kmp-compiler.ts | 70 ++++++++++++++-- .../windows-package-installer-compiler.ts | 53 ++++++++++--- .../invalid/example.qaa.sencoten.model.js | 1 + .../test/fixtures/invalid/keyman.exe | 1 + .../test/fixtures/invalid/khmer_angkor.docx | 1 + .../src/kmc-package/test/test-messages.ts | 79 ++++++++----------- .../kmc-package/test/test-package-compiler.ts | 12 ++- .../src/kmc-package/test/test-versioning.ts | 4 +- ...test-windows-package-installer-compiler.ts | 12 ++- .../commands/buildClasses/BuildModelInfo.ts | 7 +- .../src/commands/buildClasses/BuildPackage.ts | 32 ++------ .../buildWindowsPackageInstaller/index.ts | 24 ++++-- developer/src/kmc/src/kmlmp.ts | 31 +++----- 14 files changed, 207 insertions(+), 126 deletions(-) create mode 100644 developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js create mode 100644 developer/src/kmc-package/test/fixtures/invalid/keyman.exe create mode 100644 developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index beb4b88bb93..7d134e4acb1 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -85,7 +85,11 @@ export class KeyboardInfoCompiler { // .kpj work is largely in kmc at present, so that would need to move to // a separate module. - const kmpCompiler = new KmpCompiler(this.callbacks); + const kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(this.callbacks, {})) { + // Errors will have been emitted by KmpCompiler + return null; + } const kmpJsonData = kmpCompiler.transformKpsToKmpObject(sources.kpsFilename); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index 3dce88c9d5b..b9993713d03 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -2,7 +2,7 @@ import * as xml2js from 'xml2js'; import JSZip from 'jszip'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile } from '@keymanapp/common-types'; +import { KmpJsonFile, KpsFile, SchemaValidators, CompilerCallbacks, KeymanFileTypes, KvkFile, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from '@keymanapp/common-types'; import { CompilerMessages } from './messages.js'; import { PackageMetadataCollector } from './package-metadata-collector.js'; import { KmpInfWriter } from './kmp-inf-writer.js'; @@ -11,6 +11,7 @@ import { MIN_LM_FILEVERSION_KMP_JSON, PackageVersionValidator } from './package- import { PackageKeyboardTargetValidator } from './package-keyboard-target-validator.js'; import { PackageMetadataUpdater } from './package-metadata-updater.js'; import { markdownToHTML } from './markdown.js'; +import { PackageValidation } from './package-validation.js'; const KMP_JSON_FILENAME = 'kmp.json'; const KMP_INF_FILENAME = 'kmp.inf'; @@ -20,9 +21,68 @@ const KMP_INF_FILENAME = 'kmp.inf'; // this filename for existing keyboard packages. const WELCOME_HTM_FILENAME = 'welcome.htm'; -export class KmpCompiler { +export interface KmpCompilerOptions extends CompilerOptions { + // Note: WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions, so + // be careful when modifying this interface +}; - constructor(private callbacks: CompilerCallbacks) { +export interface KmpCompilerArtifacts extends KeymanCompilerArtifacts { + kmp: KeymanCompilerArtifact; +}; + +export interface KmpCompilerResult extends KeymanCompilerResult { + artifacts: KmpCompilerArtifacts; +}; + +export class KmpCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KmpCompilerOptions; + + public async init(callbacks: CompilerCallbacks, options: KmpCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = options ? {...options} : {}; + return true; + } + + public async run(inputFilename: string, outputFilename?: string): Promise { + const kmpJsonData = this.transformKpsToKmpObject(inputFilename); + if(!kmpJsonData) { + return null; + } + + // + // Validate the package file + // + + const validation = new PackageValidation(this.callbacks, this.options); + if(!validation.validate(inputFilename, kmpJsonData)) { + return null; + } + + // + // Build the .kmp package file + // + + const data = await this.buildKmpFile(inputFilename, kmpJsonData); + if(!data) { + return null; + } + + const result: KmpCompilerResult = { + artifacts: { + kmp: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.kmp') + } + } + } + + return result; + } + + public async write(artifacts: KmpCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.kmp.filename, artifacts.kmp.data); + return true; } public transformKpsToKmpObject(kpsFilename: string): KmpJsonFile.KmpJsonFile { @@ -346,7 +406,7 @@ export class KmpCompiler { * @param kpsFilename - Filename of the kps, not read, used only for calculating relative paths * @param kmpJsonData - The kmp.json Object */ - public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { + public buildKmpFile(kpsFilename: string, kmpJsonData: KmpJsonFile.KmpJsonFile): Promise { const zip = JSZip(); @@ -438,7 +498,7 @@ export class KmpCompiler { } // Generate kmp file - return zip.generateAsync({type: 'binarystring', compression:'DEFLATE'}); + return zip.generateAsync({type:'uint8array', compression:'DEFLATE'}); } private buildKmpInf(data: KmpJsonFile.KmpJsonFile): Uint8Array { diff --git a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts index 1a5f4ebe6b8..8ac54572c13 100644 --- a/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/src/compiler/windows-package-installer-compiler.ts @@ -11,9 +11,9 @@ */ import JSZip from 'jszip'; -import { CompilerCallbacks, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanFileTypes, KmpJsonFile, KpsFile } from "@keymanapp/common-types"; import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { KmpCompiler } from "./kmp-compiler.js"; +import { KmpCompiler, KmpCompilerOptions } from "./kmp-compiler.js"; import { CompilerMessages } from "./messages.js"; const SETUP_INF_FILENAME = 'setup.inf'; @@ -30,15 +30,33 @@ export interface WindowsPackageInstallerSources { startWithConfiguration: boolean; }; -export class WindowsPackageInstallerCompiler { - private kmpCompiler: KmpCompiler; +export interface WindowsPackageInstallerCompilerOptions extends KmpCompilerOptions { + sources: WindowsPackageInstallerSources; +} + +export interface WindowsPackageInstallerCompilerArtifacts extends KeymanCompilerArtifacts { + exe: KeymanCompilerArtifact; +}; + +export interface WindowsPackageInstallerCompilerResult extends KeymanCompilerResult { + artifacts: WindowsPackageInstallerCompilerArtifacts; +}; - constructor(private callbacks: CompilerCallbacks) { - this.kmpCompiler = new KmpCompiler(this.callbacks); +export class WindowsPackageInstallerCompiler implements KeymanCompiler { + private kmpCompiler: KmpCompiler; + private callbacks: CompilerCallbacks; + private options: WindowsPackageInstallerCompilerOptions; + + async init(callbacks: CompilerCallbacks, options: WindowsPackageInstallerCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + this.kmpCompiler = new KmpCompiler(); + return await this.kmpCompiler.init(callbacks, options); } - public async compile(kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { - const kps = this.kmpCompiler.loadKpsFile(kpsFilename); + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; + const kps = this.kmpCompiler.loadKpsFile(inputFilename); if(!kps) { // errors will already have been reported by loadKpsFile return null; @@ -64,7 +82,7 @@ export class WindowsPackageInstallerCompiler { // Nor do we use the MSIOptions field. // Build the zip - const zipBuffer = await this.buildZip(kps, kpsFilename, sources); + const zipBuffer = await this.buildZip(kps, inputFilename, sources); if(!zipBuffer) { // Error messages already reported by buildZip return null; @@ -72,7 +90,22 @@ export class WindowsPackageInstallerCompiler { // Build the sfx const sfxBuffer = this.buildSfx(zipBuffer, sources); - return sfxBuffer; + + const result: WindowsPackageInstallerCompilerResult = { + artifacts: { + exe: { + data: sfxBuffer, + filename: outputFilename ?? inputFilename.replace(/\.kps$/, '.exe') + } + } + }; + + return result; + } + + public async write(artifacts: WindowsPackageInstallerCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.exe.filename, artifacts.exe.data); + return true; } private async buildZip(kps: KpsFile.KpsFile, kpsFilename: string, sources: WindowsPackageInstallerSources): Promise { diff --git a/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js new file mode 100644 index 00000000000..2f309acbc00 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/example.qaa.sencoten.model.js @@ -0,0 +1 @@ +// dummy file for unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/keyman.exe b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe new file mode 100644 index 00000000000..789aa9641ad --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/keyman.exe @@ -0,0 +1 @@ +This is a dummy file for testing the unit tests \ No newline at end of file diff --git a/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx new file mode 100644 index 00000000000..0f8c58ac580 --- /dev/null +++ b/developer/src/kmc-package/test/fixtures/invalid/khmer_angkor.docx @@ -0,0 +1 @@ +This is a sample text file \ No newline at end of file diff --git a/developer/src/kmc-package/test/test-messages.ts b/developer/src/kmc-package/test/test-messages.ts index 24fb658b599..d7a7f0175e6 100644 --- a/developer/src/kmc-package/test/test-messages.ts +++ b/developer/src/kmc-package/test/test-messages.ts @@ -4,7 +4,6 @@ import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/ import { CompilerMessages } from '../src/compiler/messages.js'; import { makePathToFixture } from './helpers/index.js'; import { KmpCompiler } from '../src/compiler/kmp-compiler.js'; -import { PackageValidation } from '../src/compiler/package-validation.js'; import { CompilerErrorNamespace, CompilerOptions } from '@keymanapp/common-types'; const debug = false; @@ -20,24 +19,16 @@ describe('CompilerMessages', function () { // Message tests // - function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number, options?: CompilerOptions) { context.timeout(10000); callbacks.clear(); const kpsPath = makePathToFixture(...fixture); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, options ?? {})); - let kmpJson = kmpCompiler.transformKpsToKmpObject(kpsPath); - if(kmpJson && callbacks.messages.length == 0) { - const validator = new PackageValidation(callbacks, options ?? {}); - validator.validate(kpsPath, kmpJson); // we'll ignore return value and rely on the messages - } - - if(kmpJson && callbacks.messages.length == 0) { - // We'll try building the package if we have not yet received any messages - kmpCompiler.buildKmpFile(kpsPath, kmpJson) - } + await kmpCompiler.run(kpsPath); if(debug) callbacks.printMessages(); @@ -52,106 +43,106 @@ describe('CompilerMessages', function () { // WARN_FileIsNotABinaryKvkFile it('should generate WARN_FileIsNotABinaryKvkFile if a non-binary kvk file is included', async function() { - testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); + await testForMessage(this, ['xml_kvk_file', 'source', 'xml_kvk_file.kps'], CompilerMessages.WARN_FileIsNotABinaryKvkFile); }); it('should not warn if a binary kvk file is included', async function() { - testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); + await testForMessage(this, ['binary_kvk_file', 'source', 'binary_kvk_file.kps']); }); // ERROR_FollowKeyboardVersionNotAllowedForModelPackages it('should generate ERROR_FollowKeyboardVersionNotAllowedForModelPackages if is set for model packages', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); + await testForMessage(this, ['invalid', 'followkeyboardversion.qaa.sencoten.model.kps'], CompilerMessages.ERROR_FollowKeyboardVersionNotAllowedForModelPackages); }); // ERROR_FollowKeyboardVersionButNoKeyboards it('should generate ERROR_FollowKeyboardVersionButNoKeyboards if is set for a package with no keyboards', async function() { - testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); + await testForMessage(this, ['invalid', 'followkeyboardversion.empty.kps'], CompilerMessages.ERROR_FollowKeyboardVersionButNoKeyboards); }); // ERROR_KeyboardContentFileNotFound it('should generate ERROR_KeyboardContentFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); + await testForMessage(this, ['invalid', 'keyboardcontentfilenotfound.kps'], CompilerMessages.ERROR_KeyboardContentFileNotFound); }); // ERROR_KeyboardFileNotValid it('should generate ERROR_KeyboardFileNotValid if a .kmx is not valid in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); + await testForMessage(this, ['invalid', 'keyboardfilenotvalid.kps'], CompilerMessages.ERROR_KeyboardFileNotValid); }); // INFO_KeyboardFileHasNoKeyboardVersion it('should generate INFO_KeyboardFileHasNoKeyboardVersion if is set but keyboard has no version', async function() { - testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); + await testForMessage(this, ['invalid', 'nokeyboardversion.kps'], CompilerMessages.INFO_KeyboardFileHasNoKeyboardVersion); }); // ERROR_PackageCannotContainBothModelsAndKeyboards it('should generate ERROR_PackageCannotContainBothModelsAndKeyboards if package has both keyboards and models', async function() { - testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); + await testForMessage(this, ['invalid', 'error_package_cannot_contain_both_models_and_keyboards.kps'], CompilerMessages.ERROR_PackageCannotContainBothModelsAndKeyboards); }); // HINT_PackageShouldNotRepeatLanguages (models) it('should generate HINT_PackageShouldNotRepeatLanguages if model has same language repeated', async function() { - testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'keyman.en.hint_package_should_not_repeat_languages.model.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // HINT_PackageShouldNotRepeatLanguages (keyboards) it('should generate HINT_PackageShouldNotRepeatLanguages if keyboard has same language repeated', async function() { - testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); + await testForMessage(this, ['invalid', 'hint_package_should_not_repeat_languages.kps'], CompilerMessages.HINT_PackageShouldNotRepeatLanguages); }); // WARN_PackageNameDoesNotFollowLexicalModelConventions it('should generate WARN_PackageNameDoesNotFollowLexicalModelConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowLexicalModelConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowLexicalModelConventions); }); // WARN_PackageNameDoesNotFollowKeyboardConventions it('should generate WARN_PackageNameDoesNotFollowKeyboardConventions if filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); + await testForMessage(this, ['invalid', 'WARN_PackageNameDoesNotFollowKeyboardConventions.kps'], CompilerMessages.WARN_PackageNameDoesNotFollowKeyboardConventions); }); // WARN_FileInPackageDoesNotFollowFilenameConventions it('should generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], CompilerMessages.WARN_FileInPackageDoesNotFollowFilenameConventions, {checkFilenameConventions: true}); }); // Test the inverse -- no warning generated if checkFilenameConventions is false it('should not generate WARN_FileInPackageDoesNotFollowFilenameConventions if content filename has wrong conventions but checkFilenameConventions is false', async function() { - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); - testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions.kps'], null, {checkFilenameConventions: false}); + await testForMessage(this, ['invalid', 'warn_file_in_package_does_not_follow_filename_conventions_2.kps'], null, {checkFilenameConventions: false}); }); // ERROR_PackageNameCannotBeBlank it('should generate ERROR_PackageNameCannotBeBlank if package info has empty name', async function() { - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field - testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // blank field + await testForMessage(this, ['invalid', 'error_package_name_cannot_be_blank_2.kps'], CompilerMessages.ERROR_PackageNameCannotBeBlank); // missing field }); // ERROR_KeyboardFileNotFound it('should generate ERROR_KeyboardFileNotFound if a is listed in a package but not found in ', async function() { - testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); + await testForMessage(this, ['invalid', 'keyboardfilenotfound.kps'], CompilerMessages.ERROR_KeyboardFileNotFound); }); // WARN_KeyboardVersionsDoNotMatch it('should generate WARN_KeyboardVersionsDoNotMatch if two have different versions', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); + await testForMessage(this, ['invalid', 'warn_keyboard_versions_do_not_match.kps'], CompilerMessages.WARN_KeyboardVersionsDoNotMatch); }); // ERROR_LanguageTagIsNotValid @@ -163,73 +154,73 @@ describe('CompilerMessages', function () { // HINT_LanguageTagIsNotMinimal it('should generate HINT_LanguageTagIsNotMinimal if keyboard has a non-minimal language tag', async function() { - testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); + await testForMessage(this, ['invalid', 'hint_language_tag_is_not_minimal.kps'], CompilerMessages.HINT_LanguageTagIsNotMinimal); }); // ERROR_ModelMustHaveAtLeastOneLanguage it('should generate ERROR_MustHaveAtLeastOneLanguage if model has zero language tags', async function() { - testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], + await testForMessage(this, ['invalid', 'keyman.en.error_model_must_have_at_least_one_language.model.kps'], CompilerMessages.ERROR_ModelMustHaveAtLeastOneLanguage); }); // WARN_RedistFileShouldNotBeInPackage it('should generate WARN_RedistFileShouldNotBeInPackage if package contains a redist file', async function() { - testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'warn_redist_file_should_not_be_in_package.kps'], CompilerMessages.WARN_RedistFileShouldNotBeInPackage); }); // WARN_DocFileDangerous it('should generate WARN_DocFileDangerous if package contains a .doc file', async function() { - testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], + await testForMessage(this, ['invalid', 'warn_doc_file_dangerous.kps'], CompilerMessages.WARN_DocFileDangerous); }); // ERROR_PackageMustContainAPackageOrAKeyboard it('should generate ERROR_PackageMustContainAModelOrAKeyboard if package contains no keyboard or model', async function() { - testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], + await testForMessage(this, ['invalid', 'error_package_must_contain_a_model_or_a_keyboard.kps'], CompilerMessages.ERROR_PackageMustContainAModelOrAKeyboard); }); // WARN_JsKeyboardFileIsMissing it('should generate WARN_JsKeyboardFileIsMissing if package is missing corresponding .js for a touch .kmx', async function() { - testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], + await testForMessage(this, ['invalid', 'warn_js_keyboard_file_is_missing.kps'], CompilerMessages.WARN_JsKeyboardFileIsMissing); }); // WARN_KeyboardShouldHaveAtLeastOneLanguage it('should generate WARN_KeyboardShouldHaveAtLeastOneLanguage if keyboard has zero language tags', async function() { - testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], + await testForMessage(this, ['invalid', 'warn_keyboard_should_have_at_least_one_language.kps'], CompilerMessages.WARN_KeyboardShouldHaveAtLeastOneLanguage); }); // HINT_JsKeyboardFileHasNoTouchTargets it('should generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has no touch targets', async function() { - testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], + await testForMessage(this, ['invalid', 'hint_js_keyboard_file_has_no_touch_targets.kps'], CompilerMessages.HINT_JsKeyboardFileHasNoTouchTargets); }); it('should not generate HINT_JsKeyboardFileHasNoTouchTargets if keyboard has a touch target', async function() { - testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); + await testForMessage(this, ['khmer_angkor', 'source', 'khmer_angkor.kps'], null); }); // HINT_PackageContainsSourceFile it('should generate HINT_PackageContainsSourceFile if package contains a source file', async function() { - testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], + await testForMessage(this, ['invalid', 'hint_source_file_should_not_be_in_package.kps'], CompilerMessages.HINT_PackageContainsSourceFile); }); // ERROR_InvalidPackageFile it('should generate ERROR_InvalidPackageFile if package source file contains invalid XML', async function() { - testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], + await testForMessage(this, ['invalid', 'error_invalid_package_file.kps'], CompilerMessages.ERROR_InvalidPackageFile); }); diff --git a/developer/src/kmc-package/test/test-package-compiler.ts b/developer/src/kmc-package/test/test-package-compiler.ts index 433d730c181..3a17d321eb1 100644 --- a/developer/src/kmc-package/test/test-package-compiler.ts +++ b/developer/src/kmc-package/test/test-package-compiler.ts @@ -15,14 +15,15 @@ import { CompilerMessages } from '../src/compiler/messages.js'; const debug = false; -describe('KmpCompiler', function () { +describe('KmpCompiler', async function () { const MODELS : string[] = [ 'example.qaa.sencoten', 'withfolders.qaa.sencoten', ]; const callbacks = new TestCompilerCallbacks(); - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); for (let modelID of MODELS) { const kpsPath = modelID.includes('withfolders') ? @@ -89,7 +90,9 @@ describe('KmpCompiler', function () { const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpJsonRefPath = makePathToFixture('khmer_angkor', 'ref', 'kmp.json'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kmpJsonFixture: KmpJsonFile.KmpJsonFile = JSON.parse(fs.readFileSync(kmpJsonRefPath, 'utf-8')); // We override the fixture version so that we can compare with the compiler output @@ -193,7 +196,8 @@ describe('KmpCompiler', function () { callbacks.clear(); const kpsPath = makePathToFixture('absolute_path', 'source', 'absolute_path.kps'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); let kmpJson: KmpJsonFile.KmpJsonFile = null; diff --git a/developer/src/kmc-package/test/test-versioning.ts b/developer/src/kmc-package/test/test-versioning.ts index e9771a369ab..98d00b853ce 100644 --- a/developer/src/kmc-package/test/test-versioning.ts +++ b/developer/src/kmc-package/test/test-versioning.ts @@ -33,7 +33,9 @@ describe('package versioning', function () { for(const [ caseTitle, filename ] of cases) { it(caseTitle, async function () { const callbacks = new TestCompilerCallbacks(); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + const kpsPath = makePathToFixture('versioning', filename); const kmpJson: KmpJsonFile.KmpJsonFile = kmpCompiler.transformKpsToKmpObject(kpsPath); assert.isTrue(kmpJson !== null); diff --git a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts index 8ed2fee0e7b..e1fde656478 100644 --- a/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts +++ b/developer/src/kmc-package/test/test-windows-package-installer-compiler.ts @@ -12,9 +12,6 @@ describe('WindowsPackageInstallerCompiler', function () { it(`should build an SFX archive`, async function () { this.timeout(10000); // this test can take a little while to run - const callbacks = new TestCompilerCallbacks(); - let compiler = new WindowsPackageInstallerCompiler(callbacks); - const kpsPath = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const sources: WindowsPackageInstallerSources = { licenseFilename: makePathToFixture('windows-installer', 'license.txt'), @@ -25,12 +22,19 @@ describe('WindowsPackageInstallerCompiler', function () { appName: 'Testing', }; - const sfxBuffer = await compiler.compile(kpsPath, sources); + const callbacks = new TestCompilerCallbacks(); + let compiler = new WindowsPackageInstallerCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + + const result = await compiler.run(kpsPath, null); + assert.isNotNull(result); // This returns a buffer with a SFX loader and a zip suffix. For the sake of repository size // we actually provide a stub SFX loader and a stub MSI file, which is enough to verify that // the compiler is generating what it thinks is a valid file. + const sfxBuffer = result.artifacts.exe.data; + const zip = JSZip(); // Check that file.kmp contains just 3 files - setup.inf, keymandesktop.msi, and khmer_angkor.kmp, diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index bddfcc38db7..fe82a2185e0 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -50,7 +50,12 @@ export class BuildModelInfo extends BuildActivity { return false; } - let kmpCompiler = new KmpCompiler(callbacks); + let kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(callbacks, options)) { + // Errors will have been emitted by KmpCompiler + return false; + } + let kmpJsonData = kmpCompiler.transformKpsToKmpObject(project.resolveInputFilePath(kps)); if(!kmpJsonData) { // Errors will have been emitted by KmpCompiler diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 8adcd4a92cf..12593b0f8e1 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -1,7 +1,6 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; -import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; export class BuildPackage extends BuildActivity { public get name(): string { return 'Package'; } @@ -12,35 +11,16 @@ export class BuildPackage extends BuildActivity { const outfile = this.getOutputFilename(infile, options); - // - // Load .kps source data - // - - const kmpCompiler = new KmpCompiler(callbacks); - const kmpJsonData = kmpCompiler.transformKpsToKmpObject(infile); - if(!kmpJsonData) { - return false; - } - - // - // Validate the package file - // - - const validation = new PackageValidation(callbacks, options); - if(!validation.validate(infile, kmpJsonData)) { + const kmpCompiler = new KmpCompiler(); + if(!await kmpCompiler.init(callbacks, options)) { return false; } - // - // Build the .kmp package file - // - - const data = await kmpCompiler.buildKmpFile(infile, kmpJsonData); - if(!data) { + const result = await kmpCompiler.run(infile, outfile); + if(!result) { return false; } - fs.writeFileSync(outfile, data, 'binary'); - return true; + return await kmpCompiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts index 90bd30ce7f5..6f559708388 100644 --- a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts +++ b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts @@ -4,7 +4,7 @@ import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from ' import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package'; -interface WindowsPackageInstallerOptions extends CompilerBaseOptions { +interface WindowsPackageInstallerCommandLineOptions extends CompilerBaseOptions { msi: string; exe: string; license: string; @@ -15,7 +15,9 @@ interface WindowsPackageInstallerOptions extends CompilerBaseOptions { }; export async function buildWindowsPackageInstaller(infile: string, _options: any, commander: any) { - const options: WindowsPackageInstallerOptions = commander.optsWithGlobals(); + // TODO(lowpri): we probably should cleanup the options management here, move + // translation of command line options to kmc-* options into a separate module + const options: WindowsPackageInstallerCommandLineOptions = commander.optsWithGlobals(); const sources: WindowsPackageInstallerSources = { licenseFilename: options.license, msiFilename: options.msi, @@ -31,11 +33,8 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any infile = fs.realpathSync.native(infile); const callbacks: CompilerCallbacks = new NodeCompilerCallbacks({...defaultCompilerOptions, ...options}); - const compiler = new WindowsPackageInstallerCompiler(callbacks); - - const buffer = await compiler.compile(infile, sources); - if(!buffer) { - // errors will have been reported already + const compiler = new WindowsPackageInstallerCompiler(); + if(!await compiler.init(callbacks, {...options, sources})) { process.exit(1); } @@ -43,5 +42,14 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); const outFileDir = path.dirname(fileBaseName); const outFileExe = path.join(outFileDir, outFileBase + '.exe'); - fs.writeFileSync(outFileExe, buffer); + + const result = await compiler.run(infile, outFileExe); + if(!result) { + // errors will have been reported already + process.exit(1); + } + + if(!await compiler.write(result.artifacts)) { + process.exit(1); + } } diff --git a/developer/src/kmc/src/kmlmp.ts b/developer/src/kmc/src/kmlmp.ts index ae629f073e5..bb0cda5e53b 100644 --- a/developer/src/kmc/src/kmlmp.ts +++ b/developer/src/kmc/src/kmlmp.ts @@ -5,9 +5,8 @@ // Note: this is a deprecated package and will be removed in Keyman 18.0 -import * as fs from 'fs'; import { Command } from 'commander'; -import { PackageValidation, KmpCompiler } from '@keymanapp/kmc-package'; +import { KmpCompiler } from '@keymanapp/kmc-package'; import { SysExits } from './util/sysexits.js'; import KEYMAN_VERSION from "@keymanapp/keyman-version"; import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; @@ -34,36 +33,24 @@ if (!inputFilename) { let outputFilename: string = program.opts().outFile ? program.opts().outFile : inputFilename.replace(/\.kps$/, ".kmp"); // -// Load .kps source data +// Run the compiler // const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); -let kmpCompiler = new KmpCompiler(callbacks); -let kmpJsonData = kmpCompiler.transformKpsToKmpObject(inputFilename); -if(!kmpJsonData) { +let kmpCompiler = new KmpCompiler(); +if(!await kmpCompiler.init(callbacks, null)) { process.exit(1); } -// -// Validate the package file -// - -const validation = new PackageValidation(callbacks, {}); -if(!validation.validate(inputFilename, kmpJsonData)) { +let result = await kmpCompiler.run(inputFilename, outputFilename); +if(!result) { process.exit(1); } -// -// Build the .kmp package file -// - -let promise = kmpCompiler.buildKmpFile(inputFilename, kmpJsonData); -promise.then(data => { - fs.writeFileSync(outputFilename, data, 'binary'); -}).catch(error => { - // Consumer decides how to handle errors +if(!await kmpCompiler.write(result.artifacts)) { console.error('Failed to write kmp file'); -}); + process.exit(1); +} function exitDueToUsageError(message: string): never { console.error(`${program.name()}: ${message}`); From 13533a726a9a1b715ab4f4b97ef23fc6bb31ea65 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 07:15:06 +0700 Subject: [PATCH 06/18] feat(developer): KeyboardInfoCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of KeyboardInfoCompiler to meet KeymanCompiler, including moving file write responsibilities into the class itself. `sources` has become a property of `options`, which is perhaps a slight violation of the intent of the `init()` function being a one-time call, as it means we'll need to instantiate an new class for each file we compile. Given the use-case for the keyboard_info compiler is almost exclusively internal, I think this is acceptable. Tell me if you think otherwise! --- developer/src/kmc-keyboard-info/src/index.ts | 53 +++++++++++++++---- .../test/test-keyboard-info-compiler.ts | 30 ++++++----- .../buildClasses/BuildKeyboardInfo.ts | 22 ++++---- 3 files changed, 71 insertions(+), 34 deletions(-) diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index 7d134e4acb1..7ee90916107 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFileLanguageFont, KeyboardInfoFilePlatform } from "./keyboard-info-file.js"; -import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types"; +import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets, KeymanCompiler, CompilerOptions, KeymanCompilerResult, KeymanCompilerArtifacts, KeymanCompilerArtifact } from "@keymanapp/common-types"; import { KeyboardInfoCompilerMessages } from "./messages.js"; import langtags from "./imports/langtags.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -24,7 +24,7 @@ const HelpRoot = 'https://help.keyman.com/keyboard/'; * Build a dictionary of language tags from langtags.json */ -function init(): void { +function preinit(): void { if(langtagsByTag['en']) { // Already initialized, we can reasonably assume that 'en' will always be in // langtags.json. @@ -62,9 +62,30 @@ export interface KeyboardInfoSources { forPublishing: boolean; }; -export class KeyboardInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { - init(); +export interface KeyboardInfoCompilerOptions extends CompilerOptions { + sources: KeyboardInfoSources; +}; + +export interface KeyboardInfoCompilerArtifacts extends KeymanCompilerArtifacts { + keyboard_info: KeymanCompilerArtifact; +}; + +export interface KeyboardInfoCompilerResult extends KeymanCompilerResult { + artifacts: KeyboardInfoCompilerArtifacts; +}; + +export class KeyboardInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: KeyboardInfoCompilerOptions; + + constructor() { + preinit(); + } + + public async init(callbacks: CompilerCallbacks, options: KeyboardInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -77,9 +98,8 @@ export class KeyboardInfoCompiler { * * @param sources Details on files from which to extract metadata */ - public async writeKeyboardInfoFile( - sources: KeyboardInfoSources - ): Promise { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; // TODO(lowpri): work from .kpj and nothing else as input. Blocked because // .kpj work is largely in kmc at present, so that would need to move to @@ -311,7 +331,22 @@ export class KeyboardInfoCompiler { }, null, 2)); } - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: KeyboardInfoCompilerResult = { + artifacts: { + keyboard_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.keyboard_info') + } + } + }; + + return result; + } + + public async write(artifacts: KeyboardInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.keyboard_info.filename, artifacts.keyboard_info.data); + return true; } private mapKeymanTargetToPlatform(target: KeymanTargets.KeymanTarget): KeyboardInfoFilePlatform[] { diff --git a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts index 40ff764285b..eab1202c8cb 100644 --- a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import 'mocha'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { KeyboardInfoCompiler } from '../src/index.js'; +import { KeyboardInfoCompiler, KeyboardInfoCompilerResult } from '../src/index.js'; const callbacks = new TestCompilerCallbacks(); @@ -13,31 +13,35 @@ beforeEach(function() { describe('keyboard-info-compiler', function () { it('compile a .keyboard_info file correctly', async function() { + const kpjFilename = makePathToFixture('khmer_angkor', 'khmer_angkor.kpj'); const jsFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.js'); const kpsFilename = makePathToFixture('khmer_angkor', 'source', 'khmer_angkor.kps'); const kmpFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.kmp'); const buildKeyboardInfoFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.keyboard_info'); - const compiler = new KeyboardInfoCompiler(callbacks); - let data = null; + const sources = { + kmpFilename, + sourcePath: 'release/k/khmer_angkor', + kpsFilename, + jsFilename: jsFilename, + forPublishing: true, + }; + + const compiler = new KeyboardInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + let result: KeyboardInfoCompilerResult = null; try { - data = await compiler.writeKeyboardInfoFile({ - kmpFilename, - sourcePath: 'release/k/khmer_angkor', - kpsFilename, - jsFilename: jsFilename, - forPublishing: true, - }); + result = await compiler.run(kpjFilename, null); } catch(e) { callbacks.printMessages(); throw e; } - if(data == null) { + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.keyboard_info.data)); const expected = JSON.parse(fs.readFileSync(buildKeyboardInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 936c3585645..6ed023a1395 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -35,29 +35,27 @@ export class BuildKeyboardInfo extends BuildActivity { const keyboard = project.files.find(file => file.fileType == KeymanFileTypes.Source.KeymanKeyboard); const jsFilename = keyboard ? project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard) : null; const lastCommitDate = getLastGitCommitDate(project.projectPath); - - const compiler = new KeyboardInfoCompiler(callbacks); - const data = await compiler.writeKeyboardInfoFile({ + const sources = { kmpFilename: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), kpsFilename: project.resolveInputFilePath(kps), jsFilename: jsFilename && fs.existsSync(jsFilename) ? jsFilename : undefined, sourcePath: calculateSourcePath(infile), lastCommitDate, forPublishing: !!options.forPublishing, - }); + }; - if(data == null) { - // Error messages have already been emitted by KeyboardInfoCompiler + const compiler = new KeyboardInfoCompiler(); + if(!await compiler.init(callbacks, {sources})) { return false; } - const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); + const data = await compiler.run(infile, outputFilename); - fs.writeFileSync( - outputFilename, - data - ); + if(data == null) { + // Error messages have already been emitted by KeyboardInfoCompiler + return false; + } - return true; + return await compiler.write(data.artifacts); } } From 7b6b4f0add1d687a2e8dc8e00ee52fb43d416c0e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 9 Dec 2023 07:28:26 +0700 Subject: [PATCH 07/18] feat(developer): ModelInfoCompiler now implements KeymanCompiler Relates to #9473. Refactors the public API of ModelInfoCompiler to meet KeymanCompiler, including moving file write responsibilities into the class itself. `sources` has become a property of `options`, which is perhaps a slight violation of the intent of the `init()` function being a one-time call, as it means we'll need to instantiate an new class for each file we compile. Given the use-case for the model_info compiler is almost exclusively internal, I think this is acceptable. Tell me if you think otherwise! --- .../kmc-model-info/src/model-info-compiler.ts | 50 ++++++++++++++++--- .../test/test-model-info-compiler.ts | 19 ++++--- .../commands/buildClasses/BuildModelInfo.ts | 22 ++++---- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 8a7101e86fe..934ffe32005 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -5,7 +5,7 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { ModelInfoFile } from "./model-info-file.js"; -import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types"; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KmpJsonFile } from "@keymanapp/common-types"; import { ModelInfoCompilerMessages } from "./messages.js"; import { validateMITLicense } from "@keymanapp/developer-utils"; @@ -39,8 +39,30 @@ export class ModelInfoSources { }; /* c8 ignore stop */ -export class ModelInfoCompiler { - constructor(private callbacks: CompilerCallbacks) { +export interface ModelInfoCompilerOptions extends CompilerOptions { + sources: ModelInfoSources; +}; + +export interface ModelInfoCompilerArtifacts extends KeymanCompilerArtifacts { + model_info: KeymanCompilerArtifact; +}; + +export interface ModelInfoCompilerResult extends KeymanCompilerResult { + artifacts: ModelInfoCompilerArtifacts; +}; + + +export class ModelInfoCompiler implements KeymanCompiler { + private callbacks: CompilerCallbacks; + private options: ModelInfoCompilerOptions; + + constructor() { + } + + public async init(callbacks: CompilerCallbacks, options: ModelInfoCompilerOptions): Promise { + this.callbacks = callbacks; + this.options = {...options}; + return true; } /** @@ -51,9 +73,8 @@ export class ModelInfoCompiler { * * @param sources Details on files from which to extract additional metadata */ - writeModelMetadataFile( - sources: ModelInfoSources - ): Uint8Array { + public async run(inputFilename: string, outputFilename?: string): Promise { + const sources = this.options.sources; /* * Model info looks like this: @@ -171,7 +192,22 @@ export class ModelInfoCompiler { } const jsonOutput = JSON.stringify(model_info, null, 2); - return new TextEncoder().encode(jsonOutput); + const data = new TextEncoder().encode(jsonOutput); + const result: ModelInfoCompilerResult = { + artifacts: { + model_info: { + data, + filename: outputFilename ?? inputFilename.replace(/\.kpj$/, '.model_info') + } + } + }; + + return result; + } + + public async write(artifacts: ModelInfoCompilerArtifacts): Promise { + this.callbacks.fs.writeFileSync(artifacts.model_info.filename, artifacts.model_info.data); + return true; } private isLicenseMIT(filename: string) { diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index d74a2f5c60a..729eca3ac96 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -13,16 +13,18 @@ beforeEach(function() { }); describe('model-info-compiler', function () { - it('compile a .model_info file correctly', function() { + it('compile a .model_info file correctly', async function() { + const kpjFilename = makePathToFixture('sil.cmo.bw', 'sil.cmo.bw.model.kpj'); const kpsFilename = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp'); const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info'); - const kmpCompiler = new KmpCompiler(callbacks); + const kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, {})); const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename); const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js'); - const data = (new ModelInfoCompiler(callbacks)).writeModelMetadataFile({ + const sources = { kmpFileName, kmpJsonData, model_id: 'sil.cmo.bw', @@ -30,13 +32,16 @@ describe('model-info-compiler', function () { sourcePath: 'release/sil/sil.cmo.bw', kpsFilename, forPublishing: true, - }); - if(data == null) { + }; + const compiler = new ModelInfoCompiler(); + assert.isTrue(await compiler.init(callbacks, {sources})); + const result = await compiler.run(kpjFilename, null); + if(result == null) { callbacks.printMessages(); } - assert.isNotNull(data); + assert.isNotNull(result); - const actual = JSON.parse(new TextDecoder().decode(data)); + const actual = JSON.parse(new TextDecoder().decode(result.artifacts.model_info.data)); let expected = JSON.parse(fs.readFileSync(buildModelInfoFilename, 'utf-8')); // `lastModifiedDate` is dependent on time of run (not worth mocking) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index fe82a2185e0..322fa5707bb 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,4 +1,3 @@ -import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; @@ -63,8 +62,7 @@ export class BuildModelInfo extends BuildActivity { } const lastCommitDate = getLastGitCommitDate(project.projectPath); - const compiler = new ModelInfoCompiler(callbacks); - const data = compiler.writeModelMetadataFile({ + const sources = { model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), @@ -73,18 +71,20 @@ export class BuildModelInfo extends BuildActivity { kpsFilename: project.resolveInputFilePath(kps), lastCommitDate, forPublishing: !!options.forPublishing, - }); + }; + const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); - if(data == null) { - // Error messages have already been emitted by writeModelMetadataFile + const compiler = new ModelInfoCompiler(); + if(!await compiler.init(callbacks, {sources})) { return false; } + const result = await compiler.run(infile, outputFilename); - fs.writeFileSync( - project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo), - data - ); + if(result == null) { + // Error messages have already been emitted by writeModelMetadataFile + return false; + } - return true; + return await compiler.write(result.artifacts); } } \ No newline at end of file From f9c3be09a6e7a010cd7d6bfd2451957a5345a5af Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 09:47:23 +0700 Subject: [PATCH 08/18] refactor(developer): move outFile out of CompilerBaseOptions `outFile` is not a compiler option, and so it should not be a part of the `CompilerBaseOptions` interface. This formally separates the command-line options from the compiler options, which eliminates a number of restrictive structures. --- .../web/types/src/util/compiler-interfaces.ts | 1 - developer/src/kmc/src/commands/build.ts | 15 ++++++++---- .../commands/buildClasses/BuildActivity.ts | 7 +++--- .../buildClasses/BuildKeyboardInfo.ts | 4 +++- .../commands/buildClasses/BuildKmnKeyboard.ts | 3 +-- .../buildClasses/BuildLdmlKeyboard.ts | 24 +++++++++---------- .../src/commands/buildClasses/BuildModel.ts | 6 ++--- .../commands/buildClasses/BuildModelInfo.ts | 4 +++- .../src/commands/buildClasses/BuildPackage.ts | 4 ++-- .../src/commands/buildClasses/BuildProject.ts | 18 +++++++------- .../kmc/src/commands/buildTestData/index.ts | 5 ++-- .../buildWindowsPackageInstaller/index.ts | 9 ++++--- .../src/messages/infrastructureMessages.ts | 4 ++++ developer/src/kmc/src/util/baseOptions.ts | 19 +++++++++++++-- developer/src/kmc/test/test-build.ts | 2 +- .../kmc/test/test-infrastructureMessages.ts | 6 ++--- developer/src/kmc/test/test-project-build.ts | 2 +- 17 files changed, 80 insertions(+), 53 deletions(-) diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index fc6a33fa1a9..6409c934f5c 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -403,7 +403,6 @@ export interface CompilerBaseOptions { * Format of output for log to console */ logFormat?: CompilerLogFormat; - outFile?: string; //TODO:REMOVE /** * Colorize log output, default is detected from console */ diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index 0d091680b7c..21aa6fdc3bb 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -19,7 +19,6 @@ function commandOptionsToCompilerOptions(options: any): ExtendedCompilerOptions // CompilerOptions properties... return { // CompilerBaseOptions - outFile: options.outFile, logLevel: options.logLevel, logFormat: options.logFormat, color: options.color, @@ -66,7 +65,8 @@ File lists can be referenced with @filelist.txt. If no input file is supplied, kmc will build the current folder.`) .action(async (filenames: string[], _options: any, commander: any) => { - const options = commandOptionsToCompilerOptions(commander.optsWithGlobals()); + const commanderOptions/*:{TODO?} CommandLineCompilerOptions*/ = commander.optsWithGlobals(); + const options = commandOptionsToCompilerOptions(commanderOptions); const callbacks = new NodeCompilerCallbacks(options); if(!filenames.length) { @@ -75,12 +75,17 @@ If no input file is supplied, kmc will build the current folder.`) filenames.push('.'); } + if(filenames.length > 1 && commanderOptions.outFile) { + // -o can only be specified with a single input file + callbacks.reportMessage(InfrastructureMessages.Error_OutFileCanOnlyBeSpecifiedWithSingleInfile()); + } + if(!expandFileLists(filenames, callbacks)) { process.exit(1); } for(let filename of filenames) { - if(!await build(filename, callbacks, options)) { + if(!await build(filename, commanderOptions.outFile, callbacks, options)) { // Once a file fails to build, we bail on subsequent builds process.exit(1); } @@ -105,7 +110,7 @@ If no input file is supplied, kmc will build the current folder.`) .action(buildWindowsPackageInstaller); } -async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { +async function build(filename: string, outfile: string, parentCallbacks: NodeCompilerCallbacks, options: CompilerOptions): Promise { try { // TEST: allow command-line simulation of infrastructure fatal errors, and // also for unit tests @@ -151,7 +156,7 @@ async function build(filename: string, parentCallbacks: NodeCompilerCallbacks, o const callbacks = new CompilerFileCallbacks(buildFilename, options, parentCallbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename:buildFilename, relativeFilename})); - let result = await builder.build(filename, callbacks, options); + let result = await builder.build(filename, outfile, callbacks, options); result = result && !callbacks.hasFailureMessage(); if(result) { callbacks.reportMessage(builder instanceof BuildProject diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index dae545366f2..064eb8ffb23 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -6,10 +6,9 @@ export abstract class BuildActivity { public abstract get sourceExtension(): KeymanFileTypes.Source; public abstract get compiledExtension(): KeymanFileTypes.Binary; public abstract get description(): string; - public abstract build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; - protected getOutputFilename(infile: string, options: CompilerOptions): string { - return options.outFile ? - options.outFile : + public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; + protected getOutputFilename(infile: string, outfile?: string): string { + return outfile ?? infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension); } }; \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 6ed023a1395..1ede2decdab 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -13,7 +13,7 @@ export class BuildKeyboardInfo extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; } public get description(): string { return 'Build a keyboard metadata file'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -48,6 +48,8 @@ export class BuildKeyboardInfo extends BuildActivity { if(!await compiler.init(callbacks, {sources})) { return false; } + + // Note: should we always ignore the passed-in output filename for .keyboard_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); const data = await compiler.run(infile, outputFilename); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 519d251b61d..6965bd3b9cb 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -11,9 +11,8 @@ export class BuildKmnKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeymanKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a Keyman keyboard'; } - public async build(infile: string, /*TODO: outfile?: string,*/ callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn - let outfile = options.outFile;//TODO: remove here if(outfile) { outfile = getPosixAbsolutePath(outfile); const folderName = path.dirname(outfile); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index fbb69f921c5..f93aabcf475 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -10,10 +10,19 @@ export class BuildLdmlKeyboard extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.LdmlKeyboard; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Keyboard; } public get description(): string { return 'Build a LDML keyboard'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk // Compile: - const outfile = options.outFile; //TODO + // TODO: Consider if this mkdir should be in write() + // TODO: This pattern may be needed for kmc-kmn as well? + const outFileDir = callbacks.path.dirname(outfile ?? infile); + + try { + fs.mkdirSync(outFileDir, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); + return false; + } const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) @@ -29,17 +38,6 @@ export class BuildLdmlKeyboard extends BuildActivity { return false; } - // TODO: Consider if this mkdir should be in write() - // TODO: This pattern may be needed for kmc-kmn as well? - const outFileDir = callbacks.path.dirname(outfile ?? infile); - - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); - return false; - } - return await compiler.write(result.artifacts); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index 00614c27310..41d6fcf792f 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -7,15 +7,15 @@ export class BuildModel extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Model; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - const outputFilename: string = this.getOutputFilename(infile, options); + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + outfile = this.getOutputFilename(infile, outfile); const compiler = new LexicalModelCompiler(); if(!await compiler.init(callbacks, options)) { return false; } - const result = await compiler.run(infile, outputFilename); + const result = await compiler.run(infile, outfile); if(!result) { return false; } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 322fa5707bb..4000af68949 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -24,7 +24,7 @@ export class BuildModelInfo extends BuildActivity { * @param options * @returns */ - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, _outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -72,6 +72,8 @@ export class BuildModelInfo extends BuildActivity { lastCommitDate, forPublishing: !!options.forPublishing, }; + + // Note: should we always ignore the passed-in output filename for .model_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); const compiler = new ModelInfoCompiler(); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 12593b0f8e1..05172d4419d 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -7,9 +7,9 @@ export class BuildPackage extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Package; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; } public get description(): string { return 'Build a Keyman package'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - const outfile = this.getOutputFilename(infile, options); + outfile = this.getOutputFilename(infile, outfile); const kmpCompiler = new KmpCompiler(); if(!await kmpCompiler.init(callbacks, options)) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 5cbd3da80ee..043edbf11c6 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -12,7 +12,12 @@ export class BuildProject extends BuildActivity { public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return null; } public get description(): string { return 'Build a keyboard or lexical model project'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { + if(outfile) { + callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); + return false; + } + let builder = new ProjectBuilder(infile, callbacks, options); return builder.run(); } @@ -31,11 +36,6 @@ class ProjectBuilder { } async run(): Promise { - if(this.options.outFile) { - this.callbacks.reportMessage(InfrastructureMessages.Error_OutFileNotValidForProjects()); - return false; - } - this.project = loadProject(this.infile, this.callbacks); if(!this.project) { return false; @@ -97,7 +97,7 @@ class ProjectBuilder { async buildTarget(file: KeymanDeveloperProjectFile, activity: BuildActivity): Promise { const options = {...this.options}; - options.outFile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); + const outfile = this.project.resolveOutputFilePath(file, activity.sourceExtension, activity.compiledExtension); options.checkFilenameConventions = this.project.options.checkFilenameConventions ?? this.options.checkFilenameConventions; const infile = this.project.resolveInputFilePath(file); @@ -105,9 +105,9 @@ class ProjectBuilder { const callbacks = new CompilerFileCallbacks(buildFilename, options, this.callbacks); callbacks.reportMessage(InfrastructureMessages.Info_BuildingFile({filename: infile, relativeFilename:buildFilename})); - fs.mkdirSync(path.dirname(options.outFile), {recursive:true}); + fs.mkdirSync(path.dirname(outfile), {recursive:true}); - let result = await activity.build(infile, callbacks, options); + let result = await activity.build(infile, outfile, callbacks, options); // check if we had a message that causes the build to be a failure // note: command line option here, if set, overrides project setting diff --git a/developer/src/kmc/src/commands/buildTestData/index.ts b/developer/src/kmc/src/commands/buildTestData/index.ts index 5d0f6a2380f..4253f55cf6b 100644 --- a/developer/src/kmc/src/commands/buildTestData/index.ts +++ b/developer/src/kmc/src/commands/buildTestData/index.ts @@ -1,12 +1,13 @@ import * as fs from 'fs'; import * as path from 'path'; import * as kmcLdml from '@keymanapp/kmc-ldml'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { fileURLToPath } from 'url'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; export async function buildTestData(infile: string, _options: any, commander: any) { - const options: CompilerBaseOptions = commander.optsWithGlobals(); + const options: CommandLineBaseOptions = commander.optsWithGlobals(); let compilerOptions: kmcLdml.LdmlCompilerOptions = { ...defaultCompilerOptions, diff --git a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts index 6f559708388..5d64908856d 100644 --- a/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts +++ b/developer/src/kmc/src/commands/buildWindowsPackageInstaller/index.ts @@ -1,10 +1,11 @@ import * as fs from 'fs'; import * as path from 'path'; -import { CompilerBaseOptions, CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; +import { CompilerCallbacks, defaultCompilerOptions } from '@keymanapp/common-types'; import { NodeCompilerCallbacks } from '../../util/NodeCompilerCallbacks.js'; import { WindowsPackageInstallerCompiler, WindowsPackageInstallerSources } from '@keymanapp/kmc-package'; +import { CommandLineBaseOptions } from 'src/util/baseOptions.js'; -interface WindowsPackageInstallerCommandLineOptions extends CompilerBaseOptions { +interface WindowsPackageInstallerCommandLineOptions extends CommandLineBaseOptions { msi: string; exe: string; license: string; @@ -28,6 +29,8 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any titleImageFilename: options.titleImage } + const outfile: string = options.outFile; + // Normalize case for the filename and expand the path; this avoids false // positive case mismatches on input filenames and glommed paths infile = fs.realpathSync.native(infile); @@ -38,7 +41,7 @@ export async function buildWindowsPackageInstaller(infile: string, _options: any process.exit(1); } - const fileBaseName = options.outFile ?? infile; + const fileBaseName = outfile ?? infile; const outFileBase = path.basename(fileBaseName, path.extname(fileBaseName)); const outFileDir = path.dirname(fileBaseName); const outFileExe = path.join(outFileDir, outFileBase + '.exe'); diff --git a/developer/src/kmc/src/messages/infrastructureMessages.ts b/developer/src/kmc/src/messages/infrastructureMessages.ts index 50334a6fd64..561b81ed4c7 100644 --- a/developer/src/kmc/src/messages/infrastructureMessages.ts +++ b/developer/src/kmc/src/messages/infrastructureMessages.ts @@ -87,5 +87,9 @@ export class InfrastructureMessages { static Error_UnsupportedProjectVersion = (o:{version:string}) => m(this.ERROR_UnsupportedProjectVersion, `Project version ${o.version} is not supported by this version of Keyman Developer.`); static ERROR_UnsupportedProjectVersion = SevError | 0x0013; + + static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile, + `Parameter -out-file can only be used with a single input file.`); + static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0014; } diff --git a/developer/src/kmc/src/util/baseOptions.ts b/developer/src/kmc/src/util/baseOptions.ts index 94163769516..dd91f0bae67 100644 --- a/developer/src/kmc/src/util/baseOptions.ts +++ b/developer/src/kmc/src/util/baseOptions.ts @@ -1,8 +1,23 @@ -import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS } from "@keymanapp/common-types"; +import { ALL_COMPILER_LOG_FORMATS, ALL_COMPILER_LOG_LEVELS, CompilerLogFormat, CompilerLogLevel } from "@keymanapp/common-types"; import { Command, Option } from "commander"; -// These options map to CompilerBaseOptions +/** + * Abstract interface for compiler options + */ +export interface CommandLineBaseOptions { + // These options map to CompilerBaseOptions + logLevel?: CompilerLogLevel; + logFormat?: CompilerLogFormat; + color?: boolean; + + // This option is not in CompilerBaseOptions + outFile?:string; +} + +/** + * These options map to CompilerBaseOptions + */ export class BaseOptions { public static addLogLevel(program: Command) { return program.addOption(new Option('-l, --log-level ', 'Log level').choices(ALL_COMPILER_LOG_LEVELS).default('info')); diff --git a/developer/src/kmc/test/test-build.ts b/developer/src/kmc/test/test-build.ts index 26ef09e746b..d859e3f26dd 100644 --- a/developer/src/kmc/test/test-build.ts +++ b/developer/src/kmc/test/test-build.ts @@ -34,7 +34,7 @@ describe('compilerWarningsAsErrors', function () { const builder = new BuildProject(); const path = makePathToFixture('compiler-warnings-as-errors', `compiler_warnings_as_errors_${truth.kpj === true ? 'true' : (truth.kpj === false ? 'false' : 'undefined')}.kpj`); - const result = await builder.build(path, callbacks, { + const result = await builder.build(path, null, callbacks, { compilerWarningsAsErrors: truth.cli, }); if(truth.result != result) { diff --git a/developer/src/kmc/test/test-infrastructureMessages.ts b/developer/src/kmc/test/test-infrastructureMessages.ts index 6d543a5fd7c..34898a5ad92 100644 --- a/developer/src/kmc/test/test-infrastructureMessages.ts +++ b/developer/src/kmc/test/test-infrastructureMessages.ts @@ -24,7 +24,7 @@ describe('InfrastructureMessages', function () { const expectedMessages = [InfrastructureMessages.FATAL_UnexpectedException]; process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION = '1'; - await unitTestEndpoints.build(null, ncb, {}); + await unitTestEndpoints.build(null, null, ncb, {}); delete process.env.SENTRY_CLIENT_TEST_BUILD_EXCEPTION; assertMessagesEqual(ncb.messages, expectedMessages); @@ -79,7 +79,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.INFO_WarningsHaveFailedBuild, InfrastructureMessages.INFO_FileNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); @@ -93,7 +93,7 @@ describe('InfrastructureMessages', function () { InfrastructureMessages.ERROR_UnsupportedProjectVersion, InfrastructureMessages.INFO_ProjectNotBuiltSuccessfully ]; - await unitTestEndpoints.build(filename, ncb, {compilerWarningsAsErrors: true}); + await unitTestEndpoints.build(filename, null, ncb, {compilerWarningsAsErrors: true}); assertMessagesEqual(ncb.messages, expectedMessages); }); }); diff --git a/developer/src/kmc/test/test-project-build.ts b/developer/src/kmc/test/test-project-build.ts index 88563168805..ebcff59dc03 100644 --- a/developer/src/kmc/test/test-project-build.ts +++ b/developer/src/kmc/test/test-project-build.ts @@ -11,7 +11,7 @@ describe('BuildProject', function () { it('should build a keyboard project', async function() { const builder = new BuildProject(); const path = makePathToFixture('relative_paths', 'k_000___null_keyboard.kpj'); - let result = await builder.build(path, callbacks, { + let result = await builder.build(path, null, callbacks, { shouldAddCompilerVersion: false, compilerWarningsAsErrors: true, saveDebug: false, From f4918d36e29dc2fc250af49cae96e41238459608 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 10:10:31 +0700 Subject: [PATCH 09/18] feat(developer): refactor common compiler steps into BuildActivity.runCompiler Relates to #9473. Now that each compiler has a common interface, we can consolidate the common code into BuildActivity. This makes it very easy to see where the compiler calls have special cases, such as the model_info and keyboard_info compilers (these are primarily for internal use, so the special case work should not be a problem). --- .../commands/buildClasses/BuildActivity.ts | 37 ++++++++++++++++--- .../buildClasses/BuildKeyboardInfo.ts | 16 +------- .../commands/buildClasses/BuildKmnKeyboard.ts | 22 +---------- .../buildClasses/BuildLdmlKeyboard.ts | 26 +------------ .../src/commands/buildClasses/BuildModel.ts | 13 +------ .../commands/buildClasses/BuildModelInfo.ts | 12 +----- .../src/commands/buildClasses/BuildPackage.ts | 16 +------- developer/src/kmc/src/util/escapeRegExp.ts | 7 ---- 8 files changed, 41 insertions(+), 108 deletions(-) delete mode 100644 developer/src/kmc/src/util/escapeRegExp.ts diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index 064eb8ffb23..086333149a6 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -1,5 +1,6 @@ -import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from "@keymanapp/common-types"; -import { escapeRegExp } from "../../util/escapeRegExp.js"; +import * as fs from 'fs'; +import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types"; +import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export abstract class BuildActivity { public abstract get name(): string; @@ -7,8 +8,34 @@ export abstract class BuildActivity { public abstract get compiledExtension(): KeymanFileTypes.Binary; public abstract get description(): string; public abstract build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise; - protected getOutputFilename(infile: string, outfile?: string): string { - return outfile ?? - infile.replace(new RegExp(escapeRegExp(this.sourceExtension), "g"), this.compiledExtension); + + protected async runCompiler(compiler: KeymanCompiler, infile: string, outfile: string, callbacks: CompilerCallbacks, options: T): Promise { + if(!await compiler.init(callbacks, options)) { + return false; + } + + const result = await compiler.run(infile, outfile); + if(!result) { + return false; + } + + if(!this.createOutputFolder(outfile ?? infile, callbacks)) { + return false; + } + + return await compiler.write(result.artifacts); + } + + private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean { + const targetFolder = callbacks.path.dirname(targetFilename); + + try { + fs.mkdirSync(targetFolder, {recursive: true}); + } catch(e) { + callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:targetFolder, e})); + return false; + } + + return true; } }; \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 1ede2decdab..605870741e3 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -43,21 +43,9 @@ export class BuildKeyboardInfo extends BuildActivity { lastCommitDate, forPublishing: !!options.forPublishing, }; - - const compiler = new KeyboardInfoCompiler(); - if(!await compiler.init(callbacks, {sources})) { - return false; - } - // Note: should we always ignore the passed-in output filename for .keyboard_info? const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); - const data = await compiler.run(infile, outputFilename); - - if(data == null) { - // Error messages have already been emitted by KeyboardInfoCompiler - return false; - } - - return await compiler.write(data.artifacts); + const compiler = new KeyboardInfoCompiler(); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts index 6965bd3b9cb..e540aba4513 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKmnKeyboard.ts @@ -3,8 +3,6 @@ import { platform } from 'os'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { CompilerOptions, CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; -import * as fs from 'fs'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildKmnKeyboard extends BuildActivity { public get name(): string { return 'Keyman keyboard'; } @@ -13,29 +11,13 @@ export class BuildKmnKeyboard extends BuildActivity { public get description(): string { return 'Build a Keyman keyboard'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // We need to resolve paths to absolute paths before calling kmc-kmn + infile = getPosixAbsolutePath(infile); if(outfile) { outfile = getPosixAbsolutePath(outfile); - const folderName = path.dirname(outfile); - try { - fs.mkdirSync(folderName, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName, e})); - return false; - } } - infile = getPosixAbsolutePath(infile); const compiler = new KmnCompiler(); - if(!await compiler.init(callbacks, options)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts index f93aabcf475..0e0cfd97661 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildLdmlKeyboard.ts @@ -1,9 +1,7 @@ -import * as fs from 'fs'; import * as kmcLdml from '@keymanapp/kmc-ldml'; import { CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { fileURLToPath } from 'url'; -import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; export class BuildLdmlKeyboard extends BuildActivity { public get name(): string { return 'LDML keyboard'; } @@ -12,32 +10,10 @@ export class BuildLdmlKeyboard extends BuildActivity { public get description(): string { return 'Build a LDML keyboard'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { // TODO-LDML: consider hardware vs touch -- touch-only layout will not have a .kvk - // Compile: - // TODO: Consider if this mkdir should be in write() - // TODO: This pattern may be needed for kmc-kmn as well? - const outFileDir = callbacks.path.dirname(outfile ?? infile); - - try { - fs.mkdirSync(outFileDir, {recursive: true}); - } catch(e) { - callbacks.reportMessage(InfrastructureMessages.Error_CannotCreateFolder({folderName:outFileDir, e})); - return false; - } - const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = {...options, readerOptions: { importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) }}; - const compiler = new kmcLdml.LdmlKeyboardCompiler(); - if(!await compiler.init(callbacks, ldmlCompilerOptions)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, ldmlCompilerOptions); } } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts index 41d6fcf792f..b2afc592943 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModel.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModel.ts @@ -8,18 +8,7 @@ export class BuildModel extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Model; } public get description(): string { return 'Build a lexical model'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - outfile = this.getOutputFilename(infile, outfile); - const compiler = new LexicalModelCompiler(); - if(!await compiler.init(callbacks, options)) { - return false; - } - - const result = await compiler.run(infile, outfile); - if(!result) { - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 4000af68949..912bca986a8 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -77,16 +77,6 @@ export class BuildModelInfo extends BuildActivity { const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo); const compiler = new ModelInfoCompiler(); - if(!await compiler.init(callbacks, {sources})) { - return false; - } - const result = await compiler.run(infile, outputFilename); - - if(result == null) { - // Error messages have already been emitted by writeModelMetadataFile - return false; - } - - return await compiler.write(result.artifacts); + return await super.runCompiler(compiler, infile, outputFilename, callbacks, {...options, sources}); } } \ No newline at end of file diff --git a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts index 05172d4419d..1bb26b3b7f8 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildPackage.ts @@ -8,19 +8,7 @@ export class BuildPackage extends BuildActivity { public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.Package; } public get description(): string { return 'Build a Keyman package'; } public async build(infile: string, outfile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - - outfile = this.getOutputFilename(infile, outfile); - - const kmpCompiler = new KmpCompiler(); - if(!await kmpCompiler.init(callbacks, options)) { - return false; - } - - const result = await kmpCompiler.run(infile, outfile); - if(!result) { - return false; - } - - return await kmpCompiler.write(result.artifacts); + const compiler = new KmpCompiler(); + return await super.runCompiler(compiler, infile, outfile, callbacks, options); } } diff --git a/developer/src/kmc/src/util/escapeRegExp.ts b/developer/src/kmc/src/util/escapeRegExp.ts deleted file mode 100644 index 8972b6bfede..00000000000 --- a/developer/src/kmc/src/util/escapeRegExp.ts +++ /dev/null @@ -1,7 +0,0 @@ - -const escapedRegexp = /[.*+?^${}()|[\]\\]/g; - -// See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping -export function escapeRegExp(s: string) { - return s.replace(escapedRegexp, "\\$&"); // $& means the whole matched string -} From 52038e22aab57e5ded9e822a6d273721412c5b37 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 11:43:24 +0700 Subject: [PATCH 10/18] chore(developer): remove unintended usage of `callbacks.path`/`callbacks.fs` in kmc --- common/web/types/src/kpj/keyman-developer-project.ts | 12 ++++++++++-- developer/src/kmc/src/commands/analyze.ts | 2 +- .../kmc/src/commands/buildClasses/BuildActivity.ts | 3 ++- .../kmc/src/commands/buildClasses/BuildModelInfo.ts | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/common/web/types/src/kpj/keyman-developer-project.ts b/common/web/types/src/kpj/keyman-developer-project.ts index a53cbca20e9..fdc84664974 100644 --- a/common/web/types/src/kpj/keyman-developer-project.ts +++ b/common/web/types/src/kpj/keyman-developer-project.ts @@ -37,8 +37,16 @@ export class KeymanDeveloperProject { for(let filename of files) { let fullPath = this.callbacks.path.join(sourcePath, filename); if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Source.LdmlKeyboard)) { - if(!this.callbacks.fs.readFileSync(fullPath, 'utf-8').match(/ldmlKeyboard3\.dtd/)) { - // Skip this .xml because we assume it isn't really a keyboard .xml + try { + const data = this.callbacks.loadFile(fullPath); + const text = new TextDecoder().decode(data); + if(!text?.match(/ldmlKeyboard3\.dtd/)) { + // Skip this .xml because we assume it isn't really a keyboard .xml + continue; + } + } catch(e) { + // We'll just silently skip this file because we were not able to load it, + // so let's hope it wasn't a real LDML keyboard XML :-) continue; } } diff --git a/developer/src/kmc/src/commands/analyze.ts b/developer/src/kmc/src/commands/analyze.ts index eb2feb528ab..15c8ac3f39d 100644 --- a/developer/src/kmc/src/commands/analyze.ts +++ b/developer/src/kmc/src/commands/analyze.ts @@ -105,7 +105,7 @@ async function analyzeOskCharUse(callbacks: CompilerCallbacks, filenames: string async function analyzeOskRewritePua(callbacks: CompilerCallbacks, filenames: string[], options: AnalysisActivityOptions) { const analyzer = new AnalyzeOskRewritePua(callbacks); - const mapping: any = JSON.parse(callbacks.fs.readFileSync(options.mappingFile, 'UTF-8')); + const mapping: any = JSON.parse(fs.readFileSync(options.mappingFile, 'utf-8')); return await runOnFiles(callbacks, filenames, async (filename: string): Promise => { if(!await analyzer.analyze(filename, mapping)) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts index 086333149a6..864611c26d9 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildActivity.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as path from 'path'; import { CompilerCallbacks, CompilerOptions, KeymanCompiler, KeymanFileTypes } from "@keymanapp/common-types"; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; @@ -27,7 +28,7 @@ export abstract class BuildActivity { } private createOutputFolder(targetFilename: string, callbacks: CompilerCallbacks): boolean { - const targetFolder = callbacks.path.dirname(targetFilename); + const targetFolder = path.dirname(targetFilename); try { fs.mkdirSync(targetFolder, {recursive: true}); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 912bca986a8..470c90639de 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,3 +1,4 @@ +import * as path from 'path'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; @@ -63,7 +64,7 @@ export class BuildModelInfo extends BuildActivity { const lastCommitDate = getLastGitCommitDate(project.projectPath); const sources = { - model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), + model_id: path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), From d99c4cd669e497140a41e8f6a6eacb61dd05b276 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 12:21:20 +0700 Subject: [PATCH 11/18] chore(web): update LexicalModelCompiler call --- common/web/input-processor/tests/cases/languageProcessor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index cd7d87dda97..1cabec0ee84 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -56,8 +56,9 @@ describe('LanguageProcessor', function() { }); }); - describe('.predict', function() { - let compiler = new LexicalModelCompiler(callbacks); + describe('.predict', async function() { + let compiler = new LexicalModelCompiler(); + assert.isTrue(await compiler.init(callbacks, {})); const MODEL_ID = 'example.qaa.trivial'; // ES-module mode leaves out `__dirname`, so we rebuild it using other components. From 89254296dba520fca17350fa133d32ec25d92bea Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 11 Dec 2023 21:17:00 +0700 Subject: [PATCH 12/18] chore(developer): honor prompt to upgrade in kmc Fixes #10162. --- developer/src/common/web/utils/src/options.ts | 1 + developer/src/kmc/src/commands/buildClasses/BuildProject.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/options.ts b/developer/src/common/web/utils/src/options.ts index 9cb3048ea28..ec6b0f9a7c8 100644 --- a/developer/src/common/web/utils/src/options.ts +++ b/developer/src/common/web/utils/src/options.ts @@ -35,6 +35,7 @@ export interface KeymanDeveloperOptions { "automatically report usage"?: boolean; "toolbar visible"?: boolean; "active project"?: string; + "prompt to upgrade projects"?: boolean; }; type KeymanDeveloperOption = keyof KeymanDeveloperOptions; diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 38dd2ae6da2..e428a0c0756 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -6,6 +6,7 @@ import { buildActivities, buildKeyboardInfoActivity, buildModelInfoActivity } fr import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { loadProject } from '../../util/projectLoader.js'; import { ExtendedCompilerOptions } from 'src/util/extendedCompilerOptions.js'; +import { getOption } from '@keymanapp/developer-utils'; export class BuildProject extends BuildActivity { public get name(): string { return 'Project'; } @@ -43,7 +44,9 @@ class ProjectBuilder { // Give a hint if the project is v1.0 if(this.project.options.version != '2.0') { - this.callbacks.reportMessage(InfrastructureMessages.Hint_ProjectIsVersion10()); + if(getOption("prompt to upgrade projects", true)) { + this.callbacks.reportMessage(InfrastructureMessages.Hint_ProjectIsVersion10()); + } } // Go through the various file types and build them From 3d250fecc019a7822535901e4023b1b40f6b516e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 12 Dec 2023 07:38:13 +0700 Subject: [PATCH 13/18] chore(developer): ensure unit tests have default options set --- developer/src/common/web/utils/index.ts | 2 +- developer/src/common/web/utils/src/options.ts | 14 +++++++++++++- developer/src/kmc/test/test-build.ts | 6 +++++- .../src/kmc/test/test-infrastructureMessages.ts | 4 ++++ developer/src/kmc/test/test-project-build.ts | 2 ++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/index.ts b/developer/src/common/web/utils/index.ts index 410823bd6dc..f4414f1e44f 100644 --- a/developer/src/common/web/utils/index.ts +++ b/developer/src/common/web/utils/index.ts @@ -1,3 +1,3 @@ export { validateMITLicense } from './src/validate-mit-license.js'; export { KeymanSentry } from './src/KeymanSentry.js'; -export { getOption, loadOptions } from './src/options.js'; \ No newline at end of file +export { getOption, loadOptions, clearOptions } from './src/options.js'; diff --git a/developer/src/common/web/utils/src/options.ts b/developer/src/common/web/utils/src/options.ts index ec6b0f9a7c8..df32a4a7b9b 100644 --- a/developer/src/common/web/utils/src/options.ts +++ b/developer/src/common/web/utils/src/options.ts @@ -40,8 +40,11 @@ export interface KeymanDeveloperOptions { type KeymanDeveloperOption = keyof KeymanDeveloperOptions; +// Default has no options set, and unit tests will use the defaults (won't call +// `loadOptions()`) +let options: KeymanDeveloperOptions = {}; + // We only load the options from disk once on first use -let options: KeymanDeveloperOptions = null; let optionsLoaded = false; export async function loadOptions(): Promise { @@ -75,9 +78,18 @@ export async function loadOptions(): Promise { // low level. options = {}; } + optionsLoaded = true; return options; } export function getOption(valueName: T, defaultValue: KeymanDeveloperOptions[T]): KeymanDeveloperOptions[T] { return options[valueName] ?? defaultValue; } + +/** + * unit tests will clear options before running, for consistency + */ +export function clearOptions() { + options = {}; + optionsLoaded = true; +} \ No newline at end of file diff --git a/developer/src/kmc/test/test-build.ts b/developer/src/kmc/test/test-build.ts index 26ef09e746b..243ec77cf77 100644 --- a/developer/src/kmc/test/test-build.ts +++ b/developer/src/kmc/test/test-build.ts @@ -1,4 +1,5 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; +import { clearOptions } from '@keymanapp/developer-utils'; import { assert } from 'chai'; import 'mocha'; import { BuildProject } from '../src/commands/buildClasses/BuildProject.js'; @@ -13,7 +14,10 @@ interface CompilerWarningsAsErrorsTruthTable { }; describe('compilerWarningsAsErrors', function () { - beforeEach(() => callbacks.clear()); + beforeEach(() => { + callbacks.clear(); + clearOptions(); + }); // The CLI option should override the project setting diff --git a/developer/src/kmc/test/test-infrastructureMessages.ts b/developer/src/kmc/test/test-infrastructureMessages.ts index 6d543a5fd7c..310e5bfab45 100644 --- a/developer/src/kmc/test/test-infrastructureMessages.ts +++ b/developer/src/kmc/test/test-infrastructureMessages.ts @@ -7,8 +7,12 @@ import { NodeCompilerCallbacks } from '../src/util/NodeCompilerCallbacks.js'; import { CompilerErrorNamespace, CompilerEvent } from '@keymanapp/common-types'; import { unitTestEndpoints } from '../src/commands/build.js'; import { KmnCompilerMessages } from '@keymanapp/kmc-kmn'; +import { clearOptions } from '@keymanapp/developer-utils'; describe('InfrastructureMessages', function () { + + beforeEach(clearOptions); + it('should have a valid InfrastructureMessages object', function() { return verifyCompilerMessagesObject(InfrastructureMessages, CompilerErrorNamespace.Infrastructure); }); diff --git a/developer/src/kmc/test/test-project-build.ts b/developer/src/kmc/test/test-project-build.ts index c7e1c82bf58..41f1b736707 100644 --- a/developer/src/kmc/test/test-project-build.ts +++ b/developer/src/kmc/test/test-project-build.ts @@ -4,11 +4,13 @@ import 'mocha'; import { BuildProject } from '../src/commands/buildClasses/BuildProject.js'; import { makePathToFixture } from './helpers/index.js'; import { InfrastructureMessages } from '../src/messages/infrastructureMessages.js'; +import { clearOptions } from '@keymanapp/developer-utils'; const callbacks = new TestCompilerCallbacks(); describe('BuildProject', function () { it('should build a keyboard project', async function() { + clearOptions(); const builder = new BuildProject(); const path = makePathToFixture('relative_paths', 'k_000___null_keyboard.kpj'); let result = await builder.build(path, callbacks, { From e2ce64d9be0cf01cb25da50c23bff8ddb8ccd1d6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 05:38:51 +0700 Subject: [PATCH 14/18] fix(web): mocha describe does not do async mocha describe() does not accept an async function. Any async prep should be done in a before() function, which does support async. One helpful ref: https://github.com/mochajs/mocha/issues/2975#issuecomment-1004176440 --- .../tests/cases/inputProcessorTests.js | 2 +- .../tests/cases/languageProcessor.js | 71 +++++++++++-------- developer/src/kmc-kmn/test/test-features.ts | 2 +- .../kmc-package/test/test-package-compiler.ts | 11 +-- 4 files changed, 51 insertions(+), 35 deletions(-) diff --git a/common/web/input-processor/tests/cases/inputProcessorTests.js b/common/web/input-processor/tests/cases/inputProcessorTests.js index 4085efee4f5..939ea36004d 100644 --- a/common/web/input-processor/tests/cases/inputProcessorTests.js +++ b/common/web/input-processor/tests/cases/inputProcessorTests.js @@ -184,7 +184,7 @@ describe('InputProcessor', function() { // rest of the recorder stuff since it uses only KeyboardProcessor, not InputProcessor. let testDefinitions = new KeyboardTest(JSON.parse(testJSONtext)); - before(async function () { + this.before(async function () { // Load the keyboard. let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_8568_deadkeys.js')); diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index 1cabec0ee84..034afcf6fb1 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -56,9 +56,16 @@ describe('LanguageProcessor', function() { }); }); - describe('.predict', async function() { - let compiler = new LexicalModelCompiler(); - assert.isTrue(await compiler.init(callbacks, {})); + describe('.predict', function() { + let compiler = null; + + this.beforeAll(async function() { + compiler = new LexicalModelCompiler(); + console.dir(compiler) + console.dir(compiler.init) + assert.isTrue(await compiler.init(callbacks, {})); + }); + const MODEL_ID = 'example.qaa.trivial'; // ES-module mode leaves out `__dirname`, so we rebuild it using other components. @@ -68,20 +75,23 @@ describe('LanguageProcessor', function() { const PATH = path.join(__dirname, '../../../../../developer/src/kmc-model/test/fixtures', MODEL_ID); describe('using angle brackets for quotes', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - punctuation: { - quotesForKeepSuggestion: { open: `«`, close: `»`}, - insertAfterWord: " " , // OGHAM SPACE MARK - } - }, PATH); - - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + let modelCode = null, modelSpec = null; + this.beforeAll(function() { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + punctuation: { + quotesForKeepSuggestion: { open: `«`, close: `»`}, + insertAfterWord: " " , // OGHAM SPACE MARK + } + }, PATH); + + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); it("successfully loads the model", function(done) { let languageProcessor = new LanguageProcessor(worker); @@ -116,18 +126,21 @@ describe('LanguageProcessor', function() { }); describe('properly cases generated suggestions', function() { - let modelCode = compiler.generateLexicalModelCode(MODEL_ID, { - format: 'trie-1.0', - sources: ['wordlist.tsv'], - languageUsesCasing: true, - //applyCasing // we rely on the compiler's default implementation here. - }, PATH); - - let modelSpec = { - id: MODEL_ID, - languages: ['en'], - code: modelCode - }; + let modelCode = null, modelSpec = null; + this.beforeAll(function () { + modelCode = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + languageUsesCasing: true, + //applyCasing // we rely on the compiler's default implementation here. + }, PATH); + + modelSpec = { + id: MODEL_ID, + languages: ['en'], + code: modelCode + }; + }); describe("does not alter casing when input is lowercased", function() { it("when input is fully lowercased", function(done) { diff --git a/developer/src/kmc-kmn/test/test-features.ts b/developer/src/kmc-kmn/test/test-features.ts index 7f4be109516..a030ba984ca 100644 --- a/developer/src/kmc-kmn/test/test-features.ts +++ b/developer/src/kmc-kmn/test/test-features.ts @@ -5,7 +5,7 @@ import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; import { KMX, KmxFileReader } from '@keymanapp/common-types'; -describe('Keyboard compiler features', async function() { +describe('Keyboard compiler features', function() { let compiler: KmnCompiler = null; let callbacks: TestCompilerCallbacks = null; diff --git a/developer/src/kmc-package/test/test-package-compiler.ts b/developer/src/kmc-package/test/test-package-compiler.ts index 3a17d321eb1..1398befca2b 100644 --- a/developer/src/kmc-package/test/test-package-compiler.ts +++ b/developer/src/kmc-package/test/test-package-compiler.ts @@ -15,15 +15,18 @@ import { CompilerMessages } from '../src/compiler/messages.js'; const debug = false; -describe('KmpCompiler', async function () { +describe('KmpCompiler', function () { const MODELS : string[] = [ 'example.qaa.sencoten', 'withfolders.qaa.sencoten', ]; - const callbacks = new TestCompilerCallbacks(); - let kmpCompiler = new KmpCompiler(); - assert.isTrue(await kmpCompiler.init(callbacks, null)); + let kmpCompiler: KmpCompiler = null; + + this.beforeAll(async function() { + kmpCompiler = new KmpCompiler(); + assert.isTrue(await kmpCompiler.init(callbacks, null)); + }); for (let modelID of MODELS) { const kpsPath = modelID.includes('withfolders') ? From 4f32875a57d90be81c0dcd3dd89156d421260f4b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 08:06:37 +0700 Subject: [PATCH 15/18] chore(web): cleanup tests --- common/web/input-processor/tests/cases/inputProcessorTests.js | 2 +- common/web/input-processor/tests/cases/languageProcessor.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/common/web/input-processor/tests/cases/inputProcessorTests.js b/common/web/input-processor/tests/cases/inputProcessorTests.js index 939ea36004d..4085efee4f5 100644 --- a/common/web/input-processor/tests/cases/inputProcessorTests.js +++ b/common/web/input-processor/tests/cases/inputProcessorTests.js @@ -184,7 +184,7 @@ describe('InputProcessor', function() { // rest of the recorder stuff since it uses only KeyboardProcessor, not InputProcessor. let testDefinitions = new KeyboardTest(JSON.parse(testJSONtext)); - this.before(async function () { + before(async function () { // Load the keyboard. let keyboardLoader = new NodeKeyboardLoader(new KeyboardInterface({}, MinimalKeymanGlobal)); const keyboard = await keyboardLoader.loadKeyboardFromPath(require.resolve('@keymanapp/common-test-resources/keyboards/test_8568_deadkeys.js')); diff --git a/common/web/input-processor/tests/cases/languageProcessor.js b/common/web/input-processor/tests/cases/languageProcessor.js index 034afcf6fb1..15b16da86ad 100644 --- a/common/web/input-processor/tests/cases/languageProcessor.js +++ b/common/web/input-processor/tests/cases/languageProcessor.js @@ -61,8 +61,6 @@ describe('LanguageProcessor', function() { this.beforeAll(async function() { compiler = new LexicalModelCompiler(); - console.dir(compiler) - console.dir(compiler.init) assert.isTrue(await compiler.init(callbacks, {})); }); From 2a737258393a06deb01fdcf41262251ea9aaaad3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 13 Dec 2023 08:59:57 +0700 Subject: [PATCH 16/18] chore(developer): fix duplicated message number --- developer/src/kmc/src/messages/infrastructureMessages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmc/src/messages/infrastructureMessages.ts b/developer/src/kmc/src/messages/infrastructureMessages.ts index dd49272325b..0100bdccf7a 100644 --- a/developer/src/kmc/src/messages/infrastructureMessages.ts +++ b/developer/src/kmc/src/messages/infrastructureMessages.ts @@ -94,6 +94,6 @@ export class InfrastructureMessages { static Error_OutFileCanOnlyBeSpecifiedWithSingleInfile = () => m(this.ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile, `Parameter --out-file can only be used with a single input file.`); - static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0014; + static ERROR_OutFileCanOnlyBeSpecifiedWithSingleInfile = SevError | 0x0015; } From 4981f5f8010bd5d29fadc9dc3e22d7c335690941 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 2 Jan 2024 13:04:02 -0500 Subject: [PATCH 17/18] auto: increment master version to 17.0.238 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 04a57cf7197..7e45b97979f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 17.0.237 alpha 2024-01-02 + +* feat(developer): Consolidate public APIs for kmc modules (#10208) +* chore(developer): honor prompt to upgrade in kmc (#10218) + ## 17.0.236 alpha 2023-12-22 * fix(oem/fv): Add fv_kwadacha_tsekene (#10292) diff --git a/VERSION.md b/VERSION.md index b83abfc3d31..fa685c13a87 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -17.0.237 \ No newline at end of file +17.0.238 \ No newline at end of file From 32ba632af94c020027623f421739b34b33920ac1 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 2 Jan 2024 17:02:35 -0600 Subject: [PATCH 18/18] Update core/tests/unit/ldml/test_transforms.cpp Co-authored-by: Marc Durdin --- core/tests/unit/ldml/test_transforms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index ce2a29ed4e9..430261eb4f8 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -764,8 +764,8 @@ int test_normalize() { // from tests marker_map map; std::cout << __FILE__ << ":" << __LINE__ << " - complex test 9c" << std::endl; - const std::u32string src = U"9ce\u0300\uFFFF\b\u0002\u0320\uFFFF\b\u0001"; - const std::u32string expect = U"9ce\uFFFF\b\u0002\u0320\u0300\uFFFF\b\u0001"; + const std::u32string src = U"9ce\u0300\uFFFF\u0008\u0002\u0320\uFFFF\u0008\u0001"; + const std::u32string expect = U"9ce\uFFFF\u0008\u0002\u0320\u0300\uFFFF\u0008\u0001"; std::u32string dst = src; assert(normalize_nfd_markers(dst, map)); if (dst != expect) {