From 1aee4edaaff223fe072a439ed3fa2c5fd0a73cdf Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 3 Dec 2024 16:49:16 -0600 Subject: [PATCH] feat(developer): VSCode plugin for building - basic build of .kpj, .kps, .kmn, .xml Fixes: #12756 Co-authored-by: Marc Durdin --- developer/src/vscode-plugin/README.md | 9 +- developer/src/vscode-plugin/package.json | 16 ++ developer/src/vscode-plugin/src/extension.ts | 9 + .../vscode-plugin/src/extensionCallbacks.mts | 266 ++++++++++++++++++ developer/src/vscode-plugin/src/kpjBuild.mts | 248 ++++++++++++++++ developer/src/vscode-plugin/src/kpjTasks.ts | 131 +++++++++ 6 files changed, 675 insertions(+), 4 deletions(-) create mode 100644 developer/src/vscode-plugin/src/extensionCallbacks.mts create mode 100644 developer/src/vscode-plugin/src/kpjBuild.mts create mode 100644 developer/src/vscode-plugin/src/kpjTasks.ts diff --git a/developer/src/vscode-plugin/README.md b/developer/src/vscode-plugin/README.md index 4d0ae643b86..dda0abb95b2 100644 --- a/developer/src/vscode-plugin/README.md +++ b/developer/src/vscode-plugin/README.md @@ -1,10 +1,11 @@ # Keyman Developer for VSCode -This package is a standalone [VSCode](https://code.visualstudio.com) plugin. +This package is a standalone [VSCode](https://code.visualstudio.com) plugin which offers: -Its features include: - -- (no features yet) +- a Build Task for building .kpj files into a package +- when building the .kpj file, all .kmn and .xml (LDML keyboard) files will be compiled as well +- The Build Task assumes that the .kpj will have the same name as the directory. So, `.../some_keyboard/some_keyboard.kpj` +- The build task will also build .kps files into packages ## Building diff --git a/developer/src/vscode-plugin/package.json b/developer/src/vscode-plugin/package.json index 26a440223aa..d999dd839e1 100644 --- a/developer/src/vscode-plugin/package.json +++ b/developer/src/vscode-plugin/package.json @@ -15,6 +15,22 @@ "activationEvents": [], "main": "./out/extension.js", "contributes": { + "commands": [ + { + "command": "keyman.compileProject", + "title": "Keyman: Compile Project" + } + ], + "taskDefinitions": [ + { + "type": "kpj", + "required": [ + "task" + ], + "properties": {}, + "when": "" + } + ] }, "scripts": { "vscode:prepublish": "npm run compile", diff --git a/developer/src/vscode-plugin/src/extension.ts b/developer/src/vscode-plugin/src/extension.ts index 5cecb4fe8d2..62510318897 100644 --- a/developer/src/vscode-plugin/src/extension.ts +++ b/developer/src/vscode-plugin/src/extension.ts @@ -3,11 +3,20 @@ */ import * as vscode from 'vscode'; +import { KpjTaskProvider } from './kpjTasks'; + +/** for cleaning up the kpj provider */ +let kpjTaskProvider: vscode.Disposable | undefined; /** called when extension is activated */ export function activate(context: vscode.ExtensionContext) { + // TASK STUFF + kpjTaskProvider = vscode.tasks.registerTaskProvider('kpj', KpjTaskProvider); } /** called when extension is deactivated */ export function deactivate() { + if (kpjTaskProvider) { + kpjTaskProvider.dispose(); + } } diff --git a/developer/src/vscode-plugin/src/extensionCallbacks.mts b/developer/src/vscode-plugin/src/extensionCallbacks.mts new file mode 100644 index 00000000000..99d9ea1d59e --- /dev/null +++ b/developer/src/vscode-plugin/src/extensionCallbacks.mts @@ -0,0 +1,266 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Code related to Extension support for callbacks. + */ + +import { CompilerCallbackOptions, CompilerCallbacks, CompilerError, CompilerEvent, CompilerFileCallbacks, CompilerFileSystemAsyncCallbacks, CompilerFileSystemCallbacks, CompilerNetAsyncCallbacks, CompilerPathCallbacks, FileSystemFolderEntry } from "@keymanapp/developer-utils"; +import * as fs from 'fs'; +import * as path from 'node:path'; + + +interface ErrorWithCode { + code?: string; +} + +function resolveFilename(baseFilename: string, filename: string) { + const basePath = + baseFilename.endsWith('/') || baseFilename.endsWith('\\') ? + baseFilename : + path.dirname(baseFilename); + // Transform separators to platform separators -- we are agnostic + // in our use here but path prefers files may use + // either / or \, although older kps files were always \. + if(path.sep == '/') { + filename = filename.replace(/\\/g, '/'); + } else { + filename = filename.replace(/\//g, '\\'); + } + if(!path.isAbsolute(filename)) { + filename = path.resolve(basePath, filename); + } + return filename; +} + +class NodeCompilerFileSystemAsyncCallbacks implements CompilerFileSystemAsyncCallbacks { + async exists(filename: string): Promise { + return fs.existsSync(filename); + } + + async readFile(filename: string): Promise { + return fs.readFileSync(filename); + } + + async readdir(filename: string): Promise { + return fs.readdirSync(filename).map(item => ({ + filename: item, + type: fs.statSync(path.join(filename, item))?.isDirectory() ? 'dir' : 'file' + })); + } + + resolveFilename(baseFilename: string, filename: string): string { + return resolveFilename(baseFilename, filename); + } +} + +class NodeCompilerNetAsyncCallbacks implements CompilerNetAsyncCallbacks { + async fetchBlob(url: string, options?: RequestInit): Promise { + try { + const response = await fetch(url, options); + if(!response.ok) { + throw new Error(`HTTP error ${response.status}: ${response.statusText}`); + } + + const data = await response.blob(); + return new Uint8Array(await data.arrayBuffer()); + } catch(e) { + throw new Error(`Error downloading ${url}`, {cause:e}); + } + } + + async fetchJSON(url: string, options?: RequestInit): Promise { + try { + const response = await fetch(url, options); + if(!response.ok) { + throw new Error(`HTTP error ${response.status}: ${response.statusText}`); + } + + return await response.json(); + } catch(e) { + throw new Error(`Error downloading ${url}`, {cause:e}); + } + } +} + + +export class ExtensionCallbacks implements CompilerCallbacks { + /* from NodeCompilerCallbacks */ + + messages: CompilerEvent[] = []; + messageCount = 0; + messageFilename: string = ''; + + constructor(private options: CompilerCallbackOptions, private msg: (m: string)=>void) { + } + resolveFilename(baseFilename: string, filename: string): string { + return resolveFilename(baseFilename, filename); + } + isDirectory(filename: string): boolean { + return fs.statSync(filename)?.isDirectory(); + } + get fsAsync(): CompilerFileSystemAsyncCallbacks { + return new NodeCompilerFileSystemAsyncCallbacks(); + } + get net(): CompilerNetAsyncCallbacks { + return new NodeCompilerNetAsyncCallbacks();; + } + fileURLToPath(url: string | URL): string { + throw new Error("Method not implemented."); + } + + clear() { + this.messages = []; + this.messageCount = 0; + this.messageFilename = ''; + } + + /** + * Returns true if any message in the log is a Fatal, Error, or if we are + * treating warnings as errors, a Warning. The warning option will be taken + * from the CompilerOptions passed to the constructor, or the parameter, to + * allow for per-file overrides (as seen with projects, for example). + * @param compilerWarningsAsErrors + * @returns + */ + hasFailureMessage(compilerWarningsAsErrors?: boolean): boolean { + return CompilerFileCallbacks.hasFailureMessage( + this.messages, + // parameter overrides global option + false, // compilerWarningsAsErrors ?? this.options.compilerWarningsAsErrors + ); + } + + hasMessage(code: number): boolean { + return this.messages.find((item) => item.code == code) === undefined ? false : true; + } + + + /* CompilerCallbacks */ + + loadFile(filename: string): Uint8Array { + // this.verifyFilenameConsistency(filename); + try { + return fs.readFileSync(filename); + } catch (e) { + const { code } = e as ErrorWithCode; + if (code === 'ENOENT') { + // code smell… + return (null as unknown) as Uint8Array; + } else { + throw e; + } + } + } + + fileSize(filename: string): number { + return fs.statSync(filename)?.size; + } + + get path(): CompilerPathCallbacks { + return path; + } + + get fs(): CompilerFileSystemCallbacks { + return (fs as unknown) as CompilerFileSystemCallbacks; + } + + reportMessage(event: CompilerEvent): void { + if(!event.filename) { + event.filename = this.messageFilename; + } + + if(this.messageFilename != event.filename) { + // Reset max message limit when a new file is being processed + this.messageFilename = event.filename; + this.messageCount = 0; + } + + + this.messages.push({...event}); + + // // report fatal errors to Sentry, but don't abort; note, it won't be + // // reported if user has disabled the Sentry setting + // if(CompilerError.severity(event.code) == CompilerErrorSeverity.Fatal) { + // // this is async so returns a Promise, we'll let it resolve in its own + // // time, and it will emit a message to stderr with details at that time + // KeymanSentry.reportException(event.exceptionVar ?? event.message, false); + // } + + // if(disable || CompilerError.severity(event.code) < compilerLogLevelToSeverity[this.options.logLevel]) { + // // collect messages but don't print to console + // return; + // } + + // We don't use this.messages.length because we only want to count visible + // messages, and there's no point in recalculating the total for every + // message emitted. + + this.messageCount++; + // if(this.messageCount > 99/*MaxMessagesDefault*/) { + // return; + // } + + // if(this.messageCount == 99/*MaxMessagesDefault*/) { + // // We've hit our event limit so we'll suppress further messages, and emit + // // our little informational message so users know what's going on. Note + // // that this message will not be included in the this.messages array, and + // // that will continue to collect all messages; this only affects the + // // console emission of messages. + // event = InfrastructureMessages.Info_TooManyMessages({count: MaxMessagesDefault}); + // event.filename = this.messageFilename; + // } + + this.printMessage(event); + } + + private printMessage(event: CompilerEvent) { + if(this.options.logFormat == 'tsv') { + this.printTsvMessage(event); + } else { + this.printFormattedMessage(event); + } + } + + private printTsvMessage(event: CompilerEvent) { + this.msg([ + CompilerError.formatFilename(event.filename || '', {fullPath:true, forwardSlashes:false}), + CompilerError.formatLine(event.line || -1), + CompilerError.formatSeverity(event.code), + CompilerError.formatCode(event.code), + CompilerError.formatMessage(event.message) + ].join('\t') + '\n'); + } + + private printFormattedMessage(event: CompilerEvent) { + // const severityColor = severityColors[CompilerError.severity(event.code)] ?? color.reset; + // const messageColor = this.messageSpecialColor(event) ?? color.reset; + this.msg( + ( + event.filename + ? CompilerError.formatFilename(event.filename) + + (event.line ? ':' + CompilerError.formatLine(event.line) : '') + ' - ' + : '' + ) + + CompilerError.formatSeverity(event.code) + ' ' + + CompilerError.formatCode(event.code) + ': ' + + CompilerError.formatMessage(event.message) + '\r\n' + ); + + // if(event.code == InfrastructureMessages.INFO_ProjectBuiltSuccessfully) { + // // Special case: we'll add a blank line after project builds + // process.stdout.write('\n'); + // } + } + + debug(msg: string) { + if(this.options.logLevel == 'debug') { + console.debug(msg); + } + } + + fileExists(filename: string) { + return fs.existsSync(filename); + } + + + } diff --git a/developer/src/vscode-plugin/src/kpjBuild.mts b/developer/src/vscode-plugin/src/kpjBuild.mts new file mode 100644 index 00000000000..5cc4ab0107e --- /dev/null +++ b/developer/src/vscode-plugin/src/kpjBuild.mts @@ -0,0 +1,248 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + */ + + +/** + * Code related to building KPJ projects + */ + +/** + * TODO Some of this code may look very familiar and should be refactored from kmc ! + */ + +import { platform } from "node:os"; +import { extname, dirname, resolve } from "node:path"; +import { mkdir } from "node:fs/promises"; +import { KeymanFileTypes } from "@keymanapp/common-types"; +import { KPJFileReader, CompilerCallbackOptions, CompilerOptions, LDMLKeyboardXMLSourceFileReader } from "@keymanapp/developer-utils"; +import { ExtensionCallbacks } from "./extensionCallbacks.mjs"; +import * as kmcLdml from '@keymanapp/kmc-ldml'; +import * as kmcKmn from '@keymanapp/kmc-kmn'; +import { fileURLToPath } from 'url'; +import { KmpCompiler, KmpCompilerOptions } from "@keymanapp/kmc-package"; + + +/** + * Ensure that the parent dirs of filePath exist + * @param filePath path to some file + */ +async function mkParentDirs(filePath: string) { + const dir = dirname(filePath); + await mkdir(dir, { recursive: true }); +} + +/** + * Build a whole .kpj + * @param workspaceRoot root dir for work + * @param kpjPath path to the .kpj + * @param msg callback for writing messages + * @returns accept on OK, otherwise throws + */ +export async function buildProject(workspaceRoot: string, + kpjPath: string, msg: (m: string)=>void): Promise { + + + function getPosixAbsolutePath(filename: string): string { + if (platform() == 'win32') { + // On Win32, we need to use backslashes for path.resolve to work + filename = filename.replace(/\//g, '\\'); + } + const path = { resolve }; + // Resolve to a fully qualified absolute path relative to workspaceRoot + filename = path.resolve(workspaceRoot, filename); + + if (platform() == 'win32') { + // Ensure that we convert the result back to posix-style paths which is what + // kmc-kmn expects. On posix platforms, we assume paths have forward slashes + // already + filename = filename.replace(/\\/g, '/'); + } + return filename; + } + + const callbacks = new ExtensionCallbacks({}, msg); + + // const outfile = ''; + const coptions : CompilerCallbackOptions = { + }; + const options : CompilerOptions = { + }; + // const callbacks = new NodeCompilerCallbacks(coptions); + + // const resp = await (new BuildProject().build( + // infile, + // undefined, + // callbacks, + // options + // )); + + // // dump all + // callbacks.messages?.forEach(m => console.dir(m)); + // const resp = false; + + // return resp; + + msg(`Keyman Vancouver: Begin build of ${kpjPath}…\r\n`); + + const reader = new KPJFileReader(callbacks); + + const prj = reader.read(callbacks.fs.readFileSync(kpjPath)); + + if (!prj) { + msg(`Could not load ${kpjPath}\r\n`); + return; + } + try { + reader.validate(prj); + } catch(e) { + console.error(e); + msg(`Error validating ${kpjPath}\r\n`); + } + + // we don't need to see it. + // msg(`PRJ loaded: ${JSON.stringify(prj, null, ' ')}\r\n`); + + // this next line is important - we need the full (?) project + // otherwise we get an empty shell + const project = await reader.transform(kpjPath, prj); + // msg(`project loaded: ${JSON.stringify(project, null, ' ')}\r\n`); + + let didCompileSrc = false; + let didCompilePkg = false; + + for (const path of project.files.filter(({ filePath }) => extname(filePath) === KeymanFileTypes.Source.LdmlKeyboard)) { + const { filePath } = path; + msg(`Compiling LDML: ${filePath}\r\n`); + + const ldmlCompilerOptions: kmcLdml.LdmlCompilerOptions = { + ...options, readerOptions: { + importsPath: fileURLToPath(new URL(...LDMLKeyboardXMLSourceFileReader.defaultImportsURL)) + } + }; + const compiler = new kmcLdml.LdmlKeyboardCompiler(); + if (!await compiler.init(callbacks, ldmlCompilerOptions)) { + msg(`Compiler failed init\r\n`); + continue; + } + + const outfile = project.resolveOutputFilePath(path, KeymanFileTypes.Source.LdmlKeyboard, KeymanFileTypes.Binary.Keyboard); + msg(`.. outfile is ${outfile}\r\n`); + await mkParentDirs(outfile); + const result = await compiler.run(filePath, outfile); + if (!result) { + msg(`Compiler failed to run\r\n`); + continue; + } + msg(`.. compiled\r\n`); + // if(!this.createOutputFolder(outfile ?? infile, callbacks)) { + // return false; + // } + + if (!await compiler.write(result.artifacts)) { + msg(`Error writing ${outfile}\r\n`); + throw Error(`Error writing ${outfile}`); + } + + msg(`.. wrote\r\n`); + + msg(`\r\n\r\n`); + didCompileSrc = true; // we allow more than one xmk in each package + } + + // now, compile any .kmn + for (const path of project.files.filter(({ filePath }) => extname(filePath) === KeymanFileTypes.Source.KeymanKeyboard)) { + const { filePath } = path; + msg(`Compiling KMN: ${filePath}\r\n`); + + const compiler = new kmcKmn.KmnCompiler(); + if (!await compiler.init(callbacks, coptions)) { + msg(`Compiler failed init\r\n`); + continue; + } + + const outfile = project.resolveOutputFilePath(path, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.Keyboard); + msg(`.. outfile is ${outfile}\r\n`); + const infilePosix = getPosixAbsolutePath(filePath); + const outfilePosix = getPosixAbsolutePath(outfile); + await mkParentDirs(outfilePosix); + msg(`${infilePosix} => ${outfilePosix}\r\n`); + const result = await compiler.run(infilePosix, outfilePosix); + if (!result) { + msg(`Compiler failed to run\r\n`); + continue; + } + msg(`.. compiled\r\n`); + // if(!this.createOutputFolder(outfile ?? infile, callbacks)) { + // return false; + // } + + if (!await compiler.write(result.artifacts)) { + msg(`Error writing ${outfile}\r\n`); + throw Error(`Error writing ${outfile}`); + } + + msg(`.. wrote\r\n`); + + msg(`\r\n\r\n`); + didCompileSrc = true; // we allow more than one xmk in each package + } + + // check errs and get out + + if(callbacks.hasFailureMessage(false)) { + throw Error(`Error building ${kpjPath}`); + } + + if (!didCompileSrc) { + throw Error(`Error: no source files were compiled.`); + } + + + // now, any packaging + for (const path of project.files.filter(({ filePath }) => extname(filePath) === KeymanFileTypes.Source.Package)) { + if (didCompilePkg) { + throw Error(`Error: two packages were encountered.`); + } + + const { filePath } = path; + const kmpCompilerOptions: KmpCompilerOptions = { + ...options + }; + const outfile = project.resolveOutputFilePath(path, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package); + const infilePosix = getPosixAbsolutePath(filePath); + const outfilePosix = getPosixAbsolutePath(outfile); + await mkParentDirs(outfilePosix); + msg(`Packaging: ${filePath} into ${outfile}\r\n`); + + const compiler = new KmpCompiler(); + if (!await compiler.init(callbacks, kmpCompilerOptions)) { + msg(`Compiler failed init\r\n`); + continue; + } + + const result = await compiler.run(infilePosix, outfilePosix); + if (!result) { + msg(`Compiler failed to run\r\n`); + continue; + } + msg(`.. compiled\r\n`); + + if (!await compiler.write(result.artifacts)) { + msg(`Error writing ${outfile}\r\n`); + throw Error(`Error writing ${outfile}`); + } + + msg(`.. wrote\r\n`); + + msg(`\r\n\r\n`); + didCompilePkg = true; + } + + if (!didCompilePkg) { + throw Error(`Error: no packages were compiled.`); + } + + msg(`All done.\r\n`); + return; +} diff --git a/developer/src/vscode-plugin/src/kpjTasks.ts b/developer/src/vscode-plugin/src/kpjTasks.ts new file mode 100644 index 00000000000..ede41802fb5 --- /dev/null +++ b/developer/src/vscode-plugin/src/kpjTasks.ts @@ -0,0 +1,131 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Code related to Tasks that build KPJ projects + */ + +const path = require('node:path'); // node:path +const { existsSync } = require('node:fs'); +import * as vscode from 'vscode'; + +/** promise to list of tasks */ +let kpjPromise: Promise | undefined = undefined; + +/** Task for kpj */ +interface KpjTaskDefinition extends vscode.TaskDefinition { + // empty for now +} + +class KpjBuildTerminal implements vscode.Pseudoterminal { + private writeEmitter = new vscode.EventEmitter(); + onDidWrite: vscode.Event = this.writeEmitter.event; + private closeEmitter = new vscode.EventEmitter(); + onDidClose?: vscode.Event = this.closeEmitter.event; + + private fileWatcher: vscode.FileSystemWatcher | undefined; + + constructor(private workspaceRoot: string, private kpjPath: string, private flags: string[]) { + } + + open(initialDimensions: vscode.TerminalDimensions | undefined): void { + // At this point we can start using the terminal. + if (this.flags.indexOf('watch') > -1) { + const pattern = path.join(this.workspaceRoot, 'customBuildFile'); + this.fileWatcher = vscode.workspace.createFileSystemWatcher(pattern); + this.fileWatcher.onDidChange(() => this.doBuild()); + this.fileWatcher.onDidCreate(() => this.doBuild()); + this.fileWatcher.onDidDelete(() => this.doBuild()); + } + this.doBuild(); + } + + + close(): void { + // The terminal has been closed. Shutdown the build. + if (this.fileWatcher) { + this.fileWatcher.dispose(); + } + } + + private async doBuild(): Promise { + this.writeEmitter.fire(`Starting build of ${this.kpjPath}...\r\n`); + // esm so we can access keyman + const { buildProject } = await import('./kpjBuild.mjs'); + try { + await buildProject(this.workspaceRoot, this.kpjPath, this.writeEmitter.fire.bind(this.writeEmitter)); + this.closeEmitter.fire(0); + } catch (e) { + this.writeEmitter.fire(`Failure: ${e}\r\n\r\n`); + this.closeEmitter.fire(1); + } + // TODO: get/set sharedState + } +} + + +async function getKpjTasks(): Promise { + const workspaceFolders = vscode.workspace.workspaceFolders; + const result: vscode.Task[] = []; + if (!workspaceFolders || workspaceFolders.length === 0) { + console.log('Keyman: No workspace folders'); + return result; + } + for (const workspaceFolder of workspaceFolders) { + const folderString = workspaceFolder.uri.fsPath; + if (!folderString) { + console.log('Keyman: No folderString'); + continue; + } + const dir = path.basename(folderString); + const kpjFile = path.join(folderString, `${dir}.kpj`); + if (!existsSync(kpjFile)) { + console.log(`Keyman: No kpj file ${kpjFile}`); + continue; + } else { + console.log(`Keyman: Found kpj => ${kpjFile}`); + } + const task = new vscode.Task({ type: 'kpj' }, workspaceFolder, dir, 'kpj', + new vscode.CustomExecution(async () => new KpjBuildTerminal(folderString, kpjFile, [])) + // new vscode.ShellExecution(`npx -y @keymanapp/kmc build file ${kpjFile}`) + ); + task.group = vscode.TaskGroup.Build; + result.push(task); + } + return result; +} + +export const KpjTaskProvider = { + + // TODO should be TaskProvider subclass? + + provideTasks() { + kpjPromise = kpjPromise ?? getKpjTasks(); + kpjPromise.catch(e => { + console.error(e); + // print something + vscode.window.showErrorMessage(`Keyman: Error getting tasks: ${e}`); + }); + return kpjPromise; + }, + resolveTask(_task: vscode.Task): vscode.Task | undefined { + const task = _task.definition.task; + if (task) { + const definition: KpjTaskDefinition = _task.definition; + return new vscode.Task( + definition, + _task.scope ?? vscode.TaskScope.Workspace, + definition.type, // for now, we don't have another name to use + 'kpj', + new vscode.CustomExecution( + // TODO: Hmm. This doesn't seem to be used?? + async (): Promise => { + return new KpjBuildTerminal("something", "something/something.kpj", []); + } + ) + // OLD: shell + // new vscode.ShellExecution(`npx -y @keymanapp/kmc build`) // nothing from the definition for now + ); + } + return undefined; + } +};