From 4ec3f638fef6a4fdf0c02e754e871663cef5546f Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 29 Oct 2024 16:54:56 +0100 Subject: [PATCH 1/2] Support discarding the CST to preserve memory --- examples/requirements/test/validator.test.ts | 36 +++++----- packages/langium/src/default-module.ts | 4 +- .../src/lsp/call-hierarchy-provider.ts | 4 +- .../langium/src/lsp/definition-provider.ts | 30 +++++---- .../src/lsp/document-update-handler.ts | 26 +++++-- packages/langium/src/lsp/language-server.ts | 1 + packages/langium/src/parser/async-parser.ts | 16 ++--- .../langium/src/parser/cst-node-builder.ts | 4 +- packages/langium/src/parser/langium-parser.ts | 61 ++++++++++------- .../langium/src/references/name-provider.ts | 15 ++++- packages/langium/src/references/references.ts | 29 ++++---- .../langium/src/serializer/json-serializer.ts | 2 +- packages/langium/src/services.ts | 2 + packages/langium/src/syntax-tree.ts | 6 ++ packages/langium/src/utils/cst-utils.ts | 12 +++- .../src/validation/document-validator.ts | 3 + .../langium/src/workspace/ast-descriptions.ts | 19 +++--- packages/langium/src/workspace/documents.ts | 67 ++++++++++--------- packages/langium/src/workspace/environment.ts | 51 ++++++++++++++ .../src/workspace/workspace-manager.ts | 12 +++- .../parser/worker-thread-async-parser.test.ts | 12 ++-- packages/langium/test/parser/worker-thread.js | 4 +- 22 files changed, 278 insertions(+), 138 deletions(-) create mode 100644 packages/langium/src/workspace/environment.ts diff --git a/examples/requirements/test/validator.test.ts b/examples/requirements/test/validator.test.ts index 7b32aa7c6..e523bdb6a 100644 --- a/examples/requirements/test/validator.test.ts +++ b/examples/requirements/test/validator.test.ts @@ -13,13 +13,13 @@ import { NodeFileSystem } from 'langium/node'; describe('A requirement identifier and a test identifier shall contain a number.', () => { test('T001_good_case', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [mainDoc,allDocs] = await extractDocuments( + const [mainDoc, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'good', 'requirements.req'), services.requirements ); expect((mainDoc.diagnostics ?? [])).toEqual([]); expect(allDocs.length).toEqual(3); - allDocs.forEach(doc=>{ + allDocs.forEach(doc => { expect((doc.diagnostics ?? [])).toEqual([]); }); }); @@ -35,7 +35,7 @@ describe('A requirement identifier shall contain a number.', () => { expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Requirement name ReqIdABC_reqID should container a number'), - range: expect.objectContaining({start:expect.objectContaining({line: 2})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 2 }) }) // zero based }) ])); @@ -45,17 +45,17 @@ describe('A requirement identifier shall contain a number.', () => { describe('A test identifier shall contain a number.', () => { test('T003_badTstId: bad case', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [,allDocs] = await extractDocuments( + const [, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'bad1', 'requirements.req'), services.requirements ); - const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath)); + const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath)); expect(doc).toBeDefined(); if (!doc) throw new Error('impossible'); expect(doc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test name TA should container a number.'), - range: expect.objectContaining({start:expect.objectContaining({line: 1})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 1 }) }) // zero based }) ])); }); @@ -71,7 +71,7 @@ describe('A requirement shall be covered by at least one test.', () => { expect(mainDoc.diagnostics ?? []).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Requirement ReqId004_unicorn not covered by a test.'), - range: expect.objectContaining({start:expect.objectContaining({line: 4})}) // zero based + range: expect.objectContaining({ start: expect.objectContaining({ line: 4 }) }) // zero based }) ])); }); @@ -80,28 +80,32 @@ describe('A requirement shall be covered by at least one test.', () => { describe('A referenced environment in a test must be found in one of the referenced requirements.', () => { test('referenced environment test', async () => { const services = createRequirementsAndTestsLangServices(NodeFileSystem); - const [,allDocs] = await extractDocuments( + const [, allDocs] = await extractDocuments( path.join(__dirname, 'files', 'bad2', 'requirements.req'), services.requirements ); - const doc = allDocs.find(doc=>/tests_part1.tst/.test(doc.uri.fsPath)); + const doc = allDocs.find(doc => /tests_part1.tst/.test(doc.uri.fsPath)); expect(doc).toBeDefined(); if (!doc) throw new Error('impossible'); expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test T002_badReqId references environment Linux_x86 which is used in any referenced requirement.'), - range: expect.objectContaining({start:expect.objectContaining({ - line: 3, - character: 65 - })}) // zero based + range: expect.objectContaining({ + start: expect.objectContaining({ + line: 3, + character: 65 + }) + }) // zero based }) ])); expect((doc.diagnostics ?? [])).toEqual(expect.arrayContaining([ expect.objectContaining({ message: expect.stringMatching('Test T004_cov references environment Linux_x86 which is used in any referenced requirement.'), - range: expect.objectContaining({start:expect.objectContaining({ - line: 5 - })}) // zero based + range: expect.objectContaining({ + start: expect.objectContaining({ + line: 5 + }) + }) // zero based }) ])); diff --git a/packages/langium/src/default-module.ts b/packages/langium/src/default-module.ts index d3aab8fc4..1d3e4cdf0 100644 --- a/packages/langium/src/default-module.ts +++ b/packages/langium/src/default-module.ts @@ -35,6 +35,7 @@ import { LangiumParserErrorMessageProvider } from './parser/langium-parser.js'; import { DefaultAsyncParser } from './parser/async-parser.js'; import { DefaultWorkspaceLock } from './workspace/workspace-lock.js'; import { DefaultHydrator } from './serializer/hydrator.js'; +import { DefaultEnvironment } from './workspace/environment.js'; /** * Context required for creating the default language-specific dependency injection module. @@ -116,7 +117,8 @@ export function createDefaultSharedCoreModule(context: DefaultSharedCoreModuleCo WorkspaceManager: (services) => new DefaultWorkspaceManager(services), FileSystemProvider: (services) => context.fileSystemProvider(services), WorkspaceLock: () => new DefaultWorkspaceLock(), - ConfigurationProvider: (services) => new DefaultConfigurationProvider(services) + ConfigurationProvider: (services) => new DefaultConfigurationProvider(services), + Environment: () => new DefaultEnvironment() } }; } diff --git a/packages/langium/src/lsp/call-hierarchy-provider.ts b/packages/langium/src/lsp/call-hierarchy-provider.ts index 3c509ee43..ef75d65ff 100644 --- a/packages/langium/src/lsp/call-hierarchy-provider.ts +++ b/packages/langium/src/lsp/call-hierarchy-provider.ts @@ -54,12 +54,12 @@ export abstract class AbstractCallHierarchyProvider implements CallHierarchyProv return undefined; } - const declarationNode = this.references.findDeclarationNode(targetNode); + const declarationNode = this.references.findDeclaration(targetNode); if (!declarationNode) { return undefined; } - return this.getCallHierarchyItems(declarationNode.astNode, document); + return this.getCallHierarchyItems(declarationNode, document); } protected getCallHierarchyItems(targetNode: AstNode, document: LangiumDocument): CallHierarchyItem[] | undefined { diff --git a/packages/langium/src/lsp/definition-provider.ts b/packages/langium/src/lsp/definition-provider.ts index 603bc0ffd..800771fa3 100644 --- a/packages/langium/src/lsp/definition-provider.ts +++ b/packages/langium/src/lsp/definition-provider.ts @@ -10,7 +10,7 @@ import type { GrammarConfig } from '../languages/grammar-config.js'; import type { NameProvider } from '../references/name-provider.js'; import type { References } from '../references/references.js'; import type { LangiumServices } from './lsp-services.js'; -import type { CstNode } from '../syntax-tree.js'; +import type { AstNode, CstNode } from '../syntax-tree.js'; import type { MaybePromise } from '../utils/promise-utils.js'; import type { LangiumDocument } from '../workspace/documents.js'; import { LocationLink } from 'vscode-languageserver'; @@ -37,7 +37,7 @@ export interface DefinitionProvider { export interface GoToLink { source: CstNode - target: CstNode + target: AstNode targetDocument: LangiumDocument } @@ -67,21 +67,27 @@ export class DefaultDefinitionProvider implements DefinitionProvider { protected collectLocationLinks(sourceCstNode: CstNode, _params: DefinitionParams): MaybePromise { const goToLink = this.findLink(sourceCstNode); - if (goToLink) { - return [LocationLink.create( - goToLink.targetDocument.textDocument.uri, - (goToLink.target.astNode.$cstNode ?? goToLink.target).range, - goToLink.target.range, - goToLink.source.range - )]; + if (goToLink && goToLink.target.$segments) { + const name = this.nameProvider.getNameProperty(goToLink.target); + if (name) { + const nameSegment = goToLink.target.$segments.properties[name]?.[0]; + if (nameSegment) { + return [LocationLink.create( + goToLink.targetDocument.textDocument.uri, + goToLink.target.$segments.full.range, + nameSegment.range, + goToLink.source.range + )]; + } + } } return undefined; } protected findLink(source: CstNode): GoToLink | undefined { - const target = this.references.findDeclarationNode(source); - if (target?.astNode) { - const targetDocument = getDocument(target.astNode); + const target = this.references.findDeclaration(source); + if (target) { + const targetDocument = getDocument(target); if (target && targetDocument) { return { source, target, targetDocument }; } diff --git a/packages/langium/src/lsp/document-update-handler.ts b/packages/langium/src/lsp/document-update-handler.ts index e158cc9f0..d9ad4b9cd 100644 --- a/packages/langium/src/lsp/document-update-handler.ts +++ b/packages/langium/src/lsp/document-update-handler.ts @@ -4,17 +4,18 @@ * terms of the MIT License, which is available in the project root. ******************************************************************************/ -import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit } from 'vscode-languageserver'; +import type { TextDocumentWillSaveEvent, DidChangeWatchedFilesParams, DidChangeWatchedFilesRegistrationOptions, TextDocumentChangeEvent, TextEdit, Connection } from 'vscode-languageserver'; import { DidChangeWatchedFilesNotification, FileChangeType } from 'vscode-languageserver'; import { stream } from '../utils/stream.js'; import { URI } from '../utils/uri-utils.js'; import type { DocumentBuilder } from '../workspace/document-builder.js'; -import type { TextDocument } from '../workspace/documents.js'; +import type { LangiumDocuments, TextDocument } from '../workspace/documents.js'; import type { WorkspaceLock } from '../workspace/workspace-lock.js'; import type { LangiumSharedServices } from './lsp-services.js'; import type { WorkspaceManager } from '../workspace/workspace-manager.js'; import type { ServiceRegistry } from '../service-registry.js'; import type { MaybePromise } from '../utils/promise-utils.js'; +import { discardCst } from '../utils/cst-utils.js'; /** * Shared service for handling text document changes and watching relevant files. @@ -71,6 +72,8 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { protected readonly workspaceManager: WorkspaceManager; protected readonly documentBuilder: DocumentBuilder; protected readonly workspaceLock: WorkspaceLock; + protected readonly documents: LangiumDocuments; + protected readonly connection: Connection | undefined; protected readonly serviceRegistry: ServiceRegistry; constructor(services: LangiumSharedServices) { @@ -78,6 +81,8 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { this.documentBuilder = services.workspace.DocumentBuilder; this.workspaceLock = services.workspace.WorkspaceLock; this.serviceRegistry = services.ServiceRegistry; + this.documents = services.workspace.LangiumDocuments; + this.connection = services.lsp.Connection; let canRegisterFileWatcher = false; services.lsp.LanguageServer.onInitialize(params => { @@ -98,7 +103,6 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { .distinct() .toArray(); if (fileExtensions.length > 0) { - const connection = services.lsp.Connection; const options: DidChangeWatchedFilesRegistrationOptions = { watchers: [{ globPattern: fileExtensions.length === 1 @@ -106,7 +110,7 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { : `**/*.{${fileExtensions.join(',')}}` }] }; - connection?.client.register(DidChangeWatchedFilesNotification.type, options); + this.connection?.client.register(DidChangeWatchedFilesNotification.type, options); } } @@ -141,4 +145,18 @@ export class DefaultDocumentUpdateHandler implements DocumentUpdateHandler { .toArray(); this.fireDocumentUpdate(changedUris, deletedUris); } + + didCloseDocument(event: TextDocumentChangeEvent): void { + const document = this.documents.getDocument(URI.parse(event.document.uri)); + if (document) { + // Preserve memory by discarding the CST of the document + // Whenever the user reopens the document, the CST will be rebuilt + discardCst(document.parseResult.value); + } + // Discard the diagnostics for the closed document + this.connection?.sendDiagnostics({ + uri: event.document.uri, + diagnostics: [] + }); + } } diff --git a/packages/langium/src/lsp/language-server.ts b/packages/langium/src/lsp/language-server.ts index 76cee87ab..3cc5af199 100644 --- a/packages/langium/src/lsp/language-server.ts +++ b/packages/langium/src/lsp/language-server.ts @@ -198,6 +198,7 @@ export class DefaultLanguageServer implements LanguageServer { } protected fireInitializeOnDefaultServices(params: InitializeParams): void { + this.services.workspace.Environment.initialize(params); this.services.workspace.ConfigurationProvider.initialize(params); this.services.workspace.WorkspaceManager.initialize(params); } diff --git a/packages/langium/src/parser/async-parser.ts b/packages/langium/src/parser/async-parser.ts index 0473aa9b0..533b7faaf 100644 --- a/packages/langium/src/parser/async-parser.ts +++ b/packages/langium/src/parser/async-parser.ts @@ -7,7 +7,7 @@ import type { CancellationToken } from '../utils/cancellation.js'; import type { LangiumCoreServices } from '../services.js'; import type { AstNode } from '../syntax-tree.js'; -import type { LangiumParser, ParseResult } from './langium-parser.js'; +import type { LangiumParser, ParseResult, ParserOptions } from './langium-parser.js'; import type { Hydrator } from '../serializer/hydrator.js'; import type { Event } from '../utils/event.js'; import { Deferred, OperationCancelled } from '../utils/promise-utils.js'; @@ -30,7 +30,7 @@ export interface AsyncParser { * * @throws `OperationCancelled` if the parsing process is cancelled. */ - parse(text: string, cancelToken: CancellationToken): Promise>; + parse(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise>; } /** @@ -47,8 +47,8 @@ export class DefaultAsyncParser implements AsyncParser { this.syncParser = services.parser.LangiumParser; } - parse(text: string, _cancelToken: CancellationToken): Promise> { - return Promise.resolve(this.syncParser.parse(text)); + parse(text: string, options: ParserOptions | undefined, _cancelToken: CancellationToken): Promise> { + return Promise.resolve(this.syncParser.parse(text, options)); } } @@ -89,7 +89,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser { } } - async parse(text: string, cancelToken: CancellationToken): Promise> { + async parse(text: string, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise> { const worker = await this.acquireParserWorker(cancelToken); const deferred = new Deferred>(); let timeout: NodeJS.Timeout | undefined; @@ -101,7 +101,7 @@ export abstract class AbstractThreadedAsyncParser implements AsyncParser { this.terminateWorker(worker); }, this.terminationDelay); }); - worker.parse(text).then(result => { + worker.parse(text, options).then(result => { const hydrated = this.hydrator.hydrate(result); deferred.resolve(hydrated); }).catch(err => { @@ -194,13 +194,13 @@ export class ParserWorker { this.onReadyEmitter.fire(); } - parse(text: string): Promise { + parse(text: string, options: ParserOptions | undefined): Promise { if (this._parsing) { throw new Error('Parser worker is busy'); } this._parsing = true; this.deferred = new Deferred(); - this.sendMessage(text); + this.sendMessage([text, options]); return this.deferred.promise; } } diff --git a/packages/langium/src/parser/cst-node-builder.ts b/packages/langium/src/parser/cst-node-builder.ts index 0cc3d42b6..0e03811d6 100644 --- a/packages/langium/src/parser/cst-node-builder.ts +++ b/packages/langium/src/parser/cst-node-builder.ts @@ -55,20 +55,20 @@ export class CstNodeBuilder { } } - construct(item: { $type: string | symbol | undefined, $cstNode: CstNode }): void { + construct(item: { $type: string | symbol | undefined }): CstNode { const current: CstNode = this.current; // The specified item could be a datatype ($type is symbol) or a fragment ($type is undefined) // Only if the $type is a string, we actually assign the element if (typeof item.$type === 'string') { this.current.astNode = item; } - item.$cstNode = current; const node = this.nodeStack.pop(); // Empty composite nodes are not valid // Simply remove the node from the tree if (node?.content.length === 0) { this.removeNode(node); } + return current; } addHiddenTokens(hiddenTokens: IToken[]): void { diff --git a/packages/langium/src/parser/langium-parser.ts b/packages/langium/src/parser/langium-parser.ts index ba630176e..0f65bf873 100644 --- a/packages/langium/src/parser/langium-parser.ts +++ b/packages/langium/src/parser/langium-parser.ts @@ -20,6 +20,7 @@ import { getExplicitRuleType, isDataTypeRule } from '../utils/grammar-utils.js'; import { assignMandatoryProperties, getContainerOfType, linkContentToContainer } from '../utils/ast-utils.js'; import { CstNodeBuilder } from './cst-node-builder.js'; import type { LexingReport } from './token-builder.js'; +import { toDocumentSegment } from '../utils/cst-utils.js'; export type ParseResult = { value: T, @@ -99,10 +100,6 @@ export interface BaseParser { * Executes a grammar action that modifies the currently active AST node */ action($type: string, action: Action): void; - /** - * Finishes construction of the current AST node. Only used by the AST parser. - */ - construct(): unknown; /** * Whether the parser is currently actually in use or in "recording mode". * Recording mode is activated once when the parser is analyzing itself. @@ -163,7 +160,6 @@ export abstract class AbstractLangiumParser implements BaseParser { abstract consume(idx: number, tokenType: TokenType, feature: AbstractElement): void; abstract subrule(idx: number, rule: RuleResult, feature: AbstractElement, args: Args): void; abstract action($type: string, action: Action): void; - abstract construct(): unknown; getRule(name: string): RuleResult | undefined { return this.allRules.get(name); @@ -186,8 +182,14 @@ export abstract class AbstractLangiumParser implements BaseParser { } } +export enum CstParserMode { + Retain, + Discard +} + export interface ParserOptions { - rule?: string + rule?: string; + cst?: CstParserMode; } export class LangiumParser extends AbstractLangiumParser { @@ -197,6 +199,7 @@ export class LangiumParser extends AbstractLangiumParser { private readonly nodeBuilder = new CstNodeBuilder(); private stack: any[] = []; private assignmentMap = new Map(); + private currentMode: CstParserMode = CstParserMode.Retain; private get current(): any { return this.stack[this.stack.length - 1]; @@ -231,6 +234,7 @@ export class LangiumParser extends AbstractLangiumParser { } parse(input: string, options: ParserOptions = {}): ParseResult { + this.currentMode = options.cst ?? CstParserMode.Retain; this.nodeBuilder.buildRootNode(input); const lexerResult = this.lexer.tokenize(input); this.wrapper.input = lexerResult.tokens; @@ -239,7 +243,9 @@ export class LangiumParser extends AbstractLangiumParser { throw new Error(options.rule ? `No rule found with name '${options.rule}'` : 'No main rule available.'); } const result = ruleMethod.call(this.wrapper, {}); - this.nodeBuilder.addHiddenTokens(lexerResult.hidden); + if (this.currentMode === CstParserMode.Retain) { + this.nodeBuilder.addHiddenTokens(lexerResult.hidden); + } this.unorderedGroups.clear(); return { value: result, @@ -252,7 +258,7 @@ export class LangiumParser extends AbstractLangiumParser { private startImplementation($type: string | symbol | undefined, implementation: RuleImpl): RuleImpl { return (args) => { if (!this.isRecording()) { - const node: any = { $type }; + const node: any = { $type, $segments: { properties: {} } }; this.stack.push(node); if ($type === DatatypeSymbol) { node.value = ''; @@ -265,7 +271,7 @@ export class LangiumParser extends AbstractLangiumParser { result = undefined; } if (!this.isRecording() && result === undefined) { - result = this.construct(); + result = this.construct()[0]; } return result; }; @@ -341,33 +347,38 @@ export class LangiumParser extends AbstractLangiumParser { if (!this.isRecording()) { let last = this.current; if (action.feature && action.operator) { - last = this.construct(); - this.nodeBuilder.removeNode(last.$cstNode); + const [constructed, cstNode] = this.construct(); + last = constructed; const node = this.nodeBuilder.buildCompositeNode(action); - node.content.push(last.$cstNode); - const newItem = { $type }; + this.nodeBuilder.removeNode(cstNode); + node.content.push(cstNode); + const newItem = { $type, $segments: { properties: {} } }; this.stack.push(newItem); - this.assign(action.operator, action.feature, last, last.$cstNode, false); + this.assign(action.operator, action.feature, last, cstNode, false); } else { last.$type = $type; } } } - construct(): unknown { + construct(): [unknown, CstNode] { if (this.isRecording()) { - return undefined; + return [undefined, undefined!]; } const obj = this.current; linkContentToContainer(obj); - this.nodeBuilder.construct(obj); + const cstNode = this.nodeBuilder.construct(obj); this.stack.pop(); if (isDataTypeNode(obj)) { - return this.converter.convert(obj.value, obj.$cstNode); + return [this.converter.convert(obj.value, cstNode), cstNode]; } else { assignMandatoryProperties(this.astReflection, obj); } - return obj; + obj.$segments.full = toDocumentSegment(cstNode); + if (this.currentMode === CstParserMode.Retain) { + obj.$cstNode = cstNode; + } + return [obj, cstNode]; } private getAssignment(feature: AbstractElement): AssignmentElement { @@ -385,24 +396,29 @@ export class LangiumParser extends AbstractLangiumParser { const obj = this.current; let item: unknown; if (isCrossRef && typeof value === 'string') { - item = this.linker.buildReference(obj, feature, cstNode, value); + item = this.linker.buildReference(obj, feature, this.currentMode === CstParserMode.Retain ? cstNode : undefined, value); } else { item = value; } + const segment = toDocumentSegment(cstNode); switch (operator) { case '=': { obj[feature] = item; + obj.$segments.properties[feature] = [segment]; break; } case '?=': { obj[feature] = true; + obj.$segments.properties[feature] = [segment]; break; } case '+=': { if (!Array.isArray(obj[feature])) { obj[feature] = []; + obj.$segments.properties[feature] = []; } obj[feature].push(item); + obj.$segments.properties[feature].push(segment); } } } @@ -523,11 +539,6 @@ export class LangiumCompletionParser extends AbstractLangiumParser { // NOOP } - construct(): unknown { - // NOOP - return undefined; - } - parse(input: string): CompletionParserResult { this.resetState(); const tokens = this.lexer.tokenize(input, { mode: 'partial' }); diff --git a/packages/langium/src/references/name-provider.ts b/packages/langium/src/references/name-provider.ts index 209fd4da6..3d4604c35 100644 --- a/packages/langium/src/references/name-provider.ts +++ b/packages/langium/src/references/name-provider.ts @@ -21,9 +21,16 @@ export function isNamed(node: AstNode): node is NamedAstNode { export interface NameProvider { /** * Returns the `name` of a given AstNode. - * @param node Specified `AstNode` whose name node shall be retrieved. + * @param node Specified `AstNode` whose name shall be retrieved. */ getName(node: AstNode): string | undefined; + + /** + * Returns the property name that is used to store the name of an AstNode. + * @param node The AstNode for which the name property shall be retrieved. + */ + getNameProperty(node: AstNode): string | undefined; + /** * Returns the `CstNode` which contains the parsed value of the `name` assignment. * @param node Specified `AstNode` whose name node shall be retrieved. @@ -39,7 +46,11 @@ export class DefaultNameProvider implements NameProvider { return undefined; } + getNameProperty(_node: AstNode): string | undefined { + return 'name'; + } + getNameNode(node: AstNode): CstNode | undefined { - return findNodeForProperty(node.$cstNode, 'name'); + return findNodeForProperty(node.$cstNode, this.getNameProperty(node)); } } diff --git a/packages/langium/src/references/references.ts b/packages/langium/src/references/references.ts index 34c9089eb..f7cb80d98 100644 --- a/packages/langium/src/references/references.ts +++ b/packages/langium/src/references/references.ts @@ -15,7 +15,7 @@ import type { URI } from '../utils/uri-utils.js'; import { findAssignment } from '../utils/grammar-utils.js'; import { isReference } from '../syntax-tree.js'; import { getDocument } from '../utils/ast-utils.js'; -import { isChildNode, toDocumentSegment } from '../utils/cst-utils.js'; +import { isChildNode } from '../utils/cst-utils.js'; import { stream } from '../utils/stream.js'; import { UriUtils } from '../utils/uri-utils.js'; @@ -130,18 +130,21 @@ export class DefaultReferences implements References { } protected getReferenceToSelf(targetNode: AstNode): ReferenceDescription | undefined { - const nameNode = this.nameProvider.getNameNode(targetNode); - if (nameNode) { - const doc = getDocument(targetNode); - const path = this.nodeLocator.getAstNodePath(targetNode); - return { - sourceUri: doc.uri, - sourcePath: path, - targetUri: doc.uri, - targetPath: path, - segment: toDocumentSegment(nameNode), - local: true - }; + const nameProperty = this.nameProvider.getNameProperty(targetNode); + if (nameProperty) { + const nameSegment = targetNode.$segments?.properties[nameProperty]?.[0]; + if (nameSegment) { + const doc = getDocument(targetNode); + const path = this.nodeLocator.getAstNodePath(targetNode); + return { + sourceUri: doc.uri, + sourcePath: path, + targetUri: doc.uri, + targetPath: path, + segment: nameSegment, + local: true + }; + } } return undefined; } diff --git a/packages/langium/src/serializer/json-serializer.ts b/packages/langium/src/serializer/json-serializer.ts index e21968776..5dd2f80b6 100644 --- a/packages/langium/src/serializer/json-serializer.ts +++ b/packages/langium/src/serializer/json-serializer.ts @@ -109,7 +109,7 @@ function isIntermediateReference(obj: unknown): obj is IntermediateReference { export class DefaultJsonSerializer implements JsonSerializer { /** The set of AstNode properties to be ignored by the serializer. */ - ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode']); + ignoreProperties = new Set(['$container', '$containerProperty', '$containerIndex', '$document', '$cstNode', '$segments']); /** The document that is currently processed by the serializer; this is used by the replacer function. */ protected currentDocument: LangiumDocument | undefined; diff --git a/packages/langium/src/services.ts b/packages/langium/src/services.ts index 06627388f..bf5b82ecf 100644 --- a/packages/langium/src/services.ts +++ b/packages/langium/src/services.ts @@ -37,6 +37,7 @@ import type { IndexManager } from './workspace/index-manager.js'; import type { WorkspaceLock } from './workspace/workspace-lock.js'; import type { Hydrator } from './serializer/hydrator.js'; import type { WorkspaceManager } from './workspace/workspace-manager.js'; +import type { Environment } from './workspace/environment.js'; /** * The services generated by `langium-cli` for a specific language. These are derived from the @@ -111,6 +112,7 @@ export type LangiumGeneratedSharedCoreServices = { export type LangiumDefaultSharedCoreServices = { readonly ServiceRegistry: ServiceRegistry readonly workspace: { + readonly Environment: Environment readonly ConfigurationProvider: ConfigurationProvider readonly DocumentBuilder: DocumentBuilder readonly FileSystemProvider: FileSystemProvider diff --git a/packages/langium/src/syntax-tree.ts b/packages/langium/src/syntax-tree.ts index a318cf57a..5e62bd7d5 100644 --- a/packages/langium/src/syntax-tree.ts +++ b/packages/langium/src/syntax-tree.ts @@ -23,6 +23,7 @@ export interface AstNode { readonly $containerIndex?: number; /** The Concrete Syntax Tree (CST) node of the text range from which this node was parsed. */ readonly $cstNode?: CstNode; + readonly $segments?: AstNodeSegments; /** The document containing the AST; only the root node has a direct reference to the document. */ readonly $document?: LangiumDocument; } @@ -42,6 +43,11 @@ type SpecificNodeProperties = keyof Omit = SpecificNodeProperties extends never ? string : SpecificNodeProperties +export interface AstNodeSegments { + readonly full: DocumentSegment; + readonly properties: Record; +} + /** * A cross-reference in the AST. Cross-references may or may not be successfully resolved. */ diff --git a/packages/langium/src/utils/cst-utils.ts b/packages/langium/src/utils/cst-utils.ts index 54e2374a1..807a68209 100644 --- a/packages/langium/src/utils/cst-utils.ts +++ b/packages/langium/src/utils/cst-utils.ts @@ -6,11 +6,12 @@ import type { IToken } from '@chevrotain/types'; import type { Range } from 'vscode-languageserver-types'; -import type { CstNode, CompositeCstNode, LeafCstNode } from '../syntax-tree.js'; +import type { CstNode, CompositeCstNode, LeafCstNode, AstNode, Mutable, Reference } from '../syntax-tree.js'; import type { DocumentSegment } from '../workspace/documents.js'; import type { Stream, TreeStream } from './stream.js'; import { isCompositeCstNode, isLeafCstNode, isRootCstNode } from '../syntax-tree.js'; import { TreeStreamImpl } from './stream.js'; +import { streamAst, streamReferences } from './ast-utils.js'; /** * Create a stream of all CST nodes that are directly and indirectly contained in the given root node, @@ -339,3 +340,12 @@ interface ParentLink { parent: CompositeCstNode index: number } + +export function discardCst(node: AstNode): void { + streamAst(node).forEach(n => { + (n as Mutable).$cstNode = undefined; + streamReferences(n).forEach(r => { + (r.reference as Mutable>).$refNode = undefined; + }); + }); +} diff --git a/packages/langium/src/validation/document-validator.ts b/packages/langium/src/validation/document-validator.ts index fb3754411..4c866f4e4 100644 --- a/packages/langium/src/validation/document-validator.ts +++ b/packages/langium/src/validation/document-validator.ts @@ -61,6 +61,9 @@ export class DefaultDocumentValidator implements DocumentValidator { async validateDocument(document: LangiumDocument, options: ValidationOptions = {}, cancelToken = CancellationToken.None): Promise { const parseResult = document.parseResult; + if (!parseResult.value.$cstNode) { + return []; + } const diagnostics: Diagnostic[] = []; await interruptAndCheck(cancelToken); diff --git a/packages/langium/src/workspace/ast-descriptions.ts b/packages/langium/src/workspace/ast-descriptions.ts index 46a3743d8..a56f32524 100644 --- a/packages/langium/src/workspace/ast-descriptions.ts +++ b/packages/langium/src/workspace/ast-descriptions.ts @@ -13,7 +13,6 @@ import type { DocumentSegment, LangiumDocument } from './documents.js'; import { CancellationToken } from '../utils/cancellation.js'; import { isLinkingError } from '../syntax-tree.js'; import { getDocument, streamAst, streamReferences } from '../utils/ast-utils.js'; -import { toDocumentSegment } from '../utils/cst-utils.js'; import { interruptAndCheck } from '../utils/promise-utils.js'; import { UriUtils } from '../utils/uri-utils.js'; @@ -53,15 +52,15 @@ export class DefaultAstNodeDescriptionProvider implements AstNodeDescriptionProv if (!name) { throw new Error(`Node at path ${path} has no name.`); } - let nameNodeSegment: DocumentSegment | undefined; - const nameSegmentGetter = () => nameNodeSegment ??= toDocumentSegment(this.nameProvider.getNameNode(node) ?? node.$cstNode); + const nameProperty = this.nameProvider.getNameProperty(node); + const nameSegment: DocumentSegment | undefined = nameProperty + ? node.$segments?.properties[nameProperty]?.[0] + : undefined; return { node, name, - get nameSegment() { - return nameSegmentGetter(); - }, - selectionSegment: toDocumentSegment(node.$cstNode), + nameSegment, + selectionSegment: node.$segments?.full, type: node.$type, documentUri: doc.uri, path @@ -131,8 +130,8 @@ export class DefaultReferenceDescriptionProvider implements ReferenceDescription protected createDescription(refInfo: ReferenceInfo): ReferenceDescription | undefined { const targetNodeDescr = refInfo.reference.$nodeDescription; - const refCstNode = refInfo.reference.$refNode; - if (!targetNodeDescr || !refCstNode) { + const refSegment = refInfo.container.$segments?.properties[refInfo.property]?.[refInfo.index ?? 0]; + if (!targetNodeDescr || !refSegment) { return undefined; } const docUri = getDocument(refInfo.container).uri; @@ -141,7 +140,7 @@ export class DefaultReferenceDescriptionProvider implements ReferenceDescription sourcePath: this.nodeLocator.getAstNodePath(refInfo.container), targetUri: targetNodeDescr.documentUri, targetPath: targetNodeDescr.path, - segment: toDocumentSegment(refCstNode), + segment: refSegment, local: UriUtils.equals(targetNodeDescr.documentUri, docUri) }; } diff --git a/packages/langium/src/workspace/documents.ts b/packages/langium/src/workspace/documents.ts index a94dc3563..2cb40c693 100644 --- a/packages/langium/src/workspace/documents.ts +++ b/packages/langium/src/workspace/documents.ts @@ -15,7 +15,7 @@ export { TextDocument } from 'vscode-languageserver-textdocument'; import type { Diagnostic, Range } from 'vscode-languageserver-types'; import type { FileSystemProvider } from './file-system-provider.js'; -import type { ParseResult, ParserOptions } from '../parser/langium-parser.js'; +import { CstParserMode, type ParseResult, type ParserOptions } from '../parser/langium-parser.js'; import type { ServiceRegistry } from '../service-registry.js'; import type { LangiumSharedCoreServices } from '../services.js'; import type { AstNode, AstNodeDescription, Mutable, Reference } from '../syntax-tree.js'; @@ -130,7 +130,7 @@ export interface LangiumDocumentFactory { /** * Create a Langium document from a `TextDocument` asynchronously. This action can be cancelled if a cancellable parser implementation has been provided. */ - fromTextDocument(textDocument: TextDocument, uri: URI | undefined, cancellationToken: CancellationToken): Promise>; + fromTextDocument(textDocument: TextDocument, uri: URI | undefined, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; /** * Create an Langium document from an in-memory string. @@ -139,7 +139,7 @@ export interface LangiumDocumentFactory { /** * Create a Langium document from an in-memory string asynchronously. This action can be cancelled if a cancellable parser implementation has been provided. */ - fromString(text: string, uri: URI, cancellationToken: CancellationToken): Promise>; + fromString(text: string, uri: URI, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; /** * Create an Langium document from a model that has been constructed in memory. @@ -149,7 +149,7 @@ export interface LangiumDocumentFactory { /** * Create an Langium document from a specified `URI`. The factory will use the `FileSystemAccess` service to read the file. */ - fromUri(uri: URI, cancellationToken?: CancellationToken): Promise>; + fromUri(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise>; /** * Update the given document after changes in the corresponding textual representation. @@ -173,29 +173,29 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { this.fileSystemProvider = services.workspace.FileSystemProvider; } - async fromUri(uri: URI, cancellationToken = CancellationToken.None): Promise> { + async fromUri(uri: URI, options?: ParserOptions | undefined, cancellationToken = CancellationToken.None): Promise> { const content = await this.fileSystemProvider.readFile(uri); - return this.createAsync(uri, content, cancellationToken); + return this.createAsync(uri, content, options, cancellationToken); } fromTextDocument(textDocument: TextDocument, uri?: URI, options?: ParserOptions): LangiumDocument; - fromTextDocument(textDocument: TextDocument, uri: URI | undefined, cancellationToken: CancellationToken): Promise>; - fromTextDocument(textDocument: TextDocument, uri?: URI, token?: CancellationToken | ParserOptions): LangiumDocument | Promise> { + fromTextDocument(textDocument: TextDocument, uri: URI | undefined, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; + fromTextDocument(textDocument: TextDocument, uri?: URI, options?: ParserOptions, cancellationToken?: CancellationToken): LangiumDocument | Promise> { uri = uri ?? URI.parse(textDocument.uri); - if (CancellationToken.is(token)) { - return this.createAsync(uri, textDocument, token); + if (cancellationToken) { + return this.createAsync(uri, textDocument, options, cancellationToken); } else { - return this.create(uri, textDocument, token); + return this.create(uri, textDocument, options); } } fromString(text: string, uri: URI, options?: ParserOptions): LangiumDocument; - fromString(text: string, uri: URI, cancellationToken: CancellationToken): Promise>; - fromString(text: string, uri: URI, token?: CancellationToken | ParserOptions): LangiumDocument | Promise> { - if (CancellationToken.is(token)) { - return this.createAsync(uri, text, token); + fromString(text: string, uri: URI, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise>; + fromString(text: string, uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): LangiumDocument | Promise> { + if (cancellationToken) { + return this.createAsync(uri, text, options, cancellationToken); } else { - return this.create(uri, text, token); + return this.create(uri, text, options); } } @@ -218,12 +218,12 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { } } - protected async createAsync(uri: URI, content: string | TextDocument, cancelToken: CancellationToken): Promise> { + protected async createAsync(uri: URI, content: string | TextDocument, options: ParserOptions | undefined, cancelToken: CancellationToken): Promise> { if (typeof content === 'string') { - const parseResult = await this.parseAsync(uri, content, cancelToken); + const parseResult = await this.parseAsync(uri, content, options, cancelToken); return this.createLangiumDocument(parseResult, uri, undefined, content); } else { - const parseResult = await this.parseAsync(uri, content.getText(), cancelToken); + const parseResult = await this.parseAsync(uri, content.getText(), options, cancelToken); return this.createLangiumDocument(parseResult, uri, content); } } @@ -293,7 +293,10 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { // Some of these documents can be pretty large, so parsing them again can be quite expensive. // Therefore, we only parse if the text has actually changed. if (oldText !== text) { - document.parseResult = await this.parseAsync(document.uri, text, cancellationToken); + const options: ParserOptions = { + cst: textDocument ? CstParserMode.Retain : CstParserMode.Discard + }; + document.parseResult = await this.parseAsync(document.uri, text, options, cancellationToken); (document.parseResult.value as Mutable).$document = document; } document.state = DocumentState.Parsed; @@ -305,9 +308,9 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory { return services.parser.LangiumParser.parse(text, options); } - protected parseAsync(uri: URI, text: string, cancellationToken: CancellationToken): Promise> { + protected parseAsync(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise> { const services = this.serviceRegistry.getServices(uri); - return services.parser.AsyncParser.parse(text, cancellationToken); + return services.parser.AsyncParser.parse(text, options, cancellationToken); } protected createTextDocumentGetter(uri: URI, text?: string): () => TextDocument { @@ -346,7 +349,7 @@ export interface LangiumDocuments { * Retrieve the document with the given URI. If not present, a new one will be created using the file system access. * The new document will be added to the list of documents managed under this service. */ - getOrCreateDocument(uri: URI, cancellationToken?: CancellationToken): Promise; + getOrCreateDocument(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise; /** * Creates a new document with the given URI and text content. @@ -354,7 +357,7 @@ export interface LangiumDocuments { * * @throws an error if a document with the same URI is already present. */ - createDocument(uri: URI, text: string): LangiumDocument; + createDocument(uri: URI, text: string, options?: ParserOptions): LangiumDocument; /** * Creates a new document with the given URI and text content asynchronously. @@ -363,7 +366,7 @@ export interface LangiumDocuments { * * @throws an error if a document with the same URI is already present. */ - createDocument(uri: URI, text: string, cancellationToken: CancellationToken): Promise; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise; /** * Returns `true` if a document with the given URI is managed under this service. @@ -418,26 +421,26 @@ export class DefaultLangiumDocuments implements LangiumDocuments { return this.documentMap.get(uriString); } - async getOrCreateDocument(uri: URI, cancellationToken?: CancellationToken): Promise { + async getOrCreateDocument(uri: URI, options?: ParserOptions, cancellationToken?: CancellationToken): Promise { let document = this.getDocument(uri); if (document) { return document; } - document = await this.langiumDocumentFactory.fromUri(uri, cancellationToken); + document = await this.langiumDocumentFactory.fromUri(uri, options, cancellationToken); this.addDocument(document); return document; } - createDocument(uri: URI, text: string): LangiumDocument; - createDocument(uri: URI, text: string, cancellationToken: CancellationToken): Promise; - createDocument(uri: URI, text: string, cancellationToken?: CancellationToken): LangiumDocument | Promise { + createDocument(uri: URI, text: string, options?: ParserOptions): LangiumDocument; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken: CancellationToken): Promise; + createDocument(uri: URI, text: string, options: ParserOptions | undefined, cancellationToken?: CancellationToken): LangiumDocument | Promise { if (cancellationToken) { - return this.langiumDocumentFactory.fromString(text, uri, cancellationToken).then(document => { + return this.langiumDocumentFactory.fromString(text, uri, options, cancellationToken).then(document => { this.addDocument(document); return document; }); } else { - const document = this.langiumDocumentFactory.fromString(text, uri); + const document = this.langiumDocumentFactory.fromString(text, uri, options); this.addDocument(document); return document; } diff --git a/packages/langium/src/workspace/environment.ts b/packages/langium/src/workspace/environment.ts new file mode 100644 index 000000000..667f0e6e4 --- /dev/null +++ b/packages/langium/src/workspace/environment.ts @@ -0,0 +1,51 @@ +/****************************************************************************** + * Copyright 2024 TypeFox GmbH + * This program and the accompanying materials are made available under the + * terms of the MIT License, which is available in the project root. + ******************************************************************************/ + +import type { InitializeParams, InitializedParams } from 'vscode-languageserver-protocol'; + +export interface EnvironmentInfo { + readonly isLanguageServer: boolean; + readonly locale: string; +} + +export interface Environment extends EnvironmentInfo { + initialize(params: InitializeParams): void; + initialized(params: InitializedParams): void; + update(newEnvironment: Partial): void; +} + +export class DefaultEnvironment implements Environment { + + private _isLanguageServer: boolean = false; + private _locale: string = 'en'; + + get isLanguageServer(): boolean { + return this._isLanguageServer; + } + + get locale(): string { + return this._locale; + } + + initialize(params: InitializeParams): void { + this.update({ + isLanguageServer: true, + locale: params.locale + }); + } + + initialized(_params: InitializedParams): void { + } + + update(newEnvironment: Partial): void { + if (typeof newEnvironment.isLanguageServer === 'boolean') { + this._isLanguageServer = newEnvironment.isLanguageServer; + } + if (typeof newEnvironment.locale === 'string') { + this._locale = newEnvironment.locale; + } + } +} diff --git a/packages/langium/src/workspace/workspace-manager.ts b/packages/langium/src/workspace/workspace-manager.ts index cdc9c7c4a..02edcefec 100644 --- a/packages/langium/src/workspace/workspace-manager.ts +++ b/packages/langium/src/workspace/workspace-manager.ts @@ -15,6 +15,8 @@ import type { BuildOptions, DocumentBuilder } from './document-builder.js'; import type { LangiumDocument, LangiumDocuments } from './documents.js'; import type { FileSystemNode, FileSystemProvider } from './file-system-provider.js'; import type { WorkspaceLock } from './workspace-lock.js'; +import { CstParserMode } from '../parser/langium-parser.js'; +import type { Environment } from './environment.js'; // export type WorkspaceFolder from 'vscode-languageserver-types' for convenience, // is supposed to avoid confusion as 'WorkspaceFolder' might accidentally be imported via 'vscode-languageclient' @@ -75,6 +77,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { protected readonly langiumDocuments: LangiumDocuments; protected readonly documentBuilder: DocumentBuilder; protected readonly fileSystemProvider: FileSystemProvider; + protected readonly environment: Environment; protected readonly mutex: WorkspaceLock; protected readonly _ready = new Deferred(); protected folders?: WorkspaceFolder[]; @@ -85,6 +88,7 @@ export class DefaultWorkspaceManager implements WorkspaceManager { this.documentBuilder = services.workspace.DocumentBuilder; this.fileSystemProvider = services.workspace.FileSystemProvider; this.mutex = services.workspace.WorkspaceLock; + this.environment = services.workspace.Environment; } get ready(): Promise { @@ -167,13 +171,19 @@ export class DefaultWorkspaceManager implements WorkspaceManager { if (entry.isDirectory) { await this.traverseFolder(workspaceFolder, entry.uri, fileExtensions, collector); } else if (entry.isFile) { - const document = await this.langiumDocuments.getOrCreateDocument(entry.uri); + const document = await this.langiumDocuments.getOrCreateDocument(entry.uri, { + cst: this.getCstParserMode(entry.uri) + }); collector(document); } } })); } + protected getCstParserMode(_uri: URI): CstParserMode { + return this.environment.isLanguageServer ? CstParserMode.Discard : CstParserMode.Retain; + } + /** * Determine whether the given folder entry shall be included while indexing the workspace. */ diff --git a/packages/langium/test/parser/worker-thread-async-parser.test.ts b/packages/langium/test/parser/worker-thread-async-parser.test.ts index 96f396832..7b45fe0fe 100644 --- a/packages/langium/test/parser/worker-thread-async-parser.test.ts +++ b/packages/langium/test/parser/worker-thread-async-parser.test.ts @@ -32,7 +32,7 @@ describe('WorkerThreadAsyncParser', () => { asyncParser.setThreadCount(4); const promises: Array>> = []; for (let i = 0; i < 16; i++) { - promises.push(asyncParser.parse(file, CancellationToken.None)); + promises.push(asyncParser.parse(file, undefined, CancellationToken.None)); } const result = await Promise.all(promises); for (const parseResult of result) { @@ -50,7 +50,7 @@ describe('WorkerThreadAsyncParser', () => { setTimeout(() => cancellationTokenSource.cancel(), 50); const start = performance.now(); try { - await asyncParser.parse(file, cancellationTokenSource.token); + await asyncParser.parse(file, undefined, cancellationTokenSource.token); fail('Parsing should have been cancelled'); } catch (err) { expect(isOperationCancelled(err)).toBe(true); @@ -68,20 +68,20 @@ describe('WorkerThreadAsyncParser', () => { const cancellationTokenSource = startCancelableOperation(); setTimeout(() => cancellationTokenSource.cancel(), 50); try { - await asyncParser.parse(file, cancellationTokenSource.token); + await asyncParser.parse(file, undefined, cancellationTokenSource.token); fail('Parsing should have been cancelled'); } catch (err) { expect(isOperationCancelled(err)).toBe(true); } // Calling this method should recreate the worker and parse the file correctly - const result = await asyncParser.parse(createLargeFile(10), CancellationToken.None); + const result = await asyncParser.parse(createLargeFile(10), undefined, CancellationToken.None); expect(result.value.name).toBe('Test'); }); test('async parsing yields correct CST', async () => { const services = getServices(); const file = createLargeFile(10); - const result = await services.parser.AsyncParser.parse(file, CancellationToken.None); + const result = await services.parser.AsyncParser.parse(file, undefined, CancellationToken.None); const index = file.indexOf('TestRule'); // Assert that the CST can be found at all from the root node // This indicates that the CST is correctly linked to itself @@ -104,7 +104,7 @@ describe('WorkerThreadAsyncParser', () => { test('parser errors are correctly transmitted', async () => { const services = getServices(); const file = 'grammar Test Rule: name="Hello" // missing semicolon'; - const result = await services.parser.AsyncParser.parse(file, CancellationToken.None); + const result = await services.parser.AsyncParser.parse(file, undefined, CancellationToken.None); expect(result.parserErrors).toHaveLength(1); expect(result.parserErrors[0].name).toBe('MismatchedTokenException'); expect(result.parserErrors[0]).toHaveProperty('previousToken'); diff --git a/packages/langium/test/parser/worker-thread.js b/packages/langium/test/parser/worker-thread.js index 5922a8459..e9a95a748 100644 --- a/packages/langium/test/parser/worker-thread.js +++ b/packages/langium/test/parser/worker-thread.js @@ -12,8 +12,8 @@ const services = createLangiumGrammarServices(EmptyFileSystem).grammar; const parser = services.parser.LangiumParser; const hydrator = services.serializer.Hydrator; -parentPort.on('message', text => { - const result = parser.parse(text); +parentPort.on('message', ([text, options]) => { + const result = parser.parse(text, options); const dehydrated = hydrator.dehydrate(result); parentPort.postMessage(dehydrated); }); From 5104b2346307124109658e0381d8799b9480b877 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Wed, 6 Nov 2024 15:38:59 +0100 Subject: [PATCH 2/2] Support comments in closed files --- .../src/documentation/comment-provider.ts | 7 +++- .../langium/src/parser/cst-node-builder.ts | 8 ++-- packages/langium/src/parser/langium-parser.ts | 42 +++++++++++++++---- packages/langium/src/serializer/hydrator.ts | 5 ++- packages/langium/src/syntax-tree.ts | 5 ++- packages/langium/src/utils/cst-utils.ts | 32 +++++++------- 6 files changed, 69 insertions(+), 30 deletions(-) diff --git a/packages/langium/src/documentation/comment-provider.ts b/packages/langium/src/documentation/comment-provider.ts index 1d9c58d15..06b425d4f 100644 --- a/packages/langium/src/documentation/comment-provider.ts +++ b/packages/langium/src/documentation/comment-provider.ts @@ -28,9 +28,12 @@ export class DefaultCommentProvider implements CommentProvider { this.grammarConfig = () => services.parser.GrammarConfig; } getComment(node: AstNode): string | undefined { - if(isAstNodeWithComment(node)) { + if (isAstNodeWithComment(node)) { return node.$comment; + } else if (node.$segments && 'comment' in node.$segments) { + return node.$segments.comment; + } else { + return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text; } - return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text; } } diff --git a/packages/langium/src/parser/cst-node-builder.ts b/packages/langium/src/parser/cst-node-builder.ts index 0e03811d6..a01d17ffb 100644 --- a/packages/langium/src/parser/cst-node-builder.ts +++ b/packages/langium/src/parser/cst-node-builder.ts @@ -37,8 +37,8 @@ export class CstNodeBuilder { return compositeNode; } - buildLeafNode(token: IToken, feature: AbstractElement): LeafCstNode { - const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, false); + buildLeafNode(token: IToken, feature?: AbstractElement): LeafCstNode { + const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, !feature); leafNode.grammarSource = feature; leafNode.root = this.rootNode; this.current.content.push(leafNode); @@ -107,7 +107,7 @@ export abstract class AbstractCstNode implements CstNode { abstract get range(): Range; container?: CompositeCstNode; - grammarSource: AbstractElement; + grammarSource?: AbstractElement; root: RootCstNode; private _astNode?: AstNode; @@ -117,7 +117,7 @@ export abstract class AbstractCstNode implements CstNode { } /** @deprecated use `grammarSource` instead. */ - get feature(): AbstractElement { + get feature(): AbstractElement | undefined { return this.grammarSource; } diff --git a/packages/langium/src/parser/langium-parser.ts b/packages/langium/src/parser/langium-parser.ts index 0f65bf873..0c0906d85 100644 --- a/packages/langium/src/parser/langium-parser.ts +++ b/packages/langium/src/parser/langium-parser.ts @@ -10,7 +10,7 @@ import type { AbstractElement, Action, Assignment, ParserRule } from '../languag import type { Linker } from '../references/linker.js'; import type { LangiumCoreServices } from '../services.js'; import type { AstNode, AstReflection, CompositeCstNode, CstNode } from '../syntax-tree.js'; -import type { Lexer } from './lexer.js'; +import type { Lexer, LexerResult } from './lexer.js'; import type { IParserConfig } from './parser-config.js'; import type { ValueConverter } from './value-converter.js'; import { defaultParserErrorProvider, EmbeddedActionsParser, LLkLookaheadStrategy } from 'chevrotain'; @@ -21,6 +21,7 @@ import { assignMandatoryProperties, getContainerOfType, linkContentToContainer } import { CstNodeBuilder } from './cst-node-builder.js'; import type { LexingReport } from './token-builder.js'; import { toDocumentSegment } from '../utils/cst-utils.js'; +import type { CommentProvider } from '../documentation/comment-provider.js'; export type ParseResult = { value: T, @@ -123,6 +124,7 @@ const withRuleSuffix = (name: string): string => name.endsWith(ruleSuffix) ? nam export abstract class AbstractLangiumParser implements BaseParser { protected readonly lexer: Lexer; + protected readonly commentProvider: CommentProvider; protected readonly wrapper: ChevrotainWrapper; protected _unorderedGroups: Map = new Map(); @@ -138,6 +140,7 @@ export abstract class AbstractLangiumParser implements BaseParser { skipValidations: production, errorMessageProvider: services.parser.ParserErrorMessageProvider }); + this.commentProvider = services.documentation.CommentProvider; } alternatives(idx: number, choices: Array>): void { @@ -197,6 +200,7 @@ export class LangiumParser extends AbstractLangiumParser { private readonly converter: ValueConverter; private readonly astReflection: AstReflection; private readonly nodeBuilder = new CstNodeBuilder(); + private lexerResult?: LexerResult; private stack: any[] = []; private assignmentMap = new Map(); private currentMode: CstParserMode = CstParserMode.Retain; @@ -236,17 +240,15 @@ export class LangiumParser extends AbstractLangiumParser { parse(input: string, options: ParserOptions = {}): ParseResult { this.currentMode = options.cst ?? CstParserMode.Retain; this.nodeBuilder.buildRootNode(input); - const lexerResult = this.lexer.tokenize(input); + const lexerResult = this.lexerResult = this.lexer.tokenize(input); this.wrapper.input = lexerResult.tokens; const ruleMethod = options.rule ? this.allRules.get(options.rule) : this.mainRule; if (!ruleMethod) { throw new Error(options.rule ? `No rule found with name '${options.rule}'` : 'No main rule available.'); } const result = ruleMethod.call(this.wrapper, {}); - if (this.currentMode === CstParserMode.Retain) { - this.nodeBuilder.addHiddenTokens(lexerResult.hidden); - } this.unorderedGroups.clear(); + this.lexerResult = undefined; return { value: result, lexerErrors: lexerResult.errors, @@ -277,9 +279,32 @@ export class LangiumParser extends AbstractLangiumParser { }; } + private appendHiddenTokens(tokens: IToken[]): void { + for (const token of tokens) { + this.nodeBuilder.buildLeafNode(token); + } + } + + private getHiddenTokens(token: IToken): IToken[] { + const hiddenTokens = this.lexerResult!.hidden; + if (!hiddenTokens.length) { + return []; + } + const offset = token.startOffset; + for (let i = 0; i < hiddenTokens.length; i++) { + const token = hiddenTokens[i]; + if (token.startOffset > offset) { + return hiddenTokens.splice(0, i); + } + } + return hiddenTokens.splice(0, hiddenTokens.length); + } + consume(idx: number, tokenType: TokenType, feature: AbstractElement): void { const token = this.wrapper.wrapConsume(idx, tokenType); if (!this.isRecording() && this.isValidToken(token)) { + const hiddenTokens = this.getHiddenTokens(token); + this.appendHiddenTokens(hiddenTokens); const leafNode = this.nodeBuilder.buildLeafNode(token, feature); const { assignment, isCrossRef } = this.getAssignment(feature); const current = this.current; @@ -374,9 +399,12 @@ export class LangiumParser extends AbstractLangiumParser { } else { assignMandatoryProperties(this.astReflection, obj); } + obj.$cstNode = cstNode; + delete obj.$segments.comment; + obj.$segments.comment = this.commentProvider.getComment(obj); obj.$segments.full = toDocumentSegment(cstNode); - if (this.currentMode === CstParserMode.Retain) { - obj.$cstNode = cstNode; + if (this.currentMode === CstParserMode.Discard) { + obj.$cstNode = undefined; } return [obj, cstNode]; } diff --git a/packages/langium/src/serializer/hydrator.ts b/packages/langium/src/serializer/hydrator.ts index c18592f36..cf8ca889f 100644 --- a/packages/langium/src/serializer/hydrator.ts +++ b/packages/langium/src/serializer/hydrator.ts @@ -296,10 +296,13 @@ export class DefaultHydrator implements Hydrator { return this.lexer.definition[name]; } - protected getGrammarElementId(node: AbstractElement): number | undefined { + protected getGrammarElementId(node: AbstractElement | undefined): number | undefined { if (this.grammarElementIdMap.size === 0) { this.createGrammarElementIdMap(); } + if (!node) { + return undefined; + } return this.grammarElementIdMap.get(node); } diff --git a/packages/langium/src/syntax-tree.ts b/packages/langium/src/syntax-tree.ts index 5e62bd7d5..dd6565607 100644 --- a/packages/langium/src/syntax-tree.ts +++ b/packages/langium/src/syntax-tree.ts @@ -45,6 +45,7 @@ export type Properties = SpecificNodeProperties extends ne export interface AstNodeSegments { readonly full: DocumentSegment; + readonly comment?: string; readonly properties: Record; } @@ -240,9 +241,9 @@ export interface CstNode extends DocumentSegment { /** The root CST node */ readonly root: RootCstNode; /** The grammar element from which this node was parsed */ - readonly grammarSource: AbstractElement; + readonly grammarSource?: AbstractElement; /** @deprecated use `grammarSource` instead. */ - readonly feature: AbstractElement; + readonly feature?: AbstractElement; /** The AST node created from this CST node */ readonly astNode: AstNode; /** @deprecated use `astNode` instead. */ diff --git a/packages/langium/src/utils/cst-utils.ts b/packages/langium/src/utils/cst-utils.ts index 807a68209..c599fdeb8 100644 --- a/packages/langium/src/utils/cst-utils.ts +++ b/packages/langium/src/utils/cst-utils.ts @@ -9,7 +9,7 @@ import type { Range } from 'vscode-languageserver-types'; import type { CstNode, CompositeCstNode, LeafCstNode, AstNode, Mutable, Reference } from '../syntax-tree.js'; import type { DocumentSegment } from '../workspace/documents.js'; import type { Stream, TreeStream } from './stream.js'; -import { isCompositeCstNode, isLeafCstNode, isRootCstNode } from '../syntax-tree.js'; +import { isCompositeCstNode, isLeafCstNode } from '../syntax-tree.js'; import { TreeStreamImpl } from './stream.js'; import { streamAst, streamReferences } from './ast-utils.js'; @@ -135,20 +135,24 @@ export function findDeclarationNodeAtOffset(cstNode: CstNode | undefined, offset } export function findCommentNode(cstNode: CstNode | undefined, commentNames: string[]): CstNode | undefined { - if (cstNode) { - const previous = getPreviousNode(cstNode, true); - if (previous && isCommentNode(previous, commentNames)) { - return previous; + if (isCompositeCstNode(cstNode)) { + const firstChild = cstNode.content[0]; + if (isCompositeCstNode(firstChild)) { + return findCommentNode(firstChild, commentNames); + } + // Find the first non-hidden child + let index = -1; + for (const child of cstNode.content) { + if (!child.hidden) { + break; + } + index++; } - if (isRootCstNode(cstNode)) { - // Go from the first non-hidden node through all nodes in reverse order - // We do this to find the comment node which directly precedes the root node - const endIndex = cstNode.content.findIndex(e => !e.hidden); - for (let i = endIndex - 1; i >= 0; i--) { - const child = cstNode.content[i]; - if (isCommentNode(child, commentNames)) { - return child; - } + // Search backwards for a comment node + for (let i = index; i >= 0; i--) { + const child = cstNode.content[i]; + if (isCommentNode(child, commentNames)) { + return child; } } }