Skip to content

Commit

Permalink
Added ParserOptions to parseHelper
Browse files Browse the repository at this point in the history
  • Loading branch information
@spoenemann
spoenemann committed Apr 30, 2024
1 parent 10f216c commit 7a02166
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 32 deletions.
6 changes: 5 additions & 1 deletion packages/langium/src/parser/langium-parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ export abstract class AbstractLangiumParser implements BaseParser {
}
}

export interface ParserOptions {
rule?: string
}

export class LangiumParser extends AbstractLangiumParser {
private readonly linker: Linker;
private readonly converter: ValueConverter;
Expand Down Expand Up @@ -160,7 +164,7 @@ export class LangiumParser extends AbstractLangiumParser {
return ruleMethod;
}

parse<T extends AstNode = AstNode>(input: string, options: { rule?: string } = {}): ParseResult<T> {
parse<T extends AstNode = AstNode>(input: string, options: ParserOptions = {}): ParseResult<T> {
this.nodeBuilder.buildRootNode(input);
const lexerResult = this.lexer.tokenize(input);
this.wrapper.input = lexerResult.tokens;
Expand Down
11 changes: 8 additions & 3 deletions packages/langium/src/test/langium-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,28 @@ import type { LangiumCoreServices, LangiumSharedCoreServices } from '../services
import type { AstNode, CstNode, Properties } from '../syntax-tree.js';
import { type LangiumDocument, TextDocument } from '../workspace/documents.js';
import type { BuildOptions } from '../workspace/document-builder.js';
import type { AsyncDisposable } from '../utils/disposable.js';
import type { LangiumServices, LangiumSharedLSPServices } from '../lsp/lsp-services.js';
import type { ParserOptions } from '../parser/langium-parser.js';
import { DiagnosticSeverity, MarkupContent } from 'vscode-languageserver-types';
import { escapeRegExp } from '../utils/regexp-utils.js';
import { URI } from '../utils/uri-utils.js';
import { findNodeForProperty } from '../utils/grammar-utils.js';
import { SemanticTokensDecoder } from '../lsp/semantic-token-provider.js';
import * as assert from 'node:assert';
import { stream } from '../utils/stream.js';
import type { AsyncDisposable } from '../utils/disposable.js';
import { Disposable } from '../utils/disposable.js';
import { normalizeEOL } from '../generate/template-string.js';
import type { LangiumServices, LangiumSharedLSPServices } from '../lsp/lsp-services.js';

export interface ParseHelperOptions extends BuildOptions {
/**
* Specifies the URI of the generated document. Will use a counter variable if not specified.
*/
documentUri?: string;
/**
* Options passed to the LangiumParser.
*/
parserOptions?: ParserOptions
}

let nextDocumentId = 1;
Expand All @@ -35,7 +40,7 @@ export function parseHelper<T extends AstNode = AstNode>(services: LangiumCoreSe
const documentBuilder = services.shared.workspace.DocumentBuilder;
return async (input, options) => {
const uri = URI.parse(options?.documentUri ?? `file:///${nextDocumentId++}${metaData.fileExtensions[0] ?? ''}`);
const document = services.shared.workspace.LangiumDocumentFactory.fromString<T>(input, uri);
const document = services.shared.workspace.LangiumDocumentFactory.fromString<T>(input, uri, options?.parserOptions);
services.shared.workspace.LangiumDocuments.addDocument(document);
await documentBuilder.build([document], options);
return document;
Expand Down
36 changes: 18 additions & 18 deletions packages/langium/src/workspace/documents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export { TextDocument } from 'vscode-languageserver-textdocument';

import type { Diagnostic, Range } from 'vscode-languageserver-types';
import type { FileSystemProvider } from './file-system-provider.js';
import type { ParseResult } from '../parser/langium-parser.js';
import type { ParseResult, ParserOptions } from '../parser/langium-parser.js';
import type { ServiceRegistry } from '../service-registry.js';
import type { LangiumSharedCoreServices } from '../services.js';
import type { AstNode, AstNodeDescription, Mutable, Reference } from '../syntax-tree.js';
Expand Down Expand Up @@ -126,7 +126,7 @@ export interface LangiumDocumentFactory {
/**
* Create a Langium document from a `TextDocument` (usually associated with a file).
*/
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI): LangiumDocument<T>;
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI, options?: ParserOptions): LangiumDocument<T>;
/**
* Create a Langium document from a `TextDocument` asynchronously. This action can be cancelled if a cancellable parser implementation has been provided.
*/
Expand All @@ -135,7 +135,7 @@ export interface LangiumDocumentFactory {
/**
* Create an Langium document from an in-memory string.
*/
fromString<T extends AstNode = AstNode>(text: string, uri: URI): LangiumDocument<T>;
fromString<T extends AstNode = AstNode>(text: string, uri: URI, options?: ParserOptions): LangiumDocument<T>;
/**
* Create a Langium document from an in-memory string asynchronously. This action can be cancelled if a cancellable parser implementation has been provided.
*/
Expand Down Expand Up @@ -178,42 +178,42 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory {
return this.createAsync<T>(uri, content, cancellationToken);
}

fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI): LangiumDocument<T>;
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI, options?: ParserOptions): LangiumDocument<T>;
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri: URI | undefined, cancellationToken: CancellationToken): Promise<LangiumDocument<T>>;
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI, cancellationToken?: CancellationToken): LangiumDocument<T> | Promise<LangiumDocument<T>> {
fromTextDocument<T extends AstNode = AstNode>(textDocument: TextDocument, uri?: URI, token?: CancellationToken | ParserOptions): LangiumDocument<T> | Promise<LangiumDocument<T>> {
uri = uri ?? URI.parse(textDocument.uri);
if (cancellationToken) {
return this.createAsync<T>(uri, textDocument, cancellationToken);
if (CancellationToken.is(token)) {
return this.createAsync<T>(uri, textDocument, token);
} else {
return this.create<T>(uri, textDocument);
return this.create<T>(uri, textDocument, token);
}
}

fromString<T extends AstNode = AstNode>(text: string, uri: URI): LangiumDocument<T>;
fromString<T extends AstNode = AstNode>(text: string, uri: URI, options?: ParserOptions): LangiumDocument<T>;
fromString<T extends AstNode = AstNode>(text: string, uri: URI, cancellationToken: CancellationToken): Promise<LangiumDocument<T>>;
fromString<T extends AstNode = AstNode>(text: string, uri: URI, cancellationToken?: CancellationToken): LangiumDocument<T> | Promise<LangiumDocument<T>> {
if (cancellationToken) {
return this.createAsync<T>(uri, text, cancellationToken);
fromString<T extends AstNode = AstNode>(text: string, uri: URI, token?: CancellationToken | ParserOptions): LangiumDocument<T> | Promise<LangiumDocument<T>> {
if (CancellationToken.is(token)) {
return this.createAsync<T>(uri, text, token);
} else {
return this.create<T>(uri, text);
return this.create<T>(uri, text, token);
}
}

fromModel<T extends AstNode = AstNode>(model: T, uri: URI): LangiumDocument<T> {
return this.create<T>(uri, { $model: model });
}

protected create<T extends AstNode = AstNode>(uri: URI, content: string | TextDocument | { $model: T }): LangiumDocument<T> {
protected create<T extends AstNode = AstNode>(uri: URI, content: string | TextDocument | { $model: T }, options?: ParserOptions): LangiumDocument<T> {
if (typeof content === 'string') {
const parseResult = this.parse<T>(uri, content);
const parseResult = this.parse<T>(uri, content, options);
return this.createLangiumDocument<T>(parseResult, uri, undefined, content);

} else if ('$model' in content) {
const parseResult = { value: content.$model, parserErrors: [], lexerErrors: [] };
return this.createLangiumDocument<T>(parseResult, uri);

} else {
const parseResult = this.parse<T>(uri, content.getText());
const parseResult = this.parse<T>(uri, content.getText(), options);
return this.createLangiumDocument(parseResult, uri, content);
}
}
Expand Down Expand Up @@ -300,9 +300,9 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory {
return document;
}

protected parse<T extends AstNode>(uri: URI, text: string): ParseResult<T> {
protected parse<T extends AstNode>(uri: URI, text: string, options?: ParserOptions): ParseResult<T> {
const services = this.serviceRegistry.getServices(uri);
return services.parser.LangiumParser.parse<T>(text);
return services.parser.LangiumParser.parse<T>(text, options);
}

protected parseAsync<T extends AstNode>(uri: URI, text: string, cancellationToken: CancellationToken): Promise<ParseResult<T>> {
Expand Down
31 changes: 21 additions & 10 deletions packages/langium/test/parser/langium-parser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,57 +4,68 @@
* terms of the MIT License, which is available in the project root.
******************************************************************************/

import type { LangiumParser } from 'langium';
import type { AstNode, LangiumCoreServices } from 'langium';
import { describe, expect, test, beforeEach } from 'vitest';
import { createServicesForGrammar } from 'langium/grammar';
import { createServicesForGrammar } from 'langium/grammar';
import { parseHelper } from 'langium/test';

describe('Partial parsing', () => {
const content = `
grammar Test
entry Model: (a+=A | b+=B)*;
entry Model: 'model' (a+=A | b+=B)*;
A: 'a' name=ID;
B: 'b' name=ID;
terminal ID: /[_a-zA-Z][\\w_]*/;
hidden terminal WS: /\\s+/;
`;

let parser: LangiumParser;
let services: LangiumCoreServices;

beforeEach(async () => {
parser = await parserFromGrammar(content);
services = await createServicesForGrammar({ grammar: content });
});

// eslint-disable-next-line @typescript-eslint/no-explicit-any
function expectCorrectParse(text: string, rule?: string): any {
const result = parser.parse(text, { rule });
const result = services.parser.LangiumParser.parse(text, { rule });
expect(result.parserErrors.length).toBe(0);
return result.value;
}

function expectErrorneousParse(text: string, rule?: string): void {
const result = parser.parse(text, { rule });
const result = services.parser.LangiumParser.parse(text, { rule });
expect(result.parserErrors.length).toBeGreaterThan(0);
}

test('Should parse correctly with normal entry rule', () => {
const result = expectCorrectParse('a Foo b Bar');
const result = expectCorrectParse('model a Foo b Bar');
expect(result.a[0].name).toEqual('Foo');
expect(result.b[0].name).toEqual('Bar');
});

test('Should parse correctly with alternative entry rule A', () => {
const result = expectCorrectParse('a Foo', 'A');
expect(result.name).toEqual('Foo');
expectErrorneousParse('model a Foo', 'A');
expectErrorneousParse('b Bar', 'A');
});

test('Should parse correctly with alternative entry rule B', () => {
const result = expectCorrectParse('b Foo', 'B');
expect(result.name).toEqual('Foo');
expectErrorneousParse('model b Foo', 'B');
expectErrorneousParse('a Foo', 'B');
});

test('Parse helper supports using alternative entry rule A', async () => {
const parse = parseHelper<A>(services);
const document = await parse('a Foo', { parserOptions: { rule: 'A' } });
expect(document.parseResult.parserErrors.length).toBe(0);
expect(document.parseResult.value.name).toEqual('Foo');
});

});

async function parserFromGrammar(grammar: string): Promise<LangiumParser> {
return (await createServicesForGrammar({ grammar })).parser.LangiumParser;
interface A extends AstNode {
name: string
}

0 comments on commit 7a02166

Please sign in to comment.