Feat/mini parse 2 alpha

linker/packages/linker/src/ImportGrammar.ts

@@ -1,24 +1,26 @@
 import {
-  delimited,
-  kind,
-  NoTags,
-  opt,
-  or,
+  delimited2,
+  fn2,
+  makeTokenMatchers2,
+  opt2,
+  or2,
+  orFail2,
+  parser,
   Parser,
-  preceded,
-  repeat,
-  repeatPlus,
-  req,
-  seq,
-  seqObj,
+  Parser2,
+  ParserContext,
+  preceded2,
+  repeat2,
+  repeatPlus2,
+  seq2,
   setTraceName,
-  TagRecord,
+  Stream,
   tagScope,
-  terminated,
+  terminated2,
   tracing,
-  withSepPlus,
+  tryOr2,
+  yes2,
 } from "mini-parse";
-import { mainTokens } from "./WESLTokens.js";
 import {
   ImportCollection,
   ImportItem,
@@ -27,93 +29,141 @@ import {
 } from "./parse/ImportTree.js";
 import { ImportElem } from "./AbstractElems.js";
 import { importElem } from "./WESLCollect.js";
+import { WeslToken, WeslTokenKind } from "./parse/WeslStream.js";
+import { assertUnreachable } from "../../mini-parse/src/assert.js";
 
-const wordToken = kind(mainTokens.ident);
+function wrapParser2<O>(p: Parser2<Stream<WeslToken>, O, true>): Parser<O> {
+  return parser(
+    "wrapParser2",
+    (ctx: ParserContext) => {
+      const input: Stream<WeslToken> = ctx.lexer.stream as any;
+      const result = p.parseNext(input);
+      return result;
+    },
+    true,
+  );
+}
+const { token, tryToken, eof, tryEof } = makeTokenMatchers2<
+  Stream<WeslToken>,
+  WeslTokenKind,
+  string
+>();
+
+const wordToken = token("word");
+const tryWordToken = tryToken("word");
+const keyword = (value: string) => token("keyword", value);
+const tryKeyword = (value: string) => tryToken("keyword", value);
+const symbol = (value: string) => token("symbol", value);
+const trySymbol = (value: string) => tryToken("symbol", value);
 
-function segment(text: string) {
-  return new ImportSegment(text);
+function segment(token: WeslToken) {
+  return new ImportSegment(token.value);
 }
 function segments(
   ...values: (ImportSegment | ImportSegment[])[]
 ): ImportSegment[] {
   return values.flat();
 }
 
-/** last simple segment is allowed to have an 'as' rename */
-const item_import = seq(wordToken, opt(preceded("as", wordToken))).mapValue(
-  v => new ImportItem(v[0], v[1]),
-);
-
-// forward references for mutual recursion
-let import_collection: Parser<ImportCollection, NoTags> = null as any;
-
-const import_path = seqObj({
-  segments: repeatPlus(terminated(wordToken.mapValue(segment), "::")),
-  final: or(() => import_collection, item_import),
-}).mapValue(v => new ImportStatement(v.segments, v.final));
-
-import_collection = delimited(
-  "{",
-  withSepPlus(",", () =>
-    or(
-      import_path,
-      item_import.mapValue(v => new ImportStatement([], v)),
+const import_path_or_item: Parser2<
+  Stream<WeslToken>,
+  ImportStatement,
+  false
+> = seq2(
+  wordToken,
+  or2(
+    preceded2(
+      trySymbol("::"),
+      or2(
+        fn2(() => import_collection, true),
+        orFail2(fn2(() => import_path_or_item, false)),
+      ),
     ),
-  ).mapValue(v => new ImportCollection(v)),
-  "}",
+    preceded2(tryKeyword("as"), wordToken).map(
+      v => new ImportItem("", v.value),
+    ),
+    orFail2(yes2()),
+  ),
+).map((v): ImportStatement => {
+  const name = v[0].value;
+  const next = v[1];
+  if (next === null) {
+    // fail branch
+    return new ImportStatement([], new ImportItem(name));
+  } else if (next instanceof ImportCollection) {
+    return new ImportStatement([new ImportSegment(name)], next);
+  } else if (next instanceof ImportStatement) {
+    // more import path
+    next.segments.unshift(new ImportSegment(name));
+    return next;
+  } else if (next instanceof ImportItem) {
+    // as branch
+    next.name = name;
+    return new ImportStatement([], next);
+  } else {
+    assertUnreachable(next);
+  }
+});
+
+const import_collection: Parser2<
+  Stream<WeslToken>,
+  ImportCollection,
+  true
+> = delimited2(
+  trySymbol("{"),
+  seq2(
+    import_path_or_item,
+    repeat2(preceded2(trySymbol(","), import_path_or_item)),
+    opt2(trySymbol(",")),
+  ).map(v => new ImportCollection([v[0]].concat(v[1]))),
+  symbol("}"),
 );
 
-const import_relative = seq(
-  or("package", "super").mapValue(segment),
-  "::",
-  repeat(terminated(or("super").mapValue(segment), "::")),
-).mapValue(v => segments(v[0], v[2]));
+const import_relative = seq2(
+  tryOr2(tryKeyword("package"), tryKeyword("super")).map(segment),
+  symbol("::"),
+  repeat2(terminated2(tryKeyword("super").map(segment), symbol("::"))),
+).map(v => segments(v[0], v[2]));
 
-const import_package = terminated(wordToken.mapValue(segment), "::").mapValue(
+const import_package = terminated2(wordToken.map(segment), symbol("::")).map(
   segments,
 );
 
 /** parse a WESL style wgsl import statement. */
-export const weslImport: Parser<ImportElem, NoTags> = tagScope(
-  delimited(
-    "import",
-    req(
-      seq(
-        or(import_relative, import_package),
-        or(import_collection, import_path, item_import),
-      ),
-    ).mapValue(v => {
-      if (v[1] instanceof ImportStatement) {
-        return new ImportStatement(
-          segments(v[0], v[1].segments),
-          v[1].finalSegment,
-        );
-      } else {
-        return new ImportStatement(v[0], v[1]);
-      }
-    }),
-    req(";"),
-  )
-    .map(
+export const weslImport: Parser<ImportElem> = tagScope(
+  wrapParser2(
+    delimited2(
+      tryKeyword("import"),
+      seq2(
+        or2(import_relative, orFail2(import_package)),
+        or2(import_collection, orFail2(import_path_or_item)),
+      ).map(v => {
+        if (v[1] instanceof ImportStatement) {
+          return new ImportStatement(
+            segments(v[0], v[1].segments),
+            v[1].finalSegment,
+          );
+        } else {
+          return new ImportStatement(v[0], v[1]);
+        }
+      }),
+      symbol(";"),
+    ).map(
       (v): ImportElem => ({
         kind: "import",
         contents: [],
-        imports: v.value,
-        start: v.start,
-        end: v.end,
+        imports: v,
+        start: 0, // TODO: Put actual values here
+        end: 0,
       }),
-    )
+    ),
+  )
     .ptag("owo")
     .collect(importElem),
 );
 
 if (tracing) {
-  const names: Record<string, Parser<unknown, TagRecord>> = {
-    item_import,
-    import_path,
-    import_collection,
-    import_relative,
-    import_package,
+  const names: Record<string, Parser<unknown>> = {
     weslImport,
   };
 

linker/packages/linker/src/WESLGrammar.ts

@@ -47,21 +47,18 @@ export const word = kind(mainTokens.ident);
 const qualified_ident = withSepPlus("::", word);
 
 const diagnostic_rule_name = withSep(".", word, { requireOne: true });
-const diagnostic_control = seq(
+const diagnostic_control = delimited(
   "(",
-  word,
-  ",",
-  diagnostic_rule_name,
-  opt(","),
+  seq(word, ",", diagnostic_rule_name, opt(",")),
   ")",
 );
 
 /** list of words that we don't need to collect (e.g. for @interpolate) */
-const word_list = seq("(", withSep(",", word, { requireOne: true }), ")");
+const word_list = delimited("(", withSep(",", word, { requireOne: true }), ")");
 
 // prettier-ignore
 const attribute = tagScope(
-  seq(
+  preceded(
     "@",
     req(
       or(
@@ -105,7 +102,7 @@ const attribute = tagScope(
 )                                         .ctag("attribute");
 
 // prettier-ignore
-const attribute_argument_list = seq(
+const attribute_argument_list = delimited(
   "(",
   withSep(
     ",",

linker/packages/linker/src/parse/WeslStream.ts

@@ -64,7 +64,7 @@ export class WeslStream implements Stream<WeslToken> {
   /** Block comments */
   private blockCommentPattern = /\/\*|\*\//g;
   constructor(public src: string) {
-    this.stream = new CachingStream(new MatchersStream(src, weslMatcher));
+    this.stream = new MatchersStream(src, weslMatcher);
   }
   checkpoint(): number {
     return this.stream.checkpoint();

linker/packages/linker/src/test/TestUtil.ts

@@ -1,10 +1,4 @@
-import {
-  LexerFromStream,
-  NoTags,
-  Parser,
-  TagRecord,
-  withLogger,
-} from "mini-parse";
+import { LexerFromStream, Parser, withLogger } from "mini-parse";
 import {
   expectNoLog,
   logCatch,
@@ -17,10 +11,10 @@ import { parseWESL, syntheticWeslParseState, WeslAST } from "../ParseWESL.js";
 import { Conditions } from "../Scope.js";
 import { WeslStream } from "../parse/WeslStream.js";
 
-export function testAppParse<T, N extends TagRecord = NoTags>(
-  parser: Parser<T, N>,
+export function testAppParse<T>(
+  parser: Parser<T>,
   src: string,
-): TestParseResult<T, N, WeslAST> {
+): TestParseResult<T, WeslAST> {
   const appState = syntheticWeslParseState();
   const lexer = new LexerFromStream(new WeslStream(src), src);
   return testParseWithLexer(parser, lexer, appState);