feat: Swap custom lang with ready-made parser

package.json

@@ -6,18 +6,19 @@
   "main": "src/index.js",
   "dependencies": {
     "@popperjs/core": "^2.10.2",
-    "antlr4": "4.9.2",
-    "react": "17.0.2",
-    "react-dom": "17.0.2",
+    "antlr4": "^4.7.2",
+    "antlr4-parsers": "^1.0.20",
+    "react": "^17.0.2",
+    "react-dom": "^17.0.2",
     "react-popper": "^2.2.5"
   },
   "devDependencies": {
     "@babel/runtime": "7.13.8",
     "@types/antlr4": "^4.7.2",
-    "@types/react": "^17.0.9",
-    "@types/react-dom": "^17.0.6",
+    "@types/react": "^17.0.44",
+    "@types/react-dom": "^17.0.15",
     "react-scripts": "^4.0.3",
-    "typescript": "^4.6"
+    "typescript": "^4.6.3"
   },
   "scripts": {
     "start": "BROWSER=none react-scripts start",

src/components/CodeEditor/CodeEditor.tsx

@@ -3,10 +3,12 @@ import { useState, useRef, useEffect, FC, useMemo } from 'react';
 import TokenComponent from '../Token';
 import { Token as GenericToken, Tokenizer } from '../../types';
 import { TOKEN_TYPES } from '../../constants';
-import useEditorReducer from './useEditorReducer'
+import useEditorReducer from './useEditorReducer';
 
-function renderToken({ text, type }: GenericToken, index?: number) {
-  return <TokenComponent text={text} type={type} key={text + index} />;
+function renderToken({ text, line, type }: GenericToken, index?: number) {
+  return (
+    <TokenComponent text={text} line={line} type={type} key={text + index} />
+  );
 }
 
 function renderChar(char: string, index: number, active: boolean) {
@@ -95,8 +97,12 @@ const CodeEditor: FC<ComponentProps> = ({ code, tokenize }) => {
         />
         <div ref={output} className="CodeEditor__output">
           {tokens.map(renderToken)}
-          {tokens.length && tokens[tokens.length - 1].text.endsWith('\n')
-            ? renderToken({ text: '\n', type: TOKEN_TYPES.NONE })
+          {tokens.length && tokens[tokens.length - 1]!.text.endsWith('\n')
+            ? renderToken({
+                text: '\n',
+                type: TOKEN_TYPES.NONE,
+                line: tokens[tokens.length - 1]!.line + 1,
+              })
             : null}
         </div>
         <div ref={cursor} className="CodeEditor__cursor">

src/example/LangEditor.tsx

@@ -1,30 +1,82 @@
 import { FC } from 'react';
-import LangLexer from './antlr/LangLexer';
 import { CodeEditor } from '../components';
 import { TOKEN_TYPES } from '../constants';
 import { makeTokenizeFn } from '../support';
+import { YAMLLexer } from 'antlr4-parsers/yaml/YAMLLexer';
+import * as YAMLTokenConstants from 'antlr4-parsers/yaml/token-constants';
+import { Token } from '../types';
 
 const TOKEN_MAPPING = new Map([
-  [LangLexer.Whitespace, TOKEN_TYPES.NONE],
-  [LangLexer.Newline, TOKEN_TYPES.NONE],
-  [LangLexer.Defun, TOKEN_TYPES.KEYWORD],
-  [LangLexer.Call, TOKEN_TYPES.KEYWORD],
-  [LangLexer.Integer, TOKEN_TYPES.NUMBER],
-  [LangLexer.String, TOKEN_TYPES.NUMBER],
-  [LangLexer.Identifier, TOKEN_TYPES.IDENTIFIER],
-  [LangLexer.LeftBrace, TOKEN_TYPES.BRACE],
-  [LangLexer.RightBrace, TOKEN_TYPES.BRACE],
-  [LangLexer.Comma, TOKEN_TYPES.BRACE],
-  [LangLexer.Any, TOKEN_TYPES.RAW],
+  [YAMLTokenConstants.EOF, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.COMMENT, TOKEN_TYPES.RAW],
+  [YAMLTokenConstants.SCALAR, TOKEN_TYPES.IDENTIFIER],
+  [YAMLTokenConstants.ELEMENT, TOKEN_TYPES.IDENTIFIER],
+  [YAMLTokenConstants.MAP_START, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.LIST_START, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.MAP_END, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.LIST_END, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.COLON, TOKEN_TYPES.RAW],
+  [YAMLTokenConstants.START_DOCUMENT, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.END_DOCUMENT, TOKEN_TYPES.NONE],
+  [YAMLTokenConstants.EMPTY, TOKEN_TYPES.NONE],
 ]);
 
 type LangEditorProps = { code: string };
 
+const withSubstitution: typeof makeTokenizeFn = (...args) => {
+  return function (...finalArgs) {
+    let indentation = 0;
+    const indent = '  ';
+
+    const tokens = makeTokenizeFn(...args)(...finalArgs).reduce(
+      (all, i, idx, array) => {
+        if (
+          [
+            'MAP_START',
+            'MAP_END',
+            'LIST_START',
+            'LIST_END',
+            'START_DOCUMENT',
+            'END_DOCUMENT',
+          ].includes(i.text)
+        ) {
+          all.push({
+            ...i,
+            text: i.text.includes('START')
+              ? indent.repeat(indentation++)
+              : indent.repeat(indentation--),
+          });
+          return all;
+        }
+
+        if (['COLON'].includes(i.text)) {
+          all.push({
+            ...i,
+            text:
+              array[idx + 1] && i?.line < array[idx + 1]!.line
+                ? ':\n' + indent.repeat(indentation)
+                : ': ',
+          });
+          return all;
+        }
+
+        all.push(i);
+        return all;
+      },
+      [] as Token[]
+    );
+
+    console.log(tokens);
+
+    return tokens;
+  };
+};
+
 const LangEditor: FC<LangEditorProps> = ({ code }) => {
   return (
     <CodeEditor
       code={code}
-      tokenize={makeTokenizeFn(LangLexer, TOKEN_MAPPING)}
+      tokenize={withSubstitution(YAMLLexer, TOKEN_MAPPING)}
     />
   );
 };

src/example/index.tsx

@@ -1,12 +1,21 @@
-import React, { StrictMode } from 'react';
+import { StrictMode } from 'react';
 import ReactDOM from 'react-dom';
 import LangEditor from './LangEditor';
 
-const code = `defun inspect value {
-  call logger '[DEBUG]', value
-}
-
-call inspect 1340`;
+const code = `glossary:
+GlossDiv:
+GlossList:
+  GlossEntry:
+  Abbrev: ISO 8879:1986
+  Acronym: SGML
+  GlossDef:
+    GlossSeeAlso: [GML, XML]
+    para: A meta-markup language, used to create markup languages such as DocBook.
+  GlossSee: markup
+  GlossTerm: Standard Generalized Markup Language
+  ID: SGML
+  SortAs: SGML
+title: example glossary`;
 
 ReactDOM.render(
   <StrictMode>

src/support/antlr/makeTokenizeFn.ts

@@ -7,7 +7,11 @@ import {
 } from '../../types';
 
 type AntlrTokenType = number;
-type AntlrToken = { text: string; type: AntlrTokenType };
+interface AntlrToken {
+  text: string;
+  line: number;
+  type: AntlrTokenType;
+}
 
 const makeTokenizeFn = (
   LexerClass: Lexer,
@@ -17,10 +21,13 @@ const makeTokenizeFn = (
     const input = new antlr.InputStream(text);
     const lexer = new LexerClass(input);
 
-    const tokens = lexer.getAllTokens().map(({ text, type }: AntlrToken) => ({
-      text,
-      type: tokenMapping.get(type) || TOKEN_TYPES.NONE,
-    }));
+    const tokens = lexer
+      .getAllTokens()
+      .map(({ text, line, type }: AntlrToken) => ({
+        text,
+        line,
+        type: tokenMapping.get(type) || TOKEN_TYPES.NONE,
+      }));
 
     return tokens;
   };

src/types/lexer.ts

@@ -1,5 +1,5 @@
-import { InputStream as AntlrInputStream, Lexer as AntlrLexer } from 'antlr4';
+import antlr4 from 'antlr4';
 
 export type Lexer = {
-  new (input: AntlrInputStream): AntlrLexer;
+  new (input: antlr4.InputStream): antlr4.Lexer;
 };

src/types/token.ts

@@ -6,4 +6,8 @@ export type TokenType =
   | 'keyword'
   | 'brace';
 
-export type Token = { text: string; type: TokenType };
+export interface Token {
+  text: string;
+  line: number;
+  type: TokenType;
+}

tsconfig.base.json

@@ -0,0 +1,50 @@
+{
+    "compilerOptions": {
+      /* Basic Options */
+      "target": "es5",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
+      "lib": [
+        "dom",
+        "dom.iterable",
+        "esnext"
+      ],                                     /* Specify library files to be included in the compilation:  */
+      "module": "esnext",                    /* Specify module code generation: 'commonjs', 'amd', 'system', 'umd' or 'es2015', 'esnext'. */
+      "allowJs": true,                       /* Allow JavaScript files to be compiled. */
+      "jsx": "react-jsx",                    /* Specify JSX code generation: 'preserve', 'react-native', or 'react-jsx'. */
+      "removeComments": true,                /* Do not emit comments to output. */
+      "noEmit": true,                        /* Do not emit outputs. */
+      "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+
+      /* Strict Type-Checking Options */
+      "strict": true,                        /* Enable all strict type-checking options. */
+
+      /* Additional Checks */
+      "noUnusedLocals": true,                /* Report errors on unused locals. */
+      "noUnusedParameters": true,            /* Report errors on unused parameters. */
+      "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+      "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+      "exactOptionalPropertyTypes": true,    /* With exactOptionalPropertyTypes enabled, TypeScript applies stricter rules around how it handles properties on type or interfaces which have a ? prefix */
+      "noUncheckedIndexedAccess": true,      /* Turning on noUncheckedIndexedAccess will add undefined to any un-declared field in the type. */
+
+      /* Module Resolution Options */
+      "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+      "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+      "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+      "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+      "typeRoots": [],                       /* List of folders to include type definitions from. */
+      "types": [],                           /* Type declaration files to be included in compilation. */
+      "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+
+      /* Completeness Options */
+      "skipLibCheck": true,                  /* A common case where you might think to use skipLibCheck is when there are two copies of a library’s types in your node_modules */
+
+      /* Interop Constraints */
+      "esModuleInterop": true,                  /* By default (with esModuleInterop false or not set) TypeScript treats CommonJS/AMD/UMD modules similar to ES6 modules. */
+      "forceConsistentCasingInFileNames": true, /* TypeScript will issue an error if a program tries to include a file by a casing different from the casing on disk. */
+      "resolveJsonModule": true,                /* Allows importing modules with a ‘.json’ extension */
+
+    },
+    "include": [
+      "src/**/*"
+    ],                                       /* Specifies an array of filenames or patterns to include in the program. These filenames are resolved relative to the directory containing the tsconfig.json file. */
+  }

tsconfig.json

@@ -1,26 +1,4 @@
 {
-  "compilerOptions": {
-    "target": "es5",
-    "lib": [
-      "dom",
-      "dom.iterable",
-      "esnext"
-    ],
-    "allowJs": true,
-    "skipLibCheck": true,
-    "esModuleInterop": true,
-    "allowSyntheticDefaultImports": true,
-    "strict": true,
-    "forceConsistentCasingInFileNames": true,
-    "noFallthroughCasesInSwitch": true,
-    "module": "esnext",
-    "moduleResolution": "node",
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "noEmit": true,
-    "jsx": "react-jsx"
-  },
-  "include": [
-    "src"
-  ]
+  "extends": "./tsconfig.base.json",
+  "compilerOptions": {}
 }