Merge pull request #8 from ajmarcus/one-char-tokens

add parsing for one character tokens

.github/workflows/actions.yml

@@ -4,34 +4,33 @@
 name: Node.js CI
 
 on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
+    push:
+        branches: ["main"]
+    pull_request:
+        branches: ["main"]
 
 jobs:
-  build:
+    build:
+        runs-on: ubuntu-latest
 
-    runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                node-version: [20.x]
+                # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
-    strategy:
-      matrix:
-        node-version: [20.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Setup Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-    - name: Install Dependencies
-      run: npm ci
-    - name: Lint
-      run: npm run lint
-    - name: Build
-      run: npm run build --if-present
-    - name: Test
-      run: npm test
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v3
+            - name: Setup Node.js ${{ matrix.node-version }}
+              uses: actions/setup-node@v3
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "npm"
+            - name: Install Dependencies
+              run: npm ci
+            - name: Lint
+              run: npm run lint
+            - name: Build
+              run: npm run build --if-present
+            - name: Test
+              run: npm test

package.json

@@ -5,8 +5,8 @@
     "main": "dist/repl.js",
     "scripts": {
         "start": "node dist/repl.js",
-        "dev": "rm -Rf dist && tsc && node dist/repl.js",
-        "build": "tsc",
+        "dev": "rm -Rf dist && tsc -p tsconfig-build.json && node dist/repl.js",
+        "build": "tsc -p tsconfig-build.json",
         "clean": "rm -R dist",
         "fmt": "prettier . --write",
         "fmt-check": "prettier . --check",

src/parser.ts

@@ -34,17 +34,48 @@ const tokens = new Set([
     "WITH",
 ]);
 
-function parse(input: string): string[] {
+const ONE_CHAR_TOKENS = new Map<string, string>([
+    [",", "COMMA"],
+    [";", "SEMI"],
+    ["(", "LPAREN"],
+    [")", "RPAREN"],
+    ["*", "STAR"],
+    ["+", "PLUS"],
+    ["-", "MINUS"],
+    ["/", "SLASH"],
+    ["<", "LT"],
+    [">", "GT"],
+    ["=", "EQ"],
+]);
+
+function tokenize(input: string): string[] {
     const result = [];
     const inputTokens = input.toUpperCase().split(/\s+/);
     for (const t of inputTokens) {
         if (tokens.has(t)) {
-            result.push(t);
+            result.push(`TKN<${t}>`);
         } else {
-            result.push(`VAL<${t}>`);
+            // look for one character tokens
+            let remaining = [];
+            for (const c of t) {
+                if (ONE_CHAR_TOKENS.has(c)) {
+                    result.push(`VAL<${remaining.join("")}>`);
+                    remaining = [];
+                    result.push(`TKN<${ONE_CHAR_TOKENS.get(c)}>`);
+                } else {
+                    remaining.push(c);
+                }
+            }
+            if (remaining.length > 0) {
+                result.push(`VAL<${remaining.join("")}>`);
+            }
         }
     }
     return result;
 }
 
+function parse(input: string): string[] {
+    return tokenize(input);
+}
+
 export { parse };

test/parser.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, test } from "@jest/globals";
+import { parse } from "../src/parser";
+
+describe("select statement work in the parser", () => {
+    test('select returns ["SELECT"]', () => {
+        expect(parse("select")).toStrictEqual(["TKN<SELECT>"]);
+    });
+    test("select name from employees where tenure=5;", () => {
+        expect(
+            parse("select name from employees where tenure=5;"),
+        ).toStrictEqual([
+            "TKN<SELECT>",
+            "VAL<NAME>",
+            "TKN<FROM>",
+            "VAL<EMPLOYEES>",
+            "TKN<WHERE>",
+            "VAL<TENURE>",
+            "TKN<EQ>",
+            "VAL<5>",
+            "TKN<SEMI>",
+        ]);
+    });
+});

tsconfig-build.json

@@ -0,0 +1,4 @@
+{
+    "extends": "./tsconfig.json",
+    "exclude": ["test/**/*.test.ts"]
+}

tsconfig.json

@@ -1,6 +1,6 @@
 {
     // input files
-    "include": ["src/**/*"],
+    "include": ["src/**/*", "test/**/*.test.ts"],
     "exclude": ["node_modules", "dist"],
     "compilerOptions": {
         // output compiled js files to dist folder