Implement a simple Pratt Parser

.github/workflows/ci.yaml

@@ -0,0 +1,27 @@
+name: Go
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.22
+
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+
+      - name: Run tests
+        run: |
+            set -euo pipefail
+            go test -json -v ./... 2>&1 | tee /tmp/gotest.log | gotestfmt

.gitignore

@@ -1 +1,2 @@
-bin
+bin
+.idea

src/ast/ast.go

@@ -1,11 +1,15 @@
 package ast
 
 import (
+	"bytes"
+	"strings"
+
 	"github.com/sevenreup/chewa/src/token"
 )
 
 type Node interface {
 	TokenLiteral() string
+	String() string
 }
 
 type Statement interface {
@@ -30,8 +34,16 @@ func (p *Program) TokenLiteral() string {
 	}
 }
 
+func (p *Program) String() string {
+	var out bytes.Buffer
+	for _, s := range p.Statements {
+		out.WriteString(s.String())
+	}
+	return out.String()
+}
+
 type AssigmentStatement struct {
-	Token token.Token // the token.LET token
+	Token token.Token // the token.Nambala token
 	Name  *Identifier
 	Value Expression
 }
@@ -44,5 +56,189 @@ type Identifier struct {
 	Value string
 }
 
+func (i *Identifier) String() string { return i.Value }
+
 func (i *Identifier) expressionNode()      {}
 func (i *Identifier) TokenLiteral() string { return i.Token.Literal }
+
+func (ls *AssigmentStatement) String() string {
+	var out bytes.Buffer
+	out.WriteString(ls.TokenLiteral() + " ")
+	out.WriteString(ls.Name.Value)
+	out.WriteString(" = ")
+	if ls.Value != nil {
+		out.WriteString(ls.Value.String())
+	}
+	out.WriteString(";")
+	return out.String()
+}
+
+type ReturnStatement struct {
+	Token       token.Token // the 'return' token
+	ReturnValue Expression
+}
+
+func (rs *ReturnStatement) statementNode()       {}
+func (rs *ReturnStatement) TokenLiteral() string { return rs.Token.Literal }
+
+func (rs *ReturnStatement) String() string {
+	var out bytes.Buffer
+	out.WriteString(rs.TokenLiteral() + " ")
+	if rs.ReturnValue != nil {
+		out.WriteString(rs.ReturnValue.String())
+	}
+	out.WriteString(";")
+	return out.String()
+}
+
+type ExpressionStatement struct {
+	Token      token.Token // the first token of the expression
+	Expression Expression
+}
+
+func (es *ExpressionStatement) statementNode()       {}
+func (es *ExpressionStatement) TokenLiteral() string { return es.Token.Literal }
+
+func (es *ExpressionStatement) String() string {
+	if es.Expression != nil {
+		return es.Expression.String()
+	}
+	return ""
+}
+
+type IntegerLiteral struct {
+	Token token.Token
+	Value int64
+}
+
+func (il *IntegerLiteral) expressionNode()      {}
+func (il *IntegerLiteral) TokenLiteral() string { return il.Token.Literal }
+func (il *IntegerLiteral) String() string       { return il.Token.Literal }
+
+type PrefixExpression struct {
+	Token    token.Token // The prefix token, e.g. !
+	Operator string
+	Right    Expression
+}
+
+func (pe *PrefixExpression) expressionNode()      {}
+func (pe *PrefixExpression) TokenLiteral() string { return pe.Token.Literal }
+func (pe *PrefixExpression) String() string {
+	var out bytes.Buffer
+	out.WriteString("(")
+	out.WriteString(pe.Operator)
+	out.WriteString(pe.Right.String())
+	out.WriteString(")")
+	return out.String()
+}
+
+type InfixExpression struct {
+	Token    token.Token // The operator token, e.g. +
+	Left     Expression
+	Operator string
+	Right    Expression
+}
+
+func (oe *InfixExpression) expressionNode()      {}
+func (oe *InfixExpression) TokenLiteral() string { return oe.Token.Literal }
+
+func (oe *InfixExpression) String() string {
+	var out bytes.Buffer
+	out.WriteString("(")
+	out.WriteString(oe.Left.String())
+	out.WriteString(" " + oe.Operator + " ")
+	out.WriteString(oe.Right.String())
+	out.WriteString(")")
+	return out.String()
+}
+
+type Boolean struct {
+	Token token.Token
+	Value bool
+}
+
+func (b *Boolean) expressionNode()      {}
+func (b *Boolean) TokenLiteral() string { return b.Token.Literal }
+func (b *Boolean) String() string       { return b.Token.Literal }
+
+type IfExpression struct {
+	Token       token.Token // The 'if' token
+	Condition   Expression
+	Consequence *BlockStatement
+	Alternative *BlockStatement
+}
+
+func (ie *IfExpression) expressionNode()      {}
+func (ie *IfExpression) TokenLiteral() string { return ie.Token.Literal }
+func (ie *IfExpression) String() string {
+	var out bytes.Buffer
+	out.WriteString("ngati")
+	out.WriteString(ie.Condition.String())
+	out.WriteString(" ")
+	out.WriteString(ie.Consequence.String())
+	if ie.Alternative != nil {
+		out.WriteString("kapena ")
+		out.WriteString(ie.Alternative.String())
+	}
+	return out.String()
+}
+
+type BlockStatement struct {
+	Token      token.Token // the { token
+	Statements []Statement
+}
+
+func (bs *BlockStatement) statementNode()       {}
+func (bs *BlockStatement) TokenLiteral() string { return bs.Token.Literal }
+func (bs *BlockStatement) String() string {
+	var out bytes.Buffer
+	for _, s := range bs.Statements {
+		out.WriteString(s.String())
+	}
+	return out.String()
+}
+
+type FunctionLiteral struct {
+	Token      token.Token // The 'ndondomeko' token
+	Name       *Identifier // The name of the function
+	Parameters []*Identifier
+	Body       *BlockStatement
+}
+
+func (fl *FunctionLiteral) expressionNode()      {}
+func (fl *FunctionLiteral) TokenLiteral() string { return fl.Token.Literal }
+func (fl *FunctionLiteral) String() string {
+	var out bytes.Buffer
+	params := []string{}
+	for _, p := range fl.Parameters {
+		params = append(params, p.String())
+	}
+	out.WriteString(fl.TokenLiteral())
+	out.WriteString(fl.Name.TokenLiteral())
+	out.WriteString("(")
+	out.WriteString(strings.Join(params, ", "))
+	out.WriteString(") ")
+	out.WriteString(fl.Body.String())
+	return out.String()
+}
+
+type CallExpression struct {
+	Token     token.Token // The '(' token
+	Function  Expression  // Identifier or FunctionLiteral
+	Arguments []Expression
+}
+
+func (ce *CallExpression) expressionNode()      {}
+func (ce *CallExpression) TokenLiteral() string { return ce.Token.Literal }
+func (ce *CallExpression) String() string {
+	var out bytes.Buffer
+	args := []string{}
+	for _, a := range ce.Arguments {
+		args = append(args, a.String())
+	}
+	out.WriteString(ce.Function.String())
+	out.WriteString("(")
+	out.WriteString(strings.Join(args, ", "))
+	out.WriteString(")")
+	return out.String()
+}

src/ast/ast_test.go

@@ -0,0 +1,28 @@
+package ast
+
+import (
+	"testing"
+
+	"github.com/sevenreup/chewa/src/token"
+)
+
+func TestString(t *testing.T) {
+	program := &Program{
+		Statements: []Statement{
+			&AssigmentStatement{
+				Token: token.Token{Type: token.INTEGER, Literal: "nambala"},
+				Name: &Identifier{
+					Token: token.Token{Type: token.IDENT, Literal: "myVar"},
+					Value: "myVar",
+				},
+				Value: &Identifier{
+					Token: token.Token{Type: token.IDENT, Literal: "anotherVar"},
+					Value: "anotherVar",
+				},
+			},
+		},
+	}
+	if program.String() != "nambala myVar = anotherVar;" {
+		t.Errorf("program.String() wrong. got=%q", program.String())
+	}
+}

src/lexer/lexer.go

@@ -3,6 +3,7 @@ package lexer
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
 	"strings"
 	"unicode"
@@ -21,9 +22,9 @@ type TokenInfo struct {
 	Literal  string
 }
 
-func New(path []byte) *Lexer {
+func New(value []byte) *Lexer {
 	return &Lexer{
-		r:   bufio.NewReader(bytes.NewReader(path)),
+		r:   bufio.NewReader(bytes.NewReader(value)),
 		pos: token.Position{Line: 1, Column: 0},
 	}
 }
@@ -35,6 +36,7 @@ func (l *Lexer) NextToken() token.Token {
 			if err == io.EOF {
 				return newToken(l.pos, token.EOF, "")
 			}
+			fmt.Println("failed", l.pos.Line, l.pos.Column)
 			panic(err)
 		}
 
@@ -81,7 +83,7 @@ func (l *Lexer) NextToken() token.Token {
 				builder.WriteString("*/")
 				return newToken(l.pos, token.MULTILINE_COMMENT, builder.String())
 			}
-			return newToken(l.pos, token.DIVIDE, "/")
+			return newToken(l.pos, token.SLASH, "/")
 		case '{':
 			return newToken(l.pos, token.OPENING_BRACE, "{")
 		case '}':
@@ -121,6 +123,9 @@ func (l *Lexer) NextToken() token.Token {
 				l.Next()
 				return newToken(l.pos, token.NOT_EQUAL_TO, "!=")
 			}
+			return newToken(l.pos, token.BANG, "!")
+		case '*':
+			return newToken(l.pos, token.ASTERISK, "*")
 		default:
 			if unicode.IsSpace(r) {
 				continue
@@ -198,8 +203,9 @@ func (l *Lexer) ReadNumber(current rune) token.Token {
 		r, _, err := l.r.ReadRune()
 		if err != nil {
 			if err == io.EOF {
-				return newToken(l.pos, token.EOF, "")
+				break
 			}
+			fmt.Println("failed", current, number, l.pos.Line, l.pos.Column)
 			panic(err)
 		}
 		l.pos.Column++
@@ -210,7 +216,7 @@ func (l *Lexer) ReadNumber(current rune) token.Token {
 			break
 		}
 	}
-	return newToken(l.pos, token.INTEGER, number)
+	return newToken(l.pos, token.INT, number)
 }
 
 func (l *Lexer) ReadIdentifier(current rune) token.Token {
@@ -219,8 +225,9 @@ func (l *Lexer) ReadIdentifier(current rune) token.Token {
 		r, _, err := l.r.ReadRune()
 		if err != nil {
 			if err == io.EOF {
-				return newToken(l.pos, token.EOF, "")
+				break
 			}
+			fmt.Println("failed", current, identifier, l.pos.Line, l.pos.Column)
 			panic(err)
 		}
 		l.pos.Column++