-
Notifications
You must be signed in to change notification settings - Fork 0
/
lexer.go
101 lines (87 loc) · 2.09 KB
/
lexer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
package main
import (
"fmt"
"strings"
"unicode/utf8"
)
type lexer struct {
name string // used only for error reports.
input string // the string being scanned.
start int // start position of this item.
pos int // current position in the input.
width int // width of last rune read.
items chan item // channel of scanned items.
}
func lex(name, input string) (*lexer, chan item) {
l := &lexer{
name: name,
input: input,
items: make(chan item, 2),
}
go l.run() // Concurrently run state machine.
return l, l.items
}
// run lexes the input by executing state functions untill
// the state is nil.
func (l *lexer) run() {
for state := initState; state != nil; {
state = state(l)
}
close(l.items) // No more tokens will be delivered.
}
// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
l.items <- item{t, l.input[l.start:l.pos]}
l.start = l.pos
}
func (l *lexer) next() rune {
if l.pos >= len(l.input) {
l.width = 0
return eof
}
r, width := utf8.DecodeRuneInString(l.input[l.pos:])
l.width = width
l.pos += l.width
return r
}
// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
l.start = l.pos
}
// backup steps back one rune.
// Can be called only once per call of next.
func (l *lexer) backup() {
l.pos -= l.width
}
// peek returns but does not consume
// the next rune in the input.
func (l *lexer) peek() rune {
r := l.next()
l.backup()
return r
}
// accept consumes the next rune
// if it's from the valid set
func (l *lexer) accept(valid string) bool {
if strings.IndexRune(valid, l.next()) >= 0 {
return true
}
l.backup()
return false
}
// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
for strings.IndexRune(valid, l.next()) >= 0 {
}
l.backup()
}
// errorf returns an error token and terminates the scan
// by passing back a nil pointer that will be the next
// state, terminating l.run
func (l *lexer) errorf(format string, args ...interface{}) stateFunc {
l.items <- item{
itemError,
fmt.Sprintf(format, args...),
}
return nil
}