Skip to content

Commit

Permalink
Store tokens in BadNode instead of a raw string
Browse files Browse the repository at this point in the history
  • Loading branch information
@makenowjust
makenowjust committed Dec 18, 2024
1 parent a531ccc commit b05fe53
Show file tree
Hide file tree
Showing 16 changed files with 551 additions and 30 deletions.
12 changes: 7 additions & 5 deletions ast/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// - sqlIdentQuote x: Quotes the given identifier string if needed.
// - sqlStringQuote s: Returns the SQL quoted string of s.
// - sqlBytesQuote bs: Returns the SQL quotes bytes of bs.
// - tokenJoin toks: Concateates the string representations of tokens.
// - isnil v: Checks whether v is nil or others.
//
// Each Node's documentation has pos and end information using the following EBNF.
Expand Down Expand Up @@ -516,14 +517,14 @@ func (ChangeStreamSetOptions) isChangeStreamAlteration() {}

// BadNode is a placeholder node for a source code containing syntax errors.
//
// {{.Raw}}
// {{.Tokens | tokenJoin}}
type BadNode struct {
// pos = NodePos
// end = NodeEnd

NodePos, NodeEnd token.Pos

Raw string
Tokens []*token.Token
}

// ================================================================================
Expand Down Expand Up @@ -1258,9 +1259,10 @@ type SelectorExpr struct {

// IndexExpr is a subscript operator expression node.
// This node can be:
// - array subscript operator
// - struct subscript operator
// - JSON subscript operator
// - array subscript operator
// - struct subscript operator
// - JSON subscript operator
//
// Note: The name IndexExpr is a historical reason, maybe better to rename to SubscriptExpr.
//
// {{.Expr | sql}}[{{.Index | sql}}]
Expand Down
11 changes: 10 additions & 1 deletion ast/sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,16 @@ func paren(p prec, e Expr) string {
//
// ================================================================================

func (b *BadNode) SQL() string { return b.Raw }
func (b *BadNode) SQL() string {
var sql string
for _, tok := range b.Tokens {
if sql != "" && len(tok.Space) > 0 {
sql += " "
}
sql += tok.Raw
}
return sql
}

// ================================================================================
//
Expand Down
20 changes: 12 additions & 8 deletions parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4832,6 +4832,7 @@ func (p *Parser) handleError(r any, l *Lexer) {
func (p *Parser) handleParseStatementError(r any, l *Lexer) *ast.BadNode {
p.handleError(r, l)

var tokens []*token.Token
pos := p.Token.Pos
end := p.Token.Pos
skip:
Expand All @@ -4841,20 +4842,21 @@ skip:
break skip
}
end = p.Token.End
tokens = append(tokens, p.Token.Clone())
p.Lexer.nextToken(true)
}

raw := p.Lexer.Buffer[pos:end]
return &ast.BadNode{
NodePos: pos,
NodeEnd: end,
Raw: raw,
Tokens: tokens,
}
}

func (p *Parser) handleParseQueryExprError(simple bool, r any, l *Lexer) *ast.BadNode {
p.handleError(r, l)

var tokens []*token.Token
pos := p.Token.Pos
end := p.Token.Pos
nesting := 0
Expand All @@ -4876,20 +4878,21 @@ skip:
}
}
end = p.Token.End
tokens = append(tokens, p.Token.Clone())
p.Lexer.nextToken(true)
}

raw := p.Lexer.Buffer[pos:end]
return &ast.BadNode{
NodePos: pos,
NodeEnd: end,
Raw: raw,
Tokens: tokens,
}
}

func (p *Parser) handleParseExprError(r any, l *Lexer) *ast.BadNode {
p.handleError(r, l)

var tokens []*token.Token
pos := p.Token.Pos
end := p.Token.Pos
nesting := 0
Expand All @@ -4911,20 +4914,21 @@ skip:
}
}
end = p.Token.End
tokens = append(tokens, p.Token.Clone())
p.Lexer.nextToken(true)
}

raw := p.Lexer.Buffer[pos:end]
return &ast.BadNode{
NodePos: pos,
NodeEnd: end,
Raw: raw,
Tokens: tokens,
}
}

func (p *Parser) handleParseTypeError(r any, l *Lexer) *ast.BadNode {
p.handleError(r, l)

var tokens []*token.Token
pos := p.Token.Pos
end := p.Token.Pos
nesting := 0
Expand Down Expand Up @@ -4955,15 +4959,15 @@ skip:
break skip
}
}
tokens = append(tokens, p.Token.Clone())
end = p.Token.End
p.Lexer.nextToken(true)
}

raw := p.Lexer.Buffer[pos:end]
return &ast.BadNode{
NodePos: pos,
NodeEnd: end,
Raw: raw,
Tokens: tokens,
}
}

Expand Down
1 change: 1 addition & 0 deletions testdata/input/query/!bad_hint_select_2.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
@{hint = 1} select
116 changes: 113 additions & 3 deletions testdata/result/dml/!bad_insert.sql.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,121 @@ syntax error: testdata/input/dml/!bad_insert.sql:2:1: expected beginning of simp
Query: &ast.BadNode{
NodePos: 27,
NodeEnd: 59,
Raw: "vales (1, 2, 3),\n (4, 5, 6)",
Tokens: []*token.Token{
&token.Token{
Kind: "<ident>",
Space: "\n",
Raw: "vales",
AsString: "vales",
Pos: 27,
End: 32,
},
&token.Token{
Kind: "(",
Space: " ",
Raw: "(",
Pos: 33,
End: 34,
},
&token.Token{
Kind: "<int>",
Raw: "1",
Base: 10,
Pos: 34,
End: 35,
},
&token.Token{
Kind: ",",
Raw: ",",
Pos: 35,
End: 36,
},
&token.Token{
Kind: "<int>",
Space: " ",
Raw: "2",
Base: 10,
Pos: 37,
End: 38,
},
&token.Token{
Kind: ",",
Raw: ",",
Pos: 38,
End: 39,
},
&token.Token{
Kind: "<int>",
Space: " ",
Raw: "3",
Base: 10,
Pos: 40,
End: 41,
},
&token.Token{
Kind: ")",
Raw: ")",
Pos: 41,
End: 42,
},
&token.Token{
Kind: ",",
Raw: ",",
Pos: 42,
End: 43,
},
&token.Token{
Kind: "(",
Space: "\n ",
Raw: "(",
Pos: 50,
End: 51,
},
&token.Token{
Kind: "<int>",
Raw: "4",
Base: 10,
Pos: 51,
End: 52,
},
&token.Token{
Kind: ",",
Raw: ",",
Pos: 52,
End: 53,
},
&token.Token{
Kind: "<int>",
Space: " ",
Raw: "5",
Base: 10,
Pos: 54,
End: 55,
},
&token.Token{
Kind: ",",
Raw: ",",
Pos: 55,
End: 56,
},
&token.Token{
Kind: "<int>",
Space: " ",
Raw: "6",
Base: 10,
Pos: 57,
End: 58,
},
&token.Token{
Kind: ")",
Raw: ")",
Pos: 58,
End: 59,
},
},
},
},
}

--- SQL
INSERT INTO foo (foo, bar, baz) vales (1, 2, 3),
(4, 5, 6)
INSERT INTO foo (foo, bar, baz) vales (1, 2, 3), (4, 5, 6)
18 changes: 17 additions & 1 deletion testdata/result/expr/!bad_new_braced_constructor.sql.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,23 @@ syntax error: testdata/input/expr/!bad_new_braced_constructor.sql:1:20: unexpect
Expr: &ast.BadNode{
NodePos: 15,
NodeEnd: 18,
Raw: "1 +",
Tokens: []*token.Token{
&token.Token{
Kind: "<int>",
Space: " ",
Raw: "1",
Base: 10,
Pos: 15,
End: 16,
},
&token.Token{
Kind: "+",
Space: " ",
Raw: "+",
Pos: 17,
End: 18,
},
},
},
},
},
Expand Down
16 changes: 15 additions & 1 deletion testdata/result/expr/!bad_plus.sql.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,21 @@ syntax error: testdata/input/expr/!bad_plus.sql:2:1: unexpected token: <eof>
--- AST
&ast.BadNode{
NodeEnd: 3,
Raw: "1 +",
Tokens: []*token.Token{
&token.Token{
Kind: "<int>",
Raw: "1",
Base: 10,
End: 1,
},
&token.Token{
Kind: "+",
Space: " ",
Raw: "+",
Pos: 2,
End: 3,
},
},
}

--- SQL
Expand Down
34 changes: 32 additions & 2 deletions testdata/result/expr/!bad_plus2.sql.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,22 @@ syntax error: testdata/input/expr/!bad_plus2.sql:1:13: unexpected token: )
Expr: &ast.BadNode{
NodePos: 1,
NodeEnd: 4,
Raw: "1 +",
Tokens: []*token.Token{
&token.Token{
Kind: "<int>",
Raw: "1",
Base: 10,
Pos: 1,
End: 2,
},
&token.Token{
Kind: "+",
Space: " ",
Raw: "+",
Pos: 3,
End: 4,
},
},
},
},
Right: &ast.ParenExpr{
Expand All @@ -31,7 +46,22 @@ syntax error: testdata/input/expr/!bad_plus2.sql:1:13: unexpected token: )
Expr: &ast.BadNode{
NodePos: 9,
NodeEnd: 12,
Raw: "2 +",
Tokens: []*token.Token{
&token.Token{
Kind: "<int>",
Raw: "2",
Base: 10,
Pos: 9,
End: 10,
},
&token.Token{
Kind: "+",
Space: " ",
Raw: "+",
Pos: 11,
End: 12,
},
},
},
},
}
Expand Down
Loading

0 comments on commit b05fe53

Please sign in to comment.