diff --git a/ast/ast.go b/ast/ast.go index 0814bf77..11be102a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -14,6 +14,7 @@ // - sqlIdentQuote x: Quotes the given identifier string if needed. // - sqlStringQuote s: Returns the SQL quoted string of s. // - sqlBytesQuote bs: Returns the SQL quotes bytes of bs. +// - tokenJoin toks: Concateates the string representations of tokens. // - isnil v: Checks whether v is nil or others. // // Each Node's documentation has pos and end information using the following EBNF. @@ -516,14 +517,14 @@ func (ChangeStreamSetOptions) isChangeStreamAlteration() {} // BadNode is a placeholder node for a source code containing syntax errors. // -// {{.Raw}} +// {{.Tokens | tokenJoin}} type BadNode struct { // pos = NodePos // end = NodeEnd NodePos, NodeEnd token.Pos - Raw string + Tokens []*token.Token } // ================================================================================ @@ -1258,9 +1259,10 @@ type SelectorExpr struct { // IndexExpr is a subscript operator expression node. // This node can be: -// - array subscript operator -// - struct subscript operator -// - JSON subscript operator +// - array subscript operator +// - struct subscript operator +// - JSON subscript operator +// // Note: The name IndexExpr is a historical reason, maybe better to rename to SubscriptExpr. // // {{.Expr | sql}}[{{.Index | sql}}] diff --git a/ast/sql.go b/ast/sql.go index b2dcf9f4..56fc39b4 100644 --- a/ast/sql.go +++ b/ast/sql.go @@ -134,7 +134,16 @@ func paren(p prec, e Expr) string { // // ================================================================================ -func (b *BadNode) SQL() string { return b.Raw } +func (b *BadNode) SQL() string { + var sql string + for _, tok := range b.Tokens { + if sql != "" && len(tok.Space) > 0 { + sql += " " + } + sql += tok.Raw + } + return sql +} // ================================================================================ // diff --git a/parser.go b/parser.go index 50d7b237..3918f675 100644 --- a/parser.go +++ b/parser.go @@ -4832,6 +4832,7 @@ func (p *Parser) handleError(r any, l *Lexer) { func (p *Parser) handleParseStatementError(r any, l *Lexer) *ast.BadNode { p.handleError(r, l) + var tokens []*token.Token pos := p.Token.Pos end := p.Token.Pos skip: @@ -4841,20 +4842,21 @@ skip: break skip } end = p.Token.End + tokens = append(tokens, p.Token.Clone()) p.Lexer.nextToken(true) } - raw := p.Lexer.Buffer[pos:end] return &ast.BadNode{ NodePos: pos, NodeEnd: end, - Raw: raw, + Tokens: tokens, } } func (p *Parser) handleParseQueryExprError(simple bool, r any, l *Lexer) *ast.BadNode { p.handleError(r, l) + var tokens []*token.Token pos := p.Token.Pos end := p.Token.Pos nesting := 0 @@ -4876,20 +4878,21 @@ skip: } } end = p.Token.End + tokens = append(tokens, p.Token.Clone()) p.Lexer.nextToken(true) } - raw := p.Lexer.Buffer[pos:end] return &ast.BadNode{ NodePos: pos, NodeEnd: end, - Raw: raw, + Tokens: tokens, } } func (p *Parser) handleParseExprError(r any, l *Lexer) *ast.BadNode { p.handleError(r, l) + var tokens []*token.Token pos := p.Token.Pos end := p.Token.Pos nesting := 0 @@ -4911,20 +4914,21 @@ skip: } } end = p.Token.End + tokens = append(tokens, p.Token.Clone()) p.Lexer.nextToken(true) } - raw := p.Lexer.Buffer[pos:end] return &ast.BadNode{ NodePos: pos, NodeEnd: end, - Raw: raw, + Tokens: tokens, } } func (p *Parser) handleParseTypeError(r any, l *Lexer) *ast.BadNode { p.handleError(r, l) + var tokens []*token.Token pos := p.Token.Pos end := p.Token.Pos nesting := 0 @@ -4955,15 +4959,15 @@ skip: break skip } } + tokens = append(tokens, p.Token.Clone()) end = p.Token.End p.Lexer.nextToken(true) } - raw := p.Lexer.Buffer[pos:end] return &ast.BadNode{ NodePos: pos, NodeEnd: end, - Raw: raw, + Tokens: tokens, } } diff --git a/testdata/input/query/!bad_hint_select_2.sql b/testdata/input/query/!bad_hint_select_2.sql new file mode 100644 index 00000000..636947f2 --- /dev/null +++ b/testdata/input/query/!bad_hint_select_2.sql @@ -0,0 +1 @@ +@{hint = 1} select \ No newline at end of file diff --git a/testdata/result/dml/!bad_insert.sql.txt b/testdata/result/dml/!bad_insert.sql.txt index 9daebe42..4d2574df 100644 --- a/testdata/result/dml/!bad_insert.sql.txt +++ b/testdata/result/dml/!bad_insert.sql.txt @@ -37,11 +37,121 @@ syntax error: testdata/input/dml/!bad_insert.sql:2:1: expected beginning of simp Query: &ast.BadNode{ NodePos: 27, NodeEnd: 59, - Raw: "vales (1, 2, 3),\n (4, 5, 6)", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Space: "\n", + Raw: "vales", + AsString: "vales", + Pos: 27, + End: 32, + }, + &token.Token{ + Kind: "(", + Space: " ", + Raw: "(", + Pos: 33, + End: 34, + }, + &token.Token{ + Kind: "", + Raw: "1", + Base: 10, + Pos: 34, + End: 35, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 35, + End: 36, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "2", + Base: 10, + Pos: 37, + End: 38, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 38, + End: 39, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "3", + Base: 10, + Pos: 40, + End: 41, + }, + &token.Token{ + Kind: ")", + Raw: ")", + Pos: 41, + End: 42, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 42, + End: 43, + }, + &token.Token{ + Kind: "(", + Space: "\n ", + Raw: "(", + Pos: 50, + End: 51, + }, + &token.Token{ + Kind: "", + Raw: "4", + Base: 10, + Pos: 51, + End: 52, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 52, + End: 53, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "5", + Base: 10, + Pos: 54, + End: 55, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 55, + End: 56, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "6", + Base: 10, + Pos: 57, + End: 58, + }, + &token.Token{ + Kind: ")", + Raw: ")", + Pos: 58, + End: 59, + }, + }, }, }, } --- SQL -INSERT INTO foo (foo, bar, baz) vales (1, 2, 3), - (4, 5, 6) +INSERT INTO foo (foo, bar, baz) vales (1, 2, 3), (4, 5, 6) diff --git a/testdata/result/expr/!bad_new_braced_constructor.sql.txt b/testdata/result/expr/!bad_new_braced_constructor.sql.txt index 931e7df5..9cf390ad 100644 --- a/testdata/result/expr/!bad_new_braced_constructor.sql.txt +++ b/testdata/result/expr/!bad_new_braced_constructor.sql.txt @@ -34,7 +34,23 @@ syntax error: testdata/input/expr/!bad_new_braced_constructor.sql:1:20: unexpect Expr: &ast.BadNode{ NodePos: 15, NodeEnd: 18, - Raw: "1 +", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Space: " ", + Raw: "1", + Base: 10, + Pos: 15, + End: 16, + }, + &token.Token{ + Kind: "+", + Space: " ", + Raw: "+", + Pos: 17, + End: 18, + }, + }, }, }, }, diff --git a/testdata/result/expr/!bad_plus.sql.txt b/testdata/result/expr/!bad_plus.sql.txt index ad6bffa9..8866b00f 100644 --- a/testdata/result/expr/!bad_plus.sql.txt +++ b/testdata/result/expr/!bad_plus.sql.txt @@ -11,7 +11,21 @@ syntax error: testdata/input/expr/!bad_plus.sql:2:1: unexpected token: --- AST &ast.BadNode{ NodeEnd: 3, - Raw: "1 +", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Raw: "1", + Base: 10, + End: 1, + }, + &token.Token{ + Kind: "+", + Space: " ", + Raw: "+", + Pos: 2, + End: 3, + }, + }, } --- SQL diff --git a/testdata/result/expr/!bad_plus2.sql.txt b/testdata/result/expr/!bad_plus2.sql.txt index 4492d397..06a9d1bf 100644 --- a/testdata/result/expr/!bad_plus2.sql.txt +++ b/testdata/result/expr/!bad_plus2.sql.txt @@ -22,7 +22,22 @@ syntax error: testdata/input/expr/!bad_plus2.sql:1:13: unexpected token: ) Expr: &ast.BadNode{ NodePos: 1, NodeEnd: 4, - Raw: "1 +", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Raw: "1", + Base: 10, + Pos: 1, + End: 2, + }, + &token.Token{ + Kind: "+", + Space: " ", + Raw: "+", + Pos: 3, + End: 4, + }, + }, }, }, Right: &ast.ParenExpr{ @@ -31,7 +46,22 @@ syntax error: testdata/input/expr/!bad_plus2.sql:1:13: unexpected token: ) Expr: &ast.BadNode{ NodePos: 9, NodeEnd: 12, - Raw: "2 +", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Raw: "2", + Base: 10, + Pos: 9, + End: 10, + }, + &token.Token{ + Kind: "+", + Space: " ", + Raw: "+", + Pos: 11, + End: 12, + }, + }, }, }, } diff --git a/testdata/result/expr/!bad_typed_struct.sql.txt b/testdata/result/expr/!bad_typed_struct.sql.txt index 88ebcfa4..25558938 100644 --- a/testdata/result/expr/!bad_typed_struct.sql.txt +++ b/testdata/result/expr/!bad_typed_struct.sql.txt @@ -22,7 +22,15 @@ syntax error: testdata/input/expr/!bad_typed_struct.sql:1:14: unexpected token: Type: &ast.BadNode{ NodePos: 7, NodeEnd: 8, - Raw: "1", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Raw: "1", + Base: 10, + Pos: 7, + End: 8, + }, + }, }, }, }, @@ -30,7 +38,22 @@ syntax error: testdata/input/expr/!bad_typed_struct.sql:1:14: unexpected token: &ast.BadNode{ NodePos: 10, NodeEnd: 13, - Raw: "2 +", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Raw: "2", + Base: 10, + Pos: 10, + End: 11, + }, + &token.Token{ + Kind: "+", + Space: " ", + Raw: "+", + Pos: 12, + End: 13, + }, + }, }, }, } diff --git a/testdata/result/query/!bad_hint_select.sql.txt b/testdata/result/query/!bad_hint_select.sql.txt index 1a285960..eee910a8 100644 --- a/testdata/result/query/!bad_hint_select.sql.txt +++ b/testdata/result/query/!bad_hint_select.sql.txt @@ -12,7 +12,28 @@ syntax error: testdata/input/query/!bad_hint_select.sql:1:3: expected token: {, &ast.QueryStatement{ Query: &ast.BadNode{ NodeEnd: 10, - Raw: "@ select 1", + Tokens: []*token.Token{ + &token.Token{ + Kind: "@", + Raw: "@", + End: 1, + }, + &token.Token{ + Kind: "SELECT", + Space: " ", + Raw: "select", + Pos: 2, + End: 8, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "1", + Base: 10, + Pos: 9, + End: 10, + }, + }, }, } diff --git a/testdata/result/query/!bad_hint_select_2.sql.txt b/testdata/result/query/!bad_hint_select_2.sql.txt new file mode 100644 index 00000000..2ade3407 --- /dev/null +++ b/testdata/result/query/!bad_hint_select_2.sql.txt @@ -0,0 +1,44 @@ +--- !bad_hint_select_2.sql +@{hint = 1} select +--- Error +syntax error: testdata/input/query/!bad_hint_select_2.sql:1:19: unexpected token: + + 1: @{hint = 1} select + ^ + + +--- AST +&ast.QueryStatement{ + Hint: &ast.Hint{ + Rbrace: 10, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 2, + NameEnd: 6, + Name: "hint", + }, + Value: &ast.IntLiteral{ + ValuePos: 9, + ValueEnd: 10, + Base: 10, + Value: "1", + }, + }, + }, + }, + Query: &ast.Select{ + Select: 12, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.BadNode{ + NodePos: 18, + NodeEnd: 18, + }, + }, + }, + }, +} + +--- SQL +@{hint=1} SELECT diff --git a/testdata/result/query/!bad_select_order.sql.txt b/testdata/result/query/!bad_select_order.sql.txt index 836cfe45..e5f0bc4d 100644 --- a/testdata/result/query/!bad_select_order.sql.txt +++ b/testdata/result/query/!bad_select_order.sql.txt @@ -12,7 +12,43 @@ syntax error: testdata/input/query/!bad_select_order.sql:1:16: expected token: B &ast.QueryStatement{ Query: &ast.BadNode{ NodeEnd: 20, - Raw: "select 1 order x asc", + Tokens: []*token.Token{ + &token.Token{ + Kind: "SELECT", + Raw: "select", + End: 6, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "1", + Base: 10, + Pos: 7, + End: 8, + }, + &token.Token{ + Kind: "ORDER", + Space: " ", + Raw: "order", + Pos: 9, + End: 14, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "x", + AsString: "x", + Pos: 15, + End: 16, + }, + &token.Token{ + Kind: "ASC", + Space: " ", + Raw: "asc", + Pos: 17, + End: 20, + }, + }, }, } diff --git a/testdata/result/statement/!bad_hint_select.sql.txt b/testdata/result/statement/!bad_hint_select.sql.txt index 1a285960..eee910a8 100644 --- a/testdata/result/statement/!bad_hint_select.sql.txt +++ b/testdata/result/statement/!bad_hint_select.sql.txt @@ -12,7 +12,28 @@ syntax error: testdata/input/query/!bad_hint_select.sql:1:3: expected token: {, &ast.QueryStatement{ Query: &ast.BadNode{ NodeEnd: 10, - Raw: "@ select 1", + Tokens: []*token.Token{ + &token.Token{ + Kind: "@", + Raw: "@", + End: 1, + }, + &token.Token{ + Kind: "SELECT", + Space: " ", + Raw: "select", + Pos: 2, + End: 8, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "1", + Base: 10, + Pos: 9, + End: 10, + }, + }, }, } diff --git a/testdata/result/statement/!bad_hint_select_2.sql.txt b/testdata/result/statement/!bad_hint_select_2.sql.txt new file mode 100644 index 00000000..2ade3407 --- /dev/null +++ b/testdata/result/statement/!bad_hint_select_2.sql.txt @@ -0,0 +1,44 @@ +--- !bad_hint_select_2.sql +@{hint = 1} select +--- Error +syntax error: testdata/input/query/!bad_hint_select_2.sql:1:19: unexpected token: + + 1: @{hint = 1} select + ^ + + +--- AST +&ast.QueryStatement{ + Hint: &ast.Hint{ + Rbrace: 10, + Records: []*ast.HintRecord{ + &ast.HintRecord{ + Key: &ast.Ident{ + NamePos: 2, + NameEnd: 6, + Name: "hint", + }, + Value: &ast.IntLiteral{ + ValuePos: 9, + ValueEnd: 10, + Base: 10, + Value: "1", + }, + }, + }, + }, + Query: &ast.Select{ + Select: 12, + Results: []ast.SelectItem{ + &ast.ExprSelectItem{ + Expr: &ast.BadNode{ + NodePos: 18, + NodeEnd: 18, + }, + }, + }, + }, +} + +--- SQL +@{hint=1} SELECT diff --git a/testdata/result/statement/!bad_insert.sql.txt b/testdata/result/statement/!bad_insert.sql.txt index 9daebe42..4d2574df 100644 --- a/testdata/result/statement/!bad_insert.sql.txt +++ b/testdata/result/statement/!bad_insert.sql.txt @@ -37,11 +37,121 @@ syntax error: testdata/input/dml/!bad_insert.sql:2:1: expected beginning of simp Query: &ast.BadNode{ NodePos: 27, NodeEnd: 59, - Raw: "vales (1, 2, 3),\n (4, 5, 6)", + Tokens: []*token.Token{ + &token.Token{ + Kind: "", + Space: "\n", + Raw: "vales", + AsString: "vales", + Pos: 27, + End: 32, + }, + &token.Token{ + Kind: "(", + Space: " ", + Raw: "(", + Pos: 33, + End: 34, + }, + &token.Token{ + Kind: "", + Raw: "1", + Base: 10, + Pos: 34, + End: 35, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 35, + End: 36, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "2", + Base: 10, + Pos: 37, + End: 38, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 38, + End: 39, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "3", + Base: 10, + Pos: 40, + End: 41, + }, + &token.Token{ + Kind: ")", + Raw: ")", + Pos: 41, + End: 42, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 42, + End: 43, + }, + &token.Token{ + Kind: "(", + Space: "\n ", + Raw: "(", + Pos: 50, + End: 51, + }, + &token.Token{ + Kind: "", + Raw: "4", + Base: 10, + Pos: 51, + End: 52, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 52, + End: 53, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "5", + Base: 10, + Pos: 54, + End: 55, + }, + &token.Token{ + Kind: ",", + Raw: ",", + Pos: 55, + End: 56, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "6", + Base: 10, + Pos: 57, + End: 58, + }, + &token.Token{ + Kind: ")", + Raw: ")", + Pos: 58, + End: 59, + }, + }, }, }, } --- SQL -INSERT INTO foo (foo, bar, baz) vales (1, 2, 3), - (4, 5, 6) +INSERT INTO foo (foo, bar, baz) vales (1, 2, 3), (4, 5, 6) diff --git a/testdata/result/statement/!bad_select_order.sql.txt b/testdata/result/statement/!bad_select_order.sql.txt index 836cfe45..e5f0bc4d 100644 --- a/testdata/result/statement/!bad_select_order.sql.txt +++ b/testdata/result/statement/!bad_select_order.sql.txt @@ -12,7 +12,43 @@ syntax error: testdata/input/query/!bad_select_order.sql:1:16: expected token: B &ast.QueryStatement{ Query: &ast.BadNode{ NodeEnd: 20, - Raw: "select 1 order x asc", + Tokens: []*token.Token{ + &token.Token{ + Kind: "SELECT", + Raw: "select", + End: 6, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "1", + Base: 10, + Pos: 7, + End: 8, + }, + &token.Token{ + Kind: "ORDER", + Space: " ", + Raw: "order", + Pos: 9, + End: 14, + }, + &token.Token{ + Kind: "", + Space: " ", + Raw: "x", + AsString: "x", + Pos: 15, + End: 16, + }, + &token.Token{ + Kind: "ASC", + Space: " ", + Raw: "asc", + Pos: 17, + End: 20, + }, + }, }, }