From 1f281a875768a068a49ba1e2fd896b19dc795071 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Wed, 6 Dec 2023 00:55:46 +0300 Subject: [PATCH] fix: Revised AST --- src/lang/ast/expr.rs | 41 +++- src/lang/ast/stmt.rs | 31 ++- src/lang/parser.rs | 311 +++++++++++++++++++++----- src/lang/scanner.rs | 442 ++++++++++++++++++++++--------------- src/lang/serializer.rs | 89 ++++---- src/lang/tests/helpers.rs | 2 +- src/lang/token.rs | 27 ++- src/runtime/error.rs | 33 ++- src/runtime/eval.rs | 6 +- src/runtime/interpreter.rs | 115 ++++++---- src/runtime/mod.rs | 19 +- src/runtime/resolver.rs | 70 +++--- 12 files changed, 786 insertions(+), 400 deletions(-) diff --git a/src/lang/ast/expr.rs b/src/lang/ast/expr.rs index a85bc1b6..cf79fe95 100644 --- a/src/lang/ast/expr.rs +++ b/src/lang/ast/expr.rs @@ -1,42 +1,63 @@ use std::rc::Rc; -use crate::{lang::token::Token, runtime::types::RV}; +use crate::{ + lang::token::{Span, Token, TokenType}, + runtime::types::RV, +}; use super::{sql::SqlSelect, stmt::StmtId}; #[derive(Debug, Eq, PartialEq)] pub enum Expr { - Select(SqlSelect), - Variable(Token), - Grouping(ExprId), - Literal(RV), + Select { + query: SqlSelect, + span: Span, + }, + Variable { + name: Token, + span: Span, + }, + Grouping { + expr: ExprId, + span: Span, + }, + Literal { + value: RV, + raw: String, + span: Span, + }, Function { name: Option, parameters: Vec, body: Rc>, + span: Span, }, Binary { left: ExprId, - token: Token, + symbol: TokenType, right: ExprId, + span: Span, }, Unary { - token: Token, + symbol: TokenType, expr: ExprId, + span: Span, }, Assignment { - var_tok: Token, + dst: Token, expr: ExprId, + span: Span, }, Logical { left: ExprId, - token: Token, + symbol: TokenType, right: ExprId, + span: Span, }, Call { callee: ExprId, - paren: Token, args: Vec, + span: Span, }, } diff --git a/src/lang/ast/stmt.rs b/src/lang/ast/stmt.rs index 885e50de..40a60cd5 100644 --- a/src/lang/ast/stmt.rs +++ b/src/lang/ast/stmt.rs @@ -1,30 +1,47 @@ -use crate::lang::token::Token; +use crate::lang::token::{Span, Token}; use super::expr::ExprId; #[derive(Debug, Eq, PartialEq)] pub enum Stmt { - Expression(ExprId), - Break(Token), - Continue(Token), - Block(Vec), + Program { + stmts: Vec, + span: Span, + }, + Expression { + expr: ExprId, + span: Span, + }, + Break { + span: Span, + }, + Continue { + span: Span, + }, + Block { + stmts: Vec, + span: Span, + }, Declaration { - token: Token, + dst: Token, expr: ExprId, + span: Span, }, If { condition: ExprId, body: StmtId, r#else: Option, + span: Span, }, Loop { condition: Option, body: StmtId, post: Option, + span: Span, }, Return { - token: Token, expr: Option, + span: Span, }, } diff --git a/src/lang/parser.rs b/src/lang/parser.rs index 84174d72..4eb73c6f 100644 --- a/src/lang/parser.rs +++ b/src/lang/parser.rs @@ -1,13 +1,12 @@ -use std::rc::Rc; - use crate::lang::token::Keyword; use crate::lang::token::Keyword::*; use crate::lang::token::SqlKeyword::*; use crate::lang::token::Symbol::*; use crate::lang::token::TokenType::*; -use crate::lang::token::{Token, TokenType}; +use crate::lang::token::{Span, Token, TokenType}; use crate::runtime::types::RV; use crate::{kw, skw, sym}; +use std::rc::Rc; use super::ast::expr::Expr; use super::ast::expr::ExprId; @@ -30,13 +29,13 @@ pub struct Parser<'a> { } pub struct Parsed { - pub statements: Vec, + pub program: StmtId, pub arena: Rc, } impl Parsed { - pub fn new(statements: Vec, arena: Rc) -> Parsed { - Parsed { statements, arena } + pub fn new(program: StmtId, arena: Rc) -> Parsed { + Parsed { program, arena } } } @@ -56,7 +55,13 @@ macro_rules! binary { let token = (*$self.peek_bw(1)).clone(); let left = current_expr; let right = $self.$builder()?; - current_expr = $self.arena.expression(Expr::Binary { left, token, right }); + + current_expr = $self.arena.expression(Expr::Binary { + left, + symbol: token.tok_type, + right, + span: Span { start: 0, end: 0, line: 0, line_end: 0 } + }); } return Ok(current_expr); } @@ -93,17 +98,25 @@ impl<'a> Parser<'a> { current: 0, arena, }; - let statements = parser.program()?; - Ok(Parsed::new(statements, Rc::new(parser.arena))) + let program = parser.program()?; + Ok(Parsed::new(program, Rc::new(parser.arena))) } - fn program(&mut self) -> ParseResult> { + fn program(&mut self) -> ParseResult { let mut statements: Vec = Vec::new(); while !self.is_at_end() { statements.push(self.declaration()?); } self.expected(Eof)?; - Ok(statements) + Ok(self.arena.statement(Stmt::Program { + stmts: statements, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) } fn declaration(&mut self) -> ParseResult { @@ -135,12 +148,24 @@ impl<'a> Parser<'a> { condition, body: if_branch, r#else: Some(else_branch), + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })); } Ok(self.arena.statement(Stmt::If { condition, body: if_branch, r#else: None, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })) } @@ -150,6 +175,12 @@ impl<'a> Parser<'a> { condition: None, body: inner_stmt, post: None, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })) } @@ -163,6 +194,12 @@ impl<'a> Parser<'a> { condition: Some(condition), body: inner_stmt, post: None, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })) } @@ -174,8 +211,20 @@ impl<'a> Parser<'a> { } self.expected(sym!(Semicolon))?; + if expr.is_none() { + return Ok(self.arena.statement(Stmt::Return { + span: token.span, + expr: None, + })); + } + Ok(self.arena.statement(Stmt::Return { - token: token.clone(), + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, expr, })) } @@ -202,7 +251,15 @@ impl<'a> Parser<'a> { } else { let wrapped = self.expression()?; self.expected(sym!(RightParen))?; - Some(self.arena.statement(Stmt::Expression(wrapped))) + Some(self.arena.statement(Stmt::Expression { + expr: wrapped, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) }; let inner_stmt = self.declaration()?; @@ -212,16 +269,34 @@ impl<'a> Parser<'a> { condition, body: inner_stmt, post: increment, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })); } let loop_stmt = self.arena.statement(Stmt::Loop { condition, body: inner_stmt, post: increment, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, }); - Ok(self - .arena - .statement(Stmt::Block(vec![initializer.unwrap(), loop_stmt]))) + Ok(self.arena.statement(Stmt::Block { + stmts: vec![initializer.unwrap(), loop_stmt], + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) } fn block(&mut self) -> ParseResult { @@ -233,35 +308,69 @@ impl<'a> Parser<'a> { self.expected(sym!(RightBrace))?; - Ok(self.arena.statement(Stmt::Block(statements))) + Ok(self.arena.statement(Stmt::Block { + stmts: statements, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) } fn break_statement(&mut self) -> ParseResult { let tok = self.peek_bw(1); self.expected(sym!(Semicolon))?; - Ok(self.arena.statement(Stmt::Break(tok.clone()))) + Ok(self.arena.statement(Stmt::Break { span: tok.span })) } fn continue_statement(&mut self) -> ParseResult { let tok = self.peek_bw(1); self.expected(sym!(Semicolon))?; - Ok(self.arena.statement(Stmt::Continue(tok.clone()))) + Ok(self.arena.statement(Stmt::Continue { span: tok.span })) } fn expression_statement(&mut self) -> ParseResult { let expr = self.expression()?; self.expected(sym!(Semicolon))?; - Ok(self.arena.statement(Stmt::Expression(expr))) + Ok(self.arena.statement(Stmt::Expression { + expr, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) } fn var_declaration(&mut self) -> ParseResult { let token = self.expected(Identifier { dollar: true })?.clone(); let expr = match self.match_next(sym!(Equal)) { true => self.expression()?, - false => self.arena.expression(Expr::Literal(RV::Null)), + false => self.arena.expression(Expr::Literal { + value: RV::Undefined, + raw: "undefined".to_string(), + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + }), }; self.expected(sym!(Semicolon))?; - Ok(self.arena.statement(Stmt::Declaration { token, expr })) + Ok(self.arena.statement(Stmt::Declaration { + dst: token, + expr, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })) } fn fun_declaration(&mut self) -> ParseResult { @@ -288,7 +397,7 @@ impl<'a> Parser<'a> { let block = self.arena.get_statement(bidx); let body: Vec = match block { - Stmt::Block(stmts) => stmts.clone(), + Stmt::Block { stmts, span: _ } => stmts.clone(), _ => vec![], }; @@ -296,6 +405,12 @@ impl<'a> Parser<'a> { name: token, parameters, body: Rc::new(body), + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })) } @@ -310,10 +425,16 @@ impl<'a> Parser<'a> { if self.match_next(sym!(Equal)) { let value = self.assignment()?; match self.arena.get_expression(expr) { - Expr::Variable(tok) => { + Expr::Variable { name, span: _ } => { return Ok(self.arena.expression(Expr::Assignment { - var_tok: tok.clone(), + dst: name.clone(), expr: value, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })); } _ => { @@ -333,8 +454,14 @@ impl<'a> Parser<'a> { let right = self.and()?; return Ok(self.arena.expression(Expr::Logical { left: expr, - token: op.clone(), + symbol: op.tok_type.clone(), right, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })); } Ok(expr) @@ -347,8 +474,14 @@ impl<'a> Parser<'a> { let right = self.equality()?; return Ok(self.arena.expression(Expr::Logical { left: expr, - token: op.clone(), + symbol: op.tok_type.clone(), right, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, })); } Ok(expr) @@ -383,7 +516,16 @@ impl<'a> Parser<'a> { if self.match_next_multi(&vec![sym!(Minus), sym!(Bang)]) { let token = (*self.peek_bw(1)).clone(); let unary = self.unary()?; - return Ok(self.arena.expression(Expr::Unary { expr: unary, token })); + return Ok(self.arena.expression(Expr::Unary { + symbol: token.tok_type, + expr: unary, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + })); } self.select() } @@ -411,13 +553,21 @@ impl<'a> Parser<'a> { let secondary_core = self.select_core()?; compounds.push((compound_op, secondary_core)) } - Ok(self.arena.expression(Expr::Select(SqlSelect { - core, - compound: compounds, - order_by: None, // TODO(vck) - limit: None, // TODO(vck) - offset: None, // TODO(vck) - }))) + Ok(self.arena.expression(Expr::Select { + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + query: SqlSelect { + core, + compound: compounds, + order_by: None, // TODO(vck) + limit: None, // TODO(vck) + offset: None, // TODO(vck) + }, + })) } fn select_core(&mut self) -> ParseResult { @@ -498,7 +648,12 @@ impl<'a> Parser<'a> { Ok(self.arena.expression(Expr::Call { callee, - paren, + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, args: arguments, })) } @@ -521,17 +676,42 @@ impl<'a> Parser<'a> { let tok = self.peek_bw(0); self.current += 1; match &tok.tok_type { - True => Ok(self.arena.expression(Expr::Literal(RV::Bool(true)))), - False => Ok(self.arena.expression(Expr::Literal(RV::Bool(false)))), - TokenType::Null => Ok(self.arena.expression(Expr::Literal(RV::Null))), - Str | Num => Ok(self - .arena - .expression(Expr::Literal(tok.literal.clone().unwrap()))), - Identifier { dollar: _ } => Ok(self.arena.expression(Expr::Variable(tok.clone()))), + True => Ok(self.arena.expression(Expr::Literal { + value: RV::Bool(true), + raw: "true".to_string(), + span: tok.span, + })), + False => Ok(self.arena.expression(Expr::Literal { + value: RV::Bool(false), + raw: "false".to_string(), + span: tok.span, + })), + TokenType::Null => Ok(self.arena.expression(Expr::Literal { + value: RV::Null, + raw: "null".to_string(), + span: tok.span, + })), + Str | Num => Ok(self.arena.expression(Expr::Literal { + value: tok.literal.clone().unwrap(), + raw: tok.lexeme.clone().unwrap(), + span: tok.span, + })), + Identifier { dollar: _ } => Ok(self.arena.expression(Expr::Variable { + name: tok.clone(), + span: tok.span, + })), Symbol(LeftParen) => { let expr = self.expression()?; self.expected(sym!(RightParen))?; - Ok(self.arena.expression(Expr::Grouping(expr))) + Ok(self.arena.expression(Expr::Grouping { + span: Span { + start: 0, + end: 0, + line: 0, + line_end: 0, + }, + expr, + })) } _ => Err(ParseError::UnexpectedToken { token: tok.clone() }), } @@ -613,13 +793,14 @@ mod test { compare_parsed_to_expected( "1;", json!({ - "type": "Program", + "type": "Stmt::Program", "body": [ { "type": "Stmt::Expression", "value": { "type": "Expr::Literal", "value": "Num(1.0)", + "raw": "1" } } ] @@ -632,16 +813,19 @@ mod test { compare_parsed_to_expected( "-1;", json!({ - "type": "Program", + "type": "Stmt::Program", "body": [ { "type": "Stmt::Expression", "value": { "type": "Expr::Unary", - "operator": "-", + "operator": { + "Symbol": "Minus" + }, "value": { "type": "Expr::Literal", "value": "Num(1.0)", + "raw": "1" } } } @@ -655,7 +839,7 @@ mod test { compare_parsed_to_expected( "1 + 2;", json!({ - "type": "Program", + "type": "Stmt::Program", "body": [ { "type": "Stmt::Expression", @@ -664,11 +848,15 @@ mod test { "left": { "type": "Expr::Literal", "value": "Num(1.0)", + "raw": "1" + }, + "operator": { + "Symbol": "Plus" }, - "operator": "+", "right": { "type": "Expr::Literal", "value": "Num(2.0)", + "raw": "2" } } } @@ -682,7 +870,7 @@ mod test { compare_parsed_to_expected( "(1 + 2) * (3 / (4 - 7));", json!({ - "type": "Program", + "type": "Stmt::Program", "body": [ { "type": "Stmt::Expression", @@ -693,36 +881,49 @@ mod test { "value": { "type": "Expr::Binary", "left": { + "raw": "1", "type": "Expr::Literal", "value": "Num(1.0)", }, - "operator": "+", + "operator": { + "Symbol": "Plus" + }, "right": { + "raw": "2", "type": "Expr::Literal", "value": "Num(2.0)", } } }, - "operator": "*", + "operator": { + "Symbol": "Star" + }, "right": { "type": "Expr::Grouping", "value": { "type": "Expr::Binary", "left": { + "raw": "3", "type": "Expr::Literal", "value": "Num(3.0)", }, - "operator": "/", + "operator": { + "Symbol": "Slash" + }, "right": { "type": "Expr::Grouping", "value": { "type": "Expr::Binary", "left": { + "raw": "4", "type": "Expr::Literal", "value": "Num(4.0)", }, - "operator": "-", + "operator": { + "Symbol": "Minus" + }, "right": { + "raw": "7", "type": "Expr::Literal", "value": "Num(7.0)", } @@ -742,7 +943,7 @@ mod test { compare_parsed_to_expected( "a;", json!({ - "type": "Program", + "type": "Stmt::Program", "body": [ { "type": "Stmt::Expression", diff --git a/src/lang/scanner.rs b/src/lang/scanner.rs index 9fcf1a13..7307baa5 100644 --- a/src/lang/scanner.rs +++ b/src/lang/scanner.rs @@ -61,10 +61,26 @@ impl Scanner { self.tokens.push(Token { tok_type: token, literal: None, + lexeme: Some(lexeme.to_string()), span: Span { - line: self.line, start: self.start, - lexeme: Rc::new(lexeme.to_string()), + end: self.current - 1, + line: self.line, + line_end: self.line, + }, + }); + } + + fn finalize(&mut self) { + self.tokens.push(Token { + tok_type: Eof, + literal: None, + lexeme: None, + span: Span { + start: self.current, + end: self.current, + line: self.line, + line_end: self.line, }, }); } @@ -97,12 +113,12 @@ impl Scanner { } if self.is_at_end() { - let err_span: String = self.chars[self.start + 1..self.current].iter().collect(); return Err(ScanError::UnterminatedString { span: Span { start: self.start + 1, - lexeme: Rc::new(err_span), + end: self.current, line: self.line, + line_end: self.line, }, }); } @@ -112,15 +128,16 @@ impl Scanner { let span: String = self.chars[self.start + 1..self.current - 1] .iter() .collect(); - let value = Rc::new(span.to_string()); self.tokens.push(Token { tok_type: TokenType::Str, - literal: Some(Str(value.clone())), + literal: Some(Str(Rc::new(span.clone()))), + lexeme: Some(span), span: Span { - line: self.line, start: self.start + 1, - lexeme: value, + end: self.current - 1, + line: self.line, + line_end: self.line, }, }); Ok(()) @@ -144,12 +161,12 @@ impl Scanner { self.advance(); } if self.is_at_end() || !self.peek(0).is_ascii_digit() { - let err_span: String = self.chars[self.start..self.current].iter().collect(); return Err(ScanError::MalformedNumberLiteral { span: Span { start: self.start, - lexeme: Rc::new(err_span), + end: self.current, line: self.line, + line_end: self.line, }, }); } @@ -163,10 +180,12 @@ impl Scanner { self.tokens.push(Token { tok_type: TokenType::Num, literal: Some(Num(parsed)), + lexeme: Some(span), span: Span { start: self.start, - lexeme: Rc::new(span), + end: self.current, line: self.line, + line_end: self.line, }, }); Ok(()) @@ -188,14 +207,16 @@ impl Scanner { .clone(), ); } else { - let value = Rc::new(span.to_string()); + let value = span.to_string(); self.tokens.push(Token { tok_type: Identifier { dollar: is_dollar }, - literal: Some(Str(value.clone())), + literal: Some(Str(Rc::new(value.clone()))), + lexeme: Some(value), span: Span { start: self.start, - lexeme: value, + end: self.current, line: self.line, + line_end: self.line, }, }); } @@ -230,8 +251,9 @@ impl Scanner { return Err(ScanError::UnexpectedCharacter { span: Span { start: self.start, + end: self.current, line: self.line, - lexeme: Rc::new(String::from(c)), + line_end: self.line, }, }); } @@ -246,7 +268,7 @@ impl Scanner { self.scan_token()?; } - self.add_token(" ", Eof); + self.finalize(); Ok(()) } } @@ -274,122 +296,134 @@ mod test { vec![ Token { tok_type: sym!(LeftParen), - + lexeme: lexm!("("), literal: None, span: Span { - line: 0, start: 0, - lexeme: lexm!("("), + end: 0, + line: 0, + line_end: 0, }, }, Token { tok_type: sym!(RightParen), - + lexeme: lexm!(")"), literal: None, span: Span { - line: 0, start: 1, - lexeme: lexm!(")"), + end: 1, + line: 0, + line_end: 0, }, }, Token { tok_type: sym!(LeftBrace), - + lexeme: lexm!("{"), literal: None, span: Span { line: 0, start: 2, - lexeme: lexm!("{"), + end: 2, + line_end: 0, }, }, Token { tok_type: sym!(RightBrace), - + lexeme: lexm!("}"), literal: None, span: Span { line: 0, start: 3, - lexeme: lexm!("}"), + end: 3, + line_end: 0, }, }, Token { tok_type: sym!(Semicolon), - + lexeme: lexm!(";"), literal: None, span: Span { line: 0, start: 4, - lexeme: lexm!(";"), + end: 4, + line_end: 0, }, }, Token { + lexeme: lexm!(","), tok_type: sym!(Comma), - literal: None, span: Span { line: 0, start: 5, - lexeme: lexm!(","), + end: 5, + line_end: 0, }, }, Token { tok_type: sym!(Plus), - + lexeme: lexm!("+"), literal: None, span: Span { line: 0, start: 6, - lexeme: lexm!("+"), + end: 6, + line_end: 0, }, }, Token { tok_type: sym!(Minus), - + lexeme: lexm!("-"), literal: None, span: Span { line: 0, start: 7, - lexeme: lexm!("-"), + end: 7, + line_end: 0, }, }, Token { tok_type: sym!(Star), - + lexeme: lexm!("*"), literal: None, span: Span { line: 0, start: 8, - lexeme: lexm!("*"), + end: 8, + line_end: 0, }, }, Token { tok_type: sym!(Slash), - + lexeme: lexm!("/"), literal: None, span: Span { line: 0, start: 9, - lexeme: lexm!("/"), + end: 9, + line_end: 0, }, }, Token { tok_type: sym!(Dot), - + lexeme: lexm!("."), literal: None, span: Span { line: 0, start: 10, - lexeme: lexm!("."), + end: 10, + line_end: 0, }, }, Token { tok_type: Eof, - + lexeme: None, literal: None, span: Span { line: 0, - start: 10, - lexeme: lexm!(" "), + start: 11, + end: 11, + line_end: 0, }, }, ], @@ -404,64 +438,78 @@ mod test { Token { tok_type: TokenType::Num, literal: Some(Num(123.0)), + lexeme: lexm!("123"), span: Span { line: 0, start: 0, - lexeme: lexm!("123"), + end: 3, + line_end: 0, }, }, Token { tok_type: TokenType::Num, literal: Some(Num(123.456)), + lexeme: lexm!("123.456"), span: Span { line: 0, start: 4, - lexeme: lexm!("123.456"), + end: 11, + line_end: 0, }, }, Token { tok_type: TokenType::Str, literal: Some(Str(Rc::new("hello world".to_string()))), + lexeme: lexm!("hello world"), span: Span { line: 0, start: 13, - lexeme: lexm!("hello world"), + end: 24, + line_end: 0, }, }, Token { tok_type: TokenType::True, literal: None, + lexeme: lexm!("true"), span: Span { line: 0, start: 26, - lexeme: lexm!("true"), + end: 29, + line_end: 0, }, }, Token { tok_type: TokenType::False, literal: None, + lexeme: lexm!("false"), span: Span { line: 0, start: 31, - lexeme: lexm!("false"), + end: 35, + line_end: 0, }, }, Token { tok_type: TokenType::Identifier { dollar: false }, literal: Some(Str(Rc::new("helloIdentifier".to_string()))), + lexeme: lexm!("helloIdentifier"), span: Span { line: 0, start: 37, - lexeme: lexm!("helloIdentifier"), + end: 52, + line_end: 0, }, }, Token { tok_type: Eof, literal: None, + lexeme: None, span: Span { line: 0, - start: 37, - lexeme: lexm!(" "), + start: 53, + end: 53, + line_end: 0, }, }, ], @@ -475,46 +523,56 @@ mod test { Token { tok_type: TokenType::Identifier { dollar: true }, literal: Some(Str(Rc::new("$myPreciseVariable".to_string()))), + lexeme: lexm!("$myPreciseVariable"), span: Span { line: 0, start: 0, - lexeme: lexm!("$myPreciseVariable"), + end: 18, + line_end: 0, }, }, Token { tok_type: TokenType::Identifier { dollar: true }, literal: Some(Str(Rc::new("$my_precise_variable".to_string()))), + lexeme: lexm!("$my_precise_variable"), span: Span { line: 0, start: 19, - lexeme: lexm!("$my_precise_variable"), + end: 39, + line_end: 0, }, }, Token { tok_type: TokenType::Identifier { dollar: false }, literal: Some(Str(Rc::new("myPreciseFunction".to_string()))), + lexeme: lexm!("myPreciseFunction"), span: Span { line: 0, start: 40, - lexeme: lexm!("myPreciseFunction"), + end: 57, + line_end: 0, }, }, Token { tok_type: TokenType::Identifier { dollar: false }, literal: Some(Str(Rc::new("my_precise_function".to_string()))), + lexeme: lexm!("my_precise_function"), span: Span { line: 0, start: 58, - lexeme: lexm!("my_precise_function"), + end: 77, + line_end: 0, }, }, Token { tok_type: Eof, literal: None, + lexeme: None, span: Span { + start: 78, + end: 78, line: 0, - start: 58, - lexeme: lexm!(" "), + line_end: 0, }, }, ], @@ -528,172 +586,189 @@ mod test { vec![ Token { tok_type: TokenType::Num, - + lexeme: lexm!("0"), literal: Some(Num(0.0)), span: Span { line: 0, start: 0, - lexeme: lexm!("0"), + end: 1, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("1"), literal: Some(Num(1.0)), span: Span { line: 0, start: 2, - lexeme: lexm!("1"), + end: 3, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("2"), literal: Some(Num(2.0)), span: Span { line: 0, start: 4, - lexeme: lexm!("2"), + end: 5, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("3"), literal: Some(Num(3.0)), span: Span { line: 0, start: 6, - lexeme: lexm!("3"), + end: 7, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("4"), literal: Some(Num(4.0)), span: Span { line: 0, start: 8, - lexeme: lexm!("4"), + end: 9, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("5"), literal: Some(Num(5.0)), span: Span { line: 0, start: 10, - lexeme: lexm!("5"), + end: 11, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("6"), literal: Some(Num(6.0)), span: Span { line: 0, start: 12, - lexeme: lexm!("6"), + end: 13, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("7"), literal: Some(Num(7.0)), span: Span { line: 0, start: 14, - lexeme: lexm!("7"), + end: 15, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("8"), literal: Some(Num(8.0)), span: Span { line: 0, start: 16, - lexeme: lexm!("8"), + end: 17, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("9"), literal: Some(Num(9.0)), span: Span { line: 0, start: 18, - lexeme: lexm!("9"), + end: 19, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("10"), literal: Some(Num(10.0)), span: Span { line: 0, start: 20, - lexeme: lexm!("10"), + end: 22, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("100"), literal: Some(Num(100.0)), span: Span { line: 0, start: 23, - lexeme: lexm!("100"), + end: 26, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("500"), literal: Some(Num(500.0)), span: Span { line: 0, start: 27, - lexeme: lexm!("500"), + end: 30, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("1000"), literal: Some(Num(1000.0)), span: Span { line: 0, start: 31, - lexeme: lexm!("1000"), + end: 35, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("1.7976931348623157E+308"), literal: Some(Num(1.7976931348623157E+308)), span: Span { line: 0, start: 36, - lexeme: lexm!("1.7976931348623157E+308"), + end: 59, + line_end: 0, }, }, Token { tok_type: TokenType::Num, - + lexeme: lexm!("1.7976931348623157E-308"), literal: Some(Num(1.7976931348623157E-308)), span: Span { line: 0, start: 60, - lexeme: lexm!("1.7976931348623157E-308"), + end: 83, + line_end: 0, }, }, Token { tok_type: Eof, - + lexeme: None, literal: None, span: Span { line: 0, - start: 60, - lexeme: lexm!(" "), + start: 84, + end: 84, + line_end: 0, }, }, ], @@ -707,161 +782,178 @@ mod test { vec![ Token { tok_type: kw!(Keyword::And), + lexeme: lexm!("and"), literal: None, span: Span { line: 0, start: 0, - lexeme: lexm!("and"), + end: 2, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Or), - + lexeme: lexm!("or"), literal: None, span: Span { line: 0, start: 4, - lexeme: lexm!("or"), + end: 5, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Class), - + lexeme: lexm!("class"), literal: None, span: Span { line: 0, start: 7, - lexeme: lexm!("class"), + end: 11, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Else), - + lexeme: lexm!("else"), literal: None, span: Span { line: 0, start: 13, - lexeme: lexm!("else"), + end: 16, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::For), - + lexeme: lexm!("for"), literal: None, span: Span { line: 0, start: 18, - lexeme: lexm!("for"), + end: 20, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Fun), - + lexeme: lexm!("fun"), literal: None, span: Span { line: 0, start: 22, - lexeme: lexm!("fun"), + end: 24, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::If), - + lexeme: lexm!("if"), literal: None, span: Span { line: 0, start: 26, - lexeme: lexm!("if"), + end: 27, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Break), - + lexeme: lexm!("break"), literal: None, span: Span { line: 0, start: 29, - lexeme: lexm!("break"), + end: 33, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Continue), - + lexeme: lexm!("continue"), literal: None, span: Span { line: 0, start: 35, - lexeme: lexm!("continue"), + end: 42, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Return), - + lexeme: lexm!("return"), literal: None, span: Span { line: 0, start: 44, - lexeme: lexm!("return"), + end: 49, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Super), - + lexeme: lexm!("super"), literal: None, span: Span { line: 0, start: 51, - lexeme: lexm!("super"), + end: 55, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::This), - + lexeme: lexm!("this"), literal: None, span: Span { line: 0, start: 57, - lexeme: lexm!("this"), + end: 60, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Var), - + lexeme: lexm!("var"), literal: None, span: Span { line: 0, start: 62, - lexeme: lexm!("var"), + end: 64, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::While), - + lexeme: lexm!("while"), literal: None, span: Span { line: 0, start: 66, - lexeme: lexm!("while"), + end: 70, + line_end: 0, }, }, Token { tok_type: kw!(Keyword::Loop), - + lexeme: lexm!("loop"), literal: None, span: Span { line: 0, start: 72, - lexeme: lexm!("loop"), + end: 75, + line_end: 0, }, }, Token { tok_type: Eof, - + lexeme: None, literal: None, span: Span { line: 0, - start: 72, - lexeme: lexm!(" "), + start: 76, + end: 76, + line_end: 0, }, }, ], @@ -872,57 +964,57 @@ mod test { fn test_sql_keywords() { assert_tokens( "Begin Transaction Rollback Commit Where Having Asc Desc Order By Explain Is Not Null Offset Like Limit And Or Join Inner Outer Right Left On Create Insert Update Delete Drop Into Values Index Table Select From As Cross Default Group Key Of Only Primary References Set System Unique Read Write", vec![ - Token {tok_type: skw!(SqlKeyword::Begin), literal: None, span: Span { line: 0, start: 0, lexeme: lexm!("Begin")} }, - Token {tok_type: skw!(SqlKeyword::Transaction), literal: None, span: Span { line: 0, start: 6, lexeme: lexm!("Transaction") } }, - Token {tok_type: skw!(SqlKeyword::Rollback), literal: None, span: Span { line: 0, start: 18 , lexeme: lexm!("Rollback") }}, - Token {tok_type: skw!(SqlKeyword::Commit), literal: None,span: Span { line: 0, start: 27 , lexeme: lexm!("Commit") }}, - Token {tok_type: skw!(SqlKeyword::Where), literal: None, span: Span { line: 0, start: 34 , lexeme: lexm!("Where") }}, - Token {tok_type: skw!(SqlKeyword::Having), literal: None, span: Span { line: 0, start: 40 , lexeme: lexm!("Having") }}, - Token {tok_type: skw!(SqlKeyword::Asc), literal: None, span: Span { line: 0, start: 47 , lexeme: lexm!("Asc") }}, - Token {tok_type: skw!(SqlKeyword::Desc), literal: None, span: Span { line: 0, start: 51 , lexeme: lexm!("Desc") }}, - Token {tok_type: skw!(SqlKeyword::Order), literal: None, span: Span { line: 0, start: 56 , lexeme: lexm!("Order") }}, - Token {tok_type: skw!(SqlKeyword::By), literal: None, span: Span { line: 0, start: 62 , lexeme: lexm!("By") }}, - Token {tok_type: skw!(SqlKeyword::Explain), literal: None, span: Span { line: 0, start: 65 , lexeme: lexm!("Explain") }}, - Token {tok_type: skw!(SqlKeyword::Is), literal: None, span: Span { line: 0, start: 73 , lexeme: lexm!("Is") }}, - Token {tok_type: skw!(SqlKeyword::Not), literal: None, span: Span { line: 0, start: 76, lexeme: lexm!("Not") }}, - Token {tok_type: skw!(SqlKeyword::Null), literal: None, span: Span { line: 0, start: 80 , lexeme: lexm!("Null") }}, - Token {tok_type: skw!(SqlKeyword::Offset), literal: None, span: Span { line: 0, start: 85 , lexeme: lexm!("Offset") }}, - Token {tok_type: skw!(SqlKeyword::Like), literal: None, span: Span { line: 0, start: 92 , lexeme: lexm!("Like") }}, - Token {tok_type: skw!(SqlKeyword::Limit), literal: None, span: Span { line: 0, start: 97 , lexeme: lexm!("Limit") }}, - Token {tok_type: skw!(SqlKeyword::And), literal: None, span: Span { line: 0, start: 103, lexeme: lexm!("And")}}, - Token {tok_type: skw!(SqlKeyword::Or), literal: None, span: Span { line: 0, start: 107, lexeme: lexm!("Or")}}, - Token {tok_type: skw!(SqlKeyword::Join), literal: None, span: Span { line: 0, start: 110, lexeme: lexm!("Join") }}, - Token {tok_type: skw!(SqlKeyword::Inner), literal: None, span: Span { line: 0, start: 115, lexeme: lexm!("Inner") }}, - Token {tok_type: skw!(SqlKeyword::Outer), literal: None, span: Span { line: 0, start: 121, lexeme: lexm!("Outer") }}, - Token {tok_type: skw!(SqlKeyword::Right), literal: None, span: Span { line: 0, start: 127, lexeme: lexm!("Right") }}, - Token {tok_type: skw!(SqlKeyword::Left), literal: None, span: Span { line: 0, start: 133, lexeme: lexm!("Left") }}, - Token {tok_type: skw!(SqlKeyword::On), literal: None, span: Span { line: 0, start: 138, lexeme: lexm!("On") }}, - Token {tok_type: skw!(SqlKeyword::Create), literal: None, span: Span { line: 0, start: 141, lexeme: lexm!("Create") }}, - Token {tok_type: skw!(SqlKeyword::Insert), literal: None, span: Span { line: 0, start: 148, lexeme: lexm!("Insert") }}, - Token {tok_type: skw!(SqlKeyword::Update), literal: None, span: Span { line: 0, start: 155, lexeme: lexm!("Update") }}, - Token {tok_type: skw!(SqlKeyword::Delete), literal: None, span: Span { line: 0, start: 162, lexeme: lexm!("Delete") }}, - Token {tok_type: skw!(SqlKeyword::Drop), literal: None, span: Span { line: 0, start: 169, lexeme: lexm!("Drop") }}, - Token {tok_type: skw!(SqlKeyword::Into), literal: None, span: Span { line: 0, start: 174, lexeme: lexm!("Into") }}, - Token {tok_type: skw!(SqlKeyword::Values), literal: None, span: Span { line: 0, start: 179 , lexeme: lexm!("Values") }}, - Token {tok_type: skw!(SqlKeyword::Index), literal: None, span: Span { line: 0, start: 186, lexeme: lexm!("Index")}}, - Token {tok_type: skw!(SqlKeyword::Table), literal: None, span: Span { line: 0, start: 192, lexeme: lexm!("Table") }}, - Token {tok_type: skw!(SqlKeyword::Select), literal: None, span: Span { line: 0, start: 198, lexeme: lexm!("Select") }}, - Token {tok_type: skw!(SqlKeyword::From), literal: None, span: Span { line: 0, start: 205, lexeme: lexm!("From") }}, - Token {tok_type: skw!(SqlKeyword::As), literal: None, span: Span { line: 0, start: 210, lexeme: lexm!("As") }}, - Token {tok_type: skw!(SqlKeyword::Cross), literal: None, span: Span { line: 0, start: 213, lexeme: lexm!("Cross") }}, - Token {tok_type: skw!(SqlKeyword::Default), literal: None, span: Span { line: 0, start: 219, lexeme: lexm!("Default") }}, - Token {tok_type: skw!(SqlKeyword::Group), literal: None, span: Span { line: 0, start: 227, lexeme: lexm!("Group") }}, - Token {tok_type: skw!(SqlKeyword::Key), literal: None, span: Span { line: 0, start: 233, lexeme: lexm!("Key") }}, - Token {tok_type: skw!(SqlKeyword::Of), literal: None, span: Span { line: 0, start: 237, lexeme: lexm!("Of") }}, - Token {tok_type: skw!(SqlKeyword::Only), literal: None, span: Span { line: 0, start: 240, lexeme: lexm!("Only") }}, - Token {tok_type: skw!(SqlKeyword::Primary), literal: None, span: Span { line: 0, start: 245, lexeme: lexm!("Primary") }}, - Token {tok_type: skw!(SqlKeyword::References), literal: None, span: Span { line: 0, start: 253, lexeme: lexm!("References") }}, - Token {tok_type: skw!(SqlKeyword::Set), literal: None, span: Span { line: 0, start: 264, lexeme: lexm!("Set") }}, - Token {tok_type: skw!(SqlKeyword::System), literal: None, span: Span { line: 0, start: 268, lexeme: lexm!("System") }}, - Token {tok_type: skw!(SqlKeyword::Unique), literal: None, span: Span { line: 0, start: 275, lexeme: lexm!("Unique") }}, - Token {tok_type: skw!(SqlKeyword::Read), literal: None, span: Span { line: 0, start: 282, lexeme: lexm!("Read") }}, - Token {tok_type: skw!(SqlKeyword::Write), literal: None, span: Span { line: 0, start: 287, lexeme: lexm!("Write") }}, - Token {tok_type: Eof, literal: None, span: Span { line: 0, start: 287, lexeme: lexm!(" ")} } + Token {tok_type: skw!(SqlKeyword::Begin), literal: None, lexeme: lexm!("Begin"), span: Span { line: 0, start: 0, line_end: 0, end: 4 }}, + Token {tok_type: skw!(SqlKeyword::Transaction), literal: None, lexeme: lexm!("Transaction") , span: Span { line: 0, start: 6, line_end: 0, end: 16 }}, + Token {tok_type: skw!(SqlKeyword::Rollback), literal: None, lexeme: lexm!("Rollback"), span: Span { line: 0, start: 18, line_end: 0, end: 25 }}, + Token {tok_type: skw!(SqlKeyword::Commit), literal: None , lexeme: lexm!("Commit"), span: Span { line: 0, start: 27, line_end: 0, end: 32 }}, + Token {tok_type: skw!(SqlKeyword::Where), literal: None, lexeme: lexm!("Where"), span: Span { line: 0, start: 34, line_end: 0, end: 38 }}, + Token {tok_type: skw!(SqlKeyword::Having), literal: None , lexeme: lexm!("Having"), span: Span { line: 0, start: 40, line_end: 0, end: 45 }}, + Token {tok_type: skw!(SqlKeyword::Asc), literal: None, lexeme: lexm!("Asc") , span: Span { line: 0, start: 47, line_end: 0, end: 49 }}, + Token {tok_type: skw!(SqlKeyword::Desc), literal: None, lexeme: lexm!("Desc"), span: Span { line: 0, start: 51, line_end: 0, end: 54 }}, + Token {tok_type: skw!(SqlKeyword::Order), literal: None, lexeme: lexm!("Order"), span: Span { line: 0, start: 56, line_end: 0, end: 60 }}, + Token {tok_type: skw!(SqlKeyword::By), literal: None, lexeme: lexm!("By"), span: Span { line: 0, start: 62, line_end: 0, end: 63 }}, + Token {tok_type: skw!(SqlKeyword::Explain), literal: None, lexeme: lexm!("Explain"), span: Span { line: 0, start: 65, line_end: 0, end: 71 }}, + Token {tok_type: skw!(SqlKeyword::Is), literal: None, lexeme: lexm!("Is"), span: Span { line: 0, start: 73, line_end: 0, end: 74 }}, + Token {tok_type: skw!(SqlKeyword::Not), literal: None, lexeme: lexm!("Not"), span: Span { line: 0, start: 76, line_end: 0, end: 78 }}, + Token {tok_type: skw!(SqlKeyword::Null), literal: None, lexeme: lexm!("Null"), span: Span { line: 0, start: 80, line_end: 0, end: 83 }}, + Token {tok_type: skw!(SqlKeyword::Offset), literal: None, lexeme: lexm!("Offset"), span: Span { line: 0, start: 85, line_end: 0, end: 90 }}, + Token {tok_type: skw!(SqlKeyword::Like), literal: None, lexeme: lexm!("Like"), span: Span { line: 0, start: 92, line_end: 0, end: 95 }}, + Token {tok_type: skw!(SqlKeyword::Limit), literal: None, lexeme: lexm!("Limit"), span: Span { line: 0, start: 97, line_end: 0, end: 101 }}, + Token {tok_type: skw!(SqlKeyword::And), literal: None, lexeme: lexm!("And"), span: Span { line: 0, start: 103, line_end: 0, end: 105 }}, + Token {tok_type: skw!(SqlKeyword::Or), literal: None, lexeme: lexm!("Or"), span: Span { line: 0, start: 107, line_end: 0, end: 108 }}, + Token {tok_type: skw!(SqlKeyword::Join), literal: None, lexeme: lexm!("Join") , span: Span { line: 0, start: 110, line_end: 0, end: 113 }}, + Token {tok_type: skw!(SqlKeyword::Inner), literal: None, lexeme: lexm!("Inner") , span: Span { line: 0, start: 115, line_end: 0, end: 119 }}, + Token {tok_type: skw!(SqlKeyword::Outer), literal: None, lexeme: lexm!("Outer"), span: Span { line: 0, start: 121, line_end: 0, end: 125 }}, + Token {tok_type: skw!(SqlKeyword::Right), literal: None, lexeme: lexm!("Right"), span: Span { line: 0, start: 127, line_end: 0, end: 131 }}, + Token {tok_type: skw!(SqlKeyword::Left), literal: None, lexeme: lexm!("Left"), span: Span { line: 0, start: 133, line_end: 0, end: 136 }}, + Token {tok_type: skw!(SqlKeyword::On), literal: None, lexeme: lexm!("On") , span: Span { line: 0, start: 138, line_end: 0, end: 139 }}, + Token {tok_type: skw!(SqlKeyword::Create), literal: None, lexeme: lexm!("Create"), span: Span { line: 0, start: 141, line_end: 0, end: 146 }}, + Token {tok_type: skw!(SqlKeyword::Insert), literal: None, lexeme: lexm!("Insert"), span: Span { line: 0, start: 148, line_end: 0, end: 153 }}, + Token {tok_type: skw!(SqlKeyword::Update), literal: None, lexeme: lexm!("Update"), span: Span { line: 0, start: 155, line_end: 0, end: 160 }}, + Token {tok_type: skw!(SqlKeyword::Delete), literal: None, lexeme: lexm!("Delete"), span: Span { line: 0, start: 162, line_end: 0, end: 167 }}, + Token {tok_type: skw!(SqlKeyword::Drop), literal: None, lexeme: lexm!("Drop") , span: Span { line: 0, start: 169, line_end: 0, end: 172 }}, + Token {tok_type: skw!(SqlKeyword::Into), literal: None, lexeme: lexm!("Into") , span: Span { line: 0, start: 174, line_end: 0, end: 177 }}, + Token {tok_type: skw!(SqlKeyword::Values), literal: None , lexeme: lexm!("Values"), span: Span { line: 0, start: 179, line_end: 0, end: 184 }}, + Token {tok_type: skw!(SqlKeyword::Index), literal: None, lexeme: lexm!("Index"), span: Span { line: 0, start: 186, line_end: 0, end: 190 }}, + Token {tok_type: skw!(SqlKeyword::Table), literal: None, lexeme: lexm!("Table"), span: Span { line: 0, start: 192, line_end: 0, end: 196 }}, + Token {tok_type: skw!(SqlKeyword::Select), literal: None, lexeme: lexm!("Select"), span: Span { line: 0, start: 198, line_end: 0, end: 203 }}, + Token {tok_type: skw!(SqlKeyword::From), literal: None, lexeme: lexm!("From"), span: Span { line: 0, start: 205, line_end: 0, end: 208 }}, + Token {tok_type: skw!(SqlKeyword::As), literal: None, lexeme: lexm!("As"), span: Span { line: 0, start: 210, line_end: 0, end: 211 }}, + Token {tok_type: skw!(SqlKeyword::Cross), literal: None, lexeme: lexm!("Cross"), span: Span { line: 0, start: 213, line_end: 0, end: 217 }}, + Token {tok_type: skw!(SqlKeyword::Default), literal: None, lexeme: lexm!("Default"), span: Span { line: 0, start: 219, line_end: 0, end: 225 }}, + Token {tok_type: skw!(SqlKeyword::Group), literal: None, lexeme: lexm!("Group"), span: Span { line: 0, start: 227, line_end: 0, end: 231 }}, + Token {tok_type: skw!(SqlKeyword::Key), literal: None, lexeme: lexm!("Key"), span: Span { line: 0, start: 233, line_end: 0, end: 235 }}, + Token {tok_type: skw!(SqlKeyword::Of), literal: None, lexeme: lexm!("Of"), span: Span { line: 0, start: 237, line_end: 0, end: 238 }}, + Token {tok_type: skw!(SqlKeyword::Only), literal: None, lexeme: lexm!("Only"), span: Span { line: 0, start: 240, line_end: 0, end: 243 }}, + Token {tok_type: skw!(SqlKeyword::Primary), literal: None, lexeme: lexm!("Primary"), span: Span { line: 0, start: 245, line_end: 0, end: 251 }}, + Token {tok_type: skw!(SqlKeyword::References), literal: None, lexeme: lexm!("References"), span: Span { line: 0, start: 253, line_end: 0, end: 262 }}, + Token {tok_type: skw!(SqlKeyword::Set), literal: None, lexeme: lexm!("Set"), span: Span { line: 0, start: 264, line_end: 0, end: 266 }}, + Token {tok_type: skw!(SqlKeyword::System), literal: None, lexeme: lexm!("System"), span: Span { line: 0, start: 268, line_end: 0, end: 273 }}, + Token {tok_type: skw!(SqlKeyword::Unique), literal: None, lexeme: lexm!("Unique") , span: Span { line: 0, start: 275, line_end: 0, end: 280 }}, + Token {tok_type: skw!(SqlKeyword::Read), literal: None, lexeme: lexm!("Read"), span: Span { line: 0, start: 282, line_end: 0, end: 285 }}, + Token {tok_type: skw!(SqlKeyword::Write), literal: None, lexeme: lexm!("Write") , span: Span { line: 0, start: 287, line_end: 0, end: 291 }}, + Token {tok_type: Eof, literal: None, lexeme: None, span: Span { line: 0, start: 292, line_end: 0, end: 292 }}, ]); } } diff --git a/src/lang/serializer.rs b/src/lang/serializer.rs index 89aaa702..4405dd03 100644 --- a/src/lang/serializer.rs +++ b/src/lang/serializer.rs @@ -12,10 +12,7 @@ use std::rc::Rc; impl Parsed { pub fn to_json(&mut self) -> Value { - json!({ - "type": "Program", - "body": self.statements.clone().iter().map(|stmt| self.visit_stmt(*stmt).unwrap()).collect::>() - }) + json!(self.visit_stmt(self.program).unwrap()) } pub fn serialize(&mut self) -> String { serde_json::to_string_pretty(&self.to_json()).unwrap() @@ -29,63 +26,70 @@ impl Visitor for Parsed { let e = a.get_expression(eidx); let matched: Value = match e { - Expr::Select(val) => json!({ + Expr::Select { span: _, query: _ } => json!({ "type": "Expr::Select", // TODO(vck): Implement rest of the select }), - Expr::Literal(val) => { + Expr::Literal { raw, span, value } => { json!({ "type": "Expr::Literal", - "value": format!("{:?}", val), + "value": format!("{:?}", value), + "raw": raw, }) } - Expr::Grouping(expr) => { + Expr::Grouping { expr, span } => { json!({ "type": "Expr::Grouping", "value": self.visit_expr(*expr)?, }) } - Expr::Unary { token, expr } => { + Expr::Unary { symbol, expr, span } => { json!({ "type": "Expr::Unary", - "operator": token.span.lexeme.as_ref(), + "operator": symbol, "value": self.visit_expr(*expr)?, }) } - Expr::Binary { token, left, right } => { + Expr::Binary { + symbol, + left, + right, + span, + } => { json!({ "type": "Expr::Binary", "left": self.visit_expr(*left)?, - "operator": token.span.lexeme.as_ref(), + "operator": symbol, "right": self.visit_expr(*right)?, }) } - Expr::Variable(tok) => { + Expr::Variable { name, span } => { json!({ "type": "Expr::Variable", - "value": tok.span.lexeme.as_ref(), + "value": name.lexeme.as_ref(), }) } - Expr::Assignment { var_tok, expr } => { + Expr::Assignment { dst, expr, span } => { json!({ "type": "Expr::Assignment", - "variable": var_tok.span.lexeme.as_ref(), + "variable": dst.lexeme.as_ref(), "value": self.visit_expr(*expr)?, }) } - Expr::Logical { left, token, right } => { + Expr::Logical { + left, + symbol, + right, + span, + } => { json!({ "type": "Expr::Logical", "left": self.visit_expr(*left)?, - "operator": token.span.lexeme.as_ref(), + "operator": symbol, "right": self.visit_expr(*right)?, }) } - Expr::Call { - callee, - paren: _, - args, - } => { + Expr::Call { callee, span, args } => { json!({ "type": "Expr::Call", "callee": self.visit_expr(*callee)?, @@ -96,16 +100,17 @@ impl Visitor for Parsed { name, parameters, body, + span, } => { let fn_name = if name.is_some() { - name.as_ref().unwrap().span.lexeme.as_ref() + name.as_ref().unwrap().lexeme.as_ref().unwrap().to_string() } else { - "" + "".to_string() }; json!({ "type": "Expr::Function", "name": fn_name, - "parameters": parameters.iter().map(|param| param.span.lexeme.as_ref()).collect::>(), + "parameters": parameters.iter().map(|param| param.lexeme.as_ref()).collect::>(), "body": body.iter().map(|stmt| self.visit_stmt(*stmt).unwrap()).collect::>(), }) } @@ -119,29 +124,36 @@ impl Visitor for Parsed { let a = Rc::clone(&self.arena); let s = a.get_statement(sidx); let matched: Value = match s { - Stmt::Expression(expr) => { + Stmt::Program { stmts, span } => { json!({ - "type": "Stmt::Expression", - "value": self.visit_expr(*expr)?, + "type": "Stmt::Program", + "body": stmts.iter().map(|stmt| self.visit_stmt(*stmt).unwrap()).collect::>(), }) } - Stmt::Declaration { token, expr } => { + Stmt::Block { stmts, span } => { json!({ - "type": "Stmt::Declaration", - "variable": token.span.lexeme.as_ref(), + "type": "Stmt::Block", + "body": stmts.iter().map(|stmt| self.visit_stmt(*stmt).unwrap()).collect::>(), + }) + } + Stmt::Expression { expr, span } => { + json!({ + "type": "Stmt::Expression", "value": self.visit_expr(*expr)?, }) } - Stmt::Block(statements) => { + Stmt::Declaration { dst, expr, span } => { json!({ - "type": "Stmt::Block", - "body": statements.iter().map(|stmt| self.visit_stmt(*stmt).unwrap()).collect::>(), + "type": "Stmt::Declaration", + "variable": dst.lexeme.as_ref().unwrap(), + "value": self.visit_expr(*expr)?, }) } Stmt::If { condition, body, r#else, + span: _, } => { json!({ "type": "Stmt::If", @@ -158,6 +170,7 @@ impl Visitor for Parsed { condition, body, post, + span, } => { json!({ "type": "Stmt::Loop", @@ -174,13 +187,13 @@ impl Visitor for Parsed { }, }) } - Stmt::Break(_) => json!({ + Stmt::Break { span } => json!({ "type": "Stmt::Break", }), - Stmt::Continue(_) => json!({ + Stmt::Continue { span } => json!({ "type": "Stmt::Continue", }), - Stmt::Return { token: _, expr } => { + Stmt::Return { expr, span } => { json!({ "type": "Stmt::Return", "value": if expr.is_some() { diff --git a/src/lang/tests/helpers.rs b/src/lang/tests/helpers.rs index bdeef72f..2f3a2581 100644 --- a/src/lang/tests/helpers.rs +++ b/src/lang/tests/helpers.rs @@ -1,6 +1,6 @@ #[macro_export] macro_rules! lexm { ($a: literal) => { - Rc::new($a.to_owned()) + Some($a.to_owned()) }; } diff --git a/src/lang/token.rs b/src/lang/token.rs index d41482ae..a5663d7a 100644 --- a/src/lang/token.rs +++ b/src/lang/token.rs @@ -1,8 +1,8 @@ use crate::runtime::types::RV; use phf::phf_map; -use std::rc::Rc; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum Symbol { Comma, Semicolon, @@ -25,7 +25,7 @@ pub enum Symbol { Star, } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum TokenType { Str, Num, @@ -42,7 +42,7 @@ pub enum TokenType { Eof, } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum Keyword { And, Or, @@ -61,7 +61,7 @@ pub enum Keyword { Loop, } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum SqlKeyword { /* Bool, @@ -259,12 +259,25 @@ pub static CASE_INS_KEYWORDS: phf::Map<&'static str, TokenType> = phf_map! { pub struct Token { pub tok_type: TokenType, pub literal: Option, + pub lexeme: Option, pub span: Span, } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct Span { - pub lexeme: Rc, pub start: usize, + pub end: usize, pub line: u32, + pub line_end: u32, +} + +impl Span { + pub fn merge(&self, other: &Span) -> Span { + Span { + start: self.start.min(other.start), + end: self.end.max(other.end), + line: self.line.min(other.line), + line_end: self.line_end.min(other.line_end), + } + } } diff --git a/src/runtime/error.rs b/src/runtime/error.rs index ed6c2417..e8b3f518 100644 --- a/src/runtime/error.rs +++ b/src/runtime/error.rs @@ -23,7 +23,7 @@ pub fn report_error(filename: &str, source: &str, error: ExecutionError) { .with_code(3) .with_message(format!("{} at line {}", message, span.line + 1)) .with_label( - Label::new((filename, span.start..(span.start + span.lexeme.len()))) + Label::new((filename, span.start..(span.start + span.end))) .with_message(hint) .with_color(out), ) @@ -55,7 +55,8 @@ pub fn report_error(filename: &str, source: &str, error: ExecutionError) { "Missing token", &format!( "Add a {:?} token after \"{}\".", - expected, token.span.lexeme + expected, + token.lexeme.unwrap() ), token.span, ); @@ -63,19 +64,19 @@ pub fn report_error(filename: &str, source: &str, error: ExecutionError) { ExecutionError::Parse(ParseError::InvalidAssignmentTarget { left }) => { print( "Invalid assignment target", - &format!("No values can be assigned to {}", left.span.lexeme), + &format!("No values can be assigned to {}", left.lexeme.unwrap()), left.span, ); } ExecutionError::Parse(ParseError::UnexpectedToken { token }) => { print( "Unexpected token", - &format!("Unexpected token {}.", token.span.lexeme), + &format!("Unexpected token {}.", token.lexeme.unwrap()), token.span, ); } ExecutionError::Interpret(InterpretError::ArityMismatch { - token, + span, expected, found, }) => { @@ -85,24 +86,17 @@ pub fn report_error(filename: &str, source: &str, error: ExecutionError) { "Function expects {} arguments, while provided {}.", expected, found ), - token.span, + span, ); } - ExecutionError::Interpret(InterpretError::UnexpectedStatement { token }) => { - print( - "Unexpected statement", - &format!("Unexpected \"{}\" statement.", token.span.lexeme,), - token.span, - ); + ExecutionError::Interpret(InterpretError::UnexpectedStatement { span }) => { + print("Unexpected statement", "Remove this.", span); } - ExecutionError::Interpret(InterpretError::NotCallable { token }) => { + ExecutionError::Interpret(InterpretError::NotCallable { span }) => { print( "Not callable", - &format!( - "Expression does not yield a callable {}.", - token.span.lexeme, - ), - token.span, + "Expression does not yield a callable.", + span, ); } /*ExecutionError::Interpret(InterpretError::AssignmentToUndefined { token }) => { @@ -128,8 +122,9 @@ pub fn report_error(filename: &str, source: &str, error: ExecutionError) { "", Span { start: 0, - lexeme: Rc::new("".to_string()), + end: 0, line: 0, + line_end: 0, }, ); } diff --git a/src/runtime/eval.rs b/src/runtime/eval.rs index eecc4735..f2d903a1 100644 --- a/src/runtime/eval.rs +++ b/src/runtime/eval.rs @@ -1,7 +1,7 @@ use std::rc::Rc; use crate::lang::token::Symbol::*; -use crate::lang::token::Token; +use crate::lang::token::TokenType; use crate::lang::token::TokenType::Symbol; use crate::runtime::types::RV; @@ -37,8 +37,8 @@ pub fn coerce2number(val: RV) -> Option { } #[inline(always)] -pub fn eval_binary(left_eval: RV, right_eval: RV, tok: &Token) -> RV { - let tok_type = tok.tok_type.clone(); +pub fn eval_binary(left_eval: RV, right_eval: RV, symbol: &TokenType) -> RV { + let tok_type = symbol.clone(); let (left_coerced, right_coerced) = match (&left_eval, &tok_type, &right_eval) { (RV::Num(n), _, RV::Bool(bool)) => (RV::Num(*n), RV::Num(bool2num!(*bool))), diff --git a/src/runtime/interpreter.rs b/src/runtime/interpreter.rs index 22ca37e3..d6e0f7af 100644 --- a/src/runtime/interpreter.rs +++ b/src/runtime/interpreter.rs @@ -3,10 +3,10 @@ use super::resolver::Resolver; use crate::lang::ast::expr::{Expr, ExprId}; use crate::lang::ast::stmt::{Stmt, StmtId}; use crate::lang::ast::{ParserArena, Visitor}; -use crate::lang::token::Keyword::*; use crate::lang::token::Symbol::*; use crate::lang::token::Token; use crate::lang::token::TokenType; +use crate::lang::token::{Keyword::*, Span}; use crate::runtime::environment::Environment; use crate::runtime::types::RV::Callable; use crate::runtime::types::{Function, RV}; @@ -18,15 +18,15 @@ use std::vec; #[derive(Debug, Clone)] pub enum InterpretError { NotCallable { - token: Token, + span: Span, }, ArityMismatch { - token: Token, + span: Span, expected: usize, found: usize, }, UnexpectedStatement { - token: Token, + span: Span, }, /*AssignmentToUndefined { token: Token, @@ -121,8 +121,8 @@ impl Interpreter { } } - fn eval_unary(&mut self, tok: &Token, eidx: ExprId) -> Result { - if tok.tok_type == sym!(Minus) { + fn eval_unary(&mut self, symbol: &TokenType, eidx: ExprId) -> Result { + if *symbol == sym!(Minus) { if let Some(num) = coerce2number(self.visit_expr(eidx)?) { return Ok(RV::Num(-num)); } @@ -132,21 +132,24 @@ impl Interpreter { } } - fn eval_binary(&mut self, lidx: ExprId, ridx: ExprId, tok: &Token) -> Result { + fn eval_binary( + &mut self, + lidx: ExprId, + ridx: ExprId, + symbol: TokenType, + ) -> Result { let left_eval = self.visit_expr(lidx)?; let right_eval = self.visit_expr(ridx)?; - Ok(eval_binary(left_eval, right_eval, tok)) + Ok(eval_binary(left_eval, right_eval, &symbol)) } - fn look_up_variable(&self, name: Token, eid: ExprId) -> Result { + fn look_up_variable(&self, name: &String, eid: ExprId) -> Result { let distance = self.resolver.get_distance(eid); if let Some(unwrapped) = distance { - self.env - .borrow() - .read_at(unwrapped, &name.span.lexeme.to_owned()) + self.env.borrow().read_at(unwrapped, name) } else { - self.root_env.borrow().read(&name.span.lexeme.to_owned()) + self.root_env.borrow().read(name) } } } @@ -215,53 +218,67 @@ impl Visitor for Interpreter { let a = Rc::clone(&self.arena); let e = a.get_expression(eidx); match e { - Expr::Select(val) => Ok(RV::Str(Rc::new(format!("{:?}", val)))), - Expr::Literal(value) => Ok(value.clone()), - Expr::Grouping(expr) => self.visit_expr(*expr), - Expr::Unary { token, expr } => self.eval_unary(token, *expr), - Expr::Binary { token, left, right } => self.eval_binary(*left, *right, token), - Expr::Variable(tok) => self.look_up_variable(tok.clone(), eidx), - Expr::Assignment { var_tok, expr } => { + Expr::Select { query, span: _ } => Ok(RV::Str(Rc::new(format!("{:?}", query)))), + Expr::Literal { + value, + raw, + span: _, + } => Ok(value.clone()), + Expr::Grouping { expr, span: _ } => self.visit_expr(*expr), + Expr::Unary { + symbol, + expr, + span: _, + } => self.eval_unary(&symbol, *expr), + Expr::Binary { + symbol, + left, + right, + span: _, + } => self.eval_binary(*left, *right, symbol.clone()), + Expr::Variable { name, span: _ } => { + self.look_up_variable(name.lexeme.as_ref().unwrap(), eidx) + } + Expr::Assignment { dst, expr, span: _ } => { let distance = self.resolver.get_distance(eidx); let evaluated = self.visit_expr(*expr)?; let result = if let Some(distance_unv) = distance { self.env.borrow_mut().assign_at( distance_unv, - var_tok.span.lexeme.as_ref(), + dst.lexeme.as_ref().unwrap(), evaluated.clone(), ) } else { self.root_env .borrow_mut() - .assign(var_tok.span.lexeme.as_ref().to_string(), evaluated.clone()) + .assign(dst.lexeme.as_ref().unwrap().to_string(), evaluated.clone()) }; if result.is_err() { return Err(result.err().unwrap()); } Ok(evaluated) } - Expr::Logical { left, token, right } => { + Expr::Logical { + left, + symbol, + right, + span: _, + } => { let is_true = is_value_truthy(self.visit_expr(*left)?); - if (token.tok_type == kw!(Or) && is_true) - || (token.tok_type == kw!(And) && !is_true) - { + if (*symbol == kw!(Or) && is_true) || (*symbol == kw!(And) && !is_true) { return Ok(RV::Bool(is_true)); } Ok(RV::Bool(is_value_truthy(self.visit_expr(*right)?))) } - Expr::Call { - callee, - paren, - args, - } => { + Expr::Call { callee, args, span } => { let eval = self.visit_expr(*callee)?; if let Callable(arity, callable) = eval { if arity.is_some() && arity.unwrap() != args.len() { return Err(HaltReason::Error(InterpretError::ArityMismatch { - token: paren.clone(), + span: *span, expected: arity.unwrap(), found: args.len(), })); @@ -283,7 +300,7 @@ impl Visitor for Interpreter { } } else { Err(HaltReason::Error(InterpretError::NotCallable { - token: paren.clone(), + span: *span, })) } } @@ -291,9 +308,10 @@ impl Visitor for Interpreter { name, parameters, body, + span, } => { let fn_name = if name.is_some() { - name.as_ref().unwrap().span.lexeme.as_ref() + name.as_ref().unwrap().lexeme.as_ref().unwrap() } else { "" }; @@ -302,7 +320,7 @@ impl Visitor for Interpreter { body: Rc::clone(body), parameters: parameters .iter() - .map(|x| x.span.lexeme.as_ref().to_string()) + .map(|x| x.lexeme.as_ref().unwrap().to_string()) .collect(), closure: self.env.clone(), }; @@ -312,7 +330,7 @@ impl Visitor for Interpreter { if name.is_some() { // TODO(vck): Callable shouldn't be cloned here self.env.borrow_mut().declare( - name.as_ref().unwrap().span.lexeme.to_string(), + name.as_ref().unwrap().lexeme.as_ref().unwrap().to_string(), callable.clone(), ); } @@ -332,22 +350,26 @@ impl Visitor for Interpreter { let a = Rc::clone(&self.arena); let s = a.get_statement(sidx); match s { - Stmt::Expression(expr) => { + Stmt::Program { stmts, span: _ } => { + return self.execute_block(stmts, Some(self.env.clone())); + } + Stmt::Expression { expr, span: _ } => { return self.visit_expr(*expr); } - Stmt::Declaration { token, expr } => { + Stmt::Declaration { dst, expr, span: _ } => { let evaluated = self.visit_expr(*expr)?; self.env .borrow_mut() - .declare(token.span.lexeme.to_string(), evaluated); + .declare(dst.lexeme.as_ref().unwrap().to_string(), evaluated.clone()); } - Stmt::Block(statements) => { - return self.execute_block(statements, None); + Stmt::Block { stmts, span: _ } => { + return self.execute_block(stmts, None); } Stmt::If { condition, body, r#else, + span: _, } => { if is_value_truthy(self.visit_expr(*condition)?) { self.visit_stmt(*body)?; @@ -359,6 +381,7 @@ impl Visitor for Interpreter { condition, body, post, + span: _, } => { self.call_stack[0].push_loop(LoopState::Go); while !self.is_loop_at(LoopState::Broken) @@ -373,21 +396,21 @@ impl Visitor for Interpreter { } self.call_stack[0].pop_loop(); } - Stmt::Break(token) => { + Stmt::Break { span } => { if !self.set_loop_state(LoopState::Broken, None) { return Err(HaltReason::Error(InterpretError::UnexpectedStatement { - token: token.clone(), + span: *span, })); } } - Stmt::Continue(token) => { + Stmt::Continue { span } => { if !self.set_loop_state(LoopState::Continue, None) { return Err(HaltReason::Error(InterpretError::UnexpectedStatement { - token: token.clone(), + span: *span, })); } } - Stmt::Return { token: _, expr } => { + Stmt::Return { span: _, expr } => { if expr.is_some() { let ret = self.visit_expr(expr.unwrap())?; return Err(HaltReason::Return(ret)); diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index 2b1acfdf..99bddfb2 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -102,20 +102,15 @@ impl Runtime { let arena = Rc::clone(&parsed_unw.arena); // let mut resolver = Resolver::new(arena.clone()); - let stmts = &parsed_unw.statements.clone(); - resolver.resolve_stmts(stmts); + resolver.resolve_stmt(parsed_unw.program); // - let mut out = Ok(RV::Undefined); let mut interpreter = Interpreter::new(self.env.clone(), arena, Rc::new(resolver)); - for stmt in stmts { - out = interpreter.visit_stmt(*stmt); - if self.mode == RuntimeMode::Repl { - println!("{:?}", out); - } - if out.is_err() { - break; - } + let out = interpreter.visit_stmt(parsed_unw.program); + + if self.mode == RuntimeMode::Repl { + println!("{:?}", out); } + if let Ok(val) = out { Ok(val) } else { @@ -124,7 +119,7 @@ impl Runtime { HaltReason::Return(rv) => Ok(rv), HaltReason::Error(interpret_err) => { let error = error::ExecutionError::Interpret(interpret_err); - report_error("filename", source, error.clone()); + report_error("