From 9b1c9d08560ec88bd715d469c6f57d3151c95304 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Sun, 17 Dec 2023 11:27:11 +0300 Subject: [PATCH 01/17] fix: Loop stack optimization --- server/src/runtime/interpreter.rs | 70 +++++++++++++++++-------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/server/src/runtime/interpreter.rs b/server/src/runtime/interpreter.rs index 390714d6..1eb3038c 100644 --- a/server/src/runtime/interpreter.rs +++ b/server/src/runtime/interpreter.rs @@ -55,47 +55,54 @@ pub enum LoopState { Go, Broken, Continue, + Function } -#[derive(Debug)] -pub struct Context { - ongoing_loops: Option>, +struct LoopStack { + ongoing_loops: Vec, } -impl Context { - pub fn new() -> Context { - Context { - ongoing_loops: None, +impl LoopStack { + pub fn new() -> LoopStack { + LoopStack { + ongoing_loops: vec![], } } - pub fn push_loop(&mut self, state: LoopState) { - if self.ongoing_loops.is_none() { - self.ongoing_loops = Some(vec![]); + pub fn push_fn(&mut self) { + self.ongoing_loops.push(LoopState::Function); + } + + pub fn pop_fn(&mut self) { + if self.ongoing_loops.last() == Some(&LoopState::Function) { + self.ongoing_loops.pop(); } - self.ongoing_loops.as_mut().unwrap().push(state); + } + + pub fn push_loop(&mut self, state: LoopState) { + self.ongoing_loops.push(state); } pub fn pop_loop(&mut self) { - self.ongoing_loops.as_mut().unwrap().pop(); + if self.is_loops_empty() { + return; + } + self.ongoing_loops.pop(); } pub fn is_loops_empty(&self) -> bool { - if self.ongoing_loops.is_none() { - return true; - } - return self.ongoing_loops.as_ref().unwrap().is_empty(); + self.ongoing_loops.is_empty() || self.ongoing_loops.last() == Some(&LoopState::Function) } pub fn get_last_loop(&self) -> Option<&LoopState> { - if self.ongoing_loops.is_none() { + if self.is_loops_empty() { return None; } - return self.ongoing_loops.as_ref().unwrap().last(); + return self.ongoing_loops.last(); } pub fn set_last_loop(&mut self, to: LoopState) { - if self.ongoing_loops.is_none() { + if self.ongoing_loops.is_empty() { return; } self.pop_loop(); @@ -107,7 +114,7 @@ pub struct Interpreter { env: Shared, root_env: Shared, arena: Rc, - call_stack: Vec, + loop_stack: LoopStack, resolver: Rc, } @@ -121,7 +128,7 @@ impl Interpreter { env: env.clone(), root_env: env, arena: Rc::clone(&arena), - call_stack: vec![Context::new()], + loop_stack: LoopStack::new(), resolver, } } @@ -161,20 +168,20 @@ impl Interpreter { impl Interpreter { fn is_loop_at(&self, state: LoopState) -> bool { - let last_loop = *self.call_stack[0].get_last_loop().unwrap(); + let last_loop = *self.loop_stack.get_last_loop().unwrap(); last_loop == state } fn set_loop_state(&mut self, to: LoopState, from: Option) -> bool { if from.is_none() { - return if !self.call_stack[0].is_loops_empty() { - self.call_stack[0].set_last_loop(to); + return if !self.loop_stack.is_loops_empty() { + self.loop_stack.set_last_loop(to); true } else { false }; } else if self.is_loop_at(from.unwrap()) { - self.call_stack[0].set_last_loop(to); + self.loop_stack.set_last_loop(to); } true } @@ -316,10 +323,11 @@ impl Visitor for Interpreter { for arg in args.iter() { args_evaluated.push(self.visit_expr(*arg)?); } - self.call_stack.insert(0, Context::new()); + self.loop_stack.push_fn(); let val = callable.call(self, args_evaluated.as_slice()); - self.call_stack.remove(0); + + self.loop_stack.pop_fn(); match val { Err(HaltReason::Return(ret_val)) => Ok(ret_val), Ok(unpacked_val) => Ok(unpacked_val), @@ -410,8 +418,8 @@ impl Visitor for Interpreter { } fn visit_stmt(&mut self, sidx: StmtId) -> Result { - if !self.call_stack[0].is_loops_empty() - && *self.call_stack[0].get_last_loop().unwrap() != LoopState::Go + if !self.loop_stack.is_loops_empty() + && *self.loop_stack.get_last_loop().unwrap() != LoopState::Go { return Ok(RV::Undefined); } @@ -452,7 +460,7 @@ impl Visitor for Interpreter { post, span: _, } => { - self.call_stack[0].push_loop(LoopState::Go); + self.loop_stack.push_loop(LoopState::Go); while !self.is_loop_at(LoopState::Broken) && (condition.is_none() || is_value_truthy(self.visit_expr(condition.unwrap())?)) @@ -463,7 +471,7 @@ impl Visitor for Interpreter { self.visit_stmt(*post_id)?; } } - self.call_stack[0].pop_loop(); + self.loop_stack.pop_loop(); } Stmt::Break { span } => { if !self.set_loop_state(LoopState::Broken, None) { From a4aee9a845d0a98ece7bdd161f97a3fc21ccae53 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Sun, 17 Dec 2023 11:34:40 +0300 Subject: [PATCH 02/17] fix: Refactored Interpreter --- server/src/runtime/interpreter.rs | 48 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/server/src/runtime/interpreter.rs b/server/src/runtime/interpreter.rs index 1eb3038c..ae389667 100644 --- a/server/src/runtime/interpreter.rs +++ b/server/src/runtime/interpreter.rs @@ -108,6 +108,25 @@ impl LoopStack { self.pop_loop(); self.push_loop(to); } + + fn is_loop_at(&self, state: LoopState) -> bool { + let last_loop = *self.get_last_loop().unwrap(); + last_loop == state + } + + fn set_loop_state(&mut self, to: LoopState, from: Option) -> bool { + if from.is_none() { + return if !self.is_loops_empty() { + self.set_last_loop(to); + true + } else { + false + }; + } else if self.is_loop_at(from.unwrap()) { + self.set_last_loop(to); + } + true + } } pub struct Interpreter { @@ -164,27 +183,6 @@ impl Interpreter { self.root_env.borrow().read(name) } } -} - -impl Interpreter { - fn is_loop_at(&self, state: LoopState) -> bool { - let last_loop = *self.loop_stack.get_last_loop().unwrap(); - last_loop == state - } - - fn set_loop_state(&mut self, to: LoopState, from: Option) -> bool { - if from.is_none() { - return if !self.loop_stack.is_loops_empty() { - self.loop_stack.set_last_loop(to); - true - } else { - false - }; - } else if self.is_loop_at(from.unwrap()) { - self.loop_stack.set_last_loop(to); - } - true - } pub fn user_fn_call( &mut self, @@ -461,12 +459,12 @@ impl Visitor for Interpreter { span: _, } => { self.loop_stack.push_loop(LoopState::Go); - while !self.is_loop_at(LoopState::Broken) + while !self.loop_stack.is_loop_at(LoopState::Broken) && (condition.is_none() || is_value_truthy(self.visit_expr(condition.unwrap())?)) { self.visit_stmt(*body)?; - self.set_loop_state(LoopState::Go, Some(LoopState::Continue)); + self.loop_stack.set_loop_state(LoopState::Go, Some(LoopState::Continue)); if let Some(post_id) = post { self.visit_stmt(*post_id)?; } @@ -474,14 +472,14 @@ impl Visitor for Interpreter { self.loop_stack.pop_loop(); } Stmt::Break { span } => { - if !self.set_loop_state(LoopState::Broken, None) { + if !self.loop_stack.set_loop_state(LoopState::Broken, None) { return Err(HaltReason::Error(InterpretError::UnexpectedStatement { span: *span, })); } } Stmt::Continue { span } => { - if !self.set_loop_state(LoopState::Continue, None) { + if !self.loop_stack.set_loop_state(LoopState::Continue, None) { return Err(HaltReason::Error(InterpretError::UnexpectedStatement { span: *span, })); From ed99efc7dcb8a3fcc6be5861ef803ce429da42a9 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Sun, 17 Dec 2023 12:37:30 +0300 Subject: [PATCH 03/17] fix: Changed function keyword --- server/src/lang/scanner.rs | 46 +++++++++++++-------------- server/src/lang/token.rs | 2 +- server/src/runtime/tests/functions.rs | 32 +++++++++---------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/server/src/lang/scanner.rs b/server/src/lang/scanner.rs index 2f0f21bd..bb777e30 100644 --- a/server/src/lang/scanner.rs +++ b/server/src/lang/scanner.rs @@ -830,7 +830,7 @@ mod test { #[test] fn test_keywords() { assert_tokens( - "and or class else for fun if break continue return super this var while loop", + "and or class else for function if break continue return super this var while loop", vec![ Token { tok_type: kw!(Keyword::And), @@ -889,12 +889,12 @@ mod test { }, Token { tok_type: kw!(Keyword::Fun), - lexeme: lexm!("fun"), + lexeme: lexm!("function"), literal: None, span: Span { line: 0, start: 22, - end: 25, + end: 30, line_end: 0, }, }, @@ -904,8 +904,8 @@ mod test { literal: None, span: Span { line: 0, - start: 26, - end: 28, + start: 31, + end: 33, line_end: 0, }, }, @@ -915,8 +915,8 @@ mod test { literal: None, span: Span { line: 0, - start: 29, - end: 34, + start: 34, + end: 39, line_end: 0, }, }, @@ -926,8 +926,8 @@ mod test { literal: None, span: Span { line: 0, - start: 35, - end: 43, + start: 40, + end: 48, line_end: 0, }, }, @@ -937,8 +937,8 @@ mod test { literal: None, span: Span { line: 0, - start: 44, - end: 50, + start: 49, + end: 55, line_end: 0, }, }, @@ -948,8 +948,8 @@ mod test { literal: None, span: Span { line: 0, - start: 51, - end: 56, + start: 56, + end: 61, line_end: 0, }, }, @@ -959,8 +959,8 @@ mod test { literal: None, span: Span { line: 0, - start: 57, - end: 61, + start: 62, + end: 66, line_end: 0, }, }, @@ -970,8 +970,8 @@ mod test { literal: None, span: Span { line: 0, - start: 62, - end: 65, + start: 67, + end: 70, line_end: 0, }, }, @@ -981,8 +981,8 @@ mod test { literal: None, span: Span { line: 0, - start: 66, - end: 71, + start: 71, + end: 76, line_end: 0, }, }, @@ -992,8 +992,8 @@ mod test { literal: None, span: Span { line: 0, - start: 72, - end: 76, + start: 77, + end: 81, line_end: 0, }, }, @@ -1003,8 +1003,8 @@ mod test { literal: None, span: Span { line: 0, - start: 77, - end: 77, + start: 82, + end: 82, line_end: 0, }, }, diff --git a/server/src/lang/token.rs b/server/src/lang/token.rs index 627f09e4..4a6b3434 100644 --- a/server/src/lang/token.rs +++ b/server/src/lang/token.rs @@ -186,7 +186,7 @@ pub static CASE_SNS_KEYWORDS: phf::Map<&'static str, TokenType> = phf_map! { "class" => kw!(Keyword::Class), "else" => kw!(Keyword::Else), "for" => kw!(Keyword::For), - "fun" => kw!(Keyword::Fun), + "function" => kw!(Keyword::Fun), "if" => kw!(Keyword::If), "or" => kw!(Keyword::Or), "break" => kw!(Keyword::Break), diff --git a/server/src/runtime/tests/functions.rs b/server/src/runtime/tests/functions.rs index ec6a7045..b65d20bf 100644 --- a/server/src/runtime/tests/functions.rs +++ b/server/src/runtime/tests/functions.rs @@ -7,11 +7,11 @@ mod test { #[test] fn test_higher_order_0() { exec_assert( - "fun f($x, $q) { + "function f($x, $q) { $x($q); }; - fun g($q) { + function g($q) { TestUtils.out($q); }; @@ -36,9 +36,9 @@ mod test { #[test] fn test_high_order_1() { exec_assert( - "fun makeCounter() { + "function makeCounter() { var $i = 0; - fun count() { + function count() { $i = $i + 1; TestUtils.out($i); }; @@ -57,7 +57,7 @@ mod test { exec_assert( "var $a = \"global\"; { - fun showA() { + function showA() { TestUtils.out($a); }; @@ -77,14 +77,14 @@ mod test { exec_assert( "var $a = \"global\"; { - fun showA() { + function showA() { TestUtils.out($a); }; showA(); var $a = \"block\"; showA(); - fun showB() { + function showB() { TestUtils.out($a); }; showB(); @@ -103,7 +103,7 @@ mod test { "{ var $a = \"global\"; { - fun showA() { + function showA() { TestUtils.out($a); }; @@ -124,13 +124,13 @@ mod test { exec_assert( "var $a = \"global\"; { - fun showA() { + function showA() { TestUtils.out($a); }; var $a = \"block\"; - fun showB() { + function showB() { TestUtils.out($a); }; @@ -155,7 +155,7 @@ mod test { #[test] fn test_anonymous_fn_0() { exec_assert( - "var $pr = fun a() { + "var $pr = function a() { TestUtils.out(\"hello\"); }; @@ -172,7 +172,7 @@ mod test { #[test] fn test_anonymous_fn_1() { exec_assert( - "(fun a() { + "(function a() { TestUtils.out(\"hello\"); })(); @@ -188,7 +188,7 @@ mod test { #[test] fn test_anonymous_fn_2() { exec_assert( - "var $pr = fun() { + "var $pr = function() { TestUtils.out(\"hello\"); }; @@ -201,7 +201,7 @@ mod test { #[test] fn test_anonymous_fn_3() { exec_assert( - "(fun() { + "(function() { TestUtils.out(\"hello\"); })(); ", @@ -214,13 +214,13 @@ mod test { exec_assert( "var $a = \"global\"; { - var $showA = fun() { + var $showA = function() { TestUtils.out($a); }; var $a = \"block\"; - var $showB = fun() { + var $showB = function() { TestUtils.out($a); }; From a378a2fbd0f0c2be9747cf807a7967b170aeb0fe Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Sun, 17 Dec 2023 17:22:41 +0300 Subject: [PATCH 04/17] fix: SQL parsing improvements --- server/src/lang/ast/sql.rs | 17 +++++--------- server/src/lang/parser.rs | 46 +++++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/server/src/lang/ast/sql.rs b/server/src/lang/ast/sql.rs index 85aea957..dc85b6c0 100644 --- a/server/src/lang/ast/sql.rs +++ b/server/src/lang/ast/sql.rs @@ -42,22 +42,15 @@ pub enum SqlExpr { #[derive(Debug, Eq, PartialEq)] pub enum SqlFrom { CollectionSubquery(Vec), - JoinClause(Box), + Join(Box, Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)>), } #[derive(Debug, Eq, PartialEq)] pub enum SqlProjection { - All, - /*AllFieldsOf{ - collection: Token - },*/ - Complex { expr: SqlExpr, alias: Option }, -} - -#[derive(Debug, Eq, PartialEq)] -pub enum SqlJoinClause { - None(SqlCollectionSubquery), - Join(Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)>), + All { + collection: Option, + }, + Expr { expr: SqlExpr, alias: Option }, } #[derive(Debug, Eq, PartialEq)] diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index d0675259..d02c907e 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -42,7 +42,7 @@ type ParseResult = Result; macro_rules! binary { ($self: ident, [$($operator:expr),*], $builder: ident) => { let mut current_expr: ExprId = $self.$builder()?; - while $self.match_next_multi(&vec![$($operator,)*]) { + while $self.match_next_one_of(&vec![$($operator,)*]) { let token = (*$self.peek_bw(1)).clone(); let left = current_expr; let right = $self.$builder()?; @@ -460,7 +460,7 @@ impl<'a> Parser<'a> { } fn unary(&mut self) -> ParseResult { - if self.match_next_multi(&vec![sym!(Minus), sym!(Bang)]) { + if self.match_next_one_of(&vec![sym!(Minus), sym!(Bang)]) { let token = (*self.peek_bw(1)).clone(); let unary = self.unary()?; return Ok(self.arena.expression(Expr::Unary { @@ -479,7 +479,7 @@ impl<'a> Parser<'a> { } let core = self.select_core()?; let mut compounds: Vec<(SqlCompoundOperator, SelectCore)> = vec![]; - while self.match_next_multi(&vec![skw!(Union), skw!(Intersect), skw!(Except)]) { + while self.match_next_one_of(&vec![skw!(Union), skw!(Intersect), skw!(Except)]) { let op = self.peek_bw(1); let compound_op = if op.tok_type == skw!(Union) && self.match_next(skw!(All)) { SqlCompoundOperator::UnionAll @@ -552,13 +552,12 @@ impl<'a> Parser<'a> { self.expected(skw!(Select))?; let distinct = if self.match_next(skw!(Distinct)) { SqlDistinct::Distinct + } else if self.match_next(skw!(All)) { + SqlDistinct::All } else { SqlDistinct::All }; - /* else if self.match_next(skw!(All)) { - SqlDistinct::All - }*/ - + let projection = self.sql_projection(); let from = self.sql_from()?; let r#where = self.sql_where()?; @@ -601,12 +600,18 @@ impl<'a> Parser<'a> { let mut projections: Vec = vec![]; loop { if self.match_next(sym!(Star)) { - projections.push(SqlProjection::All); + projections.push(SqlProjection::All { + collection: None, + }); + } else if self.match_next_all_of(&vec![Identifier { dollar: false }, sym!(Dot), sym!(Star)]) { + projections.push(SqlProjection::All { + collection: Some(self.peek_bw(3).clone()), + }); } else { let expr = self.expression().unwrap(); let alias: Option = optional_with_expected!(self, skw!(As), Identifier { dollar: false }); - projections.push(SqlProjection::Complex { + projections.push(SqlProjection::Expr { expr: SqlExpr::Default(expr), alias, }); @@ -615,7 +620,6 @@ impl<'a> Parser<'a> { break; } } - // TODO(vck): Add support for collection selectors projections } @@ -630,7 +634,7 @@ impl<'a> Parser<'a> { }, ]))); } - // TODO(vck): Joins + // TODO(vck): CollectionSubquery, SelectStmt, Join Ok(None) } @@ -789,6 +793,10 @@ impl<'a> Parser<'a> { &self.tokens[self.current - offset] } + fn peek_fw(&self, offset: usize) -> &'a Token { + &self.tokens[self.current + offset] + } + fn cmp_tok(&self, t: &TokenType) -> bool { let current = self.peek_bw(0); current.tok_type == *t @@ -802,8 +810,20 @@ impl<'a> Parser<'a> { false } - fn match_next_multi(&mut self, types: &Vec) -> bool { - for t in types { + fn match_next_all_of(&mut self, tokens: &Vec) -> bool { + for (i, t) in tokens.iter().enumerate() { + if self.peek_fw(i).tok_type != *t { + return false; + } + } + for _ in 0..tokens.len() { + self.advance(); + } + return true; + } + + fn match_next_one_of(&mut self, tokens: &Vec) -> bool { + for t in tokens { if self.cmp_tok(t) { self.advance(); return true; From 672e6e7e89e9e4263296ef7235991f6ec6babc33 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Sun, 17 Dec 2023 17:25:07 +0300 Subject: [PATCH 05/17] fix: Examples and benches --- server/benches/scripts/fib.ly | 2 +- server/benches/scripts/while.ly | 3 +-- server/examples/fib.ly | 2 +- server/examples/fn.ly | 2 +- server/examples/scan_err.ly | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/benches/scripts/fib.ly b/server/benches/scripts/fib.ly index ead6f524..78781219 100644 --- a/server/benches/scripts/fib.ly +++ b/server/benches/scripts/fib.ly @@ -1,5 +1,5 @@ // Define recursive Fibonacci function -fun fib($n) { +function fib($n) { if ($n < 2) return $n; return fib($n - 2) + fib($n - 1); } diff --git a/server/benches/scripts/while.ly b/server/benches/scripts/while.ly index 8984f421..389a92e0 100644 --- a/server/benches/scripts/while.ly +++ b/server/benches/scripts/while.ly @@ -10,7 +10,6 @@ while ($i < 10000000) { } $i = 0; - while ($i < 10000000) { $i = $i + 1; @@ -18,4 +17,4 @@ while ($i < 10000000) { null == null; null == 1; null == "str"; null == true; true == true; true == 1; true == false; true == "str"; true == null; "str" == "str"; "str" == "stru"; "str" == 1; "str" == null; "str" == true; -} \ No newline at end of file +} diff --git a/server/examples/fib.ly b/server/examples/fib.ly index e9ae733c..ddeae2db 100644 --- a/server/examples/fib.ly +++ b/server/examples/fib.ly @@ -1,5 +1,5 @@ // Define recursive Fibonacci function -fun fib($n) { +function fib($n) { if ($n < 2) return $n; return fib($n - 2) + fib($n - 1); }; diff --git a/server/examples/fn.ly b/server/examples/fn.ly index 43a65694..90f2177e 100644 --- a/server/examples/fn.ly +++ b/server/examples/fn.ly @@ -1,4 +1,4 @@ -fun helloWorld ($message) { +function helloWorld ($message) { print("Hello world!", $message); { { diff --git a/server/examples/scan_err.ly b/server/examples/scan_err.ly index 0c72e47c..abfe9c87 100644 --- a/server/examples/scan_err.ly +++ b/server/examples/scan_err.ly @@ -1,5 +1,5 @@ // Define recursive Fibonacci function -fun fib($n) { +function fib($n) { if ($n < 2) return $n; return fib($n - 2) + fib($n - 1); }; From 4e88efc4938bc1d9e61ba0eee07ad4a0d1d20869 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Mon, 18 Dec 2023 13:42:53 +0300 Subject: [PATCH 06/17] feat: Collection and subquery parsing --- server/src/lang/ast/sql.rs | 26 +++---- server/src/lang/parser.rs | 115 +++++++++++++++++++++++++----- server/src/lang/serializer.rs | 18 ++++- server/src/runtime/interpreter.rs | 7 +- 4 files changed, 128 insertions(+), 38 deletions(-) diff --git a/server/src/lang/ast/sql.rs b/server/src/lang/ast/sql.rs index dc85b6c0..54483837 100644 --- a/server/src/lang/ast/sql.rs +++ b/server/src/lang/ast/sql.rs @@ -39,32 +39,26 @@ pub enum SqlExpr { Default(ExprId), } -#[derive(Debug, Eq, PartialEq)] -pub enum SqlFrom { - CollectionSubquery(Vec), - Join(Box, Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)>), -} - #[derive(Debug, Eq, PartialEq)] pub enum SqlProjection { - All { - collection: Option, - }, + All { collection: Option }, Expr { expr: SqlExpr, alias: Option }, } #[derive(Debug, Eq, PartialEq)] pub enum SqlCollectionSubquery { - Simple { + Recursive(Vec), + Join( + Box, + Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)>, + ), + Collection { namespace: Option, - collection: Token, + name: Token, alias: Option, }, - From { - from: SqlFrom, - }, Select { - stmt: SqlSelect, + expr: ExprId, alias: Option, }, } @@ -73,7 +67,7 @@ pub enum SqlCollectionSubquery { pub struct SelectCore { pub distinct: SqlDistinct, pub projection: Vec, - pub from: Option, + pub from: Option, pub r#where: Option, pub group_by: Option>, pub having: Option, diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index d02c907e..63626ef9 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -1,6 +1,6 @@ use super::ast::expr::{Expr, ExprId}; use super::ast::sql::{ - SelectCore, SqlCollectionSubquery, SqlCompoundOperator, SqlDistinct, SqlExpr, SqlFrom, + SelectCore, SqlCollectionSubquery, SqlCompoundOperator, SqlDistinct, SqlExpr, SqlJoinType, SqlOrdering, SqlProjection, SqlSelect, }; use super::ast::stmt::{Stmt, StmtId}; @@ -557,7 +557,7 @@ impl<'a> Parser<'a> { } else { SqlDistinct::All }; - + let projection = self.sql_projection(); let from = self.sql_from()?; let r#where = self.sql_where()?; @@ -600,10 +600,12 @@ impl<'a> Parser<'a> { let mut projections: Vec = vec![]; loop { if self.match_next(sym!(Star)) { - projections.push(SqlProjection::All { - collection: None, - }); - } else if self.match_next_all_of(&vec![Identifier { dollar: false }, sym!(Dot), sym!(Star)]) { + projections.push(SqlProjection::All { collection: None }); + } else if self.match_next_all_of(&vec![ + Identifier { dollar: false }, + sym!(Dot), + sym!(Star), + ]) { projections.push(SqlProjection::All { collection: Some(self.peek_bw(3).clone()), }); @@ -623,21 +625,102 @@ impl<'a> Parser<'a> { projections } - fn sql_from(&mut self) -> ParseResult> { + fn sql_from(&mut self) -> ParseResult> { if self.match_next(skw!(From)) { - let token = self.expected(Identifier { dollar: false }); - return Ok(Some(SqlFrom::CollectionSubquery(vec![ - SqlCollectionSubquery::Simple { - namespace: None, - collection: token.unwrap().clone(), - alias: None, - }, - ]))); + return Ok(Some(self.sql_collection_subquery()?)); } - // TODO(vck): CollectionSubquery, SelectStmt, Join Ok(None) } + fn sql_collection_subquery(&mut self) -> ParseResult { + if self.cmp_tok(&sym!(LeftParen)) { + // If the next token is a left paren, then it must be either a select statement or a recursive subquery + self.advance(); + if self.cmp_tok(&skw!(Select)) { + let expr = self.select()?; + let alias = optional_with_expected!(self, skw!(As), Identifier { dollar: false }); + self.expected(sym!(RightParen))?; // closing paren + return Ok(SqlCollectionSubquery::Select { expr, alias }); + } + + let mut recursive: Vec = vec![]; + + loop { + let subquery = self.sql_collection_subquery()?; + recursive.push(subquery); + if !self.match_next(sym!(Comma)) { + break; + } + } + + let mut joins: Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)> = vec![]; + + while self.match_next_one_of(&vec![skw!(Left), skw!(Right), skw!(Inner), skw!(Join)]) { + // If the next token is a join keyword, then it must be a join subquery + let peek = self.peek_bw(1); + let join_type = if peek.tok_type == skw!(Inner) { + self.expected(skw!(Join))?; + SqlJoinType::Inner + } else if peek.tok_type == skw!(Left) { + if self.match_next(skw!(Outer)) { + self.advance(); + } + self.expected(skw!(Join))?; + SqlJoinType::Left + } else if peek.tok_type == skw!(Right) { + if self.match_next(skw!(Outer)) { + self.advance(); + } + self.expected(skw!(Join))?; + SqlJoinType::Right + } else if peek.tok_type == skw!(Join) { + SqlJoinType::Inner + } else { + return Err(ParseError::UnexpectedToken { + token: peek.clone(), + }); + }; + let join_subquery = self.sql_collection_subquery()?; + self.expected(skw!(On))?; + let join_expr = self.expression()?; + joins.push((join_type, join_subquery, SqlExpr::Default(join_expr))); + } + + self.expected(sym!(RightParen))?; // closing paren + + if !joins.is_empty() { + return Ok(SqlCollectionSubquery::Join( + Box::new(SqlCollectionSubquery::Recursive(recursive)), + joins, + )); + } + + return Ok(SqlCollectionSubquery::Recursive(recursive)); + } else if self.cmp_tok(&Identifier { dollar: false }) { + // If the next token is not a left paren, then it must be a collection getter + if self.match_next_all_of(&vec![ + Identifier { dollar: false }, + sym!(Dot), + Identifier { dollar: false }, + ]) { + return Ok(SqlCollectionSubquery::Collection { + namespace: Some(self.peek_bw(3).clone()), + name: self.peek_bw(1).clone(), + alias: optional_with_expected!(self, skw!(As), Identifier { dollar: false }), + }); + } + return Ok(SqlCollectionSubquery::Collection { + namespace: None, + name: self.expected(Identifier { dollar: false })?.clone(), + alias: optional_with_expected!(self, skw!(As), Identifier { dollar: false }), + }); + } else { + Err(ParseError::UnexpectedToken { + token: self.peek_bw(0).clone(), + }) + } + } + fn sql_where(&mut self) -> ParseResult> { if self.match_next(skw!(Where)) { let expr = self.expression()?; diff --git a/server/src/lang/serializer.rs b/server/src/lang/serializer.rs index 05bdff02..b19a9199 100644 --- a/server/src/lang/serializer.rs +++ b/server/src/lang/serializer.rs @@ -30,7 +30,11 @@ impl Visitor for Parsed { "type": "Expr::Select", // TODO(vck): Implement rest of the select }), - Expr::Literal { raw, span: _, value } => { + Expr::Literal { + raw, + span: _, + value, + } => { json!({ "type": "Expr::Literal", "value": format!("{:?}", value), @@ -43,7 +47,11 @@ impl Visitor for Parsed { "value": self.visit_expr(*expr)?, }) } - Expr::Unary { symbol, expr, span: _ } => { + Expr::Unary { + symbol, + expr, + span: _, + } => { json!({ "type": "Expr::Unary", "operator": symbol, @@ -89,7 +97,11 @@ impl Visitor for Parsed { "right": self.visit_expr(*right)?, }) } - Expr::Call { callee, span: _, args } => { + Expr::Call { + callee, + span: _, + args, + } => { json!({ "type": "Expr::Call", "callee": self.visit_expr(*callee)?, diff --git a/server/src/runtime/interpreter.rs b/server/src/runtime/interpreter.rs index ae389667..0612c241 100644 --- a/server/src/runtime/interpreter.rs +++ b/server/src/runtime/interpreter.rs @@ -55,7 +55,7 @@ pub enum LoopState { Go, Broken, Continue, - Function + Function, } struct LoopStack { @@ -324,7 +324,7 @@ impl Visitor for Interpreter { self.loop_stack.push_fn(); let val = callable.call(self, args_evaluated.as_slice()); - + self.loop_stack.pop_fn(); match val { Err(HaltReason::Return(ret_val)) => Ok(ret_val), @@ -464,7 +464,8 @@ impl Visitor for Interpreter { || is_value_truthy(self.visit_expr(condition.unwrap())?)) { self.visit_stmt(*body)?; - self.loop_stack.set_loop_state(LoopState::Go, Some(LoopState::Continue)); + self.loop_stack + .set_loop_state(LoopState::Go, Some(LoopState::Continue)); if let Some(post_id) = post { self.visit_stmt(*post_id)?; } From 5baa6e5c7ddec1e1222fc83899a11204ef880eae Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Mon, 18 Dec 2023 19:26:25 +0300 Subject: [PATCH 07/17] fix: Changed parsing order --- server/src/lang/ast/sql.rs | 2 +- server/src/lang/parser.rs | 119 +++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 58 deletions(-) diff --git a/server/src/lang/ast/sql.rs b/server/src/lang/ast/sql.rs index 54483837..c3b70b03 100644 --- a/server/src/lang/ast/sql.rs +++ b/server/src/lang/ast/sql.rs @@ -50,7 +50,7 @@ pub enum SqlCollectionSubquery { Recursive(Vec), Join( Box, - Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)>, + Vec<(SqlJoinType, SqlCollectionSubquery, Option)>, ), Collection { namespace: Option, diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 63626ef9..eef2b489 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -627,75 +627,80 @@ impl<'a> Parser<'a> { fn sql_from(&mut self) -> ParseResult> { if self.match_next(skw!(From)) { - return Ok(Some(self.sql_collection_subquery()?)); + return Ok(Some(self.sql_subquery_join()?)); } Ok(None) } - fn sql_collection_subquery(&mut self) -> ParseResult { - if self.cmp_tok(&sym!(LeftParen)) { - // If the next token is a left paren, then it must be either a select statement or a recursive subquery - self.advance(); - if self.cmp_tok(&skw!(Select)) { - let expr = self.select()?; - let alias = optional_with_expected!(self, skw!(As), Identifier { dollar: false }); - self.expected(sym!(RightParen))?; // closing paren - return Ok(SqlCollectionSubquery::Select { expr, alias }); - } + fn sql_subquery_join(&mut self) -> ParseResult { + let mut recursive: Vec = vec![]; - let mut recursive: Vec = vec![]; + loop { + let subquery = self.sql_subquery_collection()?; + recursive.push(subquery); + if !self.match_next(sym!(Comma)) { + break; + } + } - loop { - let subquery = self.sql_collection_subquery()?; - recursive.push(subquery); - if !self.match_next(sym!(Comma)) { - break; + let mut joins: Vec<(SqlJoinType, SqlCollectionSubquery, Option)> = vec![]; + + while self.match_next_one_of(&vec![skw!(Left), skw!(Right), skw!(Inner), skw!(Join)]) { + // If the next token is a join keyword, then it must be a join subquery + let peek = self.peek_bw(1); + let join_type = if peek.tok_type == skw!(Inner) { + self.expected(skw!(Join))?; + SqlJoinType::Inner + } else if peek.tok_type == skw!(Left) { + if self.match_next(skw!(Outer)) { + self.advance(); } + self.expected(skw!(Join))?; + SqlJoinType::Left + } else if peek.tok_type == skw!(Right) { + if self.match_next(skw!(Outer)) { + self.advance(); + } + self.expected(skw!(Join))?; + SqlJoinType::Right + } else if peek.tok_type == skw!(Join) { + SqlJoinType::Inner + } else { + return Err(ParseError::UnexpectedToken { + token: peek.clone(), + }); + }; + let join_subquery = self.sql_subquery_collection()?; + let mut join_expr: Option = None; + if self.match_next(skw!(On)) { + join_expr = Some(SqlExpr::Default(self.expression()?)); } + joins.push((join_type, join_subquery, join_expr)); + } - let mut joins: Vec<(SqlJoinType, SqlCollectionSubquery, SqlExpr)> = vec![]; - - while self.match_next_one_of(&vec![skw!(Left), skw!(Right), skw!(Inner), skw!(Join)]) { - // If the next token is a join keyword, then it must be a join subquery - let peek = self.peek_bw(1); - let join_type = if peek.tok_type == skw!(Inner) { - self.expected(skw!(Join))?; - SqlJoinType::Inner - } else if peek.tok_type == skw!(Left) { - if self.match_next(skw!(Outer)) { - self.advance(); - } - self.expected(skw!(Join))?; - SqlJoinType::Left - } else if peek.tok_type == skw!(Right) { - if self.match_next(skw!(Outer)) { - self.advance(); - } - self.expected(skw!(Join))?; - SqlJoinType::Right - } else if peek.tok_type == skw!(Join) { - SqlJoinType::Inner - } else { - return Err(ParseError::UnexpectedToken { - token: peek.clone(), - }); - }; - let join_subquery = self.sql_collection_subquery()?; - self.expected(skw!(On))?; - let join_expr = self.expression()?; - joins.push((join_type, join_subquery, SqlExpr::Default(join_expr))); - } + if !joins.is_empty() { + return Ok(SqlCollectionSubquery::Join( + Box::new(SqlCollectionSubquery::Recursive(recursive)), + joins, + )); + } - self.expected(sym!(RightParen))?; // closing paren + return Ok(SqlCollectionSubquery::Recursive(recursive)); + } - if !joins.is_empty() { - return Ok(SqlCollectionSubquery::Join( - Box::new(SqlCollectionSubquery::Recursive(recursive)), - joins, - )); + fn sql_subquery_collection(&mut self) -> ParseResult { + if self.match_next(sym!(LeftParen)) { + if self.cmp_tok(&skw!(Select)) { + let expr = self.select()?; + self.expected(sym!(RightParen))?; // closing paren + let alias: Option = + optional_with_expected!(self, skw!(As), Identifier { dollar: false }); + return Ok(SqlCollectionSubquery::Select { expr, alias }); } - - return Ok(SqlCollectionSubquery::Recursive(recursive)); + // If the next token is a left paren, then it must be either a select statement or a recursive subquery + let parsed = self.sql_subquery_collection()?; + self.expected(sym!(RightParen))?; // closing paren + return Ok(parsed); } else if self.cmp_tok(&Identifier { dollar: false }) { // If the next token is not a left paren, then it must be a collection getter if self.match_next_all_of(&vec![ From 0054556c9c5d4288cd57f981f5549b32a283c6d5 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Mon, 18 Dec 2023 20:19:58 +0300 Subject: [PATCH 08/17] fix: Parsing bug --- server/src/lang/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index eef2b489..68ba6b73 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -698,7 +698,7 @@ impl<'a> Parser<'a> { return Ok(SqlCollectionSubquery::Select { expr, alias }); } // If the next token is a left paren, then it must be either a select statement or a recursive subquery - let parsed = self.sql_subquery_collection()?; + let parsed = self.sql_subquery_join()?; // TODO(vck): Check if using _collection variant makes sense. self.expected(sym!(RightParen))?; // closing paren return Ok(parsed); } else if self.cmp_tok(&Identifier { dollar: false }) { From f2aa656dc72915032f1b4e7f94a8b5531ac603c0 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 11:19:23 +0300 Subject: [PATCH 09/17] fix: Bug fix --- server/src/lang/parser.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 68ba6b73..a4dffbf6 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -652,16 +652,10 @@ impl<'a> Parser<'a> { self.expected(skw!(Join))?; SqlJoinType::Inner } else if peek.tok_type == skw!(Left) { - if self.match_next(skw!(Outer)) { - self.advance(); - } - self.expected(skw!(Join))?; + optional_with_expected!(self, skw!(Outer), skw!(Join)); SqlJoinType::Left } else if peek.tok_type == skw!(Right) { - if self.match_next(skw!(Outer)) { - self.advance(); - } - self.expected(skw!(Join))?; + optional_with_expected!(self, skw!(Outer), skw!(Join)); SqlJoinType::Right } else if peek.tok_type == skw!(Join) { SqlJoinType::Inner From 8845c3f4498e8cbab245300763cdaf29330e2923 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 11:45:43 +0300 Subject: [PATCH 10/17] fix: Refactored SQL join --- server/src/lang/ast/sql.rs | 14 +++++++++----- server/src/lang/parser.rs | 26 +++++++++++++++----------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/server/src/lang/ast/sql.rs b/server/src/lang/ast/sql.rs index c3b70b03..f600fe2d 100644 --- a/server/src/lang/ast/sql.rs +++ b/server/src/lang/ast/sql.rs @@ -45,13 +45,17 @@ pub enum SqlProjection { Expr { expr: SqlExpr, alias: Option }, } +#[derive(Debug, Eq, PartialEq)] +pub struct SqlJoin { + pub join_type: SqlJoinType, + pub subquery: SqlCollectionSubquery, + pub join_constraint: Option, +} + #[derive(Debug, Eq, PartialEq)] pub enum SqlCollectionSubquery { - Recursive(Vec), - Join( - Box, - Vec<(SqlJoinType, SqlCollectionSubquery, Option)>, - ), + Group(Vec), + Join(Box, Vec), Collection { namespace: Option, name: Token, diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index a4dffbf6..4e201728 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -1,7 +1,7 @@ use super::ast::expr::{Expr, ExprId}; use super::ast::sql::{ - SelectCore, SqlCollectionSubquery, SqlCompoundOperator, SqlDistinct, SqlExpr, SqlJoinType, - SqlOrdering, SqlProjection, SqlSelect, + SelectCore, SqlCollectionSubquery, SqlCompoundOperator, SqlDistinct, SqlExpr, SqlJoin, + SqlJoinType, SqlOrdering, SqlProjection, SqlSelect, }; use super::ast::stmt::{Stmt, StmtId}; use super::ast::{Literal, ParserArena}; @@ -633,17 +633,17 @@ impl<'a> Parser<'a> { } fn sql_subquery_join(&mut self) -> ParseResult { - let mut recursive: Vec = vec![]; + let mut subquery_group: Vec = vec![]; loop { let subquery = self.sql_subquery_collection()?; - recursive.push(subquery); + subquery_group.push(subquery); if !self.match_next(sym!(Comma)) { break; } } - let mut joins: Vec<(SqlJoinType, SqlCollectionSubquery, Option)> = vec![]; + let mut joins: Vec = vec![]; while self.match_next_one_of(&vec![skw!(Left), skw!(Right), skw!(Inner), skw!(Join)]) { // If the next token is a join keyword, then it must be a join subquery @@ -664,22 +664,26 @@ impl<'a> Parser<'a> { token: peek.clone(), }); }; - let join_subquery = self.sql_subquery_collection()?; - let mut join_expr: Option = None; + let subquery = self.sql_subquery_collection()?; + let mut join_constraint: Option = None; if self.match_next(skw!(On)) { - join_expr = Some(SqlExpr::Default(self.expression()?)); + join_constraint = Some(SqlExpr::Default(self.expression()?)); } - joins.push((join_type, join_subquery, join_expr)); + joins.push(SqlJoin { + join_type, + subquery, + join_constraint, + }); } if !joins.is_empty() { return Ok(SqlCollectionSubquery::Join( - Box::new(SqlCollectionSubquery::Recursive(recursive)), + Box::new(SqlCollectionSubquery::Group(subquery_group)), joins, )); } - return Ok(SqlCollectionSubquery::Recursive(recursive)); + return Ok(SqlCollectionSubquery::Group(subquery_group)); } fn sql_subquery_collection(&mut self) -> ParseResult { From eddc368a2886ba2b4f620d713f0eee4a4dc07d40 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 11:58:08 +0300 Subject: [PATCH 11/17] fix: Method names --- server/src/lang/parser.rs | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 4e201728..559e488b 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -470,14 +470,14 @@ impl<'a> Parser<'a> { .get_merged_span(&token.span, &self.arena.get_expression(unary).get_span()), })); } - self.select() + self.sql_select() } - fn select(&mut self) -> ParseResult { + fn sql_select(&mut self) -> ParseResult { if !self.cmp_tok(&skw!(Select)) { return self.call(); } - let core = self.select_core()?; + let core = self.sql_select_core()?; let mut compounds: Vec<(SqlCompoundOperator, SelectCore)> = vec![]; while self.match_next_one_of(&vec![skw!(Union), skw!(Intersect), skw!(Except)]) { let op = self.peek_bw(1); @@ -493,7 +493,7 @@ impl<'a> Parser<'a> { } } }; - let secondary_core = self.select_core()?; + let secondary_core = self.sql_select_core()?; compounds.push((compound_op, secondary_core)) } let order_by = if self.match_next(skw!(Order)) { @@ -548,7 +548,7 @@ impl<'a> Parser<'a> { })) } - fn select_core(&mut self) -> ParseResult { + fn sql_select_core(&mut self) -> ParseResult { self.expected(skw!(Select))?; let distinct = if self.match_next(skw!(Distinct)) { SqlDistinct::Distinct @@ -558,10 +558,10 @@ impl<'a> Parser<'a> { SqlDistinct::All }; - let projection = self.sql_projection(); - let from = self.sql_from()?; - let r#where = self.sql_where()?; - let group_by = self.sql_group_by()?; + let projection = self.sql_select_projection(); + let from = self.sql_select_from()?; + let r#where = self.sql_select_where()?; + let group_by = self.sql_select_group_by()?; let having = if group_by.is_some() && self.match_next(skw!(Having)) { Some(SqlExpr::Default(self.expression()?)) } else { @@ -578,7 +578,7 @@ impl<'a> Parser<'a> { }) } - fn sql_group_by(&mut self) -> ParseResult>> { + fn sql_select_group_by(&mut self) -> ParseResult>> { if self.match_next(skw!(Group)) { self.expected(skw!(By))?; let mut groups: Vec = vec![]; @@ -596,7 +596,7 @@ impl<'a> Parser<'a> { } } - fn sql_projection(&mut self) -> Vec { + fn sql_select_projection(&mut self) -> Vec { let mut projections: Vec = vec![]; loop { if self.match_next(sym!(Star)) { @@ -625,18 +625,18 @@ impl<'a> Parser<'a> { projections } - fn sql_from(&mut self) -> ParseResult> { + fn sql_select_from(&mut self) -> ParseResult> { if self.match_next(skw!(From)) { - return Ok(Some(self.sql_subquery_join()?)); + return Ok(Some(self.sql_select_subquery_join()?)); } Ok(None) } - fn sql_subquery_join(&mut self) -> ParseResult { + fn sql_select_subquery_join(&mut self) -> ParseResult { let mut subquery_group: Vec = vec![]; loop { - let subquery = self.sql_subquery_collection()?; + let subquery = self.sql_select_subquery_collection()?; subquery_group.push(subquery); if !self.match_next(sym!(Comma)) { break; @@ -664,7 +664,7 @@ impl<'a> Parser<'a> { token: peek.clone(), }); }; - let subquery = self.sql_subquery_collection()?; + let subquery = self.sql_select_subquery_collection()?; let mut join_constraint: Option = None; if self.match_next(skw!(On)) { join_constraint = Some(SqlExpr::Default(self.expression()?)); @@ -686,17 +686,17 @@ impl<'a> Parser<'a> { return Ok(SqlCollectionSubquery::Group(subquery_group)); } - fn sql_subquery_collection(&mut self) -> ParseResult { + fn sql_select_subquery_collection(&mut self) -> ParseResult { if self.match_next(sym!(LeftParen)) { if self.cmp_tok(&skw!(Select)) { - let expr = self.select()?; + let expr = self.sql_select()?; self.expected(sym!(RightParen))?; // closing paren let alias: Option = optional_with_expected!(self, skw!(As), Identifier { dollar: false }); return Ok(SqlCollectionSubquery::Select { expr, alias }); } // If the next token is a left paren, then it must be either a select statement or a recursive subquery - let parsed = self.sql_subquery_join()?; // TODO(vck): Check if using _collection variant makes sense. + let parsed = self.sql_select_subquery_join()?; // TODO(vck): Check if using _collection variant makes sense. self.expected(sym!(RightParen))?; // closing paren return Ok(parsed); } else if self.cmp_tok(&Identifier { dollar: false }) { @@ -724,7 +724,7 @@ impl<'a> Parser<'a> { } } - fn sql_where(&mut self) -> ParseResult> { + fn sql_select_where(&mut self) -> ParseResult> { if self.match_next(skw!(Where)) { let expr = self.expression()?; return Ok(Some(SqlExpr::Default(expr))); From 0402faba7c877e7f2a10c4bcde2f803567e82690 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 22:23:24 +0300 Subject: [PATCH 12/17] fix: Testing improvements --- server/src/lang/parser.rs | 190 +----------------- server/src/lang/tests/generic/binary.json | 27 +++ server/src/lang/tests/generic/grouping.json | 69 +++++++ .../lang/tests/generic/number_literal.json | 16 ++ server/src/lang/tests/generic/unary.json | 22 ++ server/src/lang/tests/generic/variable.json | 15 ++ server/src/lang/tests/helpers.rs | 41 ++++ 7 files changed, 196 insertions(+), 184 deletions(-) create mode 100644 server/src/lang/tests/generic/binary.json create mode 100644 server/src/lang/tests/generic/grouping.json create mode 100644 server/src/lang/tests/generic/number_literal.json create mode 100644 server/src/lang/tests/generic/unary.json create mode 100644 server/src/lang/tests/generic/variable.json diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 559e488b..1d53cc1f 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -928,190 +928,12 @@ impl<'a> Parser<'a> { #[cfg(test)] mod test { - use assert_json_diff::assert_json_eq; - use serde_json::{json, Value}; + mod generic { + use crate::generate_test_cases; + use crate::lang::tests::helpers::compare_parsed_to_expected; + use serde_json::{from_str, Value}; + use std::fs; - use crate::lang::{scanner::Scanner, token::Token}; - - use super::*; - - fn get_tokens(source: &str) -> Vec { - return Scanner::scan(source).unwrap(); - } - - fn compare_parsed_to_expected(source: &str, expected: Value) { - let tokens = get_tokens(source); - let mut parsed = Parser::parse(&tokens).unwrap(); - let actual = parsed.to_json(); - assert_json_eq!(actual, expected); - } - - #[test] - fn test_parse_literal_expression() { - compare_parsed_to_expected( - "1;", - json!({ - "type": "Stmt::Program", - "body": [ - { - "type": "Stmt::Expression", - "value": { - "type": "Expr::Literal", - "value": "Num(1.0)", - "raw": "1" - } - } - ] - }), - ); - } - - #[test] - fn test_parse_unary_expression() { - compare_parsed_to_expected( - "-1;", - json!({ - "type": "Stmt::Program", - "body": [ - { - "type": "Stmt::Expression", - "value": { - "type": "Expr::Unary", - "operator": { - "Symbol": "Minus" - }, - "value": { - "type": "Expr::Literal", - "value": "Num(1.0)", - "raw": "1" - } - } - } - ] - }), - ); - } - - #[test] - fn test_parse_binary_expression() { - compare_parsed_to_expected( - "1 + 2;", - json!({ - "type": "Stmt::Program", - "body": [ - { - "type": "Stmt::Expression", - "value": { - "type": "Expr::Binary", - "left": { - "type": "Expr::Literal", - "value": "Num(1.0)", - "raw": "1" - }, - "operator": { - "Symbol": "Plus" - }, - "right": { - "type": "Expr::Literal", - "value": "Num(2.0)", - "raw": "2" - } - } - } - ] - }), - ); - } - - #[test] - fn test_parse_grouping_expression() { - compare_parsed_to_expected( - "(1 + 2) * (3 / (4 - 7));", - json!({ - "type": "Stmt::Program", - "body": [ - { - "type": "Stmt::Expression", - "value": { - "type": "Expr::Binary", - "left": { - "type": "Expr::Grouping", - "value": { - "type": "Expr::Binary", - "left": { - "raw": "1", - "type": "Expr::Literal", - "value": "Num(1.0)", - }, - "operator": { - "Symbol": "Plus" - }, - "right": { - "raw": "2", - "type": "Expr::Literal", - "value": "Num(2.0)", - } - } - }, - "operator": { - "Symbol": "Star" - }, - "right": { - "type": "Expr::Grouping", - "value": { - "type": "Expr::Binary", - "left": { - "raw": "3", - "type": "Expr::Literal", - "value": "Num(3.0)", - }, - "operator": { - "Symbol": "Slash" - }, - "right": { - "type": "Expr::Grouping", - "value": { - "type": "Expr::Binary", - "left": { - "raw": "4", - "type": "Expr::Literal", - "value": "Num(4.0)", - }, - "operator": { - "Symbol": "Minus" - }, - "right": { - "raw": "7", - "type": "Expr::Literal", - "value": "Num(7.0)", - } - } - } - } - } - } - } - ] - }), - ); - } - - #[test] - fn test_parse_variable_expression() { - compare_parsed_to_expected( - "$a;", - json!({ - "type": "Stmt::Program", - "body": [ - { - "type": "Stmt::Expression", - "value": { - "type": "Expr::Variable", - "value": "$a", - } - } - ] - }), - ); + generate_test_cases!("generic", binary, unary, grouping, number_literal, variable); } } diff --git a/server/src/lang/tests/generic/binary.json b/server/src/lang/tests/generic/binary.json new file mode 100644 index 00000000..7f2e2dfb --- /dev/null +++ b/server/src/lang/tests/generic/binary.json @@ -0,0 +1,27 @@ +{ + "source": "1 + 2;", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Binary", + "left": { + "type": "Expr::Literal", + "value": "Num(1.0)", + "raw": "1" + }, + "operator": { + "Symbol": "Plus" + }, + "right": { + "type": "Expr::Literal", + "value": "Num(2.0)", + "raw": "2" + } + } + } + ] + } +} \ No newline at end of file diff --git a/server/src/lang/tests/generic/grouping.json b/server/src/lang/tests/generic/grouping.json new file mode 100644 index 00000000..f219d0ec --- /dev/null +++ b/server/src/lang/tests/generic/grouping.json @@ -0,0 +1,69 @@ +{ + "source": "(1 + 2) * (3 / (4 - 7));", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Binary", + "left": { + "type": "Expr::Grouping", + "value": { + "type": "Expr::Binary", + "left": { + "raw": "1", + "type": "Expr::Literal", + "value": "Num(1.0)" + }, + "operator": { + "Symbol": "Plus" + }, + "right": { + "raw": "2", + "type": "Expr::Literal", + "value": "Num(2.0)" + } + } + }, + "operator": { + "Symbol": "Star" + }, + "right": { + "type": "Expr::Grouping", + "value": { + "type": "Expr::Binary", + "left": { + "raw": "3", + "type": "Expr::Literal", + "value": "Num(3.0)" + }, + "operator": { + "Symbol": "Slash" + }, + "right": { + "type": "Expr::Grouping", + "value": { + "type": "Expr::Binary", + "left": { + "raw": "4", + "type": "Expr::Literal", + "value": "Num(4.0)" + }, + "operator": { + "Symbol": "Minus" + }, + "right": { + "raw": "7", + "type": "Expr::Literal", + "value": "Num(7.0)" + } + } + } + } + } + } + } + ] + } +} \ No newline at end of file diff --git a/server/src/lang/tests/generic/number_literal.json b/server/src/lang/tests/generic/number_literal.json new file mode 100644 index 00000000..e567643f --- /dev/null +++ b/server/src/lang/tests/generic/number_literal.json @@ -0,0 +1,16 @@ +{ + "source": "1;", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Literal", + "value": "Num(1.0)", + "raw": "1" + } + } + ] + } +} \ No newline at end of file diff --git a/server/src/lang/tests/generic/unary.json b/server/src/lang/tests/generic/unary.json new file mode 100644 index 00000000..4c613844 --- /dev/null +++ b/server/src/lang/tests/generic/unary.json @@ -0,0 +1,22 @@ +{ + "source": "-1;", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Unary", + "operator": { + "Symbol": "Minus" + }, + "value": { + "type": "Expr::Literal", + "value": "Num(1.0)", + "raw": "1" + } + } + } + ] + } +} \ No newline at end of file diff --git a/server/src/lang/tests/generic/variable.json b/server/src/lang/tests/generic/variable.json new file mode 100644 index 00000000..e05b6e94 --- /dev/null +++ b/server/src/lang/tests/generic/variable.json @@ -0,0 +1,15 @@ +{ + "source": "$a;", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Variable", + "value": "$a" + } + } + ] + } +} \ No newline at end of file diff --git a/server/src/lang/tests/helpers.rs b/server/src/lang/tests/helpers.rs index 2f3a2581..ce207005 100644 --- a/server/src/lang/tests/helpers.rs +++ b/server/src/lang/tests/helpers.rs @@ -1,6 +1,47 @@ +#[cfg(test)] +use assert_json_diff::assert_json_eq; + +#[cfg(test)] +use serde_json::Value; + +#[cfg(test)] +use crate::lang::{parser::Parser, scanner::Scanner, token::Token}; + #[macro_export] macro_rules! lexm { ($a: literal) => { Some($a.to_owned()) }; } + +#[cfg(test)] +pub fn get_tokens(source: &str) -> Vec { + return Scanner::scan(source).unwrap(); +} + +#[cfg(test)] +pub fn compare_parsed_to_expected(source: &str, expected: Value) { + let tokens = get_tokens(source); + let mut parsed = Parser::parse(&tokens).unwrap(); + let actual = parsed.to_json(); + assert_json_eq!(actual, expected); +} + +#[macro_export] +#[cfg(test)] +macro_rules! generate_test_cases { + ($dir:expr, $($file:ident),*) => { + $( + #[test] + fn $file() { + let path = format!("src/lang/tests/{}/{}.json", $dir, stringify!($file)); + let content_json = fs::read_to_string(&path).unwrap(); + + let content: Value = from_str(&content_json).unwrap(); + let source = content["source"].as_str().unwrap(); + let expected = content["expected"].clone(); + compare_parsed_to_expected(source, expected); + } + )* + }; +} From f7f4c3f37ed833499b5c4965be3c4ebbb1f6bf35 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 22:45:19 +0300 Subject: [PATCH 13/17] fix: Testing sugar --- server/src/lang/parser.rs | 10 ++-------- server/src/lang/tests/helpers.rs | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 1d53cc1f..8779936f 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -927,13 +927,7 @@ impl<'a> Parser<'a> { #[cfg(test)] mod test { + use crate::parse_tests; - mod generic { - use crate::generate_test_cases; - use crate::lang::tests::helpers::compare_parsed_to_expected; - use serde_json::{from_str, Value}; - use std::fs; - - generate_test_cases!("generic", binary, unary, grouping, number_literal, variable); - } + parse_tests!(generic, binary, unary, grouping, number_literal, variable); } diff --git a/server/src/lang/tests/helpers.rs b/server/src/lang/tests/helpers.rs index ce207005..ad353432 100644 --- a/server/src/lang/tests/helpers.rs +++ b/server/src/lang/tests/helpers.rs @@ -29,12 +29,12 @@ pub fn compare_parsed_to_expected(source: &str, expected: Value) { #[macro_export] #[cfg(test)] -macro_rules! generate_test_cases { - ($dir:expr, $($file:ident),*) => { +macro_rules! generate_parse_test_cases { + ($dir:ident, $($file:ident),*) => { $( #[test] fn $file() { - let path = format!("src/lang/tests/{}/{}.json", $dir, stringify!($file)); + let path = format!("src/lang/tests/{}/{}.json", stringify!($dir), stringify!($file)); let content_json = fs::read_to_string(&path).unwrap(); let content: Value = from_str(&content_json).unwrap(); @@ -45,3 +45,19 @@ macro_rules! generate_test_cases { )* }; } + +#[macro_export] +#[cfg(test)] +macro_rules! parse_tests { + ($package:ident, $($file:ident),*) => { + + mod $package { + use crate::generate_parse_test_cases; + use crate::lang::tests::helpers::compare_parsed_to_expected; + use serde_json::{from_str, Value}; + use std::fs; + + generate_parse_test_cases!($package, $($file),*); + } + } +} From 0bde5ebd9d4894083a83701f5d72138bda3f93dd Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Tue, 19 Dec 2023 22:46:36 +0300 Subject: [PATCH 14/17] fix: Parse testing macro --- server/src/lang/parser.rs | 2 +- server/src/lang/tests/helpers.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 8779936f..96a2fa67 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -929,5 +929,5 @@ impl<'a> Parser<'a> { mod test { use crate::parse_tests; - parse_tests!(generic, binary, unary, grouping, number_literal, variable); + parse_tests!(generic / binary, unary, grouping, number_literal, variable); } diff --git a/server/src/lang/tests/helpers.rs b/server/src/lang/tests/helpers.rs index ad353432..cfa9334c 100644 --- a/server/src/lang/tests/helpers.rs +++ b/server/src/lang/tests/helpers.rs @@ -49,7 +49,7 @@ macro_rules! generate_parse_test_cases { #[macro_export] #[cfg(test)] macro_rules! parse_tests { - ($package:ident, $($file:ident),*) => { + ($package:ident / $($file:ident),*) => { mod $package { use crate::generate_parse_test_cases; From 6116b165cdf5847bdc19b5e0ce54d218899af623 Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Wed, 20 Dec 2023 10:18:20 +0300 Subject: [PATCH 15/17] chore: Codecov --- .github/workflows/codecov.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/codecov.yml diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 00000000..0f90f608 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,22 @@ +name: Coverage + +on: [pull_request, push] + +jobs: + coverage: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: rustup update stable + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Generate code coverage + run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: lcov.info + fail_ci_if_error: true \ No newline at end of file From 8dd44ac63a26711a83f46b9d40cf40645d539d1a Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Wed, 20 Dec 2023 10:25:47 +0300 Subject: [PATCH 16/17] chore: Codecov badge --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 3181714a..87ed232b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,17 @@ # λykiaDB +
+
+ [![CI](https://github.com/lykia-rs/lykiadb/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/lykia-rs/lykiadb/actions/workflows/ci.yml) +
+
+ +[![codecov](https://codecov.io/gh/lykia-rs/lykiadb/graph/badge.svg?token=DGIK7BE3K1)](https://codecov.io/gh/lykia-rs/lykiadb) + +
+
+

LykiaDB logo

From dd02f63c49ec269ad714e1ec9c9add3212cd7e0e Mon Sep 17 00:00:00 2001 From: Vedat Can Keklik Date: Wed, 20 Dec 2023 10:35:07 +0300 Subject: [PATCH 17/17] feat: Function call parser test --- server/src/lang/parser.rs | 2 +- .../src/lang/tests/generic/function_call.json | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 server/src/lang/tests/generic/function_call.json diff --git a/server/src/lang/parser.rs b/server/src/lang/parser.rs index 96a2fa67..e1af48f7 100644 --- a/server/src/lang/parser.rs +++ b/server/src/lang/parser.rs @@ -929,5 +929,5 @@ impl<'a> Parser<'a> { mod test { use crate::parse_tests; - parse_tests!(generic / binary, unary, grouping, number_literal, variable); + parse_tests!(generic / binary, unary, grouping, number_literal, variable, function_call); } diff --git a/server/src/lang/tests/generic/function_call.json b/server/src/lang/tests/generic/function_call.json new file mode 100644 index 00000000..82d7f350 --- /dev/null +++ b/server/src/lang/tests/generic/function_call.json @@ -0,0 +1,25 @@ +{ + "source": "print(50);", + "expected": { + "type": "Stmt::Program", + "body": [ + { + "type": "Stmt::Expression", + "value": { + "type": "Expr::Call", + "callee": { + "type": "Expr::Variable", + "value": "print" + }, + "args": [ + { + "type": "Expr::Literal", + "value": "Num(50.0)", + "raw": "50" + } + ] + } + } + ] + } +} \ No newline at end of file