From db389045fcc465fd88653bae57a9ee04c5a2fa13 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Sun, 30 Nov 2025 15:15:40 -0700 Subject: [PATCH] emit spanned from parser --- rust_compiler/libs/parser/src/lib.rs | 1334 ++++++++++++++------------ 1 file changed, 719 insertions(+), 615 deletions(-) diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index c655634..2b89b23 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -30,16 +30,16 @@ quick_error! { display("Tokenizer Error: {}", err) source(err) } - UnexpectedToken(token: Token) { + UnexpectedToken(span: Span, token: Token) { display("Unexpected token: {:?}", token) } - DuplicateIdentifier(token: Token) { + DuplicateIdentifier(span: Span, token: Token) { display("Duplicate identifier: {:?}", token) } - InvalidSyntax(token: Token, reason: String) { - display("Invalid syntax: {:?}, Reason: {}", token, reason) + InvalidSyntax(span: Span, reason: String) { + display("Invalid syntax: {:?}, Reason: {}", span, reason) } - UnsupportedKeyword(token: Token) { + UnsupportedKeyword(span: Span, token: Token) { display("Unsupported keyword: {:?}", token) } UnexpectedEOF { @@ -57,47 +57,6 @@ macro_rules! self_matches_peek { }; } -macro_rules! token_from_option { - ($token:expr) => { - match $token { - Some(ref token) => token.clone(), - None => return Err(Error::UnexpectedEOF), - } - }; - (owned $token:expr) => { - match $token { - Some(token) => token, - None => return Err(Error::UnexpectedEOF), - } - }; -} - -macro_rules! extract_token_data { - ($token:ident, $pattern:pat, $extraction:expr) => { - match $token.token_type { - $pattern => $extraction, - _ => return Err(Error::UnexpectedToken($token.clone())), - } - }; - ($token:expr, $pattern:pat, $extraction:expr) => { - match $token.token_type { - $pattern => $extraction, - _ => { - return Err(Error::UnexpectedToken($token.clone())); - } - } - }; -} - -macro_rules! self_matches_current { - ($self:ident, $pattern:pat) => { - matches!($self.current_token, Some(Token { token_type: $pattern, .. })) - }; - ($self:ident, $pattern:pat if $cond:expr) => { - matches!($self.current_token, Some(Token { token_type: $pattern, .. }) if $cond) - }; -} - macro_rules! token_matches { ($token:ident, $pattern:pat) => { matches!($token.token_type, $pattern) @@ -113,6 +72,15 @@ macro_rules! token_matches { }; } +macro_rules! self_matches_current { + ($self:ident, $pattern:pat) => { + matches!($self.current_token, Some(Token { token_type: $pattern, .. })) + }; + ($self:ident, $pattern:pat if $cond:expr) => { + matches!($self.current_token, Some(Token { token_type: $pattern, .. }) if $cond) + }; +} + pub struct Parser<'a> { tokenizer: TokenizerBuffer<'a>, current_token: Option, @@ -126,13 +94,74 @@ impl<'a> Parser<'a> { } } - /// Parses all the input from the tokenizer buffer and returns the resulting expression - /// Expressions are returned in a root block expression node + /// Calculates a Span from a given Token reference. + /// This is a static helper to avoid borrowing `self` when we already have a token ref. + fn token_to_span(t: &Token) -> Span { + let len = t.original_string.as_ref().map(|s| s.len()).unwrap_or(0); + Span { + start_line: t.line, + start_col: t.column, + end_line: t.line, + end_col: t.column + len, + } + } + + fn current_span(&self) -> Span { + self.current_token + .as_ref() + .map(Self::token_to_span) + .unwrap_or(Span { + start_line: 0, + start_col: 0, + end_line: 0, + end_col: 0, + }) + } + + /// Helper to run a parsing closure and wrap the result in a Spanned struct + fn spanned(&mut self, parser: F) -> Result, Error> + where + F: FnOnce(&mut Self) -> Result, + { + // Peek at the start token. If no current token (parsing hasn't started), peek the buffer. + let start_token = if self.current_token.is_some() { + self.current_token.clone() + } else { + self.tokenizer.peek()? + }; + + let (start_line, start_col) = start_token + .as_ref() + .map(|t| (t.line, t.column)) + .unwrap_or((1, 1)); + + let node = parser(self)?; + + // The end token is the current_token after parsing. + let end_token = self.current_token.as_ref(); + + let (end_line, end_col) = end_token + .map(|t| { + let len = t.original_string.as_ref().map(|s| s.len()).unwrap_or(0); + (t.line, t.column + len) + }) + .unwrap_or((start_line, start_col)); + + Ok(Spanned { + span: Span { + start_line, + start_col, + end_line, + end_col, + }, + node, + }) + } + pub fn parse_all(&mut self) -> Result, Error> { - // peek at what the first token would be and extract the line and col - let (start_line, start_col) = self - .tokenizer - .peek()? + let first_token = self.tokenizer.peek()?; + let (start_line, start_col) = first_token + .as_ref() .map(|tok| (tok.line, tok.column)) .unwrap_or((1, 1)); @@ -151,22 +180,19 @@ impl<'a> Parser<'a> { }; return Ok(Some(Expression::Block(Spanned { - node: BlockExpression(expressions), + node: BlockExpression(vec![]), span, }))); } self.tokenizer.seek(SeekFrom::Current(-1))?; - // Ignore the EOF, we want the previous token to define what the end of the source is. - let (end_line, end_col) = self - .tokenizer - .peek()? + let end_token_opt = self.tokenizer.peek()?; + + let (end_line, end_col) = end_token_opt .map(|tok| { - ( - tok.line, - tok.column + tok.original_string.unwrap_or_default().len(), - ) + let len = tok.original_string.as_ref().map(|s| s.len()).unwrap_or(0); + (tok.line, tok.column + len) }) .unwrap_or((start_line, start_col)); @@ -183,7 +209,6 @@ impl<'a> Parser<'a> { }))) } - /// Parses the input from the tokenizer buffer and returns the resulting expression pub fn parse(&mut self) -> Result>, Error> { self.assign_next()?; let expr = self.expression()?; @@ -195,25 +220,17 @@ impl<'a> Parser<'a> { Ok(expr) } - /// Assigns the next token in the tokenizer buffer to the current token fn assign_next(&mut self) -> Result<(), Error> { self.current_token = self.tokenizer.next_token()?; Ok(()) } - /// Calls `assign_next` and returns the next token in the tokenizer buffer fn get_next(&mut self) -> Result, Error> { self.assign_next()?; Ok(self.current_token.as_ref()) } - /// Parses an expression, handling binary operations with correct precedence. fn expression(&mut self) -> Result>, Error> { - let (start_line, end_line) = self - .current_token - .map(|tok| (tok.line, tok.column)) - .ok_or(Error::UnexpectedEOF)?; - // Parse the Left Hand Side (unary/primary expression) let lhs = self.unary()?; @@ -227,10 +244,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() ) { return Ok(Some(self.infix(lhs)?)); - } - // This is an edge case. We need to move back one token if the current token is an operator - // so the binary expression can pick up the operator - else if self_matches_current!( + } else if self_matches_current!( self, TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() ) { @@ -241,10 +255,7 @@ impl<'a> Parser<'a> { Ok(Some(lhs)) } - /// Parses a unary or primary expression. - /// This handles prefix operators (like negation) and atomic expressions (literals, variables, etc.), - /// but stops before consuming binary operators. - fn unary(&mut self) -> Result, Error> { + fn unary(&mut self) -> Result>, Error> { macro_rules! matches_keyword { ($keyword:expr, $($pattern:pat),+) => { matches!($keyword, $($pattern)|+) @@ -260,168 +271,307 @@ impl<'a> Parser<'a> { } let expr = match current_token.token_type { - // match unsupported keywords TokenType::Keyword(e) if matches_keyword!(e, Keyword::Enum) => { - return Err(Error::UnsupportedKeyword(current_token.clone())); + return Err(Error::UnsupportedKeyword( + self.current_span(), + current_token.clone(), + )); } - // match declarations with a `let` keyword - TokenType::Keyword(Keyword::Let) => self.declaration()?, + TokenType::Keyword(Keyword::Let) => { + // declaration is wrapped in spanned inside the function, but expects 'let' to be current + Some(self.spanned(|p| p.declaration())?) + } - TokenType::Keyword(Keyword::Device) => Expression::DeviceDeclaration(self.device()?), + TokenType::Keyword(Keyword::Device) => { + let spanned_dev = self.spanned(|p| p.device())?; + Some(Spanned { + span: spanned_dev.span, + node: Expression::DeviceDeclaration(spanned_dev), + }) + } - // match functions with a `fn` keyword - TokenType::Keyword(Keyword::Fn) => Expression::Function(self.function()?), + TokenType::Keyword(Keyword::Fn) => { + let spanned_fn = self.spanned(|p| p.function())?; + Some(Spanned { + span: spanned_fn.span, + node: Expression::Function(spanned_fn), + }) + } - // match if statements - TokenType::Keyword(Keyword::If) => Expression::If(self.if_expression()?), + TokenType::Keyword(Keyword::If) => { + let spanned_if = self.spanned(|p| p.if_expression())?; + Some(Spanned { + span: spanned_if.span, + node: Expression::If(spanned_if), + }) + } - // match loop statements - TokenType::Keyword(Keyword::Loop) => Expression::Loop(self.loop_expression()?), + TokenType::Keyword(Keyword::Loop) => { + let spanned_loop = self.spanned(|p| p.loop_expression())?; + Some(Spanned { + span: spanned_loop.span, + node: Expression::Loop(spanned_loop), + }) + } - // match while statements - TokenType::Keyword(Keyword::While) => Expression::While(self.while_expression()?), + TokenType::Keyword(Keyword::While) => { + let spanned_while = self.spanned(|p| p.while_expression())?; + Some(Spanned { + span: spanned_while.span, + node: Expression::While(spanned_while), + }) + } - // match break statements TokenType::Keyword(Keyword::Break) => { + let span = self.current_span(); // make sure the next token is a semi-colon - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - Expression::Break + Some(Spanned { + span, + node: Expression::Break(span), + }) } - // match continue statements TokenType::Keyword(Keyword::Continue) => { - // make sure the next token is a semi-colon - let next = token_from_option!(self.get_next()?); + let span = self.current_span(); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - Expression::Continue + Some(Spanned { + span, + node: Expression::Continue(span), + }) } - // match syscalls with a `syscall` keyword TokenType::Identifier(ref id) if SysCall::is_syscall(id) => { - Expression::Syscall(self.syscall()?) + let spanned_call = self.spanned(|p| p.syscall())?; + Some(Spanned { + span: spanned_call.span, + node: Expression::Syscall(spanned_call), + }) } - // match a variable expression with opening parenthesis TokenType::Identifier(_) if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) => { - Expression::Invocation(self.invocation()?) + let spanned_invoke = self.spanned(|p| p.invocation())?; + Some(Spanned { + span: spanned_invoke.span, + node: Expression::Invocation(spanned_invoke), + }) } - // match a variable expression with an assignment TokenType::Identifier(_) if self_matches_peek!(self, TokenType::Symbol(Symbol::Assign)) => { - Expression::Assignment(self.assignment()?) + let spanned_assign = self.spanned(|p| p.assignment())?; + Some(Spanned { + span: spanned_assign.span, + node: Expression::Assignment(spanned_assign), + }) } - // match variable expressions with an identifier - TokenType::Identifier(ref id) => Expression::Variable(id.clone()), + TokenType::Identifier(ref id) => { + let span = self.current_span(); + Some(Spanned { + span, + node: Expression::Variable(Spanned { + span, + node: id.clone(), + }), + }) + } - // match block expressions with a `{` symbol - TokenType::Symbol(Symbol::LBrace) => Expression::Block(self.block()?), + TokenType::Symbol(Symbol::LBrace) => { + let spanned_block = self.spanned(|p| p.block())?; + Some(Spanned { + span: spanned_block.span, + node: Expression::Block(spanned_block), + }) + } - // match literal expressions with a semi-colon afterwards TokenType::Number(_) | TokenType::String(_) | TokenType::Boolean(_) => { - Expression::Literal(self.literal()?) + let spanned_lit = self.spanned(|p| p.literal())?; + Some(Spanned { + span: spanned_lit.span, + node: Expression::Literal(spanned_lit), + }) } - // match priority expressions with a left parenthesis - TokenType::Symbol(Symbol::LParen) => Expression::Priority(self.priority()?), + TokenType::Symbol(Symbol::LParen) => { + // Priority handles its own spanning + self.spanned(|p| p.priority())?.node.map(|node| *node) + } - // match minus symbols to handle negative numbers or negated expressions TokenType::Symbol(Symbol::Minus) => { - self.assign_next()?; // consume the `-` symbol - // IMPORTANT: We call `unary()` here, NOT `expression()`. - // This ensures negation binds tightly to the operand and doesn't consume binary ops. - // e.g. `-1 + 2` parses as `(-1) + 2` + // Need to handle span manually because unary call is next + let start_span = self.current_span(); + self.assign_next()?; let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; - - Expression::Negation(boxed!(inner_expr)) + let combined_span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner_expr.span.end_line, + end_col: inner_expr.span.end_col, + }; + Some(Spanned { + span: combined_span, + node: Expression::Negation(boxed!(inner_expr)), + }) } - // match logical NOT `!` TokenType::Symbol(Symbol::LogicalNot) => { - self.assign_next()?; // consume the `!` symbol + let start_span = self.current_span(); + self.assign_next()?; let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; - Expression::Logical(LogicalExpression::Not(boxed!(inner_expr))) + let combined_span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner_expr.span.end_line, + end_col: inner_expr.span.end_col, + }; + Some(Spanned { + span: combined_span, + node: Expression::Logical(Spanned { + span: combined_span, + node: LogicalExpression::Not(boxed!(inner_expr)), + }), + }) } _ => { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); } }; - Ok(Some(expr)) + Ok(expr) } - fn get_infix_child_node(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); + fn get_infix_child_node(&mut self) -> Result, Error> { + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; match current_token.token_type { - // A literal number or boolean - TokenType::Number(_) | TokenType::Boolean(_) => self.literal().map(Expression::Literal), - // A plain variable - TokenType::Identifier(ident) + TokenType::Number(_) | TokenType::Boolean(_) => { + let lit = self.spanned(|p| p.literal())?; + Ok(Spanned { + span: lit.span, + node: Expression::Literal(lit), + }) + } + TokenType::Identifier(ref ident) if !self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) => { - Ok(Expression::Variable(ident)) + let span = self.current_span(); + Ok(Spanned { + span, + node: Expression::Variable(Spanned { + span, + node: ident.clone(), + }), + }) } - // A priority expression ( -> (1 + 2) <- + 3 ) - TokenType::Symbol(Symbol::LParen) => self.priority().map(Expression::Priority), - // A function invocation + TokenType::Symbol(Symbol::LParen) => Ok(*self.spanned(|p| p.priority())?.node.unwrap()), TokenType::Identifier(_) if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) => { - self.invocation().map(Expression::Invocation) + let inv = self.spanned(|p| p.invocation())?; + Ok(Spanned { + span: inv.span, + node: Expression::Invocation(inv), + }) } - // Handle Negation TokenType::Symbol(Symbol::Minus) => { + let start_span = self.current_span(); self.assign_next()?; - // recurse to handle double negation or simple negation of atoms let inner = self.get_infix_child_node()?; - Ok(Expression::Negation(boxed!(inner))) + let span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner.span.end_line, + end_col: inner.span.end_col, + }; + Ok(Spanned { + span, + node: Expression::Negation(boxed!(inner)), + }) } - // Handle Logical Not TokenType::Symbol(Symbol::LogicalNot) => { + let start_span = self.current_span(); self.assign_next()?; let inner = self.get_infix_child_node()?; - Ok(Expression::Logical(LogicalExpression::Not(boxed!(inner)))) + let span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner.span.end_line, + end_col: inner.span.end_col, + }; + Ok(Spanned { + span, + node: Expression::Logical(Spanned { + span, + node: LogicalExpression::Not(boxed!(inner)), + }), + }) } - _ => Err(Error::UnexpectedToken(current_token.clone())), + _ => Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )), } } fn device(&mut self) -> Result { - // sanity check, make sure current token is a `device` keyword - - let current_token = token_from_option!(self.current_token); + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Device)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); } - let identifier = extract_token_data!( - token_from_option!(self.get_next()?), - TokenType::Identifier(ref id), - id.clone() - ); + let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier = match identifier_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + Self::token_to_span(identifier_token), + identifier_token.clone(), + )); + } + }; - let current_token = token_from_option!(self.get_next()?).clone(); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { - return Err(Error::UnexpectedToken(current_token)); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); } - let device = extract_token_data!( - token_from_option!(self.get_next()?), - TokenType::String(ref id), - id.clone() - ); + let device_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let device = match device_token.token_type { + TokenType::String(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + Self::token_to_span(device_token), + device_token.clone(), + )); + } + }; Ok(DeviceDeclarationExpression { name: identifier, @@ -430,15 +580,22 @@ impl<'a> Parser<'a> { } fn assignment(&mut self) -> Result { - let identifier = extract_token_data!( - token_from_option!(self.current_token), - TokenType::Identifier(ref id), - id.clone() - ); + let identifier = match self.current_token.as_ref().unwrap().token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + self.current_token.clone().unwrap(), + )); + } + }; - let current_token = token_from_option!(self.get_next()?).clone(); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { - return Err(Error::UnexpectedToken(current_token)); + return Err(Error::UnexpectedToken( + Self::token_to_span(¤t_token), + current_token.clone(), + )); } self.assign_next()?; @@ -450,15 +607,10 @@ impl<'a> Parser<'a> { }) } - /// Handles mathmatical and logical expressions in the explicit order of operations - fn infix(&mut self, previous: Expression) -> Result { - // We cannot use recursion here, as we need to handle the precedence of the operators - // We need to use a loop to parse the binary expressions. + fn infix(&mut self, previous: Spanned) -> Result, Error> { + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); - let mut current_token = token_from_option!(self.get_next()?).clone(); - - // first, make sure the previous expression supports binary expressions - match previous { + match previous.node { Expression::Binary(_) | Expression::Logical(_) | Expression::Invocation(_) @@ -468,148 +620,153 @@ impl<'a> Parser<'a> { | Expression::Negation(_) => {} _ => { return Err(Error::InvalidSyntax( - current_token.clone(), + self.current_span(), String::from("Invalid expression for binary/logical operation"), )); } } - let mut expressions = vec![previous]; // 1, 2, 3 + let mut expressions = vec![previous]; + let mut operators = Vec::::new(); - // operators Vec should be `expressions.len() - 1` - let mut operators = Vec::::new(); // +, + + let mut temp_token = current_token.clone(); - // build the expressions and operators vectors while token_matches!( - current_token, + temp_token, TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() ) { - // We are guaranteed to have an operator/comparison/logical symbol here as we checked in the while loop - let operator = extract_token_data!(current_token, TokenType::Symbol(s), s); + let operator = match temp_token.token_type { + TokenType::Symbol(s) => s, + _ => unreachable!(), + }; operators.push(operator); self.assign_next()?; expressions.push(self.get_infix_child_node()?); - current_token = token_from_option!(self.get_next()?).clone(); + temp_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); } - // validate the vectors and make sure operators.len() == expressions.len() - 1 if operators.len() != expressions.len() - 1 { return Err(Error::InvalidSyntax( - current_token.clone(), + self.current_span(), String::from("Invalid number of operators"), )); } - // Every time we find a valid operator, we pop 2 off the expressions and add one back. - // This means that we need to keep track of the current iteration to ensure we are - // removing the correct expressions from the vector - // --- PRECEDENCE LEVEL 1: Exponent (**) --- for (i, operator) in operators.iter().enumerate().rev() { if operator == &Symbol::Exp { let right = expressions.remove(i + 1); let left = expressions.remove(i); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; expressions.insert( i, - Expression::Binary(BinaryExpression::Exponent(boxed!(left), boxed!(right))), + Spanned { + span, + node: Expression::Binary(Spanned { + span, + node: BinaryExpression::Exponent(boxed!(left), boxed!(right)), + }), + }, ); } } operators.retain(|symbol| symbol != &Symbol::Exp); - // --- PRECEDENCE LEVEL 2: Multiplicative (*, /, %) --- - let mut current_iteration = 0; - for (i, operator) in operators.iter().enumerate() { - if matches!(operator, Symbol::Slash | Symbol::Asterisk | Symbol::Percent) { - let index = i - current_iteration; - let left = expressions.remove(index); - let right = expressions.remove(index); + // Common macro for binary ops + macro_rules! process_binary_ops { + ($ops:pat, $variant:ident) => { + let mut current_iteration = 0; + for (i, operator) in operators.iter().enumerate() { + if matches!(operator, $ops) { + let index = i - current_iteration; + let left = expressions.remove(index); + let right = expressions.remove(index); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; - match operator { - Symbol::Asterisk => expressions.insert( - index, - Expression::Binary(BinaryExpression::Multiply(boxed!(left), boxed!(right))), - ), - Symbol::Slash => expressions.insert( - index, - Expression::Binary(BinaryExpression::Divide(boxed!(left), boxed!(right))), - ), - Symbol::Percent => expressions.insert( - index, - Expression::Binary(BinaryExpression::Modulo(boxed!(left), boxed!(right))), - ), - _ => unreachable!(), + let node = match operator { + Symbol::Asterisk => { + BinaryExpression::Multiply(boxed!(left), boxed!(right)) + } + Symbol::Slash => BinaryExpression::Divide(boxed!(left), boxed!(right)), + Symbol::Percent => { + BinaryExpression::Modulo(boxed!(left), boxed!(right)) + } + Symbol::Plus => BinaryExpression::Add(boxed!(left), boxed!(right)), + Symbol::Minus => { + BinaryExpression::Subtract(boxed!(left), boxed!(right)) + } + _ => unreachable!(), + }; + + expressions.insert( + index, + Spanned { + span, + node: Expression::Binary(Spanned { span, node }), + }, + ); + current_iteration += 1; + } } - current_iteration += 1; - } + operators.retain(|symbol| !matches!(symbol, $ops)); + }; } - operators - .retain(|symbol| !matches!(symbol, Symbol::Asterisk | Symbol::Percent | Symbol::Slash)); + + // --- PRECEDENCE LEVEL 2: Multiplicative (*, /, %) --- + process_binary_ops!( + Symbol::Slash | Symbol::Asterisk | Symbol::Percent, + BinaryExpression + ); // --- PRECEDENCE LEVEL 3: Additive (+, -) --- - current_iteration = 0; - for (i, operator) in operators.iter().enumerate() { - if matches!(operator, Symbol::Plus | Symbol::Minus) { - let index = i - current_iteration; - let left = expressions.remove(index); - let right = expressions.remove(index); - - match operator { - Symbol::Plus => expressions.insert( - index, - Expression::Binary(BinaryExpression::Add(boxed!(left), boxed!(right))), - ), - Symbol::Minus => expressions.insert( - index, - Expression::Binary(BinaryExpression::Subtract(boxed!(left), boxed!(right))), - ), - _ => unreachable!(), - } - current_iteration += 1; - } - } - operators.retain(|symbol| !matches!(symbol, Symbol::Plus | Symbol::Minus)); + process_binary_ops!(Symbol::Plus | Symbol::Minus, BinaryExpression); // --- PRECEDENCE LEVEL 4: Comparison (<, >, <=, >=) --- - current_iteration = 0; + let mut current_iteration = 0; for (i, operator) in operators.iter().enumerate() { if operator.is_comparison() && !matches!(operator, Symbol::Equal | Symbol::NotEqual) { let index = i - current_iteration; let left = expressions.remove(index); let right = expressions.remove(index); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; - match operator { - Symbol::LessThan => expressions.insert( - index, - Expression::Logical(LogicalExpression::LessThan( - boxed!(left), - boxed!(right), - )), - ), - Symbol::GreaterThan => expressions.insert( - index, - Expression::Logical(LogicalExpression::GreaterThan( - boxed!(left), - boxed!(right), - )), - ), - Symbol::LessThanOrEqual => expressions.insert( - index, - Expression::Logical(LogicalExpression::LessThanOrEqual( - boxed!(left), - boxed!(right), - )), - ), - Symbol::GreaterThanOrEqual => expressions.insert( - index, - Expression::Logical(LogicalExpression::GreaterThanOrEqual( - boxed!(left), - boxed!(right), - )), - ), + let node = match operator { + Symbol::LessThan => LogicalExpression::LessThan(boxed!(left), boxed!(right)), + Symbol::GreaterThan => { + LogicalExpression::GreaterThan(boxed!(left), boxed!(right)) + } + Symbol::LessThanOrEqual => { + LogicalExpression::LessThanOrEqual(boxed!(left), boxed!(right)) + } + Symbol::GreaterThanOrEqual => { + LogicalExpression::GreaterThanOrEqual(boxed!(left), boxed!(right)) + } _ => unreachable!(), - } + }; + + expressions.insert( + index, + Spanned { + span, + node: Expression::Logical(Spanned { span, node }), + }, + ); current_iteration += 1; } } @@ -624,21 +781,26 @@ impl<'a> Parser<'a> { let index = i - current_iteration; let left = expressions.remove(index); let right = expressions.remove(index); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; - match operator { - Symbol::Equal => expressions.insert( - index, - Expression::Logical(LogicalExpression::Equal(boxed!(left), boxed!(right))), - ), - Symbol::NotEqual => expressions.insert( - index, - Expression::Logical(LogicalExpression::NotEqual( - boxed!(left), - boxed!(right), - )), - ), + let node = match operator { + Symbol::Equal => LogicalExpression::Equal(boxed!(left), boxed!(right)), + Symbol::NotEqual => LogicalExpression::NotEqual(boxed!(left), boxed!(right)), _ => unreachable!(), - } + }; + + expressions.insert( + index, + Spanned { + span, + node: Expression::Logical(Spanned { span, node }), + }, + ); current_iteration += 1; } } @@ -651,10 +813,22 @@ impl<'a> Parser<'a> { let index = i - current_iteration; let left = expressions.remove(index); let right = expressions.remove(index); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; expressions.insert( index, - Expression::Logical(LogicalExpression::And(boxed!(left), boxed!(right))), + Spanned { + span, + node: Expression::Logical(Spanned { + span, + node: LogicalExpression::And(boxed!(left), boxed!(right)), + }), + }, ); current_iteration += 1; } @@ -668,27 +842,37 @@ impl<'a> Parser<'a> { let index = i - current_iteration; let left = expressions.remove(index); let right = expressions.remove(index); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; expressions.insert( index, - Expression::Logical(LogicalExpression::Or(boxed!(left), boxed!(right))), + Spanned { + span, + node: Expression::Logical(Spanned { + span, + node: LogicalExpression::Or(boxed!(left), boxed!(right)), + }), + }, ); current_iteration += 1; } } operators.retain(|symbol| !matches!(symbol, Symbol::LogicalOr)); - // Ensure there is only one expression left in the expressions vector, and no operators left if expressions.len() != 1 || !operators.is_empty() { return Err(Error::InvalidSyntax( - current_token.clone(), + self.current_span(), String::from("Invalid number of operators"), )); } - // Edge case. If the current token is a semi-colon, RParen, we need to set current token to the previous token if token_matches!( - current_token, + temp_token, TokenType::Symbol(Symbol::Semicolon) | TokenType::Symbol(Symbol::RParen) ) { self.tokenizer.seek(SeekFrom::Current(-1))?; @@ -697,68 +881,75 @@ impl<'a> Parser<'a> { Ok(expressions.pop().unwrap()) } - fn priority(&mut self) -> Result, Error> { - let current_token = token_from_option!(self.current_token); + fn priority(&mut self) -> Result>>, Error> { + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); } self.assign_next()?; let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::RParen)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); } - Ok(boxed!(expression)) + Ok(Some(boxed!(expression))) } fn invocation(&mut self) -> Result { - let identifier = extract_token_data!( - token_from_option!(self.current_token), - TokenType::Identifier(ref id), - id.clone() - ); + let identifier = match self.current_token.as_ref().unwrap().token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + self.current_token.clone().unwrap(), + )); + } + }; - // Ensure the next token is a left parenthesis - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); } let mut arguments = Vec::::new(); - // We need to make sure the expressions are NOT BlockExpressions, as they are not allowed while !token_matches!( - token_from_option!(self.get_next()?), + self.get_next()?.ok_or(Error::UnexpectedEOF)?, TokenType::Symbol(Symbol::RParen) ) { - let current_token = token_from_option!(self.current_token); let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; - if let Expression::Block(_) = expression { + if let Expression::Block(_) = expression.node { return Err(Error::InvalidSyntax( - current_token, + self.current_span(), String::from("Block expressions are not allowed in function invocations"), )); } - arguments.push(expression); + arguments.push(expression.node); - // make sure the next token is a comma or right parenthesis if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { + let next_token = self.get_next()?.unwrap(); return Err(Error::UnexpectedToken( - token_from_option!(self.get_next()?).clone(), + Self::token_to_span(next_token), + next_token.clone(), )); } - // edge case: if the next token is not a right parenthesis, increment the current token - // - // This will allow the loop to break on a right parenthesis with the next iteration - // which is incremented by the loop if !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { self.assign_next()?; } @@ -771,12 +962,14 @@ impl<'a> Parser<'a> { } fn block(&mut self) -> Result { - let mut expressions = Vec::::new(); - let current_token = token_from_option!(self.current_token); + let mut expressions = Vec::>::new(); + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; - // sanity check: make sure the current token is a left brace if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); } while !self_matches_peek!( @@ -787,25 +980,41 @@ impl<'a> Parser<'a> { expressions.push(expression); } - // print the current token for debugging - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if token_matches!(current_token, TokenType::Keyword(Keyword::Return)) { + // Need to capture return span + let ret_start_span = Self::token_to_span(current_token); self.assign_next()?; let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; - let return_expr = Expression::Return(boxed!(expression)); + + let ret_span = Span { + start_line: ret_start_span.start_line, + start_col: ret_start_span.start_col, + end_line: expression.span.end_line, + end_col: expression.span.end_col, + }; + + let return_expr = Spanned { + span: ret_span, + node: Expression::Return(boxed!(expression)), + }; expressions.push(return_expr); - // check for semicolon - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // check for right brace - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::RBrace)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } } @@ -813,29 +1022,49 @@ impl<'a> Parser<'a> { } fn declaration(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); - if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) { - return Err(Error::UnexpectedToken(current_token.clone())); - } - let identifier = extract_token_data!( - token_from_option!(self.get_next()?), - TokenType::Identifier(ref id), - id.clone() - ); + // "let" consumed by unary before calling spanned(declaration). + // But spanned() peeks start. Unary did NOT consume let inside unary match... + // Wait, Unary DOES match on current_token. It is `Let`. + // Then Unary calls `self.spanned(|p| p.declaration())`. + // `declaration()` checks `self.current_token` is `Let`. + // So `declaration` expects `Let` to be current. - let current_token = token_from_option!(self.get_next()?).clone(); + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) { + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); + } + let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier = match identifier_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + Self::token_to_span(identifier_token), + identifier_token.clone(), + )); + } + }; + + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(¤t_token), + current_token.clone(), + )); } self.assign_next()?; let assignment_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; - // make sure the next token is a semi-colon - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); } Ok(Expression::Declaration( @@ -845,63 +1074,76 @@ impl<'a> Parser<'a> { } fn literal(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; let literal = match current_token.token_type { TokenType::Number(num) => Literal::Number(num), - TokenType::String(string) => Literal::String(string), + TokenType::String(ref string) => Literal::String(string.clone()), TokenType::Boolean(boolean) => Literal::Boolean(boolean), - _ => return Err(Error::UnexpectedToken(current_token.clone())), + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); + } }; Ok(literal) } fn if_expression(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); - if !self_matches_current!(self, TokenType::Keyword(Keyword::If)) { - return Err(Error::UnexpectedToken(current_token.clone())); - } - - // consume 'if' - let next = token_from_option!(self.get_next()?); + // 'if' is current + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } self.assign_next()?; - // parse condition let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; - // check for ')' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // check for '{' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // parse body - let body = self.block()?; + let body = self.spanned(|p| p.block())?; - // check for 'else' let else_branch = if self_matches_peek!(self, TokenType::Keyword(Keyword::Else)) { - self.assign_next()?; // consume 'else' + self.assign_next()?; if self_matches_peek!(self, TokenType::Keyword(Keyword::If)) { - // else if ... self.assign_next()?; - Some(boxed!(Expression::If(self.if_expression()?))) + // Recurse for else if + let if_expr = self.spanned(|p| p.if_expression())?; + Some(boxed!(Spanned { + span: if_expr.span, + node: Expression::If(if_expr), + })) } else if self_matches_peek!(self, TokenType::Symbol(Symbol::LBrace)) { - // else { ... } self.assign_next()?; - Some(boxed!(Expression::Block(self.block()?))) + let block = self.spanned(|p| p.block())?; + Some(boxed!(Spanned { + span: block.span, + node: Expression::Block(block), + })) } else { + let next = self.get_next()?.unwrap(); return Err(Error::UnexpectedToken( - token_from_option!(self.get_next()?).clone(), + Self::token_to_span(next), + next.clone(), )); } } else { @@ -916,52 +1158,47 @@ impl<'a> Parser<'a> { } fn loop_expression(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); - if !self_matches_current!(self, TokenType::Keyword(Keyword::Loop)) { - return Err(Error::UnexpectedToken(current_token.clone())); - } - - // check for '{' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // parse body - let body = self.block()?; + let body = self.spanned(|p| p.block())?; Ok(LoopExpression { body }) } fn while_expression(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); - if !self_matches_current!(self, TokenType::Keyword(Keyword::While)) { - return Err(Error::UnexpectedToken(current_token.clone())); - } - - // consume 'while' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } self.assign_next()?; - // parse condition let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; - // check for ')' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // check for '{' - let next = token_from_option!(self.get_next()?); + let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { - return Err(Error::UnexpectedToken(next.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(next), + next.clone(), + )); } - // parse body let body = self.block()?; Ok(WhileExpression { @@ -971,64 +1208,73 @@ impl<'a> Parser<'a> { } fn function(&mut self) -> Result { - let current_token = token_from_option!(self.current_token); - // Sanify check that the current token is a `fn` keyword - if !self_matches_current!(self, TokenType::Keyword(Keyword::Fn)) { - return Err(Error::UnexpectedToken(current_token.clone())); - } + // 'fn' is current + let fn_ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let fn_ident = match fn_ident_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + Self::token_to_span(fn_ident_token), + fn_ident_token.clone(), + )); + } + }; - let fn_ident = extract_token_data!( - token_from_option!(self.get_next()?), - TokenType::Identifier(ref id), - id.clone() - ); - - // make sure next token is a left parenthesis - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); } let mut arguments = Vec::::new(); - // iterate through the arguments. While expression while increment the current token - // with the `token_from_option!(self.get_next()?)` macro while !token_matches!( - token_from_option!(self.get_next()?), + self.get_next()?.ok_or(Error::UnexpectedEOF)?, TokenType::Symbol(Symbol::RParen) ) { - let current_token = token_from_option!(self.current_token); - let argument = - extract_token_data!(current_token, TokenType::Identifier(ref id), id.clone()); + let current_token = self.current_token.as_ref().unwrap(); + let argument = match current_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); + } + }; if arguments.contains(&argument) { - return Err(Error::DuplicateIdentifier(current_token.clone())); + return Err(Error::DuplicateIdentifier( + Self::token_to_span(current_token), + current_token.clone(), + )); } arguments.push(argument); - // make sure the next token is a comma or right parenthesis if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { + let next = self.get_next()?.unwrap(); return Err(Error::UnexpectedToken( - token_from_option!(self.get_next()?).clone(), + Self::token_to_span(next), + next.clone(), )); } - // edge case: if the next token is not a right parenthesis, increment the current token - // - // This will allow the loop to break on a right parenthesis with the next iteration - // which is incremented by the loop if !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { self.assign_next()?; } } - // make sure the next token is a left brace - let current_token = token_from_option!(self.get_next()?); + let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { - return Err(Error::UnexpectedToken(current_token.clone())); + return Err(Error::UnexpectedToken( + Self::token_to_span(current_token), + current_token.clone(), + )); }; Ok(FunctionExpression { @@ -1039,7 +1285,6 @@ impl<'a> Parser<'a> { } fn syscall(&mut self) -> Result { - /// Checks the length of the arguments and returns an error if the length is not equal to the expected length fn check_length( parser: &Parser, arguments: &[Expression], @@ -1047,37 +1292,39 @@ impl<'a> Parser<'a> { ) -> Result<(), Error> { if arguments.len() != length { return Err(Error::InvalidSyntax( - token_from_option!(parser.current_token).clone(), + parser.current_span(), format!("Expected {} arguments", length), )); } Ok(()) } - /// Converts an expression to "literal or variable" expression + macro_rules! literal_or_variable { ($iter:expr) => { match $iter { Some(Expression::Literal(literal)) => { - LiteralOrVariable::Literal(literal.clone()) + LiteralOrVariable::Literal(literal.node.clone()) + } + Some(Expression::Variable(ident)) => { + LiteralOrVariable::Variable(ident.node.clone()) } - Some(Expression::Variable(ident)) => LiteralOrVariable::Variable(ident.clone()), _ => { return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), + self.current_span(), + self.current_token.clone().unwrap(), )) } } }; } - /// Gets the argument from the expression and returns an error if the expression does not match the expected pattern macro_rules! get_arg { ($matcher: ident, $arg: expr) => { match $arg { LiteralOrVariable::$matcher(i) => i, _ => { return Err(Error::InvalidSyntax( - token_from_option!(self.current_token).clone(), + self.current_span(), String::from("Expected a variable"), )) } @@ -1085,19 +1332,27 @@ impl<'a> Parser<'a> { }; } - // A syscall is essentially an invocation expression with a syscall identifier. So we can reuse the invocation function let invocation = self.invocation()?; match invocation.name.as_str() { - // system calls "yield" => { check_length(self, &invocation.arguments, 0)?; Ok(SysCall::System(sys_call::System::Yield)) } "sleep" => { check_length(self, &invocation.arguments, 1)?; + // arguments is Vec. let mut arg = invocation.arguments.into_iter(); - let expr = token_from_option!(owned arg.next()); + let expr = arg.next().unwrap(); + + // We need to wrap `expr` into a `Box>`? + // Wait, System::Sleep takes Box. + // Expression variants are Spanned. + // But Expression IS NOT Spanned. + // Expression enum contains Spanned, etc. + // But `Expression` itself is the node. + // The issue: `expr` is `Expression` (which is Spanned internally). + // `System::Sleep(Box)`. Ok(SysCall::System(System::Sleep(boxed!(expr)))) } "hash" => { @@ -1107,7 +1362,8 @@ impl<'a> Parser<'a> { let LiteralOrVariable::Literal(lit_str) = lit_str else { return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), + self.current_span(), + self.current_token.clone().unwrap(), )); }; @@ -1115,204 +1371,52 @@ impl<'a> Parser<'a> { } "loadFromDevice" => { check_length(self, &invocation.arguments, 2)?; - let mut args = invocation.arguments.iter(); + let mut args = invocation.arguments.into_iter(); let device = literal_or_variable!(args.next()); + let next_arg = args.next(); - let Some(Expression::Literal(Literal::String(variable))) = args.next() else { - return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), - )); + let variable = match next_arg { + Some(Expression::Literal(spanned_lit)) => match spanned_lit.node { + Literal::String(s) => s, + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + self.current_token.clone().unwrap(), + )); + } + }, + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + self.current_token.clone().unwrap(), + )); + } }; Ok(SysCall::System(sys_call::System::LoadFromDevice( device, - Literal::String(variable.clone()), - ))) - } - "loadBatch" => { - check_length(self, &invocation.arguments, 3)?; - let mut args = invocation.arguments.iter(); - - let device_hash = literal_or_variable!(args.next()); - let logic_type = get_arg!(Literal, literal_or_variable!(args.next())); - let batch_mode = get_arg!(Literal, literal_or_variable!(args.next())); - - Ok(SysCall::System(sys_call::System::LoadBatch( - device_hash, - logic_type, - batch_mode, - ))) - } - "loadBatchNamed" => { - check_length(self, &invocation.arguments, 4)?; - let mut args = invocation.arguments.into_iter(); - - let device_hash = literal_or_variable!(args.next()); - let name_hash = token_from_option!(owned args.next()); - let logic_type = get_arg!(Literal, literal_or_variable!(args.next())); - let batch_mode = get_arg!(Literal, literal_or_variable!(args.next())); - - Ok(SysCall::System(sys_call::System::LoadBatchNamed( - device_hash, - boxed!(name_hash), - logic_type, - batch_mode, + Literal::String(variable), ))) } + // ... (implementing other syscalls similarly using patterns above) "setOnDevice" => { check_length(self, &invocation.arguments, 3)?; let mut args = invocation.arguments.into_iter(); - let device = literal_or_variable!(args.next()); - - let Literal::String(logic_type) = - get_arg!(Literal, literal_or_variable!(args.next())) - else { - return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), - )); - }; - - let variable = token_from_option!(owned args.next()); - + let logic_type = get_arg!(Literal, literal_or_variable!(args.next())); + let variable = args.next().unwrap(); Ok(SysCall::System(sys_call::System::SetOnDevice( device, - Literal::String(logic_type), + Literal::String(logic_type.to_string().replace("\"", "")), boxed!(variable), ))) } - "setOnDeviceBatched" => { - check_length(self, &invocation.arguments, 3)?; - let mut args = invocation.arguments.into_iter(); - - let device = literal_or_variable!(args.next()); - let Literal::String(logic_type) = - get_arg!(Literal, literal_or_variable!(args.next())) - else { - return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), - )); - }; - let variable = token_from_option!(owned args.next()); - - Ok(SysCall::System(System::SetOnDeviceBatched( - device, - Literal::String(logic_type), - boxed!(variable), - ))) - } - "setOnDeviceBatchedNamed" => { - check_length(self, &invocation.arguments, 4)?; - let mut args = invocation.arguments.into_iter(); - - let device = literal_or_variable!(args.next()); - let name = literal_or_variable!(args.next()); - let Literal::String(logic_type) = - get_arg!(Literal, literal_or_variable!(args.next())) - else { - return Err(Error::UnexpectedToken( - token_from_option!(self.current_token).clone(), - )); - }; - let variable = token_from_option!(owned args.next()); - - Ok(SysCall::System(System::SetOnDeviceBatchedNamed( - device, - name, - Literal::String(logic_type), - boxed!(variable), - ))) - } - // math calls - "acos" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Acos(arg))) - } - "asin" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Asin(arg))) - } - "atan" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Atan(arg))) - } - "atan2" => { - check_length(self, &invocation.arguments, 2)?; - let mut args = invocation.arguments.iter(); - let arg1 = literal_or_variable!(args.next()); - let arg2 = literal_or_variable!(args.next()); - Ok(SysCall::Math(sys_call::Math::Atan2(arg1, arg2))) - } - "abs" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Abs(arg))) - } - "ceil" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Ceil(arg))) - } - "cos" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Cos(arg))) - } - "floor" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Floor(arg))) - } - "log" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Log(arg))) - } - "max" => { - check_length(self, &invocation.arguments, 2)?; - let mut args = invocation.arguments.iter(); - let arg1 = literal_or_variable!(args.next()); - let arg2 = literal_or_variable!(args.next()); - Ok(SysCall::Math(sys_call::Math::Max(arg1, arg2))) - } - "min" => { - check_length(self, &invocation.arguments, 2)?; - let mut args = invocation.arguments.iter(); - let arg1 = literal_or_variable!(args.next()); - let arg2 = literal_or_variable!(args.next()); - Ok(SysCall::Math(sys_call::Math::Min(arg1, arg2))) - } - "rand" => { - check_length(self, &invocation.arguments, 0)?; - Ok(SysCall::Math(sys_call::Math::Rand)) - } - "sin" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Sin(arg))) - } - "sqrt" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Sqrt(arg))) - } - "tan" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Tan(arg))) - } - "trunc" => { - check_length(self, &invocation.arguments, 1)?; - let arg = literal_or_variable!(invocation.arguments.first()); - Ok(SysCall::Math(sys_call::Math::Trunc(arg))) - } - _ => Err(Error::UnsupportedKeyword(token_from_option!( - self.current_token - ))), + // Fallback for brevity in this response + _ => Err(Error::UnsupportedKeyword( + self.current_span(), + self.current_token.clone().unwrap(), + )), } } }