diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index 12ed6fd..435319f 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -43,8 +43,11 @@ pub enum Error<'a> { #[error("Unsupported Keyword: {1}")] UnsupportedKeyword(Span, Token<'a>), + #[error("Expected semicolon")] + MissingSemicolon(Span), + #[error("Unexpected End of File")] - UnexpectedEOF, + UnexpectedEOF(Option), } impl<'a> From> for lsp_types::Diagnostic { @@ -56,17 +59,22 @@ impl<'a> From> for lsp_types::Diagnostic { UnexpectedToken(span, _) | DuplicateIdentifier(span, _) | InvalidSyntax(span, _) - | UnsupportedKeyword(span, _) => Diagnostic { + | UnsupportedKeyword(span, _) + | MissingSemicolon(span) => Diagnostic { message: value.to_string(), severity: Some(DiagnosticSeverity::ERROR), range: span.into(), ..Default::default() }, - UnexpectedEOF => Diagnostic { - message: value.to_string(), - severity: Some(DiagnosticSeverity::ERROR), - ..Default::default() - }, + UnexpectedEOF(span) => { + let range = span.map(|s| s.into()).unwrap_or_default(); + Diagnostic { + message: value.to_string(), + severity: Some(DiagnosticSeverity::ERROR), + range, + ..Default::default() + } + } } } } @@ -107,6 +115,7 @@ macro_rules! self_matches_current { pub struct Parser<'a> { tokenizer: TokenizerBuffer<'a>, current_token: Option>, + last_token_span: Option, pub errors: Vec>, } @@ -115,6 +124,7 @@ impl<'a> Parser<'a> { Parser { tokenizer: TokenizerBuffer::new(tokenizer), current_token: None, + last_token_span: None, errors: Vec::new(), } } @@ -141,6 +151,10 @@ impl<'a> Parser<'a> { }) } + fn unexpected_eof(&self) -> Error<'a> { + Error::UnexpectedEOF(self.last_token_span) + } + /// Helper to run a parsing closure and wrap the result in a Spanned struct fn spanned(&mut self, parser: F) -> Result, Error<'a>> where @@ -271,6 +285,9 @@ impl<'a> Parser<'a> { } fn assign_next(&mut self) -> Result<(), Error<'a>> { + if let Some(token) = &self.current_token { + self.last_token_span = Some(Self::token_to_span(token)); + } self.current_token = self.tokenizer.next_token()?; Ok(()) } @@ -317,7 +334,7 @@ impl<'a> Parser<'a> { if self_matches_peek!(self, TokenType::Symbol(Symbol::Dot)) { self.assign_next()?; // consume Dot - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -336,10 +353,10 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; // Block expressions not allowed in args if let Expression::Block(_) = expression.node { @@ -353,7 +370,8 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next_token = + self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken( Self::token_to_span(&next_token), next_token, @@ -415,11 +433,11 @@ impl<'a> Parser<'a> { // Index Access self.assign_next()?; // consume '[', now current is '[' self.assign_next()?; // advance to the expression, now current is first token of index expr - let index = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let index = self.expression()?.ok_or_else(|| self.unexpected_eof())?; // After expression(), current is still on the last token of the index expression // We need to get the next token and verify it's ']' - let rbracket_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let rbracket_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(rbracket_token, TokenType::Symbol(Symbol::RBracket)) { return Err(Error::UnexpectedToken( Self::token_to_span(&rbracket_token), @@ -534,9 +552,9 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Break) => { let span = self.current_span(); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(span)); } Some(Spanned { span, @@ -546,9 +564,9 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Continue) => { let span = self.current_span(); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(span)); } Some(Spanned { span, @@ -606,7 +624,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::Minus) => { let start_span = self.current_span(); self.assign_next()?; - let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; let inner_with_postfix = self.parse_postfix(inner_expr)?; @@ -625,7 +643,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::LogicalNot) => { let start_span = self.current_span(); self.assign_next()?; - let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; let inner_with_postfix = self.parse_postfix(inner_expr)?; let combined_span = Span { start_line: start_span.start_line, @@ -645,7 +663,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::BitwiseNot) => { let start_span = self.current_span(); self.assign_next()?; - let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; let inner_with_postfix = self.parse_postfix(inner_expr)?; let combined_span = Span { start_line: start_span.start_line, @@ -671,7 +689,10 @@ impl<'a> Parser<'a> { } fn get_infix_child_node(&mut self) -> Result>, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let start_span = self.current_span(); @@ -699,7 +720,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::LParen) => *self .parenthesized_or_tuple()? .map(Box::new) - .ok_or(Error::UnexpectedEOF)?, + .ok_or_else(|| self.unexpected_eof())?, TokenType::Identifier(ref id) if SysCall::is_syscall(id) => { let spanned_call = self.spanned(|p| p.syscall())?; @@ -778,7 +799,10 @@ impl<'a> Parser<'a> { } fn device(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Device)) { return Err(Error::UnexpectedToken( self.current_span(), @@ -786,7 +810,7 @@ impl<'a> Parser<'a> { )); } - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -798,7 +822,7 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -806,7 +830,7 @@ impl<'a> Parser<'a> { )); } - let device_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let device_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let device = match device_token.token_type { TokenType::String(ref id) => id.clone(), _ => { @@ -830,7 +854,10 @@ impl<'a> Parser<'a> { &mut self, previous: Spanned>, ) -> Result>, Error<'a>> { - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let current_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); match previous.node { Expression::Binary(_) @@ -873,7 +900,10 @@ impl<'a> Parser<'a> { self.assign_next()?; expressions.push(self.get_infix_child_node()?); - temp_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + temp_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); } if operators.len() != expressions.len() - 1 { @@ -1239,14 +1269,17 @@ impl<'a> Parser<'a> { self.tokenizer.seek(SeekFrom::Current(-1))?; } - expressions.pop().ok_or(Error::UnexpectedEOF) + expressions.pop().ok_or_else(|| self.unexpected_eof()) } fn parenthesized_or_tuple( &mut self, ) -> Result>>, Error<'a>> { let start_span = self.current_span(); - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( @@ -1273,7 +1306,7 @@ impl<'a> Parser<'a> { })); } - let first_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let first_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; if self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) { // It is a tuple @@ -1282,10 +1315,10 @@ impl<'a> Parser<'a> { // Next toekn is a comma, we need to consume it and advance 1 more time. self.assign_next()?; self.assign_next()?; - items.push(self.expression()?.ok_or(Error::UnexpectedEOF)?); + items.push(self.expression()?.ok_or_else(|| self.unexpected_eof())?); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1304,7 +1337,7 @@ impl<'a> Parser<'a> { })) } else { // It is just priority - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1319,14 +1352,14 @@ impl<'a> Parser<'a> { fn tuple_declaration(&mut self) -> Result, Error<'a>> { // 'let' is consumed before this call // expect '(' - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } let mut names = Vec::new(); while !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let span = Self::token_to_span(&token); if let TokenType::Identifier(id) = token.token_type { names.push(Spanned { span, node: id }); @@ -1340,7 +1373,7 @@ impl<'a> Parser<'a> { } self.assign_next()?; // consume ')' - let assign = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let assign = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(assign, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken(Self::token_to_span(&assign), assign)); @@ -1348,11 +1381,12 @@ impl<'a> Parser<'a> { self.assign_next()?; // Consume the `=` - let value = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let value = self.expression()?.ok_or_else(|| self.unexpected_eof())?; + let value_span = value.span; - let semi = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let semi = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(semi, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&semi), semi)); + return Err(Error::MissingSemicolon(value_span)); } Ok(Expression::TupleDeclaration(Spanned { @@ -1365,19 +1399,24 @@ impl<'a> Parser<'a> { } fn invocation(&mut self) -> Result, Error<'a>> { - let identifier_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let identifier_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1388,10 +1427,10 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; if let Expression::Block(_) = expression.node { return Err(Error::InvalidSyntax( @@ -1405,7 +1444,7 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken( Self::token_to_span(&next_token), next_token, @@ -1428,7 +1467,10 @@ impl<'a> Parser<'a> { fn block(&mut self) -> Result, Error<'a>> { let mut expressions = Vec::>::new(); - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken( @@ -1441,11 +1483,11 @@ impl<'a> Parser<'a> { self, TokenType::Symbol(Symbol::RBrace) | TokenType::Keyword(Keyword::Return) ) { - let expression = self.parse()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.parse()?.ok_or_else(|| self.unexpected_eof())?; expressions.push(expression); } - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if token_matches!(current_token, TokenType::Keyword(Keyword::Return)) { // Need to capture return span @@ -1453,14 +1495,16 @@ impl<'a> Parser<'a> { self.assign_next()?; let expr = if token_matches!( - self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?, + self.current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::Semicolon) ) { // rewind 1 token so we can check for the semicolon at the bottom of this function. self.tokenizer.seek(SeekFrom::Current(-1))?; None } else { - Some(self.expression()?.ok_or(Error::UnexpectedEOF)?) + Some(self.expression()?.ok_or_else(|| self.unexpected_eof())?) }; let ret_span = Span { @@ -1482,12 +1526,12 @@ impl<'a> Parser<'a> { }; expressions.push(return_expr); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(ret_span)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1498,7 +1542,10 @@ impl<'a> Parser<'a> { fn const_declaration(&mut self) -> Result, Error<'a>> { // const - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Const)) { return Err(Error::UnexpectedToken( self.current_span(), @@ -1507,7 +1554,7 @@ impl<'a> Parser<'a> { } // variable_name - let ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let ident_span = Self::token_to_span(&ident_token); let ident = match ident_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1515,7 +1562,10 @@ impl<'a> Parser<'a> { }; // `=` - let assign_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let assign_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); if !token_matches!(assign_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( Self::token_to_span(&assign_token), @@ -1552,7 +1602,9 @@ impl<'a> Parser<'a> { ) { return Err(Error::UnexpectedToken( syscall.span, - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } @@ -1567,14 +1619,17 @@ impl<'a> Parser<'a> { } fn declaration(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) { return Err(Error::UnexpectedToken( self.current_span(), current_token.clone(), )); } - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1586,7 +1641,10 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let current_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( @@ -1596,14 +1654,12 @@ impl<'a> Parser<'a> { } self.assign_next()?; - let assignment_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let assignment_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; + let expr_span = assignment_expression.span; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken( - Self::token_to_span(¤t_token), - current_token, - )); + return Err(Error::MissingSemicolon(expr_span)); } Ok(Expression::Declaration( @@ -1616,7 +1672,10 @@ impl<'a> Parser<'a> { } fn literal(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.clone().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?; let literal = match current_token.token_type { TokenType::Number(num) => Literal::Number(num), TokenType::String(ref string) => Literal::String(string.clone()), @@ -1632,7 +1691,7 @@ impl<'a> Parser<'a> { wrong_token, )); } - None => return Err(Error::UnexpectedEOF), + None => return Err(self.unexpected_eof()), }, _ => { return Err(Error::UnexpectedToken( @@ -1647,20 +1706,20 @@ impl<'a> Parser<'a> { fn if_expression(&mut self) -> Result, Error<'a>> { // 'if' is current - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } self.assign_next()?; - let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?; - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1686,7 +1745,7 @@ impl<'a> Parser<'a> { node: Expression::Block(block), })) } else { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } } else { @@ -1701,7 +1760,7 @@ impl<'a> Parser<'a> { } fn loop_expression(&mut self) -> Result, Error<'a>> { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1712,20 +1771,20 @@ impl<'a> Parser<'a> { } fn while_expression(&mut self) -> Result, Error<'a>> { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } self.assign_next()?; - let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?; - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1740,7 +1799,7 @@ impl<'a> Parser<'a> { fn function(&mut self) -> Result, Error<'a>> { // 'fn' is current - let fn_ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let fn_ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let fn_ident_span = Self::token_to_span(&fn_ident_token); let fn_ident = match fn_ident_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1749,7 +1808,7 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1760,10 +1819,13 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let arg_span = Self::token_to_span(current_token); let argument = match current_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1789,7 +1851,7 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1798,7 +1860,7 @@ impl<'a> Parser<'a> { } } - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1861,7 +1923,9 @@ impl<'a> Parser<'a> { _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )) } } @@ -1893,7 +1957,7 @@ impl<'a> Parser<'a> { } "sleep" => { let mut args = args!(1); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::Sleep(boxed!(expr)))) } "hash" => { @@ -1947,7 +2011,9 @@ impl<'a> Parser<'a> { _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } }; @@ -2003,7 +2069,7 @@ impl<'a> Parser<'a> { let tmp = args.next(); let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); - let variable = args.next().ok_or(Error::UnexpectedEOF)?; + let variable = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(sys_call::System::SetOnDevice( device, Spanned { @@ -2022,7 +2088,7 @@ impl<'a> Parser<'a> { let tmp = args.next(); let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); - let variable = args.next().ok_or(Error::UnexpectedEOF)?; + let variable = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(sys_call::System::SetOnDeviceBatched( device_hash, @@ -2045,7 +2111,7 @@ impl<'a> Parser<'a> { let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); let tmp = args.next(); - let expr = Box::new(tmp.ok_or(Error::UnexpectedEOF)?); + let expr = Box::new(tmp.ok_or_else(|| self.unexpected_eof())?); Ok(SysCall::System(System::SetOnDeviceBatchedNamed( device_hash, @@ -2058,7 +2124,7 @@ impl<'a> Parser<'a> { let mut args = args!(3); let next = args.next(); let dev_name = literal_or_variable!(next); - let slot_index = args.next().ok_or(Error::UnexpectedEOF)?; + let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?; let next = args.next(); let slot_logic = get_arg!(Literal, literal_or_variable!(next)); @@ -2085,7 +2151,7 @@ impl<'a> Parser<'a> { let mut args = args!(4); let next = args.next(); let dev_name = literal_or_variable!(next); - let slot_index = args.next().ok_or(Error::UnexpectedEOF)?; + let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?; let next = args.next(); let slot_logic = get_arg!(Literal, literal_or_variable!(next)); @@ -2102,7 +2168,7 @@ impl<'a> Parser<'a> { )); } let next = args.next(); - let expr = next.ok_or(Error::UnexpectedEOF)?; + let expr = next.ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::SetSlot( dev_name, @@ -2117,7 +2183,7 @@ impl<'a> Parser<'a> { let device = literal_or_variable!(next); let next = args.next(); let reagent_mode = get_arg!(Literal, literal_or_variable!(next)); - let reagent_hash = args.next().ok_or(Error::UnexpectedEOF)?; + let reagent_hash = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::LoadReagent( device, @@ -2130,80 +2196,80 @@ impl<'a> Parser<'a> { "acos" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let tmp = args.next().ok_or(Error::UnexpectedEOF)?; + let tmp = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Acos(boxed!(tmp)))) } "asin" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let tmp = args.next().ok_or(Error::UnexpectedEOF)?; + let tmp = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Asin(boxed!(tmp)))) } "atan" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Atan(boxed!(expr)))) } "atan2" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Atan2(boxed!(arg1), boxed!(arg2)))) } "abs" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Abs(boxed!(expr)))) } "ceil" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Ceil(boxed!(arg)))) } "cos" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Cos(boxed!(arg)))) } "floor" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Floor(boxed!(arg)))) } "log" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Log(boxed!(arg)))) } "max" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Max(boxed!(arg1), boxed!(arg2)))) } "min" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Min(boxed!(arg1), boxed!(arg2)))) } @@ -2214,34 +2280,36 @@ impl<'a> Parser<'a> { "sin" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Sin(boxed!(arg)))) } "sqrt" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Sqrt(boxed!(arg)))) } "tan" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Tan(boxed!(arg)))) } "trunc" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Trunc(boxed!(arg)))) } _ => Err(Error::UnsupportedKeyword( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )), } } diff --git a/rust_compiler/libs/parser/src/test/mod.rs b/rust_compiler/libs/parser/src/test/mod.rs index a08f6ce..c889d7f 100644 --- a/rust_compiler/libs/parser/src/test/mod.rs +++ b/rust_compiler/libs/parser/src/test/mod.rs @@ -287,3 +287,36 @@ fn test_tuple_declaration_all_complex_expressions() -> Result<()> { Ok(()) } +#[test] +fn test_eof_error_has_span() -> Result<()> { + // Test that UnexpectedEOF errors capture the span of the last token + let mut parser = parser!("let x = 5"); + let result = parser.parse(); + + // Should have an error + assert!(result.is_err()); + + let err = result.unwrap_err(); + + // Check that it's an UnexpectedEOF error + match err { + super::Error::UnexpectedEOF(Some(span)) => { + // Verify the span points to somewhere in the code (not zero defaults) + assert!( + span.start_line > 0 || span.start_col > 0 || span.end_line > 0 || span.end_col > 0, + "Span should not be all zeros: {:?}", + span + ); + } + super::Error::UnexpectedEOF(None) => { + eprintln!("ERROR: UnexpectedEOF captured None span instead of previous token span"); + eprintln!("This means unexpected_eof() is being called when current_token is None"); + panic!("UnexpectedEOF should have captured the previous token's span"); + } + other => { + panic!("Expected UnexpectedEOF error, got: {:?}", other); + } + } + + Ok(()) +}