dbidwell/stationeers_lang
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects 1 Releases 10 Wiki Activity

bitwise #15

Open
dbidwell wants to merge 17 commits from bitwise into master
pull from: bitwise
merge into: dbidwell:master
Conversation 0 Commits 17 Files Changed 64 +215 -114
2 changed files with 215 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3c7300d2e1 - Show all commits

296
rust_compiler/libs/parser/src/lib.rs
View File

@@ -43,8 +43,11 @@ pub enum Error<'a> {
#[error("Unsupported Keyword: {1}")]
UnsupportedKeyword(Span, Token<'a>),
#[error("Expected semicolon")]
MissingSemicolon(Span),
#[error("Unexpected End of File")]
UnexpectedEOF,
UnexpectedEOF(Option<Span>),
}
impl<'a> From<Error<'a>> for lsp_types::Diagnostic {
@@ -56,17 +59,22 @@ impl<'a> From<Error<'a>> for lsp_types::Diagnostic {
UnexpectedToken(span, _)
| DuplicateIdentifier(span, _)
| InvalidSyntax(span, _)
| UnsupportedKeyword(span, _) => Diagnostic {
| UnsupportedKeyword(span, _)
| MissingSemicolon(span) => Diagnostic {
message: value.to_string(),
severity: Some(DiagnosticSeverity::ERROR),
range: span.into(),
..Default::default()
},
UnexpectedEOF => Diagnostic {
message: value.to_string(),
severity: Some(DiagnosticSeverity::ERROR),
..Default::default()
},
UnexpectedEOF(span) => {
let range = span.map(|s| s.into()).unwrap_or_default();
Diagnostic {
message: value.to_string(),
severity: Some(DiagnosticSeverity::ERROR),
range,
..Default::default()
}
}
}
}
}
@@ -107,6 +115,7 @@ macro_rules! self_matches_current {
pub struct Parser<'a> {
tokenizer: TokenizerBuffer<'a>,
current_token: Option<Token<'a>>,
last_token_span: Option<Span>,
pub errors: Vec<Error<'a>>,
}
@@ -115,6 +124,7 @@ impl<'a> Parser<'a> {
Parser {
tokenizer: TokenizerBuffer::new(tokenizer),
current_token: None,
last_token_span: None,
errors: Vec::new(),
}
}
@@ -141,6 +151,10 @@ impl<'a> Parser<'a> {
})
}
fn unexpected_eof(&self) -> Error<'a> {
Error::UnexpectedEOF(self.last_token_span)
}
/// Helper to run a parsing closure and wrap the result in a Spanned struct
fn spanned<F, T>(&mut self, parser: F) -> Result<Spanned<T>, Error<'a>>
where
@@ -271,6 +285,9 @@ impl<'a> Parser<'a> {
}
fn assign_next(&mut self) -> Result<(), Error<'a>> {
if let Some(token) = &self.current_token {
self.last_token_span = Some(Self::token_to_span(token));
}
self.current_token = self.tokenizer.next_token()?;
Ok(())
}
@@ -317,7 +334,7 @@ impl<'a> Parser<'a> {
if self_matches_peek!(self, TokenType::Symbol(Symbol::Dot)) {
self.assign_next()?; // consume Dot
let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let identifier_span = Self::token_to_span(&identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -336,10 +353,10 @@ impl<'a> Parser<'a> {
let mut arguments = Vec::<Spanned<Expression<'a>>>::new();
while !token_matches!(
self.get_next()?.ok_or(Error::UnexpectedEOF)?,
self.get_next()?.ok_or_else(|| self.unexpected_eof())?,
TokenType::Symbol(Symbol::RParen)
) {
let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
// Block expressions not allowed in args
if let Expression::Block(_) = expression.node {
@@ -353,7 +370,8 @@ impl<'a> Parser<'a> {
if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma))
&& !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen))
{
let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next_token =
self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
return Err(Error::UnexpectedToken(
Self::token_to_span(&next_token),
next_token,
@@ -415,11 +433,11 @@ impl<'a> Parser<'a> {
// Index Access
self.assign_next()?; // consume '[', now current is '['
self.assign_next()?; // advance to the expression, now current is first token of index expr
let index = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let index = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
// After expression(), current is still on the last token of the index expression
// We need to get the next token and verify it's ']'
let rbracket_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let rbracket_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(rbracket_token, TokenType::Symbol(Symbol::RBracket)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&rbracket_token),
@@ -534,9 +552,9 @@ impl<'a> Parser<'a> {
TokenType::Keyword(Keyword::Break) => {
let span = self.current_span();
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
return Err(Error::MissingSemicolon(span));
}
Some(Spanned {
span,
@@ -546,9 +564,9 @@ impl<'a> Parser<'a> {
TokenType::Keyword(Keyword::Continue) => {
let span = self.current_span();
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
return Err(Error::MissingSemicolon(span));
}
Some(Spanned {
span,
@@ -606,7 +624,7 @@ impl<'a> Parser<'a> {
TokenType::Symbol(Symbol::Minus) => {
let start_span = self.current_span();
self.assign_next()?;
let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?;
let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?;
let inner_with_postfix = self.parse_postfix(inner_expr)?;
@@ -625,7 +643,7 @@ impl<'a> Parser<'a> {
TokenType::Symbol(Symbol::LogicalNot) => {
let start_span = self.current_span();
self.assign_next()?;
let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?;
let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?;
let inner_with_postfix = self.parse_postfix(inner_expr)?;
let combined_span = Span {
start_line: start_span.start_line,
@@ -645,7 +663,7 @@ impl<'a> Parser<'a> {
TokenType::Symbol(Symbol::BitwiseNot) => {
let start_span = self.current_span();
self.assign_next()?;
let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?;
let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?;
let inner_with_postfix = self.parse_postfix(inner_expr)?;
let combined_span = Span {
start_line: start_span.start_line,
@@ -671,7 +689,10 @@ impl<'a> Parser<'a> {
}
fn get_infix_child_node(&mut self) -> Result<Spanned<tree_node::Expression<'a>>, Error<'a>> {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
let start_span = self.current_span();
@@ -699,7 +720,7 @@ impl<'a> Parser<'a> {
TokenType::Symbol(Symbol::LParen) => *self
.parenthesized_or_tuple()?
.map(Box::new)
.ok_or(Error::UnexpectedEOF)?,
.ok_or_else(|| self.unexpected_eof())?,
TokenType::Identifier(ref id) if SysCall::is_syscall(id) => {
let spanned_call = self.spanned(|p| p.syscall())?;
@@ -778,7 +799,10 @@ impl<'a> Parser<'a> {
}
fn device(&mut self) -> Result<DeviceDeclarationExpression<'a>, Error<'a>> {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
if !self_matches_current!(self, TokenType::Keyword(Keyword::Device)) {
return Err(Error::UnexpectedToken(
self.current_span(),
@@ -786,7 +810,7 @@ impl<'a> Parser<'a> {
));
}
let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let identifier_span = Self::token_to_span(&identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -798,7 +822,7 @@ impl<'a> Parser<'a> {
}
};
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
@@ -806,7 +830,7 @@ impl<'a> Parser<'a> {
));
}
let device_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let device_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let device = match device_token.token_type {
TokenType::String(ref id) => id.clone(),
_ => {
@@ -830,7 +854,10 @@ impl<'a> Parser<'a> {
&mut self,
previous: Spanned<Expression<'a>>,
) -> Result<Spanned<Expression<'a>>, Error<'a>> {
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
let current_token = self
.get_next()?
.ok_or_else(|| self.unexpected_eof())?
.clone();
match previous.node {
Expression::Binary(_)
@@ -873,7 +900,10 @@ impl<'a> Parser<'a> {
self.assign_next()?;
expressions.push(self.get_infix_child_node()?);
temp_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
temp_token = self
.get_next()?
.ok_or_else(|| self.unexpected_eof())?
.clone();
}
if operators.len() != expressions.len() - 1 {
@@ -1239,14 +1269,17 @@ impl<'a> Parser<'a> {
self.tokenizer.seek(SeekFrom::Current(-1))?;
}
expressions.pop().ok_or(Error::UnexpectedEOF)
expressions.pop().ok_or_else(|| self.unexpected_eof())
}
fn parenthesized_or_tuple(
&mut self,
) -> Result<Option<Spanned<tree_node::Expression<'a>>>, Error<'a>> {
let start_span = self.current_span();
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(
@@ -1273,7 +1306,7 @@ impl<'a> Parser<'a> {
}));
}
let first_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let first_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
if self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) {
// It is a tuple
@@ -1282,10 +1315,10 @@ impl<'a> Parser<'a> {
// Next toekn is a comma, we need to consume it and advance 1 more time.
self.assign_next()?;
self.assign_next()?;
items.push(self.expression()?.ok_or(Error::UnexpectedEOF)?);
items.push(self.expression()?.ok_or_else(|| self.unexpected_eof())?);
}
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1304,7 +1337,7 @@ impl<'a> Parser<'a> {
}))
} else {
// It is just priority
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1319,14 +1352,14 @@ impl<'a> Parser<'a> {
fn tuple_declaration(&mut self) -> Result<Expression<'a>, Error<'a>> {
// 'let' is consumed before this call
// expect '('
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
let mut names = Vec::new();
while !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) {
let token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let span = Self::token_to_span(&token);
if let TokenType::Identifier(id) = token.token_type {
names.push(Spanned { span, node: id });
@@ -1340,7 +1373,7 @@ impl<'a> Parser<'a> {
}
self.assign_next()?; // consume ')'
let assign = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let assign = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(assign, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&assign), assign));
@@ -1348,11 +1381,12 @@ impl<'a> Parser<'a> {
self.assign_next()?; // Consume the `=`
let value = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let value = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
let value_span = value.span;
let semi = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let semi = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(semi, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&semi), semi));
return Err(Error::MissingSemicolon(value_span));
}
Ok(Expression::TupleDeclaration(Spanned {
@@ -1365,19 +1399,24 @@ impl<'a> Parser<'a> {
}
fn invocation(&mut self) -> Result<InvocationExpression<'a>, Error<'a>> {
let identifier_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let identifier_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
let identifier_span = Self::token_to_span(identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
_ => {
return Err(Error::UnexpectedToken(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
self.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?,
));
}
};
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
@@ -1388,10 +1427,10 @@ impl<'a> Parser<'a> {
let mut arguments = Vec::<Spanned<Expression>>::new();
while !token_matches!(
self.get_next()?.ok_or(Error::UnexpectedEOF)?,
self.get_next()?.ok_or_else(|| self.unexpected_eof())?,
TokenType::Symbol(Symbol::RParen)
) {
let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
if let Expression::Block(_) = expression.node {
return Err(Error::InvalidSyntax(
@@ -1405,7 +1444,7 @@ impl<'a> Parser<'a> {
if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma))
&& !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen))
{
let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
return Err(Error::UnexpectedToken(
Self::token_to_span(&next_token),
next_token,
@@ -1428,7 +1467,10 @@ impl<'a> Parser<'a> {
fn block(&mut self) -> Result<BlockExpression<'a>, Error<'a>> {
let mut expressions = Vec::<Spanned<Expression>>::new();
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) {
return Err(Error::UnexpectedToken(
@@ -1441,11 +1483,11 @@ impl<'a> Parser<'a> {
self,
TokenType::Symbol(Symbol::RBrace) | TokenType::Keyword(Keyword::Return)
) {
let expression = self.parse()?.ok_or(Error::UnexpectedEOF)?;
let expression = self.parse()?.ok_or_else(|| self.unexpected_eof())?;
expressions.push(expression);
}
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if token_matches!(current_token, TokenType::Keyword(Keyword::Return)) {
// Need to capture return span
@@ -1453,14 +1495,16 @@ impl<'a> Parser<'a> {
self.assign_next()?;
let expr = if token_matches!(
self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?,
self.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?,
TokenType::Symbol(Symbol::Semicolon)
) {
// rewind 1 token so we can check for the semicolon at the bottom of this function.
self.tokenizer.seek(SeekFrom::Current(-1))?;
None
} else {
Some(self.expression()?.ok_or(Error::UnexpectedEOF)?)
Some(self.expression()?.ok_or_else(|| self.unexpected_eof())?)
};
let ret_span = Span {
@@ -1482,12 +1526,12 @@ impl<'a> Parser<'a> {
};
expressions.push(return_expr);
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
return Err(Error::MissingSemicolon(ret_span));
}
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::RBrace)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1498,7 +1542,10 @@ impl<'a> Parser<'a> {
fn const_declaration(&mut self) -> Result<ConstDeclarationExpression<'a>, Error<'a>> {
// const
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
if !self_matches_current!(self, TokenType::Keyword(Keyword::Const)) {
return Err(Error::UnexpectedToken(
self.current_span(),
@@ -1507,7 +1554,7 @@ impl<'a> Parser<'a> {
}
// variable_name
let ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let ident_span = Self::token_to_span(&ident_token);
let ident = match ident_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -1515,7 +1562,10 @@ impl<'a> Parser<'a> {
};
// `=`
let assign_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
let assign_token = self
.get_next()?
.ok_or_else(|| self.unexpected_eof())?
.clone();
if !token_matches!(assign_token, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&assign_token),
@@ -1552,7 +1602,9 @@ impl<'a> Parser<'a> {
) {
return Err(Error::UnexpectedToken(
syscall.span,
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
self.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?,
));
}
@@ -1567,14 +1619,17 @@ impl<'a> Parser<'a> {
}
fn declaration(&mut self) -> Result<Expression<'a>, Error<'a>> {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) {
return Err(Error::UnexpectedToken(
self.current_span(),
current_token.clone(),
));
}
let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let identifier_span = Self::token_to_span(&identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -1586,7 +1641,10 @@ impl<'a> Parser<'a> {
}
};
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
let current_token = self
.get_next()?
.ok_or_else(|| self.unexpected_eof())?
.clone();
if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(
@@ -1596,14 +1654,12 @@ impl<'a> Parser<'a> {
}
self.assign_next()?;
let assignment_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let assignment_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
let expr_span = assignment_expression.span;
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
current_token,
));
return Err(Error::MissingSemicolon(expr_span));
}
Ok(Expression::Declaration(
@@ -1616,7 +1672,10 @@ impl<'a> Parser<'a> {
}
fn literal(&mut self) -> Result<Literal<'a>, Error<'a>> {
let current_token = self.current_token.clone().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?;
let literal = match current_token.token_type {
TokenType::Number(num) => Literal::Number(num),
TokenType::String(ref string) => Literal::String(string.clone()),
@@ -1632,7 +1691,7 @@ impl<'a> Parser<'a> {
wrong_token,
));
}
None => return Err(Error::UnexpectedEOF),
None => return Err(self.unexpected_eof()),
},
_ => {
return Err(Error::UnexpectedToken(
@@ -1647,20 +1706,20 @@ impl<'a> Parser<'a> {
fn if_expression(&mut self) -> Result<IfExpression<'a>, Error<'a>> {
// 'if' is current
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
self.assign_next()?;
let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1686,7 +1745,7 @@ impl<'a> Parser<'a> {
node: Expression::Block(block),
}))
} else {
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
} else {
@@ -1701,7 +1760,7 @@ impl<'a> Parser<'a> {
}
fn loop_expression(&mut self) -> Result<LoopExpression<'a>, Error<'a>> {
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1712,20 +1771,20 @@ impl<'a> Parser<'a> {
}
fn while_expression(&mut self) -> Result<WhileExpression<'a>, Error<'a>> {
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
self.assign_next()?;
let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?;
let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?;
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) {
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1740,7 +1799,7 @@ impl<'a> Parser<'a> {
fn function(&mut self) -> Result<FunctionExpression<'a>, Error<'a>> {
// 'fn' is current
let fn_ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let fn_ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
let fn_ident_span = Self::token_to_span(&fn_ident_token);
let fn_ident = match fn_ident_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -1749,7 +1808,7 @@ impl<'a> Parser<'a> {
}
};
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
@@ -1760,10 +1819,13 @@ impl<'a> Parser<'a> {
let mut arguments = Vec::<Spanned<Cow<'a, str>>>::new();
while !token_matches!(
self.get_next()?.ok_or(Error::UnexpectedEOF)?,
self.get_next()?.ok_or_else(|| self.unexpected_eof())?,
TokenType::Symbol(Symbol::RParen)
) {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let current_token = self
.current_token
.as_ref()
.ok_or_else(|| self.unexpected_eof())?;
let arg_span = Self::token_to_span(current_token);
let argument = match current_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
@@ -1789,7 +1851,7 @@ impl<'a> Parser<'a> {
if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma))
&& !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen))
{
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
return Err(Error::UnexpectedToken(Self::token_to_span(&next), next));
}
@@ -1798,7 +1860,7 @@ impl<'a> Parser<'a> {
}
}
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?;
if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
@@ -1861,7 +1923,9 @@ impl<'a> Parser<'a> {
_ => {
return Err(Error::UnexpectedToken(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
self.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?,
))
}
}
@@ -1893,7 +1957,7 @@ impl<'a> Parser<'a> {
}
"sleep" => {
let mut args = args!(1);
let expr = args.next().ok_or(Error::UnexpectedEOF)?;
let expr = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::System(System::Sleep(boxed!(expr))))
}
"hash" => {
@@ -1947,7 +2011,9 @@ impl<'a> Parser<'a> {
_ => {
return Err(Error::UnexpectedToken(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
self.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?,
));
}
};
@@ -2003,7 +2069,7 @@ impl<'a> Parser<'a> {
let tmp = args.next();
let logic_type = get_arg!(Literal, literal_or_variable!(tmp));
let variable = args.next().ok_or(Error::UnexpectedEOF)?;
let variable = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::System(sys_call::System::SetOnDevice(
device,
Spanned {
@@ -2022,7 +2088,7 @@ impl<'a> Parser<'a> {
let tmp = args.next();
let logic_type = get_arg!(Literal, literal_or_variable!(tmp));
let variable = args.next().ok_or(Error::UnexpectedEOF)?;
let variable = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::System(sys_call::System::SetOnDeviceBatched(
device_hash,
@@ -2045,7 +2111,7 @@ impl<'a> Parser<'a> {
let logic_type = get_arg!(Literal, literal_or_variable!(tmp));
let tmp = args.next();
let expr = Box::new(tmp.ok_or(Error::UnexpectedEOF)?);
let expr = Box::new(tmp.ok_or_else(|| self.unexpected_eof())?);
Ok(SysCall::System(System::SetOnDeviceBatchedNamed(
device_hash,
@@ -2058,7 +2124,7 @@ impl<'a> Parser<'a> {
let mut args = args!(3);
let next = args.next();
let dev_name = literal_or_variable!(next);
let slot_index = args.next().ok_or(Error::UnexpectedEOF)?;
let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?;
let next = args.next();
let slot_logic = get_arg!(Literal, literal_or_variable!(next));
@@ -2085,7 +2151,7 @@ impl<'a> Parser<'a> {
let mut args = args!(4);
let next = args.next();
let dev_name = literal_or_variable!(next);
let slot_index = args.next().ok_or(Error::UnexpectedEOF)?;
let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?;
let next = args.next();
let slot_logic = get_arg!(Literal, literal_or_variable!(next));
@@ -2102,7 +2168,7 @@ impl<'a> Parser<'a> {
));
}
let next = args.next();
let expr = next.ok_or(Error::UnexpectedEOF)?;
let expr = next.ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::System(System::SetSlot(
dev_name,
@@ -2117,7 +2183,7 @@ impl<'a> Parser<'a> {
let device = literal_or_variable!(next);
let next = args.next();
let reagent_mode = get_arg!(Literal, literal_or_variable!(next));
let reagent_hash = args.next().ok_or(Error::UnexpectedEOF)?;
let reagent_hash = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::System(System::LoadReagent(
device,
@@ -2130,80 +2196,80 @@ impl<'a> Parser<'a> {
"acos" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let tmp = args.next().ok_or(Error::UnexpectedEOF)?;
let tmp = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Acos(boxed!(tmp))))
}
"asin" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let tmp = args.next().ok_or(Error::UnexpectedEOF)?;
let tmp = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Asin(boxed!(tmp))))
}
"atan" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let expr = args.next().ok_or(Error::UnexpectedEOF)?;
let expr = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Atan(boxed!(expr))))
}
"atan2" => {
check_length(2)?;
let mut args = invocation.arguments.into_iter();
let arg1 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg2 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?;
let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Atan2(boxed!(arg1), boxed!(arg2))))
}
"abs" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let expr = args.next().ok_or(Error::UnexpectedEOF)?;
let expr = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Abs(boxed!(expr))))
}
"ceil" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Ceil(boxed!(arg))))
}
"cos" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Cos(boxed!(arg))))
}
"floor" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Floor(boxed!(arg))))
}
"log" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Log(boxed!(arg))))
}
"max" => {
check_length(2)?;
let mut args = invocation.arguments.into_iter();
let arg1 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg2 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?;
let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Max(boxed!(arg1), boxed!(arg2))))
}
"min" => {
check_length(2)?;
let mut args = invocation.arguments.into_iter();
let arg1 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg2 = args.next().ok_or(Error::UnexpectedEOF)?;
let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?;
let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Min(boxed!(arg1), boxed!(arg2))))
}
@@ -2214,34 +2280,36 @@ impl<'a> Parser<'a> {
"sin" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Sin(boxed!(arg))))
}
"sqrt" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Sqrt(boxed!(arg))))
}
"tan" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Tan(boxed!(arg))))
}
"trunc" => {
check_length(1)?;
let mut args = invocation.arguments.into_iter();
let arg = args.next().ok_or(Error::UnexpectedEOF)?;
let arg = args.next().ok_or_else(|| self.unexpected_eof())?;
Ok(SysCall::Math(Math::Trunc(boxed!(arg))))
}
_ => Err(Error::UnsupportedKeyword(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
self.current_token
.clone()
.ok_or_else(|| self.unexpected_eof())?,
)),
}
}

33
rust_compiler/libs/parser/src/test/mod.rs
View File

@@ -287,3 +287,36 @@ fn test_tuple_declaration_all_complex_expressions() -> Result<()> {
Ok(())
}
#[test]
fn test_eof_error_has_span() -> Result<()> {
// Test that UnexpectedEOF errors capture the span of the last token
let mut parser = parser!("let x = 5");
let result = parser.parse();
// Should have an error
assert!(result.is_err());
let err = result.unwrap_err();
// Check that it's an UnexpectedEOF error
match err {
super::Error::UnexpectedEOF(Some(span)) => {
// Verify the span points to somewhere in the code (not zero defaults)
assert!(
span.start_line > 0 || span.start_col > 0 || span.end_line > 0 || span.end_col > 0,
"Span should not be all zeros: {:?}",
span
);
}
super::Error::UnexpectedEOF(None) => {
eprintln!("ERROR: UnexpectedEOF captured None span instead of previous token span");
eprintln!("This means unexpected_eof() is being called when current_token is None");
panic!("UnexpectedEOF should have captured the previous token's span");
}
other => {
panic!("Expected UnexpectedEOF error, got: {:?}", other);
}
}
Ok(())
}