still WIP

2024-11-21 12:12:06 -07:00
parent f756e3e29f
commit 62bf83e277
3 changed files with 145 additions and 136 deletions
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -2,11 +2,11 @@ mod tree_node;

 use crate::tokenizer::{
    token::{Symbol, Token, TokenType},
-    Tokenizer, TokenizerError,
+    Tokenizer, TokenizerBuffer, TokenizerError,
 };
 use std::{
    collections::VecDeque,
-    io::{Read, Seek},
+    io::{Read, Seek, SeekFrom},
 };
 use thiserror::Error;
 use tree_node::*;
@@ -24,7 +24,7 @@ pub enum ParseError {
 }

 pub struct Parser<R: Read + Seek> {
-    tokenizer: Tokenizer<R>,
+    tokenizer: TokenizerBuffer<R>,
    current_token: Option<Token>,
 }

@@ -34,40 +34,64 @@ where
 {
    pub fn new(tokenizer: Tokenizer<R>) -> Self {
        Parser {
-            tokenizer,
+            tokenizer: TokenizerBuffer::new(tokenizer),
            current_token: None,
        }
    }

-    pub fn parse(&mut self) -> Result<tree_node::Expression, ParseError> {
-        self.current_token = self.tokenizer.next_token()?;
+    pub fn parse(&mut self) -> Result<Option<tree_node::Expression>, ParseError> {
+        self.current_token = self.tokenizer.next()?;
        self.expression()
    }

-    fn expression(&mut self) -> Result<tree_node::Expression, ParseError> {
-        let current_token = self
-            .current_token
-            .as_ref()
-            .ok_or(ParseError::UnknownError)?;
+    fn expression(&mut self) -> Result<Option<tree_node::Expression>, ParseError> {
+        /// Helper macro to match the next token in the tokenizer buffer to a pattern
+        /// with an optional if condition. The token is peeked and not consumed.
+        macro_rules! matches_peek {
+            ($pattern:pat) => {
+                matches!(self.tokenizer.peek()?, Some(Token { token_type: $pattern, .. }))
+            };
+            ($pattern:pat if $cond:expr) => {
+                matches!(self.tokenizer.peek()?, Some(Token { token_type: $pattern, .. }) if $cond)
+            };
+        }
+
+        let Some(current_token) = self.current_token.as_ref() else {
+            return Ok(None);
+        };

        Ok(match current_token.token_type {
-            // Match a number or string literal as long as the next token is not an operator
+            // Assignment expression
+            TokenType::Identifier(_) if matches_peek!(TokenType::Symbol(Symbol::Assign)) => {
+                Some(Expression::AssignmentExpression(self.assignment()?))
+            }
+
+            // Negation expression
+            TokenType::Symbol(Symbol::Minus) if matches_peek!(TokenType::Number(_)) => {
+                self.tokenizer.next()?;
+                Some(Expression::Negation(Box::new(
+                    self.parse()?.ok_or(ParseError::UnexpectedEOF)?,
+                )))
+            }
+
+            // Literal expression
            TokenType::Number(_) | TokenType::String(_)
-                if !matches!(
-                    self.tokenizer.peek_next()?, Some(Token { token_type: TokenType::Symbol(e), .. }) if e.is_operator()
+                if !matches_peek!(
+                    TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical()
                ) =>
            {
-                Expression::Literal(self.literal()?)
+                Some(Expression::Literal(self.literal()?))
            }

-            // Match a negation operator
-            TokenType::Symbol(Symbol::Minus) => Expression::Negation(Box::new(self.parse()?)),
-
-            _ if matches!(self.tokenizer.peek_next()?, Some(Token { token_type: TokenType::Symbol(e), .. }) if e.is_operator()) => {
-                Expression::BinaryExpression(self.binary()?)
+            // Logical expression
+            TokenType::Number(_) | TokenType::String(_)
+                if matches_peek!(
+                    TokenType::Symbol(s) if s.is_comparison() || s.is_logical()
+                ) =>
+            {
+                Some(Expression::LogicalExpression(self.logical()?))
            }

-            // Something went wrong. Return an error
            _ => {
                return Err(ParseError::UnexpectedToken {
                    token: current_token.clone(),
@@ -76,56 +100,61 @@ where
        })
    }

+    fn assignment(&mut self) -> Result<tree_node::AssignmentExpression, ParseError> {
+        let Some(Token {
+            token_type: TokenType::Identifier(identifier),
+            ..
+        }) = self.current_token.as_ref()
+        else {
+            return Err(ParseError::UnexpectedToken {
+                // Safety: We have already checked that `self.current_token` is `Some` in the `parse()` function
+                token: self.current_token.clone().unwrap(),
+            });
+        };
+
+        // make sure the next token is `=` for sanity
+        if let Some(Token {
+            token_type: TokenType::Symbol(Symbol::Assign),
+            ..
+        }) = self.tokenizer.next()?
+        {
+        } else {
+            self.tokenizer.seek(SeekFrom::Current(-1))?;
+            return Err(ParseError::UnexpectedToken {
+                token: self.tokenizer.next()?.unwrap(),
+            });
+        };
+
+        Ok(AssignmentExpression {
+            identifier: identifier.clone(),
+            expression: Box::new(self.parse()?.ok_or(ParseError::UnexpectedEOF)?),
+        })
+    }
+
+    fn logical(&mut self) -> Result<tree_node::LogicalExpression, ParseError> {
+        let Some(current_token) = self.current_token.as_ref() else {
+            return Err(ParseError::UnexpectedEOF);
+        };
+
+        todo!()
+    }
+
    fn binary(&mut self) -> Result<tree_node::BinaryExpression, ParseError> {
-        let literal = self.literal()?;
-
-        let Some(Token {
-            token_type: TokenType::Symbol(operator),
-            ..
-        }) = self.current_token
-        else {
-            return Err(ParseError::UnknownError);
+        let Some(current_token) = self.current_token.as_ref() else {
+            return Err(ParseError::UnexpectedEOF);
        };
-        self.current_token = self.tokenizer.next_token()?;

-        Ok(match operator {
-            Symbol::Plus => BinaryExpression::Add(
-                Box::new(Expression::Literal(literal)),
-                Box::new(self.expression()?),
-            ),
-            Symbol::Asterisk => BinaryExpression::Multiply(
-                Box::new(Expression::Literal(literal)),
-                Box::new(self.expression()?),
-            ),
-            Symbol::Slash => BinaryExpression::Divide(
-                Box::new(Expression::Literal(literal)),
-                Box::new(self.expression()?),
-            ),
-            Symbol::Minus => BinaryExpression::Subtract(
-                Box::new(Expression::Literal(literal)),
-                Box::new(self.expression()?),
-            ),
-            _ => {
-                return Err(ParseError::UnexpectedToken {
-                    token: Token {
-                        token_type: TokenType::Symbol(operator),
-                        line: 0,
-                        column: 0,
-                    },
-                })
-            }
-        })
+        todo!()
    }

    fn literal(&mut self) -> Result<tree_node::Literal, ParseError> {
-        let current_token = self
-            .current_token
-            .as_ref()
-            .ok_or(ParseError::UnknownError)?;
+        let Some(current_token) = self.current_token.as_ref() else {
+            return Err(ParseError::UnexpectedEOF);
+        };

        let to_return = match current_token.token_type {
-            TokenType::Number(ref number) => tree_node::Literal::Number(number.clone()),
-            TokenType::String(ref string) => tree_node::Literal::String(string.clone()),
+            TokenType::Number(n) => Literal::Number(n),
+            TokenType::String(ref s) => Literal::String(s.clone()),
            _ => {
                return Err(ParseError::UnexpectedToken {
                    token: current_token.clone(),
@@ -133,78 +162,59 @@ where
            }
        };

-        self.current_token = self.tokenizer.next_token()?;
+        // Advance the tokenizer if the next token is a semicolon
+        if let Some(Token {
+            token_type: TokenType::Symbol(Symbol::Semicolon),
+            ..
+        }) = self.tokenizer.peek()?
+        {
+            self.tokenizer.next()?;
+        }
+
        Ok(to_return)
    }
 }

 #[cfg(test)]
 mod tests {
-    use super::tree_node::*;
    use super::*;
    use anyhow::Result;

    #[test]
-    fn test_add_expr() -> Result<()> {
-        let input = "123 + 456";
+    fn test_assignment() -> Result<()> {
+        let input = r#"
+        x = 10;
+        y = "testing";
+        "#;
+        let tokenizer = Tokenizer::from(input.to_owned());
+        let mut parser = Parser::new(tokenizer);

-        let mut parser = Parser::new(Tokenizer::from(input.to_owned()));
+        let expr = parser.parse()?.unwrap();

-        let result = parser.parse()?;
-        let formatted_output = format!("{}", result);
+        assert_eq!("x = 10", format!("{}", expr));

-        assert_eq!(formatted_output, "(123 + 456)");
-        Ok(())
-    }
+        let expr = parser.parse()?.unwrap();

-    #[test]
-    fn test_parse_number() -> Result<()> {
-        let input = "123";
-        let mut parser = Parser::new(Tokenizer::from(input.to_owned()));
-        let result = parser.parse()?;
-
-        let formatted_output = format!("{}", result);
-
-        assert_eq!(formatted_output, "123");
+        assert_eq!("y = \"testing\"", format!("{}", expr));

        Ok(())
    }

    #[test]
-    fn test_parse_negation() -> Result<()> {
-        let input = "-123";
-        let mut parser = Parser::new(Tokenizer::from(input.to_owned()));
-        let result = parser.parse()?;
+    fn test_literal() -> Result<()> {
+        let input = r#"
+            10;
+            "testing";
+        "#;

-        let formatted_output = format!("{}", result);
+        let tokenizer = Tokenizer::from(input.to_owned());
+        let mut parser = Parser::new(tokenizer);

-        assert_eq!(formatted_output, "(-123)");
+        let expr = parser.parse()?.unwrap();
+        assert_eq!("10", format!("{}", expr));

-        Ok(())
-    }
-
-    #[test]
-    fn test_order_of_operations() -> Result<()> {
-        let input = "123 - 456 + 789";
-
-        let mut parser = Parser::new(Tokenizer::from(input.to_owned()));
-        let result = parser.parse()?;
-
-        let formatted_output = format!("{}", result);
-        println!("{}", formatted_output);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_chained_operators() -> Result<()> {
-        let input = "123 + 456 * 789";
-        let mut parser = Parser::new(Tokenizer::from(input.to_owned()));
-        let result = parser.parse()?;
-
-        let formatted_output = format!("{}", result);
-
-        assert_eq!(formatted_output, "(123 + (456 * 789))");
+        let expr = parser.parse()?.unwrap();
+        assert_eq!("\"testing\"", format!("{}", expr));

        Ok(())
    }