Logos plugged into Parser

rust_compiler/libs/tokenizer/src/lib.rs

@@ -66,13 +66,13 @@ impl<'a> Tokenizer<'a> {
     }
 
     pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
-        let to_return = self
-            .lexer
-            .next()
-            .transpose()
-            .map(|t| t.map(|t| self.get_token(t)))?;
+        let mut current = self.lexer.next().transpose();
 
-        Ok(to_return)
+        while matches!(current, Ok(Some(TokenType::Comment(_)))) {
+            current = self.lexer.next().transpose();
+        }
+
+        Ok(current.map(|t| t.map(|t| self.get_token(t)))?)
     }
 }
 

rust_compiler/libs/tokenizer/src/token.rs

@@ -233,8 +233,8 @@ pub enum TokenType {
     /// Represents a symbol token
     Symbol(Symbol),
 
-    #[regex(r"///[\n]*", |val| Comment::Doc(val.slice()[3..].trim().to_string()))]
-    #[regex(r"//[\n]*", |val| Comment::Line(val.slice()[2..].trim().to_string()))]
+    #[token("//", |lex| Comment::Line(read_line(lex)))]
+    #[token("///", |lex| Comment::Doc(read_line(lex)))]
     /// Represents a comment, both a line comment and a doc comment
     Comment(Comment),
 
@@ -243,6 +243,15 @@ pub enum TokenType {
     EOF,
 }
 
+fn read_line<'a>(lexer: &mut Lexer<'a, TokenType>) -> String {
+    let rem = lexer.remainder();
+    let len = rem.find('\n').unwrap_or(rem.len());
+    let content = rem[..len].trim().to_string();
+
+    lexer.bump(len);
+    content
+}
+
 #[derive(Hash, Debug, Eq, PartialEq, Clone)]
 pub enum Comment {
     Line(String),