From 7b7c1f7d29ebcf66022c152087fb4759e03ebf84 Mon Sep 17 00:00:00 2001
From: Devin Bidwell <dbidwell94@biddydev.com>
Date: Tue, 9 Dec 2025 02:15:43 -0700
Subject: [PATCH] Logos plugged into Parser

---
 rust_compiler/libs/tokenizer/src/lib.rs   | 12 ++++++------
 rust_compiler/libs/tokenizer/src/token.rs | 13 +++++++++++--
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs
index 1e2b342..1e76e36 100644
--- a/rust_compiler/libs/tokenizer/src/lib.rs
+++ b/rust_compiler/libs/tokenizer/src/lib.rs
@@ -66,13 +66,13 @@ impl<'a> Tokenizer<'a> {
     }
 
     pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
-        let to_return = self
-            .lexer
-            .next()
-            .transpose()
-            .map(|t| t.map(|t| self.get_token(t)))?;
+        let mut current = self.lexer.next().transpose();
 
-        Ok(to_return)
+        while matches!(current, Ok(Some(TokenType::Comment(_)))) {
+            current = self.lexer.next().transpose();
+        }
+
+        Ok(current.map(|t| t.map(|t| self.get_token(t)))?)
     }
 }
 
diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs
index e8ffd04..74f00b1 100644
--- a/rust_compiler/libs/tokenizer/src/token.rs
+++ b/rust_compiler/libs/tokenizer/src/token.rs
@@ -233,8 +233,8 @@ pub enum TokenType {
     /// Represents a symbol token
     Symbol(Symbol),
 
-    #[regex(r"///[\n]*", |val| Comment::Doc(val.slice()[3..].trim().to_string()))]
-    #[regex(r"//[\n]*", |val| Comment::Line(val.slice()[2..].trim().to_string()))]
+    #[token("//", |lex| Comment::Line(read_line(lex)))]
+    #[token("///", |lex| Comment::Doc(read_line(lex)))]
     /// Represents a comment, both a line comment and a doc comment
     Comment(Comment),
 
@@ -243,6 +243,15 @@ pub enum TokenType {
     EOF,
 }
 
+fn read_line<'a>(lexer: &mut Lexer<'a, TokenType>) -> String {
+    let rem = lexer.remainder();
+    let len = rem.find('\n').unwrap_or(rem.len());
+    let content = rem[..len].trim().to_string();
+
+    lexer.bump(len);
+    content
+}
+
 #[derive(Hash, Debug, Eq, PartialEq, Clone)]
 pub enum Comment {
     Line(String),