TEST -- use Cow instead of String for tokens

rust_compiler/libs/parser/src/lib.rs

@@ -26,27 +26,27 @@ macro_rules! boxed {
 }
 
 #[derive(Error, Debug)]
-pub enum Error {
+pub enum Error<'a> {
     #[error(transparent)]
     Tokenizer(#[from] tokenizer::Error),
 
     #[error("Unexpected token: {1}")]
-    UnexpectedToken(Span, Token),
+    UnexpectedToken(Span, Token<'a>),
 
     #[error("Duplicate identifier: {1}")]
-    DuplicateIdentifier(Span, Token),
+    DuplicateIdentifier(Span, Token<'a>),
 
     #[error("Invalid Syntax: {1}")]
     InvalidSyntax(Span, String),
 
     #[error("Unsupported Keyword: {1}")]
-    UnsupportedKeyword(Span, Token),
+    UnsupportedKeyword(Span, Token<'a>),
 
     #[error("Unexpected End of File")]
     UnexpectedEOF,
 }
 
-impl From<Error> for lsp_types::Diagnostic {
+impl<'a> From<Error<'a>> for lsp_types::Diagnostic {
     fn from(value: Error) -> Self {
         use Error::*;
         use lsp_types::*;
@@ -105,8 +105,8 @@ macro_rules! self_matches_current {
 
 pub struct Parser<'a> {
     tokenizer: TokenizerBuffer<'a>,
-    current_token: Option<Token>,
-    pub errors: Vec<Error>,
+    current_token: Option<Token<'a>>,
+    pub errors: Vec<Error<'a>>,
 }
 
 impl<'a> Parser<'a> {
@@ -119,7 +119,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Calculates a Span from a given Token reference.
-    fn token_to_span(t: &Token) -> Span {
+    fn token_to_span<'t>(t: &'t Token<'a>) -> Span {
         Span {
             start_line: t.line,
             start_col: t.span.start,
@@ -269,14 +269,14 @@ impl<'a> Parser<'a> {
         Ok(expr)
     }
 
-    fn assign_next(&mut self) -> Result<(), Error> {
+    fn assign_next(&'a mut self) -> Result<(), Error> {
         self.current_token = self.tokenizer.next_token()?;
         Ok(())
     }
 
-    fn get_next(&mut self) -> Result<Option<&Token>, Error> {
+    fn get_next(&'a mut self) -> Result<Option<Token>, Error> {
         self.assign_next()?;
-        Ok(self.current_token.as_ref())
+        Ok(self.current_token.clone())
     }
 
     fn expression(&mut self) -> Result<Option<Spanned<tree_node::Expression>>, Error> {

rust_compiler/libs/parser/src/tree_node.rs

@@ -1,22 +1,22 @@
 use super::sys_call::SysCall;
 use crate::sys_call;
-use std::ops::Deref;
+use std::{borrow::Cow, ops::Deref};
 use tokenizer::token::Number;
 
 #[derive(Debug, Eq, PartialEq, Clone)]
-pub enum Literal {
+pub enum Literal<'a> {
     Number(Number),
-    String(String),
+    String(Cow<'a, str>),
     Boolean(bool),
 }
 
 #[derive(Debug, Eq, PartialEq, Clone)]
-pub enum LiteralOr<T> {
-    Literal(Spanned<Literal>),
+pub enum LiteralOr<'a, T> {
+    Literal(Spanned<Literal<'a>>),
     Or(Spanned<T>),
 }
 
-impl<T: std::fmt::Display> std::fmt::Display for LiteralOr<T> {
+impl<'a, T: std::fmt::Display> std::fmt::Display for LiteralOr<'a, T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Literal(l) => write!(f, "{l}"),
@@ -25,7 +25,7 @@ impl<T: std::fmt::Display> std::fmt::Display for LiteralOr<T> {
     }
 }
 
-impl std::fmt::Display for Literal {
+impl<'a> std::fmt::Display for Literal<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Literal::Number(n) => write!(f, "{}", n),
@@ -36,16 +36,16 @@ impl std::fmt::Display for Literal {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub enum BinaryExpression {
-    Add(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Multiply(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Divide(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Subtract(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Exponent(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Modulo(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
+pub enum BinaryExpression<'a> {
+    Add(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Multiply(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Divide(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Subtract(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Exponent(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Modulo(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
 }
 
-impl std::fmt::Display for BinaryExpression {
+impl<'a> std::fmt::Display for BinaryExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             BinaryExpression::Add(l, r) => write!(f, "({} + {})", l, r),
@@ -59,19 +59,19 @@ impl std::fmt::Display for BinaryExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub enum LogicalExpression {
-    And(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Or(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    Not(Box<Spanned<Expression>>),
-    Equal(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    NotEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    GreaterThan(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    GreaterThanOrEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    LessThan(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
-    LessThanOrEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>),
+pub enum LogicalExpression<'a> {
+    And(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Or(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    Not(Box<Spanned<Expression<'a>>>),
+    Equal(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    NotEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    GreaterThan(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    GreaterThanOrEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    LessThan(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
+    LessThanOrEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
 }
 
-impl std::fmt::Display for LogicalExpression {
+impl<'a> std::fmt::Display for LogicalExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             LogicalExpression::And(l, r) => write!(f, "({} && {})", l, r),
@@ -88,25 +88,25 @@ impl std::fmt::Display for LogicalExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct AssignmentExpression {
-    pub assignee: Box<Spanned<Expression>>,
-    pub expression: Box<Spanned<Expression>>,
+pub struct AssignmentExpression<'a> {
+    pub assignee: Box<Spanned<Expression<'a>>>,
+    pub expression: Box<Spanned<Expression<'a>>>,
 }
 
-impl std::fmt::Display for AssignmentExpression {
+impl<'a> std::fmt::Display for AssignmentExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "({} = {})", self.assignee, self.expression)
     }
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct FunctionExpression {
-    pub name: Spanned<String>,
-    pub arguments: Vec<Spanned<String>>,
-    pub body: BlockExpression,
+pub struct FunctionExpression<'a> {
+    pub name: Spanned<Cow<'a, str>>,
+    pub arguments: Vec<Spanned<Cow<'a, str>>>,
+    pub body: BlockExpression<'a>,
 }
 
-impl std::fmt::Display for FunctionExpression {
+impl<'a> std::fmt::Display for FunctionExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
@@ -123,9 +123,9 @@ impl std::fmt::Display for FunctionExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct BlockExpression(pub Vec<Spanned<Expression>>);
+pub struct BlockExpression<'a>(pub Vec<Spanned<Expression<'a>>>);
 
-impl std::fmt::Display for BlockExpression {
+impl<'a> std::fmt::Display for BlockExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
@@ -140,12 +140,12 @@ impl std::fmt::Display for BlockExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct InvocationExpression {
-    pub name: Spanned<String>,
-    pub arguments: Vec<Spanned<Expression>>,
+pub struct InvocationExpression<'a> {
+    pub name: Spanned<Cow<'a, str>>,
+    pub arguments: Vec<Spanned<Expression<'a>>>,
 }
 
-impl std::fmt::Display for InvocationExpression {
+impl<'a> std::fmt::Display for InvocationExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
@@ -161,25 +161,25 @@ impl std::fmt::Display for InvocationExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct MemberAccessExpression {
-    pub object: Box<Spanned<Expression>>,
-    pub member: Spanned<String>,
+pub struct MemberAccessExpression<'a> {
+    pub object: Box<Spanned<Expression<'a>>>,
+    pub member: Spanned<Cow<'a, str>>,
 }
 
-impl std::fmt::Display for MemberAccessExpression {
+impl<'a> std::fmt::Display for MemberAccessExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}.{}", self.object, self.member)
     }
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct MethodCallExpression {
-    pub object: Box<Spanned<Expression>>,
-    pub method: Spanned<String>,
-    pub arguments: Vec<Spanned<Expression>>,
+pub struct MethodCallExpression<'a> {
+    pub object: Box<Spanned<Expression<'a>>>,
+    pub method: Spanned<Cow<'a, str>>,
+    pub arguments: Vec<Spanned<Expression<'a>>>,
 }
 
-impl std::fmt::Display for MethodCallExpression {
+impl<'a> std::fmt::Display for MethodCallExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
@@ -196,12 +196,12 @@ impl std::fmt::Display for MethodCallExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub enum LiteralOrVariable {
-    Literal(Literal),
-    Variable(Spanned<String>),
+pub enum LiteralOrVariable<'a> {
+    Literal(Literal<'a>),
+    Variable(Spanned<Cow<'a, str>>),
 }
 
-impl std::fmt::Display for LiteralOrVariable {
+impl<'a> std::fmt::Display for LiteralOrVariable<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             LiteralOrVariable::Literal(l) => write!(f, "{}", l),
@@ -211,46 +211,46 @@ impl std::fmt::Display for LiteralOrVariable {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct ConstDeclarationExpression {
-    pub name: Spanned<String>,
-    pub value: LiteralOr<SysCall>,
+pub struct ConstDeclarationExpression<'a> {
+    pub name: Spanned<Cow<'a, str>>,
+    pub value: LiteralOr<'a, SysCall>,
 }
 
-impl ConstDeclarationExpression {
+impl<'a> ConstDeclarationExpression<'a> {
     pub fn is_syscall_supported(call: &SysCall) -> bool {
         use sys_call::System;
         matches!(call, SysCall::System(sys) if matches!(sys, System::Hash(_)))
     }
 }
 
-impl std::fmt::Display for ConstDeclarationExpression {
+impl<'a> std::fmt::Display for ConstDeclarationExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "(const {} = {})", self.name, self.value)
     }
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct DeviceDeclarationExpression {
+pub struct DeviceDeclarationExpression<'a> {
     /// any variable-like name
-    pub name: Spanned<String>,
+    pub name: Spanned<Cow<'a, str>>,
     /// The device port, ex. (db, d0, d1, d2, d3, d4, d5)
-    pub device: String,
+    pub device: Cow<'a, str>,
 }
 
-impl std::fmt::Display for DeviceDeclarationExpression {
+impl<'a> std::fmt::Display for DeviceDeclarationExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "(device {} = {})", self.name, self.device)
     }
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct IfExpression {
-    pub condition: Box<Spanned<Expression>>,
-    pub body: Spanned<BlockExpression>,
-    pub else_branch: Option<Box<Spanned<Expression>>>,
+pub struct IfExpression<'a> {
+    pub condition: Box<Spanned<Expression<'a>>>,
+    pub body: Spanned<BlockExpression<'a>>,
+    pub else_branch: Option<Box<Spanned<Expression<'a>>>>,
 }
 
-impl std::fmt::Display for IfExpression {
+impl<'a> std::fmt::Display for IfExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "(if ({}) {}", self.condition, self.body)?;
         if let Some(else_branch) = &self.else_branch {
@@ -261,23 +261,23 @@ impl std::fmt::Display for IfExpression {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct LoopExpression {
-    pub body: Spanned<BlockExpression>,
+pub struct LoopExpression<'a> {
+    pub body: Spanned<BlockExpression<'a>>,
 }
 
-impl std::fmt::Display for LoopExpression {
+impl<'a> std::fmt::Display for LoopExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "(loop {})", self.body)
     }
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct WhileExpression {
-    pub condition: Box<Spanned<Expression>>,
-    pub body: BlockExpression,
+pub struct WhileExpression<'a> {
+    pub condition: Box<Spanned<Expression<'a>>>,
+    pub body: BlockExpression<'a>,
 }
 
-impl std::fmt::Display for WhileExpression {
+impl<'a> std::fmt::Display for WhileExpression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "(while {} {})", self.condition, self.body)
     }
@@ -345,32 +345,32 @@ impl<T> Deref for Spanned<T> {
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub enum Expression {
-    Assignment(Spanned<AssignmentExpression>),
-    Binary(Spanned<BinaryExpression>),
-    Block(Spanned<BlockExpression>),
+pub enum Expression<'a> {
+    Assignment(Spanned<AssignmentExpression<'a>>),
+    Binary(Spanned<BinaryExpression<'a>>),
+    Block(Spanned<BlockExpression<'a>>),
     Break(Span),
-    ConstDeclaration(Spanned<ConstDeclarationExpression>),
+    ConstDeclaration(Spanned<ConstDeclarationExpression<'a>>),
     Continue(Span),
-    Declaration(Spanned<String>, Box<Spanned<Expression>>),
-    DeviceDeclaration(Spanned<DeviceDeclarationExpression>),
-    Function(Spanned<FunctionExpression>),
-    If(Spanned<IfExpression>),
-    Invocation(Spanned<InvocationExpression>),
-    Literal(Spanned<Literal>),
-    Logical(Spanned<LogicalExpression>),
-    Loop(Spanned<LoopExpression>),
-    MemberAccess(Spanned<MemberAccessExpression>),
-    MethodCall(Spanned<MethodCallExpression>),
-    Negation(Box<Spanned<Expression>>),
-    Priority(Box<Spanned<Expression>>),
-    Return(Box<Spanned<Expression>>),
+    Declaration(Spanned<Cow<'a, str>>, Box<Spanned<Expression<'a>>>),
+    DeviceDeclaration(Spanned<DeviceDeclarationExpression<'a>>),
+    Function(Spanned<FunctionExpression<'a>>),
+    If(Spanned<IfExpression<'a>>),
+    Invocation(Spanned<InvocationExpression<'a>>),
+    Literal(Spanned<Literal<'a>>),
+    Logical(Spanned<LogicalExpression<'a>>),
+    Loop(Spanned<LoopExpression<'a>>),
+    MemberAccess(Spanned<MemberAccessExpression<'a>>),
+    MethodCall(Spanned<MethodCallExpression<'a>>),
+    Negation(Box<Spanned<Expression<'a>>>),
+    Priority(Box<Spanned<Expression<'a>>>),
+    Return(Box<Spanned<Expression<'a>>>),
     Syscall(Spanned<SysCall>),
     Variable(Spanned<String>),
-    While(Spanned<WhileExpression>),
+    While(Spanned<WhileExpression<'a>>),
 }
 
-impl std::fmt::Display for Expression {
+impl<'a> std::fmt::Display for Expression<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Expression::Assignment(e) => write!(f, "{}", e),

rust_compiler/libs/tokenizer/src/lib.rs

@@ -38,7 +38,7 @@ pub trait Tokenize: Read + Seek {}
 impl<T> Tokenize for T where T: Read + Seek {}
 
 pub struct Tokenizer<'a> {
-    lexer: Lexer<'a, TokenType>,
+    lexer: Lexer<'a, TokenType<'a>>,
     returned_eof: bool,
 }
 
@@ -52,14 +52,14 @@ impl<'a> From<&'a str> for Tokenizer<'a> {
 }
 
 impl<'a> Tokenizer<'a> {
-    fn get_token(&mut self, t_type: TokenType) -> Token {
+    fn get_token(&mut self, t_type: TokenType<'a>) -> Token<'a> {
         let mut span = self.lexer.span();
         span.start -= self.lexer.extras.line_start_index;
         span.end -= self.lexer.extras.line_start_index;
         Token::new(t_type, self.lexer.extras.line_count, span)
     }
 
-    pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
+    pub fn next_token(&mut self) -> Result<Option<Token<'a>>, Error> {
         let mut current = self.lexer.next().transpose();
 
         while matches!(current, Ok(Some(TokenType::Comment(_)))) {
@@ -73,7 +73,7 @@ impl<'a> Tokenizer<'a> {
 // ... Iterator and TokenizerBuffer implementations remain unchanged ...
 // They just call the methods above which now use the passed-in start coordinates.
 impl<'a> Iterator for Tokenizer<'a> {
-    type Item = Result<Token, Error>;
+    type Item = Result<Token<'a>, Error>;
     fn next(&mut self) -> Option<Self::Item> {
         match self.lexer.next() {
             None => {
@@ -98,8 +98,8 @@ impl<'a> Iterator for Tokenizer<'a> {
 
 pub struct TokenizerBuffer<'a> {
     tokenizer: Tokenizer<'a>,
-    buffer: VecDeque<Token>,
-    history: VecDeque<Token>,
+    buffer: VecDeque<Token<'a>>,
+    history: VecDeque<Token<'a>>,
     index: i64,
 }
 
@@ -112,7 +112,7 @@ impl<'a> TokenizerBuffer<'a> {
             index: 0,
         }
     }
-    pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
+    pub fn next_token(&mut self) -> Result<Option<Token<'a>>, Error> {
         if let Some(token) = self.buffer.pop_front() {
             self.history.push_back(token.clone());
             self.index += 1;
@@ -127,7 +127,7 @@ impl<'a> TokenizerBuffer<'a> {
         self.index += 1;
         Ok(token)
     }
-    pub fn peek(&mut self) -> Result<Option<Token>, Error> {
+    pub fn peek(&mut self) -> Result<Option<Token<'a>>, Error> {
         if let Some(token) = self.buffer.front() {
             return Ok(Some(token.clone()));
         }

rust_compiler/libs/tokenizer/src/token.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
 use helpers::prelude::*;
 use logos::{Lexer, Logos, Skip, Span};
 use lsp_types::{Diagnostic, DiagnosticSeverity, Position, Range};
@@ -43,7 +45,7 @@ impl From<LexError> for Diagnostic {
 }
 
 impl LexError {
-    pub fn from_lexer<'a>(lex: &mut Lexer<'a, TokenType>) -> Self {
+    pub fn from_lexer<'a>(lex: &mut Lexer<'a, TokenType<'a>>) -> Self {
         let mut span = lex.span();
         let line = lex.extras.line_count;
         span.start -= lex.extras.line_start_index;
@@ -68,30 +70,30 @@ pub struct Extras {
     pub line_start_index: usize,
 }
 
-fn update_line_index<'a>(lex: &mut Lexer<'a, TokenType>) -> Skip {
+fn update_line_index<'a>(lex: &mut Lexer<'a, TokenType<'a>>) -> Skip {
     lex.extras.line_count += 1;
     lex.extras.line_start_index = lex.span().end;
     Skip
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
-pub struct Token {
+pub struct Token<'a> {
     /// The type of the token
-    pub token_type: TokenType,
+    pub token_type: TokenType<'a>,
     /// The line where the token was found
     pub line: usize,
     /// The span where the token starts and ends
     pub span: Span,
 }
 
-impl std::fmt::Display for Token {
+impl<'a> std::fmt::Display for Token<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.token_type)
     }
 }
 
-impl Token {
-    pub fn new(token_type: TokenType, line: usize, span: Span) -> Self {
+impl<'a> Token<'a> {
+    pub fn new(token_type: TokenType<'a>, line: usize, span: Span) -> Self {
         Self {
             token_type,
             line,
@@ -158,22 +160,22 @@ macro_rules! keyword {
 #[logos(skip r"[ \t\f]+")]
 #[logos(extras = Extras)]
 #[logos(error(LexError, LexError::from_lexer))]
-pub enum TokenType {
+pub enum TokenType<'a> {
     #[regex(r"\n", update_line_index)]
     Newline,
 
     // matches strings with double quotes
     #[regex(r#""(?:[^"\\]|\\.)*""#, |v| {
         let str = v.slice();
-        str[1..str.len() - 1].to_string()
+        Cow::from(&str[1..str.len() - 1])
     })]
     // matches strings with single quotes
     #[regex(r#"'(?:[^'\\]|\\.)*'"#, |v| {
         let str = v.slice();
-        str[1..str.len() - 1].to_string()
+        Cow::from(&str[1..str.len() - 1])
     })]
     /// Represents a string token
-    String(String),
+    String(Cow<'a, str>),
 
     #[regex(r"[0-9][0-9_]*(\.[0-9][0-9_]*)?([cfk])?", parse_number)]
     /// Represents a number token
@@ -199,9 +201,9 @@ pub enum TokenType {
     /// Represents a keyword token
     Keyword(Keyword),
 
-    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |v| v.slice().to_string())]
+    #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |v| Cow::from(v.slice()))]
     /// Represents an identifier token
-    Identifier(String),
+    Identifier(Cow<'a, str>),
 
     #[token("(", symbol!(LParen))]
     #[token(")", symbol!(RParen))]
@@ -236,29 +238,29 @@ pub enum TokenType {
     #[token("//", |lex| Comment::Line(read_line(lex)))]
     #[token("///", |lex| Comment::Doc(read_line(lex)))]
     /// Represents a comment, both a line comment and a doc comment
-    Comment(Comment),
+    Comment(Comment<'a>),
 
     #[end]
     /// Represents an end of file token
     EOF,
 }
 
-fn read_line<'a>(lexer: &mut Lexer<'a, TokenType>) -> String {
+fn read_line<'a>(lexer: &mut Lexer<'a, TokenType<'a>>) -> Cow<'a, str> {
     let rem = lexer.remainder();
     let len = rem.find('\n').unwrap_or(rem.len());
     let content = rem[..len].trim().to_string();
 
     lexer.bump(len);
-    content
+    Cow::from(content)
 }
 
 #[derive(Hash, Debug, Eq, PartialEq, Clone)]
-pub enum Comment {
-    Line(String),
-    Doc(String),
+pub enum Comment<'a> {
+    Line(Cow<'a, str>),
+    Doc(Cow<'a, str>),
 }
 
-fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType>) -> Result<Number, LexError> {
+fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType<'a>>) -> Result<Number, LexError> {
     let slice = lexer.slice();
     let last_char = slice.chars().last().unwrap_or_default();
     let (num_str, suffix) = match last_char {
@@ -304,7 +306,7 @@ fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType>) -> Result<Number, LexError
     }
 }
 
-impl std::fmt::Display for Comment {
+impl<'a> std::fmt::Display for Comment<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Line(c) => write!(f, "// {}", c),
@@ -321,7 +323,7 @@ impl std::fmt::Display for Comment {
     }
 }
 
-impl Documentation for TokenType {
+impl<'a> Documentation for TokenType<'a> {
     fn docs(&self) -> String {
         match self {
             Self::Keyword(k) => k.docs(),
@@ -336,7 +338,7 @@ impl Documentation for TokenType {
 
 helpers::with_syscalls!(generate_check);
 
-impl From<TokenType> for u32 {
+impl<'a> From<TokenType<'a>> for u32 {
     fn from(value: TokenType) -> Self {
         match value {
             TokenType::String(_) => 1,
@@ -376,7 +378,7 @@ impl From<TokenType> for u32 {
     }
 }
 
-impl std::fmt::Display for TokenType {
+impl<'a> std::fmt::Display for TokenType<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             TokenType::String(s) => write!(f, "{}", s),