dbidwell/stationeers_lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

TEST -- use Cow instead of String for tokens

Browse Source
This commit is contained in:
Devin Bidwell
2025-12-09 13:17:35 -07:00
parent 080b5320f7
commit d40b759442
4 changed files with 140 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
rust_compiler/libs/parser/src/lib.rs
View File

@@ -26,27 +26,27 @@ macro_rules! boxed {
} }
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum Error { pub enum Error<'a> {
#[error(transparent)] #[error(transparent)]
Tokenizer(#[from] tokenizer::Error), Tokenizer(#[from] tokenizer::Error),
#[error("Unexpected token: {1}")] #[error("Unexpected token: {1}")]
UnexpectedToken(Span, Token), UnexpectedToken(Span, Token<'a>),
#[error("Duplicate identifier: {1}")] #[error("Duplicate identifier: {1}")]
DuplicateIdentifier(Span, Token), DuplicateIdentifier(Span, Token<'a>),
#[error("Invalid Syntax: {1}")] #[error("Invalid Syntax: {1}")]
InvalidSyntax(Span, String), InvalidSyntax(Span, String),
#[error("Unsupported Keyword: {1}")] #[error("Unsupported Keyword: {1}")]
UnsupportedKeyword(Span, Token), UnsupportedKeyword(Span, Token<'a>),
#[error("Unexpected End of File")] #[error("Unexpected End of File")]
UnexpectedEOF, UnexpectedEOF,
} }
impl From<Error> for lsp_types::Diagnostic { impl<'a> From<Error<'a>> for lsp_types::Diagnostic {
fn from(value: Error) -> Self { fn from(value: Error) -> Self {
use Error::*; use Error::*;
use lsp_types::*; use lsp_types::*;
@@ -105,8 +105,8 @@ macro_rules! self_matches_current {
pub struct Parser<'a> { pub struct Parser<'a> {
tokenizer: TokenizerBuffer<'a>, tokenizer: TokenizerBuffer<'a>,
current_token: Option<Token>, current_token: Option<Token<'a>>,
pub errors: Vec<Error>, pub errors: Vec<Error<'a>>,
} }
impl<'a> Parser<'a> { impl<'a> Parser<'a> {
@@ -119,7 +119,7 @@ impl<'a> Parser<'a> {
} }
/// Calculates a Span from a given Token reference. /// Calculates a Span from a given Token reference.
fn token_to_span(t: &Token) -> Span { fn token_to_span<'t>(t: &'t Token<'a>) -> Span {
Span { Span {
start_line: t.line, start_line: t.line,
start_col: t.span.start, start_col: t.span.start,
@@ -269,14 +269,14 @@ impl<'a> Parser<'a> {
Ok(expr) Ok(expr)
} }
fn assign_next(&mut self) -> Result<(), Error> { fn assign_next(&'a mut self) -> Result<(), Error> {
self.current_token = self.tokenizer.next_token()?; self.current_token = self.tokenizer.next_token()?;
Ok(()) Ok(())
} }
fn get_next(&mut self) -> Result<Option<&Token>, Error> { fn get_next(&'a mut self) -> Result<Option<Token>, Error> {
self.assign_next()?; self.assign_next()?;
Ok(self.current_token.as_ref()) Ok(self.current_token.clone())
} }
fn expression(&mut self) -> Result<Option<Spanned<tree_node::Expression>>, Error> { fn expression(&mut self) -> Result<Option<Spanned<tree_node::Expression>>, Error> {

190
rust_compiler/libs/parser/src/tree_node.rs
View File

@@ -1,22 +1,22 @@
use super::sys_call::SysCall; use super::sys_call::SysCall;
use crate::sys_call; use crate::sys_call;
use std::ops::Deref; use std::{borrow::Cow, ops::Deref};
use tokenizer::token::Number; use tokenizer::token::Number;
#[derive(Debug, Eq, PartialEq, Clone)] #[derive(Debug, Eq, PartialEq, Clone)]
pub enum Literal { pub enum Literal<'a> {
Number(Number), Number(Number),
String(String), String(Cow<'a, str>),
Boolean(bool), Boolean(bool),
} }
#[derive(Debug, Eq, PartialEq, Clone)] #[derive(Debug, Eq, PartialEq, Clone)]
pub enum LiteralOr<T> { pub enum LiteralOr<'a, T> {
Literal(Spanned<Literal>), Literal(Spanned<Literal<'a>>),
Or(Spanned<T>), Or(Spanned<T>),
} }
impl<T: std::fmt::Display> std::fmt::Display for LiteralOr<T> { impl<'a, T: std::fmt::Display> std::fmt::Display for LiteralOr<'a, T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Self::Literal(l) => write!(f, "{l}"), Self::Literal(l) => write!(f, "{l}"),
@@ -25,7 +25,7 @@ impl<T: std::fmt::Display> std::fmt::Display for LiteralOr<T> {
} }
} }
impl std::fmt::Display for Literal { impl<'a> std::fmt::Display for Literal<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Literal::Number(n) => write!(f, "{}", n), Literal::Number(n) => write!(f, "{}", n),
@@ -36,16 +36,16 @@ impl std::fmt::Display for Literal {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum BinaryExpression { pub enum BinaryExpression<'a> {
Add(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Add(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Multiply(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Multiply(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Divide(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Divide(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Subtract(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Subtract(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Exponent(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Exponent(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Modulo(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Modulo(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
} }
impl std::fmt::Display for BinaryExpression { impl<'a> std::fmt::Display for BinaryExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
BinaryExpression::Add(l, r) => write!(f, "({} + {})", l, r), BinaryExpression::Add(l, r) => write!(f, "({} + {})", l, r),
@@ -59,19 +59,19 @@ impl std::fmt::Display for BinaryExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum LogicalExpression { pub enum LogicalExpression<'a> {
And(Box<Spanned<Expression>>, Box<Spanned<Expression>>), And(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Or(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Or(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
Not(Box<Spanned<Expression>>), Not(Box<Spanned<Expression<'a>>>),
Equal(Box<Spanned<Expression>>, Box<Spanned<Expression>>), Equal(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
NotEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>), NotEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
GreaterThan(Box<Spanned<Expression>>, Box<Spanned<Expression>>), GreaterThan(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
GreaterThanOrEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>), GreaterThanOrEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
LessThan(Box<Spanned<Expression>>, Box<Spanned<Expression>>), LessThan(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
LessThanOrEqual(Box<Spanned<Expression>>, Box<Spanned<Expression>>), LessThanOrEqual(Box<Spanned<Expression<'a>>>, Box<Spanned<Expression<'a>>>),
} }
impl std::fmt::Display for LogicalExpression { impl<'a> std::fmt::Display for LogicalExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
LogicalExpression::And(l, r) => write!(f, "({} && {})", l, r), LogicalExpression::And(l, r) => write!(f, "({} && {})", l, r),
@@ -88,25 +88,25 @@ impl std::fmt::Display for LogicalExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct AssignmentExpression { pub struct AssignmentExpression<'a> {
pub assignee: Box<Spanned<Expression>>, pub assignee: Box<Spanned<Expression<'a>>>,
pub expression: Box<Spanned<Expression>>, pub expression: Box<Spanned<Expression<'a>>>,
} }
impl std::fmt::Display for AssignmentExpression { impl<'a> std::fmt::Display for AssignmentExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "({} = {})", self.assignee, self.expression) write!(f, "({} = {})", self.assignee, self.expression)
} }
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct FunctionExpression { pub struct FunctionExpression<'a> {
pub name: Spanned<String>, pub name: Spanned<Cow<'a, str>>,
pub arguments: Vec<Spanned<String>>, pub arguments: Vec<Spanned<Cow<'a, str>>>,
pub body: BlockExpression, pub body: BlockExpression<'a>,
} }
impl std::fmt::Display for FunctionExpression { impl<'a> std::fmt::Display for FunctionExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!( write!(
f, f,
@@ -123,9 +123,9 @@ impl std::fmt::Display for FunctionExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct BlockExpression(pub Vec<Spanned<Expression>>); pub struct BlockExpression<'a>(pub Vec<Spanned<Expression<'a>>>);
impl std::fmt::Display for BlockExpression { impl<'a> std::fmt::Display for BlockExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!( write!(
f, f,
@@ -140,12 +140,12 @@ impl std::fmt::Display for BlockExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct InvocationExpression { pub struct InvocationExpression<'a> {
pub name: Spanned<String>, pub name: Spanned<Cow<'a, str>>,
pub arguments: Vec<Spanned<Expression>>, pub arguments: Vec<Spanned<Expression<'a>>>,
} }
impl std::fmt::Display for InvocationExpression { impl<'a> std::fmt::Display for InvocationExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!( write!(
f, f,
@@ -161,25 +161,25 @@ impl std::fmt::Display for InvocationExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct MemberAccessExpression { pub struct MemberAccessExpression<'a> {
pub object: Box<Spanned<Expression>>, pub object: Box<Spanned<Expression<'a>>>,
pub member: Spanned<String>, pub member: Spanned<Cow<'a, str>>,
} }
impl std::fmt::Display for MemberAccessExpression { impl<'a> std::fmt::Display for MemberAccessExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.object, self.member) write!(f, "{}.{}", self.object, self.member)
} }
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct MethodCallExpression { pub struct MethodCallExpression<'a> {
pub object: Box<Spanned<Expression>>, pub object: Box<Spanned<Expression<'a>>>,
pub method: Spanned<String>, pub method: Spanned<Cow<'a, str>>,
pub arguments: Vec<Spanned<Expression>>, pub arguments: Vec<Spanned<Expression<'a>>>,
} }
impl std::fmt::Display for MethodCallExpression { impl<'a> std::fmt::Display for MethodCallExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!( write!(
f, f,
@@ -196,12 +196,12 @@ impl std::fmt::Display for MethodCallExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum LiteralOrVariable { pub enum LiteralOrVariable<'a> {
Literal(Literal), Literal(Literal<'a>),
Variable(Spanned<String>), Variable(Spanned<Cow<'a, str>>),
} }
impl std::fmt::Display for LiteralOrVariable { impl<'a> std::fmt::Display for LiteralOrVariable<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
LiteralOrVariable::Literal(l) => write!(f, "{}", l), LiteralOrVariable::Literal(l) => write!(f, "{}", l),
@@ -211,46 +211,46 @@ impl std::fmt::Display for LiteralOrVariable {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct ConstDeclarationExpression { pub struct ConstDeclarationExpression<'a> {
pub name: Spanned<String>, pub name: Spanned<Cow<'a, str>>,
pub value: LiteralOr<SysCall>, pub value: LiteralOr<'a, SysCall>,
} }
impl ConstDeclarationExpression { impl<'a> ConstDeclarationExpression<'a> {
pub fn is_syscall_supported(call: &SysCall) -> bool { pub fn is_syscall_supported(call: &SysCall) -> bool {
use sys_call::System; use sys_call::System;
matches!(call, SysCall::System(sys) if matches!(sys, System::Hash(_))) matches!(call, SysCall::System(sys) if matches!(sys, System::Hash(_)))
} }
} }
impl std::fmt::Display for ConstDeclarationExpression { impl<'a> std::fmt::Display for ConstDeclarationExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(const {} = {})", self.name, self.value) write!(f, "(const {} = {})", self.name, self.value)
} }
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct DeviceDeclarationExpression { pub struct DeviceDeclarationExpression<'a> {
/// any variable-like name /// any variable-like name
pub name: Spanned<String>, pub name: Spanned<Cow<'a, str>>,
/// The device port, ex. (db, d0, d1, d2, d3, d4, d5) /// The device port, ex. (db, d0, d1, d2, d3, d4, d5)
pub device: String, pub device: Cow<'a, str>,
} }
impl std::fmt::Display for DeviceDeclarationExpression { impl<'a> std::fmt::Display for DeviceDeclarationExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(device {} = {})", self.name, self.device) write!(f, "(device {} = {})", self.name, self.device)
} }
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct IfExpression { pub struct IfExpression<'a> {
pub condition: Box<Spanned<Expression>>, pub condition: Box<Spanned<Expression<'a>>>,
pub body: Spanned<BlockExpression>, pub body: Spanned<BlockExpression<'a>>,
pub else_branch: Option<Box<Spanned<Expression>>>, pub else_branch: Option<Box<Spanned<Expression<'a>>>>,
} }
impl std::fmt::Display for IfExpression { impl<'a> std::fmt::Display for IfExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(if ({}) {}", self.condition, self.body)?; write!(f, "(if ({}) {}", self.condition, self.body)?;
if let Some(else_branch) = &self.else_branch { if let Some(else_branch) = &self.else_branch {
@@ -261,23 +261,23 @@ impl std::fmt::Display for IfExpression {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct LoopExpression { pub struct LoopExpression<'a> {
pub body: Spanned<BlockExpression>, pub body: Spanned<BlockExpression<'a>>,
} }
impl std::fmt::Display for LoopExpression { impl<'a> std::fmt::Display for LoopExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(loop {})", self.body) write!(f, "(loop {})", self.body)
} }
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct WhileExpression { pub struct WhileExpression<'a> {
pub condition: Box<Spanned<Expression>>, pub condition: Box<Spanned<Expression<'a>>>,
pub body: BlockExpression, pub body: BlockExpression<'a>,
} }
impl std::fmt::Display for WhileExpression { impl<'a> std::fmt::Display for WhileExpression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(while {} {})", self.condition, self.body) write!(f, "(while {} {})", self.condition, self.body)
} }
@@ -345,32 +345,32 @@ impl<T> Deref for Spanned<T> {
} }
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum Expression { pub enum Expression<'a> {
Assignment(Spanned<AssignmentExpression>), Assignment(Spanned<AssignmentExpression<'a>>),
Binary(Spanned<BinaryExpression>), Binary(Spanned<BinaryExpression<'a>>),
Block(Spanned<BlockExpression>), Block(Spanned<BlockExpression<'a>>),
Break(Span), Break(Span),
ConstDeclaration(Spanned<ConstDeclarationExpression>), ConstDeclaration(Spanned<ConstDeclarationExpression<'a>>),
Continue(Span), Continue(Span),
Declaration(Spanned<String>, Box<Spanned<Expression>>), Declaration(Spanned<Cow<'a, str>>, Box<Spanned<Expression<'a>>>),
DeviceDeclaration(Spanned<DeviceDeclarationExpression>), DeviceDeclaration(Spanned<DeviceDeclarationExpression<'a>>),
Function(Spanned<FunctionExpression>), Function(Spanned<FunctionExpression<'a>>),
If(Spanned<IfExpression>), If(Spanned<IfExpression<'a>>),
Invocation(Spanned<InvocationExpression>), Invocation(Spanned<InvocationExpression<'a>>),
Literal(Spanned<Literal>), Literal(Spanned<Literal<'a>>),
Logical(Spanned<LogicalExpression>), Logical(Spanned<LogicalExpression<'a>>),
Loop(Spanned<LoopExpression>), Loop(Spanned<LoopExpression<'a>>),
MemberAccess(Spanned<MemberAccessExpression>), MemberAccess(Spanned<MemberAccessExpression<'a>>),
MethodCall(Spanned<MethodCallExpression>), MethodCall(Spanned<MethodCallExpression<'a>>),
Negation(Box<Spanned<Expression>>), Negation(Box<Spanned<Expression<'a>>>),
Priority(Box<Spanned<Expression>>), Priority(Box<Spanned<Expression<'a>>>),
Return(Box<Spanned<Expression>>), Return(Box<Spanned<Expression<'a>>>),
Syscall(Spanned<SysCall>), Syscall(Spanned<SysCall>),
Variable(Spanned<String>), Variable(Spanned<String>),
While(Spanned<WhileExpression>), While(Spanned<WhileExpression<'a>>),
} }
impl std::fmt::Display for Expression { impl<'a> std::fmt::Display for Expression<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Expression::Assignment(e) => write!(f, "{}", e), Expression::Assignment(e) => write!(f, "{}", e),

16
rust_compiler/libs/tokenizer/src/lib.rs
View File

@@ -38,7 +38,7 @@ pub trait Tokenize: Read + Seek {}
impl<T> Tokenize for T where T: Read + Seek {} impl<T> Tokenize for T where T: Read + Seek {}
pub struct Tokenizer<'a> { pub struct Tokenizer<'a> {
lexer: Lexer<'a, TokenType>, lexer: Lexer<'a, TokenType<'a>>,
returned_eof: bool, returned_eof: bool,
} }
@@ -52,14 +52,14 @@ impl<'a> From<&'a str> for Tokenizer<'a> {
} }
impl<'a> Tokenizer<'a> { impl<'a> Tokenizer<'a> {
fn get_token(&mut self, t_type: TokenType) -> Token { fn get_token(&mut self, t_type: TokenType<'a>) -> Token<'a> {
let mut span = self.lexer.span(); let mut span = self.lexer.span();
span.start -= self.lexer.extras.line_start_index; span.start -= self.lexer.extras.line_start_index;
span.end -= self.lexer.extras.line_start_index; span.end -= self.lexer.extras.line_start_index;
Token::new(t_type, self.lexer.extras.line_count, span) Token::new(t_type, self.lexer.extras.line_count, span)
} }
pub fn next_token(&mut self) -> Result<Option<Token>, Error> { pub fn next_token(&mut self) -> Result<Option<Token<'a>>, Error> {
let mut current = self.lexer.next().transpose(); let mut current = self.lexer.next().transpose();
while matches!(current, Ok(Some(TokenType::Comment(_)))) { while matches!(current, Ok(Some(TokenType::Comment(_)))) {
@@ -73,7 +73,7 @@ impl<'a> Tokenizer<'a> {
// ... Iterator and TokenizerBuffer implementations remain unchanged ... // ... Iterator and TokenizerBuffer implementations remain unchanged ...
// They just call the methods above which now use the passed-in start coordinates. // They just call the methods above which now use the passed-in start coordinates.
impl<'a> Iterator for Tokenizer<'a> { impl<'a> Iterator for Tokenizer<'a> {
type Item = Result<Token, Error>; type Item = Result<Token<'a>, Error>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
match self.lexer.next() { match self.lexer.next() {
None => { None => {
@@ -98,8 +98,8 @@ impl<'a> Iterator for Tokenizer<'a> {
pub struct TokenizerBuffer<'a> { pub struct TokenizerBuffer<'a> {
tokenizer: Tokenizer<'a>, tokenizer: Tokenizer<'a>,
buffer: VecDeque<Token>, buffer: VecDeque<Token<'a>>,
history: VecDeque<Token>, history: VecDeque<Token<'a>>,
index: i64, index: i64,
} }
@@ -112,7 +112,7 @@ impl<'a> TokenizerBuffer<'a> {
index: 0, index: 0,
} }
} }
pub fn next_token(&mut self) -> Result<Option<Token>, Error> { pub fn next_token(&mut self) -> Result<Option<Token<'a>>, Error> {
if let Some(token) = self.buffer.pop_front() { if let Some(token) = self.buffer.pop_front() {
self.history.push_back(token.clone()); self.history.push_back(token.clone());
self.index += 1; self.index += 1;
@@ -127,7 +127,7 @@ impl<'a> TokenizerBuffer<'a> {
self.index += 1; self.index += 1;
Ok(token) Ok(token)
} }
pub fn peek(&mut self) -> Result<Option<Token>, Error> { pub fn peek(&mut self) -> Result<Option<Token<'a>>, Error> {
if let Some(token) = self.buffer.front() { if let Some(token) = self.buffer.front() {
return Ok(Some(token.clone())); return Ok(Some(token.clone()));
} }

50
rust_compiler/libs/tokenizer/src/token.rs
View File

@@ -1,3 +1,5 @@
use std::borrow::Cow;
use helpers::prelude::*; use helpers::prelude::*;
use logos::{Lexer, Logos, Skip, Span}; use logos::{Lexer, Logos, Skip, Span};
use lsp_types::{Diagnostic, DiagnosticSeverity, Position, Range}; use lsp_types::{Diagnostic, DiagnosticSeverity, Position, Range};
@@ -43,7 +45,7 @@ impl From<LexError> for Diagnostic {
} }
impl LexError { impl LexError {
pub fn from_lexer<'a>(lex: &mut Lexer<'a, TokenType>) -> Self { pub fn from_lexer<'a>(lex: &mut Lexer<'a, TokenType<'a>>) -> Self {
let mut span = lex.span(); let mut span = lex.span();
let line = lex.extras.line_count; let line = lex.extras.line_count;
span.start -= lex.extras.line_start_index; span.start -= lex.extras.line_start_index;
@@ -68,30 +70,30 @@ pub struct Extras {
pub line_start_index: usize, pub line_start_index: usize,
} }
fn update_line_index<'a>(lex: &mut Lexer<'a, TokenType>) -> Skip { fn update_line_index<'a>(lex: &mut Lexer<'a, TokenType<'a>>) -> Skip {
lex.extras.line_count += 1; lex.extras.line_count += 1;
lex.extras.line_start_index = lex.span().end; lex.extras.line_start_index = lex.span().end;
Skip Skip
} }
#[derive(Debug, PartialEq, Eq, Clone)] #[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token { pub struct Token<'a> {
/// The type of the token /// The type of the token
pub token_type: TokenType, pub token_type: TokenType<'a>,
/// The line where the token was found /// The line where the token was found
pub line: usize, pub line: usize,
/// The span where the token starts and ends /// The span where the token starts and ends
pub span: Span, pub span: Span,
} }
impl std::fmt::Display for Token { impl<'a> std::fmt::Display for Token<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.token_type) write!(f, "{}", self.token_type)
} }
} }
impl Token { impl<'a> Token<'a> {
pub fn new(token_type: TokenType, line: usize, span: Span) -> Self { pub fn new(token_type: TokenType<'a>, line: usize, span: Span) -> Self {
Self { Self {
token_type, token_type,
line, line,
@@ -158,22 +160,22 @@ macro_rules! keyword {
#[logos(skip r"[ \t\f]+")] #[logos(skip r"[ \t\f]+")]
#[logos(extras = Extras)] #[logos(extras = Extras)]
#[logos(error(LexError, LexError::from_lexer))] #[logos(error(LexError, LexError::from_lexer))]
pub enum TokenType { pub enum TokenType<'a> {
#[regex(r"\n", update_line_index)] #[regex(r"\n", update_line_index)]
Newline, Newline,
// matches strings with double quotes // matches strings with double quotes
#[regex(r#""(?:[^"\\]|\\.)*""#, |v| { #[regex(r#""(?:[^"\\]|\\.)*""#, |v| {
let str = v.slice(); let str = v.slice();
str[1..str.len() - 1].to_string() Cow::from(&str[1..str.len() - 1])
})] })]
// matches strings with single quotes // matches strings with single quotes
#[regex(r#"'(?:[^'\\]|\\.)*'"#, |v| { #[regex(r#"'(?:[^'\\]|\\.)*'"#, |v| {
let str = v.slice(); let str = v.slice();
str[1..str.len() - 1].to_string() Cow::from(&str[1..str.len() - 1])
})] })]
/// Represents a string token /// Represents a string token
String(String), String(Cow<'a, str>),
#[regex(r"[0-9][0-9_]*(\.[0-9][0-9_]*)?([cfk])?", parse_number)] #[regex(r"[0-9][0-9_]*(\.[0-9][0-9_]*)?([cfk])?", parse_number)]
/// Represents a number token /// Represents a number token
@@ -199,9 +201,9 @@ pub enum TokenType {
/// Represents a keyword token /// Represents a keyword token
Keyword(Keyword), Keyword(Keyword),
#[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |v| v.slice().to_string())] #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*", |v| Cow::from(v.slice()))]
/// Represents an identifier token /// Represents an identifier token
Identifier(String), Identifier(Cow<'a, str>),
#[token("(", symbol!(LParen))] #[token("(", symbol!(LParen))]
#[token(")", symbol!(RParen))] #[token(")", symbol!(RParen))]
@@ -236,29 +238,29 @@ pub enum TokenType {
#[token("//", |lex| Comment::Line(read_line(lex)))] #[token("//", |lex| Comment::Line(read_line(lex)))]
#[token("///", |lex| Comment::Doc(read_line(lex)))] #[token("///", |lex| Comment::Doc(read_line(lex)))]
/// Represents a comment, both a line comment and a doc comment /// Represents a comment, both a line comment and a doc comment
Comment(Comment), Comment(Comment<'a>),
#[end] #[end]
/// Represents an end of file token /// Represents an end of file token
EOF, EOF,
} }
fn read_line<'a>(lexer: &mut Lexer<'a, TokenType>) -> String { fn read_line<'a>(lexer: &mut Lexer<'a, TokenType<'a>>) -> Cow<'a, str> {
let rem = lexer.remainder(); let rem = lexer.remainder();
let len = rem.find('\n').unwrap_or(rem.len()); let len = rem.find('\n').unwrap_or(rem.len());
let content = rem[..len].trim().to_string(); let content = rem[..len].trim().to_string();
lexer.bump(len); lexer.bump(len);
content Cow::from(content)
} }
#[derive(Hash, Debug, Eq, PartialEq, Clone)] #[derive(Hash, Debug, Eq, PartialEq, Clone)]
pub enum Comment { pub enum Comment<'a> {
Line(String), Line(Cow<'a, str>),
Doc(String), Doc(Cow<'a, str>),
} }
fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType>) -> Result<Number, LexError> { fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType<'a>>) -> Result<Number, LexError> {
let slice = lexer.slice(); let slice = lexer.slice();
let last_char = slice.chars().last().unwrap_or_default(); let last_char = slice.chars().last().unwrap_or_default();
let (num_str, suffix) = match last_char { let (num_str, suffix) = match last_char {
@@ -304,7 +306,7 @@ fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType>) -> Result<Number, LexError
} }
} }
impl std::fmt::Display for Comment { impl<'a> std::fmt::Display for Comment<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
Self::Line(c) => write!(f, "// {}", c), Self::Line(c) => write!(f, "// {}", c),
@@ -321,7 +323,7 @@ impl std::fmt::Display for Comment {
} }
} }
impl Documentation for TokenType { impl<'a> Documentation for TokenType<'a> {
fn docs(&self) -> String { fn docs(&self) -> String {
match self { match self {
Self::Keyword(k) => k.docs(), Self::Keyword(k) => k.docs(),
@@ -336,7 +338,7 @@ impl Documentation for TokenType {
helpers::with_syscalls!(generate_check); helpers::with_syscalls!(generate_check);
impl From<TokenType> for u32 { impl<'a> From<TokenType<'a>> for u32 {
fn from(value: TokenType) -> Self { fn from(value: TokenType) -> Self {
match value { match value {
TokenType::String(_) => 1, TokenType::String(_) => 1,
@@ -376,7 +378,7 @@ impl From<TokenType> for u32 {
} }
} }
impl std::fmt::Display for TokenType { impl<'a> std::fmt::Display for TokenType<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
TokenType::String(s) => write!(f, "{}", s), TokenType::String(s) => write!(f, "{}", s),