dbidwell/stationeers_lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Unified the C# mod and the Rust compiler into a monorepo

Browse Source
This commit is contained in:
Devin Bidwell
2025-11-26 16:02:00 -07:00
parent 346b6e49e6
commit 353dc16944
34 changed files with 253 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
rust_compiler/libs/compiler/Cargo.toml Normal file
View File

@@ -0,0 +1,14 @@
[package]
name = "compiler"
version = "0.1.0"
edition = "2024"
[dependencies]
quick-error = { workspace = true }
parser = { path = "../parser" }
tokenizer = { path = "../tokenizer" }
[dev-dependencies]
anyhow = { version = "1.0" }
indoc = { version = "2.0" }
pretty_assertions = "1"

6
rust_compiler/libs/compiler/src/lib.rs Normal file
View File

@@ -0,0 +1,6 @@
#[cfg(test)]
mod test;
mod v1;
mod variable_manager;
pub use v1::{Compiler, CompilerConfig, Error};

100
rust_compiler/libs/compiler/src/test/binary_expression.rs Normal file
View File

@@ -0,0 +1,100 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn simple_binary_expression() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let i = 1 + 2;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
add r1 1 2
move r8 r1 #i
"
}
);
Ok(())
}
#[test]
fn nested_binary_expressions() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn calculateArgs(arg1, arg2, arg3) {
return (arg1 + arg2) * arg3;
};
let returned = calculateArgs(10, 20, 30) + 100;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
calculateArgs:
pop r8 #arg3
pop r9 #arg2
pop r10 #arg1
push ra
add r1 r10 r9
mul r2 r1 r8
move r15 r2
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
push 10
push 20
push 30
jal calculateArgs
move r1 r15 #__binary_temp_3
add r2 r1 100
move r8 r2 #returned
"
}
);
Ok(())
}
#[test]
fn stress_test_negation_with_stack_spillover() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let negationHell = (-1 + -2) * (-3 + (-4 * (-5 + -6)));
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
add r1 -1 -2
add r2 -5 -6
mul r3 -4 r2
add r4 -3 r3
mul r5 r1 r4
move r8 r5 #negationHell
"
}
);
Ok(())
}

158
rust_compiler/libs/compiler/src/test/branching.rs Normal file
View File

@@ -0,0 +1,158 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn test_if_statement() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 10;
if (a > 5) {
a = 20;
}
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 10 #a
sgt r1 r8 5
beq r1 0 L1
move r8 20 #a
L1:
"
}
);
Ok(())
}
#[test]
fn test_if_else_statement() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 0;
if (10 > 5) {
a = 1;
} else {
a = 2;
}
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
sgt r1 10 5
beq r1 0 L2
move r8 1 #a
j L1
L2:
move r8 2 #a
L1:
"
}
);
Ok(())
}
#[test]
fn test_if_else_if_statement() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 0;
if (a == 1) {
a = 10;
} else if (a == 2) {
a = 20;
} else {
a = 30;
}
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
seq r1 r8 1
beq r1 0 L2
move r8 10 #a
j L1
L2:
seq r2 r8 2
beq r2 0 L4
move r8 20 #a
j L3
L4:
move r8 30 #a
L3:
L1:
"
}
);
Ok(())
}
#[test]
fn test_spilled_variable_update_in_branch() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 1;
let b = 2;
let c = 3;
let d = 4;
let e = 5;
let f = 6;
let g = 7;
let h = 8; // Spilled to stack (offset 0)
if (a == 1) {
h = 99;
}
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 1 #a
move r9 2 #b
move r10 3 #c
move r11 4 #d
move r12 5 #e
move r13 6 #f
move r14 7 #g
push 8 #h
seq r1 r8 1
beq r1 0 L1
sub r0 sp 1
put db r0 99 #h
L1:
"
}
);
Ok(())
}

243
rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs Normal file
View File

@@ -0,0 +1,243 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn no_arguments() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn doSomething() {};
let i = doSomething();
"
};
let to_test = indoc! {
"
j main
doSomething:
push ra
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
jal doSomething
move r8 r15 #i
"
};
assert_eq!(compiled, to_test);
Ok(())
}
#[test]
fn let_var_args() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn doSomething(arg1) {};
let arg1 = 123;
let i = doSomething(arg1);
"
};
assert_eq!(
compiled,
indoc! {
"
j main
doSomething:
pop r8 #arg1
push ra
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
move r8 123 #arg1
push r8
push r8
jal doSomething
sub r0 sp 1
get r8 db r0
sub sp sp 1
move r9 r15 #i
"
}
);
Ok(())
}
#[test]
fn incorrect_args_count() -> anyhow::Result<()> {
let compiled = compile! {
result
"
fn doSomething(arg1, arg2){};
let i = doSomething();
"
};
assert!(matches!(
compiled,
Err(super::super::Error::AgrumentMismatch(_))
));
Ok(())
}
#[test]
fn inline_literal_args() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn doSomething(arg1, arg2) {};
let thisVariableShouldStayInPlace = 123;
let returnedValue = doSomething(12, 34);
"
};
assert_eq!(
compiled,
indoc! {
"
j main
doSomething:
pop r8 #arg2
pop r9 #arg1
push ra
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
move r8 123 #thisVariableShouldStayInPlace
push r8
push 12
push 34
jal doSomething
sub r0 sp 1
get r8 db r0
sub sp sp 1
move r9 r15 #returnedValue
"
}
);
Ok(())
}
#[test]
fn mixed_args() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let arg1 = 123;
let returnValue = doSomething(arg1, 456);
fn doSomething(arg1, arg2) {};
"
};
assert_eq!(
compiled,
indoc! {
"
j main
doSomething:
pop r8 #arg2
pop r9 #arg1
push ra
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
move r8 123 #arg1
push r8
push r8
push 456
jal doSomething
sub r0 sp 1
get r8 db r0
sub sp sp 1
move r9 r15 #returnValue
"
}
);
Ok(())
}
#[test]
fn with_return_statement() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn doSomething(arg1) {
return 456;
};
let returned = doSomething(123);
"
};
assert_eq!(
compiled,
indoc! {
"
j main
doSomething:
pop r8 #arg1
push ra
move r15 456 #returnValue
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
push 123
jal doSomething
move r8 r15 #returned
"
}
);
Ok(())
}
#[test]
fn with_negative_return_literal() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn doSomething() {
return -1;
};
let i = doSomething();
"
};
assert_eq!(
compiled,
indoc! {
"
j main
doSomething:
push ra
move r15 -1 #returnValue
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
jal doSomething
move r8 r15 #i
"
}
);
Ok(())
}

147
rust_compiler/libs/compiler/src/test/declaration_literal.rs Normal file
View File

@@ -0,0 +1,147 @@
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn variable_declaration_numeric_literal() -> anyhow::Result<()> {
let compiled = crate::compile! {
debug r#"
let i = 20c;
"#
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 293.15 #i
"
}
);
Ok(())
}
#[test]
fn variable_declaration_numeric_literal_stack_spillover() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
let a = 0;
let b = 1;
let c = 2;
let d = 3;
let e = 4;
let f = 5;
let g = 6;
let h = 7;
let i = 8;
let j = 9;
"#};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
move r9 1 #b
move r10 2 #c
move r11 3 #d
move r12 4 #e
move r13 5 #f
move r14 6 #g
push 7 #h
push 8 #i
push 9 #j
"
}
);
Ok(())
}
#[test]
fn variable_declaration_negative() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let i = -1;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 -1 #i
"
}
);
Ok(())
}
#[test]
fn test_boolean_declaration() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let t = true;
let f = false;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 1 #t
move r9 0 #f
"
}
);
Ok(())
}
#[test]
fn test_boolean_return() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
fn getTrue() {
return true;
};
let val = getTrue();
"
};
assert_eq!(
compiled,
indoc! {
"
j main
getTrue:
push ra
move r15 1 #returnValue
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
main:
jal getTrue
move r8 r15 #val
"
}
);
Ok(())
}

58
rust_compiler/libs/compiler/src/test/function_declaration.rs Normal file
View File

@@ -0,0 +1,58 @@
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn test_function_declaration_with_spillover_params() -> anyhow::Result<()> {
let compiled = compile!(debug r#"
// we need more than 4 params to 'spill' into a stack var
fn doSomething(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) {};
"#);
assert_eq!(
compiled,
indoc! {"
j main
doSomething:
pop r8 #arg9
pop r9 #arg8
pop r10 #arg7
pop r11 #arg6
pop r12 #arg5
pop r13 #arg4
pop r14 #arg3
push ra
sub r0 sp 1
get ra db r0
sub sp sp 3
j ra
"}
);
Ok(())
}
#[test]
fn test_function_declaration_with_register_params() -> anyhow::Result<()> {
let compiled = compile!(debug r#"
// This is a test function declaration with no body
fn doSomething(arg1, arg2) {
};
"#);
assert_eq!(
compiled,
indoc! {"
j main
doSomething:
pop r8 #arg2
pop r9 #arg1
push ra
sub r0 sp 1
get ra db r0
sub sp sp 1
j ra
"}
);
Ok(())
}

177
rust_compiler/libs/compiler/src/test/logic_expression.rs Normal file
View File

@@ -0,0 +1,177 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn test_comparison_expressions() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let isGreater = 10 > 5;
let isLess = 5 < 10;
let isEqual = 5 == 5;
let isNotEqual = 5 != 10;
let isGreaterOrEqual = 10 >= 10;
let isLessOrEqual = 5 <= 5;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
sgt r1 10 5
move r8 r1 #isGreater
slt r2 5 10
move r9 r2 #isLess
seq r3 5 5
move r10 r3 #isEqual
sne r4 5 10
move r11 r4 #isNotEqual
sge r5 10 10
move r12 r5 #isGreaterOrEqual
sle r6 5 5
move r13 r6 #isLessOrEqual
"
}
);
Ok(())
}
#[test]
fn test_logical_and_or_not() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let logic1 = 1 && 1;
let logic2 = 1 || 0;
let logic3 = !1;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
and r1 1 1
move r8 r1 #logic1
or r2 1 0
move r9 r2 #logic2
seq r3 1 0
move r10 r3 #logic3
"
}
);
Ok(())
}
#[test]
fn test_complex_logic() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let logic = (10 > 5) && (5 < 10);
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
sgt r1 10 5
slt r2 5 10
and r3 r1 r2
move r8 r3 #logic
"
}
);
Ok(())
}
#[test]
fn test_math_with_logic() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let logic = (1 + 2) > 1;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
add r1 1 2
sgt r2 r1 1
move r8 r2 #logic
"
}
);
Ok(())
}
#[test]
fn test_boolean_in_logic() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let res = true && false;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
and r1 1 0
move r8 r1 #res
"
}
);
Ok(())
}
#[test]
fn test_invert_a_boolean() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let i = true;
let y = !i;
let result = y == false;
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 1 #i
seq r1 r8 0
move r9 r1 #y
seq r2 r9 0
move r10 r2 #result
"
}
);
Ok(())
}

149
rust_compiler/libs/compiler/src/test/loops.rs Normal file
View File

@@ -0,0 +1,149 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn test_infinite_loop() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 0;
loop {
a = a + 1;
}
"
};
// Labels: L1 (start), L2 (end)
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
L1:
add r1 r8 1
move r8 r1 #a
j L1
L2:
"
}
);
Ok(())
}
#[test]
fn test_loop_break() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 0;
loop {
a = a + 1;
if (a > 10) {
break;
}
}
"
};
// Labels: L1 (start), L2 (end), L3 (if end - implicit else label)
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
L1:
add r1 r8 1
move r8 r1 #a
sgt r2 r8 10
beq r2 0 L3
j L2
L3:
j L1
L2:
"
}
);
Ok(())
}
#[test]
fn test_while_loop() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
let a = 0;
while (a < 10) {
a = a + 1;
}
"
};
// Labels: L1 (start), L2 (end)
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
L1:
slt r1 r8 10
beq r1 0 L2
add r2 r8 1
move r8 r2 #a
j L1
L2:
"
}
);
Ok(())
}
#[test]
fn test_loop_continue() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
let a = 0;
loop {
a = a + 1;
if (a < 5) {
continue;
}
break;
}
"#
};
// Labels: L1 (start), L2 (end), L3 (if end)
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 0 #a
L1:
add r1 r8 1
move r8 r1 #a
slt r2 r8 5
beq r2 0 L3
j L1
L3:
j L2
j L1
L2:
"
}
);
Ok(())
}

50
rust_compiler/libs/compiler/src/test/mod.rs Normal file
View File

@@ -0,0 +1,50 @@
#![allow(clippy::crate_in_macro_def)]
macro_rules! output {
($input:expr) => {
String::from_utf8($input.into_inner()?)?
};
}
#[cfg_attr(test, macro_export)]
macro_rules! compile {
($source:expr) => {{
let mut writer = std::io::BufWriter::new(Vec::new());
let compiler = ::Compiler::new(
parser::Parser::new(tokenizer::Tokenizer::from(String::from($source))),
&mut writer,
None,
);
compiler.compile()?;
output!(writer)
}};
(result $source:expr) => {{
let mut writer = std::io::BufWriter::new(Vec::new());
let compiler = crate::Compiler::new(
parser::Parser::new(tokenizer::Tokenizer::from(String::from($source))),
&mut writer,
Some(crate::CompilerConfig { debug: true }),
);
compiler.compile()
}};
(debug $source:expr) => {{
let mut writer = std::io::BufWriter::new(Vec::new());
let compiler = crate::Compiler::new(
parser::Parser::new(tokenizer::Tokenizer::from(String::from($source))),
&mut writer,
Some(crate::CompilerConfig { debug: true }),
);
compiler.compile()?;
output!(writer)
}};
}
mod binary_expression;
mod branching;
mod declaration_function_invocation;
mod declaration_literal;
mod function_declaration;
mod logic_expression;
mod loops;
mod syscall;

159
rust_compiler/libs/compiler/src/test/syscall.rs Normal file
View File

@@ -0,0 +1,159 @@
use crate::compile;
use indoc::indoc;
use pretty_assertions::assert_eq;
#[test]
fn test_yield() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
yield();
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
yield
"
}
);
Ok(())
}
#[test]
fn test_sleep() -> anyhow::Result<()> {
let compiled = compile! {
debug
"
sleep(3);
let sleepAmount = 15;
sleep(sleepAmount);
sleep(sleepAmount * 2);
"
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
sleep 3
move r8 15 #sleepAmount
sleep r8
mul r1 r8 2
sleep r1
"
}
);
Ok(())
}
#[test]
fn test_set_on_device() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
device airConditioner = "d0";
let internalTemp = 20c;
setOnDevice(airConditioner, "On", internalTemp > 25c);
"#
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
move r8 293.15 #internalTemp
sgt r1 r8 298.15
s d0 On r1
"
}
);
Ok(())
}
#[test]
fn test_set_on_device_batched() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
let doorHash = hash("Door");
setOnDeviceBatched(doorHash, "Lock", true);
"#
};
assert_eq!(
compiled,
indoc! {
r#"
j main
main:
move r15 HASH("Door") #hash_ret
move r8 r15 #doorHash
sb r8 Lock 1
"#
}
);
Ok(())
}
#[test]
fn test_load_from_device() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
device airCon = "d0";
let setting = loadFromDevice(airCon, "On");
"#
};
assert_eq!(
compiled,
indoc! {
"
j main
main:
l r15 d0 On
move r8 r15 #setting
"
}
);
Ok(())
}
#[test]
fn test_hash() -> anyhow::Result<()> {
let compiled = compile! {
debug
r#"
let nameHash = hash("testValue");
"#
};
assert_eq!(
compiled,
indoc! {
r#"
j main
main:
move r15 HASH("testValue") #hash_ret
move r8 r15 #nameHash
"#
}
);
Ok(())
}

1247
rust_compiler/libs/compiler/src/v1.rs Normal file
View File

File diff suppressed because it is too large Load Diff

188
rust_compiler/libs/compiler/src/variable_manager.rs Normal file
View File

@@ -0,0 +1,188 @@
// r15 : Return Value
// r0 : Unmanaged temp variable
// r1 - r7 : Temporary Variables
// r8 - r14 : Persistant Variables
use quick_error::quick_error;
use std::collections::{HashMap, VecDeque};
const TEMP: [u8; 7] = [1, 2, 3, 4, 5, 6, 7];
const PERSIST: [u8; 7] = [8, 9, 10, 11, 12, 13, 14];
quick_error! {
#[derive(Debug)]
pub enum Error {
DuplicateVariable(var: String) {
display("{var} already exists.")
}
UnknownVariable(var: String) {
display("{var} does not exist.")
}
Unknown(reason: String) {
display("{reason}")
}
}
}
/// A request to store a variable at a specific register type
pub enum LocationRequest {
#[allow(dead_code)]
/// Request to store a variable in a temprary register.
Temp,
/// Request to store a variable in a persistant register.
Persist,
/// Request to store a variable in the stack.
Stack,
}
#[derive(Clone)]
pub enum VariableLocation {
/// Represents a temporary register (r1 - r7)
Temporary(u8),
/// Represents a persistant register (r8 - r14)
Persistant(u8),
/// Represents a a stack offset (current stack - offset = variable loc)
Stack(u16),
}
pub struct VariableScope<'a> {
temporary_vars: VecDeque<u8>,
persistant_vars: VecDeque<u8>,
var_lookup_table: HashMap<String, VariableLocation>,
stack_offset: u16,
parent: Option<&'a VariableScope<'a>>,
}
impl<'a> Default for VariableScope<'a> {
fn default() -> Self {
Self {
parent: None,
stack_offset: 0,
persistant_vars: PERSIST.to_vec().into(),
temporary_vars: TEMP.to_vec().into(),
var_lookup_table: HashMap::new(),
}
}
}
impl<'a> VariableScope<'a> {
#[allow(dead_code)]
pub const TEMP_REGISTER_COUNT: u8 = 7;
pub const PERSIST_REGISTER_COUNT: u8 = 7;
pub const RETURN_REGISTER: u8 = 15;
pub const TEMP_STACK_REGISTER: u8 = 0;
pub fn registers(&self) -> impl Iterator<Item = &u8> {
self.var_lookup_table
.values()
.filter(|val| {
matches!(
val,
VariableLocation::Temporary(_) | VariableLocation::Persistant(_)
)
})
.map(|loc| match loc {
VariableLocation::Persistant(reg) | VariableLocation::Temporary(reg) => reg,
_ => unreachable!(),
})
}
pub fn scoped(parent: &'a VariableScope<'a>) -> Self {
Self {
parent: Option::Some(parent),
..Default::default()
}
}
pub fn stack_offset(&self) -> u16 {
self.stack_offset
}
/// Adds and tracks a new scoped variable. If the location you request is full, will fall back
/// to the stack.
pub fn add_variable(
&mut self,
var_name: impl Into<String>,
location: LocationRequest,
) -> Result<VariableLocation, Error> {
let var_name = var_name.into();
if self.var_lookup_table.contains_key(var_name.as_str()) {
return Err(Error::DuplicateVariable(var_name));
}
let var_location = match location {
LocationRequest::Temp => {
if let Some(next_var) = self.temporary_vars.pop_front() {
VariableLocation::Temporary(next_var)
} else {
let loc = VariableLocation::Stack(self.stack_offset);
self.stack_offset += 1;
loc
}
}
LocationRequest::Persist => {
if let Some(next_var) = self.persistant_vars.pop_front() {
VariableLocation::Persistant(next_var)
} else {
let loc = VariableLocation::Stack(self.stack_offset);
self.stack_offset += 1;
loc
}
}
LocationRequest::Stack => {
let loc = VariableLocation::Stack(self.stack_offset);
self.stack_offset += 1;
loc
}
};
self.var_lookup_table.insert(var_name, var_location.clone());
Ok(var_location)
}
pub fn get_location_of(
&mut self,
var_name: impl Into<String>,
) -> Result<VariableLocation, Error> {
let var_name = var_name.into();
let var = self
.var_lookup_table
.get(var_name.as_str())
.cloned()
.ok_or(Error::UnknownVariable(var_name))?;
if let VariableLocation::Stack(inserted_at_offset) = var {
Ok(VariableLocation::Stack(
self.stack_offset - inserted_at_offset,
))
} else {
Ok(var)
}
}
pub fn has_parent(&self) -> bool {
self.parent.is_some()
}
#[allow(dead_code)]
pub fn free_temp(&mut self, var_name: impl Into<String>) -> Result<(), Error> {
let var_name = var_name.into();
let Some(location) = self.var_lookup_table.remove(var_name.as_str()) else {
return Err(Error::UnknownVariable(var_name));
};
match location {
VariableLocation::Temporary(t) => {
self.temporary_vars.push_back(t);
}
VariableLocation::Persistant(_) => {
return Err(Error::UnknownVariable(String::from(
"Attempted to free a `let` variable.",
)));
}
VariableLocation::Stack(_) => {}
};
Ok(())
}
}

9
rust_compiler/libs/compiler/test_files/deviceIo.slang Normal file
View File

@@ -0,0 +1,9 @@
device airConditioner = "d0";
device gasSensor = "d1";
loop {
yield();
let indoorTemp = loadFromDevice(gasSensor, "Temperature");
let shouldSet = indoorTemp > 30c;
setOnDevice(airConditioner, "On", shouldSet);
}

12
rust_compiler/libs/parser/Cargo.toml Normal file
View File

@@ -0,0 +1,12 @@
[package]
name = "parser"
version = "0.1.0"
edition = "2024"
[dependencies]
quick-error = { workspace = true }
tokenizer = { path = "../tokenizer" }
[dev-dependencies]
anyhow = { version = "1" }

1266
rust_compiler/libs/parser/src/lib.rs Normal file
View File

File diff suppressed because it is too large Load Diff

226
rust_compiler/libs/parser/src/sys_call.rs Normal file
View File

@@ -0,0 +1,226 @@
use crate::tree_node::{Expression, Literal};
use super::LiteralOrVariable;
#[derive(Debug, PartialEq, Eq)]
pub enum Math {
/// Returns the angle in radians whose cosine is the specified number.
/// ## In Game
/// `acos r? a(r?|num)`
Acos(LiteralOrVariable),
/// Returns the angle in radians whose sine is the specified number.
/// ## In Game
/// `asin r? a(r?|num)`
Asin(LiteralOrVariable),
/// Returns the angle in radians whose tangent is the specified number.
/// ## In Game
/// `atan r? a(r?|num)`
Atan(LiteralOrVariable),
/// Returns the angle in radians whose tangent is the quotient of the specified numbers.
/// ## In Game
/// `atan2 r? a(r?|num) b(r?|num)`
Atan2(LiteralOrVariable, LiteralOrVariable),
/// Gets the absolute value of a number.
/// ## In Game
/// `abs r? a(r?|num)`
Abs(LiteralOrVariable),
/// Rounds a number up to the nearest whole number.
/// ## In Game
/// `ceil r? a(r?|num)`
Ceil(LiteralOrVariable),
/// Returns the cosine of the specified angle in radians.
/// ## In Game
/// cos r? a(r?|num)
Cos(LiteralOrVariable),
/// Rounds a number down to the nearest whole number.
/// ## In Game
/// `floor r? a(r?|num)`
Floor(LiteralOrVariable),
/// Computes the natural logarithm of a number.
/// ## In Game
/// `log r? a(r?|num)`
Log(LiteralOrVariable),
/// Computes the maximum of two numbers.
/// ## In Game
/// `max r? a(r?|num) b(r?|num)`
Max(LiteralOrVariable, LiteralOrVariable),
/// Computes the minimum of two numbers.
/// ## In Game
/// `min r? a(r?|num) b(r?|num)`
Min(LiteralOrVariable, LiteralOrVariable),
/// Gets a random number between 0 and 1.
/// ## In Game
/// `rand r?`
Rand,
/// Returns the sine of the specified angle in radians.
/// ## In Game
/// `sin r? a(r?|num)`
Sin(LiteralOrVariable),
/// Computes the square root of a number.
/// ## In Game
/// `sqrt r? a(r?|num)`
Sqrt(LiteralOrVariable),
/// Returns the tangent of the specified angle in radians.
/// ## In Game
/// `tan r? a(r?|num)`
Tan(LiteralOrVariable),
/// Truncates a number by removing the decimal portion.
/// ## In Game
/// `trunc r? a(r?|num)`
Trunc(LiteralOrVariable),
}
impl std::fmt::Display for Math {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Math::Acos(a) => write!(f, "acos({})", a),
Math::Asin(a) => write!(f, "asin({})", a),
Math::Atan(a) => write!(f, "atan({})", a),
Math::Atan2(a, b) => write!(f, "atan2({}, {})", a, b),
Math::Abs(a) => write!(f, "abs({})", a),
Math::Ceil(a) => write!(f, "ceil({})", a),
Math::Cos(a) => write!(f, "cos({})", a),
Math::Floor(a) => write!(f, "floor({})", a),
Math::Log(a) => write!(f, "log({})", a),
Math::Max(a, b) => write!(f, "max({}, {})", a, b),
Math::Min(a, b) => write!(f, "min({}, {})", a, b),
Math::Rand => write!(f, "rand()"),
Math::Sin(a) => write!(f, "sin({})", a),
Math::Sqrt(a) => write!(f, "sqrt({})", a),
Math::Tan(a) => write!(f, "tan({})", a),
Math::Trunc(a) => write!(f, "trunc({})", a),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum System {
/// Pauses execution for exactly 1 tick and then resumes.
/// ## In Game
/// yield
Yield,
/// Represents a function that can be called to sleep for a certain amount of time.
/// ## In Game
/// `sleep a(r?|num)`
Sleep(Box<Expression>),
/// Gets the in-game hash for a specific prefab name.
/// ## In Game
/// `HASH("prefabName")`
Hash(Literal),
/// Represents a function which loads a device variable into a register.
/// ## In Game
/// `l r? d? var`
/// ## Examples
/// `l r0 d0 Setting`
/// `l r1 d5 Pressure`
LoadFromDevice(LiteralOrVariable, Literal),
/// Function which gets a LogicType from all connected network devices that match
/// the provided device hash and name, aggregating them via a batchMode
/// ## In Game
/// lbn r? deviceHash nameHash logicType batchMode
/// ## Examples
/// lbn r0 HASH("StructureWallLight") HASH("wallLight") On Minimum
LoadBatchNamed(LiteralOrVariable, Box<Expression>, Literal, Literal),
/// Loads a LogicType from all connected network devices, aggregating them via a
/// batchMode
/// ## In Game
/// lb r? deviceHash loggicType batchMode
/// ## Examples
/// lb r0 HASH("StructureWallLight") On Minimum
LoadBatch(LiteralOrVariable, Literal, Literal),
/// Represents a function which stores a setting into a specific device.
/// ## In Game
/// `s d? logicType r?`
/// ## Example
/// `s d0 Setting r0`
SetOnDevice(LiteralOrVariable, Literal, Box<Expression>),
/// Represents a function which stores a setting to all devices that match
/// the given deviceHash
/// ## In Game
/// `sb deviceHash logictype r?`
/// ## Example
/// `sb HASH("Doors") Lock 1`
SetOnDeviceBatched(LiteralOrVariable, Literal, Box<Expression>),
/// Represents a function which stores a setting to all devices that match
/// both the given deviceHash AND the given nameHash
/// ## In Game
/// `sbn deviceHash nameHash logicType r?`
/// ## Example
/// `sbn HASH("Doors") HASH("Exterior") Lock 1`
SetOnDeviceBatchedNamed(
LiteralOrVariable,
LiteralOrVariable,
Literal,
Box<Expression>,
),
}
impl std::fmt::Display for System {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
System::Yield => write!(f, "yield()"),
System::Sleep(a) => write!(f, "sleep({})", a),
System::Hash(a) => write!(f, "hash({})", a),
System::LoadFromDevice(a, b) => write!(f, "loadFromDevice({}, {})", a, b),
System::LoadBatch(a, b, c) => write!(f, "loadBatch({}, {}, {})", a, b, c),
System::LoadBatchNamed(a, b, c, d) => {
write!(f, "loadBatchNamed({}, {}, {}, {})", a, b, c, d)
}
System::SetOnDevice(a, b, c) => write!(f, "setOnDevice({}, {}, {})", a, b, c),
System::SetOnDeviceBatched(a, b, c) => {
write!(f, "setOnDeviceBatched({}, {}, {})", a, b, c)
}
System::SetOnDeviceBatchedNamed(a, b, c, d) => {
write!(f, "setOnDeviceBatchedNamed({}, {}, {}, {})", a, b, c, d)
}
}
}
}
#[derive(Debug, PartialEq, Eq)]
/// This represents built in functions that cannot be overwritten, but can be invoked by the user as functions.
pub enum SysCall {
System(System),
/// Represents any mathmatical function that can be called.
Math(Math),
}
impl std::fmt::Display for SysCall {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SysCall::System(s) => write!(f, "{}", s),
SysCall::Math(m) => write!(f, "{}", m),
}
}
}
impl SysCall {
pub fn is_syscall(identifier: &str) -> bool {
matches!(
identifier,
"yield"
| "sleep"
| "hash"
| "loadFromDevice"
| "setOnDevice"
| "setOnDeviceBatched"
| "setOnDeviceBatchedNamed"
| "acos"
| "asin"
| "atan"
| "atan2"
| "abs"
| "ceil"
| "cos"
| "floor"
| "log"
| "max"
| "min"
| "rand"
| "sin"
| "sqrt"
| "tan"
| "trunc"
)
}
}

21
rust_compiler/libs/parser/src/test/blocks.rs Normal file
View File

@@ -0,0 +1,21 @@
use tokenizer::Tokenizer;
use crate::Parser;
#[test]
fn test_block() -> anyhow::Result<()> {
let mut parser = crate::parser!(
r#"
{
let x = 5;
let y = 10;
}
"#
);
let expression = parser.parse()?.unwrap();
assert_eq!("{ (let x = 5); (let y = 10); }", expression.to_string());
Ok(())
}

115
rust_compiler/libs/parser/src/test/mod.rs Normal file
View File

@@ -0,0 +1,115 @@
#[macro_export]
macro_rules! parser {
($input:expr) => {
Parser::new(Tokenizer::from($input.to_owned()))
};
}
mod blocks;
use super::Parser;
use super::Tokenizer;
use anyhow::Result;
#[test]
fn test_unsupported_keywords() -> Result<()> {
let mut parser = parser!("enum x;");
assert!(parser.parse().is_err());
let mut parser = parser!("if x {}");
assert!(parser.parse().is_err());
let mut parser = parser!("else {}");
assert!(parser.parse().is_err());
Ok(())
}
#[test]
fn test_declarations() -> Result<()> {
let input = r#"
let x = 5;
// The below line should fail
let y = 234
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
assert_eq!("(let x = 5)", expression.to_string());
assert!(parser.parse().is_err());
Ok(())
}
#[test]
fn test_function_expression() -> Result<()> {
let input = r#"
// This is a function. The parser is starting to get more complex
fn add(x, y) {
let z = x;
}
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
assert_eq!(
"(fn add(x, y) { { (let z = x); } })",
expression.to_string()
);
Ok(())
}
#[test]
fn test_function_invocation() -> Result<()> {
let input = r#"
add();
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
assert_eq!("add()", expression.to_string());
Ok(())
}
#[test]
fn test_priority_expression() -> Result<()> {
let input = r#"
let x = (4);
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
assert_eq!("(let x = (4))", expression.to_string());
Ok(())
}
#[test]
fn test_binary_expression() -> Result<()> {
let expr = parser!("4 ** 2 + 5 ** 2").parse()?.unwrap();
assert_eq!("((4 ** 2) + (5 ** 2))", expr.to_string());
let expr = parser!("2 ** 3 ** 4").parse()?.unwrap();
assert_eq!("(2 ** (3 ** 4))", expr.to_string());
let expr = parser!("45 * 2 - 15 / 5 + 5 ** 2").parse()?.unwrap();
assert_eq!("(((45 * 2) - (15 / 5)) + (5 ** 2))", expr.to_string());
let expr = parser!("(5 - 2) * 10").parse()?.unwrap();
assert_eq!("(((5 - 2)) * 10)", expr.to_string());
Ok(())
}

258
rust_compiler/libs/parser/src/tree_node.rs Normal file
View File

@@ -0,0 +1,258 @@
use super::sys_call::SysCall;
use tokenizer::token::Number;
#[derive(Debug, Eq, PartialEq, Clone)]
pub enum Literal {
Number(Number),
String(String),
Boolean(bool),
}
impl std::fmt::Display for Literal {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Literal::Number(n) => write!(f, "{}", n),
Literal::String(s) => write!(f, "\"{}\"", s),
Literal::Boolean(b) => write!(f, "{}", if *b { 1 } else { 0 }),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum BinaryExpression {
Add(Box<Expression>, Box<Expression>),
Multiply(Box<Expression>, Box<Expression>),
Divide(Box<Expression>, Box<Expression>),
Subtract(Box<Expression>, Box<Expression>),
Exponent(Box<Expression>, Box<Expression>),
Modulo(Box<Expression>, Box<Expression>),
}
impl std::fmt::Display for BinaryExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
BinaryExpression::Add(l, r) => write!(f, "({} + {})", l, r),
BinaryExpression::Multiply(l, r) => write!(f, "({} * {})", l, r),
BinaryExpression::Divide(l, r) => write!(f, "({} / {})", l, r),
BinaryExpression::Subtract(l, r) => write!(f, "({} - {})", l, r),
BinaryExpression::Exponent(l, r) => write!(f, "({} ** {})", l, r),
BinaryExpression::Modulo(l, r) => write!(f, "({} % {})", l, r),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum LogicalExpression {
And(Box<Expression>, Box<Expression>),
Or(Box<Expression>, Box<Expression>),
Not(Box<Expression>),
Equal(Box<Expression>, Box<Expression>),
NotEqual(Box<Expression>, Box<Expression>),
GreaterThan(Box<Expression>, Box<Expression>),
GreaterThanOrEqual(Box<Expression>, Box<Expression>),
LessThan(Box<Expression>, Box<Expression>),
LessThanOrEqual(Box<Expression>, Box<Expression>),
}
impl std::fmt::Display for LogicalExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
LogicalExpression::And(l, r) => write!(f, "({} && {})", l, r),
LogicalExpression::Or(l, r) => write!(f, "({} || {})", l, r),
LogicalExpression::Not(e) => write!(f, "(!{})", e),
LogicalExpression::Equal(l, r) => write!(f, "({} == {})", l, r),
LogicalExpression::NotEqual(l, r) => write!(f, "({} != {})", l, r),
LogicalExpression::GreaterThan(l, r) => write!(f, "({} > {})", l, r),
LogicalExpression::GreaterThanOrEqual(l, r) => write!(f, "({} >= {})", l, r),
LogicalExpression::LessThan(l, r) => write!(f, "({} < {})", l, r),
LogicalExpression::LessThanOrEqual(l, r) => write!(f, "({} <= {})", l, r),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct AssignmentExpression {
pub identifier: String,
pub expression: Box<Expression>,
}
impl std::fmt::Display for AssignmentExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "({} = {})", self.identifier, self.expression)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct FunctionExpression {
pub name: String,
pub arguments: Vec<String>,
pub body: BlockExpression,
}
impl std::fmt::Display for FunctionExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"(fn {}({}) {{ {} }})",
self.name,
self.arguments.to_vec().join(", "),
self.body
)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct BlockExpression(pub Vec<Expression>);
impl std::fmt::Display for BlockExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{{ {}; }}",
self.0
.iter()
.map(|e| e.to_string())
.collect::<Vec<String>>()
.join("; ")
)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct InvocationExpression {
pub name: String,
pub arguments: Vec<Expression>,
}
impl std::fmt::Display for InvocationExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}({})",
self.name,
self.arguments
.iter()
.map(|e| e.to_string())
.collect::<Vec<String>>()
.join(", ")
)
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum LiteralOrVariable {
Literal(Literal),
Variable(String),
}
impl std::fmt::Display for LiteralOrVariable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
LiteralOrVariable::Literal(l) => write!(f, "{}", l),
LiteralOrVariable::Variable(v) => write!(f, "{}", v),
}
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct DeviceDeclarationExpression {
/// any variable-like name
pub name: String,
/// The device port, ex. (db, d0, d1, d2, d3, d4, d5)
pub device: String,
}
impl std::fmt::Display for DeviceDeclarationExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(device {} = {})", self.name, self.device)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct IfExpression {
pub condition: Box<Expression>,
pub body: BlockExpression,
pub else_branch: Option<Box<Expression>>,
}
impl std::fmt::Display for IfExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(if ({}) {}", self.condition, self.body)?;
if let Some(else_branch) = &self.else_branch {
write!(f, " else {}", else_branch)?;
}
write!(f, ")")
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct LoopExpression {
pub body: BlockExpression,
}
impl std::fmt::Display for LoopExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(loop {})", self.body)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct WhileExpression {
pub condition: Box<Expression>,
pub body: BlockExpression,
}
impl std::fmt::Display for WhileExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(while {} {})", self.condition, self.body)
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum Expression {
Assignment(AssignmentExpression),
Binary(BinaryExpression),
Block(BlockExpression),
Break,
Continue,
Declaration(String, Box<Expression>),
DeviceDeclaration(DeviceDeclarationExpression),
Function(FunctionExpression),
If(IfExpression),
Invocation(InvocationExpression),
Literal(Literal),
Logical(LogicalExpression),
Loop(LoopExpression),
Negation(Box<Expression>),
Priority(Box<Expression>),
Return(Box<Expression>),
Syscall(SysCall),
Variable(String),
While(WhileExpression),
}
impl std::fmt::Display for Expression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Expression::Assignment(e) => write!(f, "{}", e),
Expression::Binary(e) => write!(f, "{}", e),
Expression::Block(e) => write!(f, "{}", e),
Expression::Break => write!(f, "break"),
Expression::Continue => write!(f, "continue"),
Expression::Declaration(id, e) => write!(f, "(let {} = {})", id, e),
Expression::DeviceDeclaration(e) => write!(f, "{}", e),
Expression::Function(e) => write!(f, "{}", e),
Expression::If(e) => write!(f, "{}", e),
Expression::Invocation(e) => write!(f, "{}", e),
Expression::Literal(l) => write!(f, "{}", l),
Expression::Logical(e) => write!(f, "{}", e),
Expression::Loop(e) => write!(f, "{}", e),
Expression::Negation(e) => write!(f, "(-{})", e),
Expression::Priority(e) => write!(f, "({})", e),
Expression::Return(e) => write!(f, "(return {})", e),
Expression::Syscall(e) => write!(f, "{}", e),
Expression::Variable(id) => write!(f, "{}", id),
Expression::While(e) => write!(f, "{}", e),
}
}
}

11
rust_compiler/libs/tokenizer/Cargo.toml Normal file
View File

@@ -0,0 +1,11 @@
[package]
name = "tokenizer"
version = "0.1.0"
edition = "2024"
[dependencies]
rust_decimal = { workspace = true }
quick-error = { workspace = true }
[dev-dependencies]
anyhow = { version = "^1" }

932
rust_compiler/libs/tokenizer/src/lib.rs Normal file
View File

@@ -0,0 +1,932 @@
pub mod token;
use quick_error::quick_error;
use rust_decimal::Decimal;
use std::{
cmp::Ordering,
collections::VecDeque,
io::{BufReader, Cursor, Read, Seek, SeekFrom},
path::PathBuf,
};
use token::{Keyword, Number, Symbol, Temperature, Token, TokenType};
quick_error! {
#[derive(Debug)]
pub enum Error {
IOError(err: std::io::Error) {
from()
display("IO Error: {}", err)
source(err)
}
NumberParseError(err: std::num::ParseIntError, line: usize, column: usize) {
display("Number Parse Error: {}\nLine: {}, Column: {}", err, line, column)
source(err)
}
DecimalParseError(err: rust_decimal::Error, line: usize, column: usize) {
display("Decimal Parse Error: {}\nLine: {}, Column: {}", err, line, column)
source(err)
}
UnknownSymbolError(char: char, line: usize, column: usize) {
display("Unknown Symbol: {}\nLine: {}, Column: {}", char, line, column)
}
UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize) {
display("Unknown Keyword or Identifier: {}\nLine: {}, Column: {}", val, line, column)
}
}
}
pub trait Tokenize: Read + Seek {}
impl<T> Tokenize for T where T: Read + Seek {}
pub struct Tokenizer {
reader: BufReader<Box<dyn Tokenize>>,
char_buffer: [u8; 1],
line: usize,
column: usize,
returned_eof: bool,
}
impl Tokenizer {
pub fn from_path(input_file: impl Into<PathBuf>) -> Result<Self, Error> {
let file = std::fs::File::open(input_file.into())?;
let reader = BufReader::new(Box::new(file) as Box<dyn Tokenize>);
Ok(Self {
reader,
line: 1,
column: 1,
char_buffer: [0],
returned_eof: false,
})
}
}
impl From<String> for Tokenizer {
fn from(input: String) -> Self {
let reader = BufReader::new(Box::new(Cursor::new(input)) as Box<dyn Tokenize>);
Self {
reader,
line: 1,
column: 1,
char_buffer: [0],
returned_eof: false,
}
}
}
impl From<&str> for Tokenizer {
fn from(value: &str) -> Self {
Self::from(value.to_string())
}
}
impl Tokenizer {
/// Consumes the tokenizer and returns the next token in the stream
/// If there are no more tokens in the stream, this function returns None
/// If there is an error reading the stream, this function returns an error
///
/// # Important
/// This function will increment the line and column counters
fn next_char(&mut self) -> Result<Option<char>, Error> {
let bytes_read = self.reader.read(&mut self.char_buffer)?;
if bytes_read == 0 {
return Ok(None);
}
// Safety: The buffer is guaranteed to have 1 value as it is initialized with a size of 1
let c = self.char_buffer[0] as char;
if c == '\n' {
self.line += 1;
self.column = 1;
} else {
self.column += 1;
}
Ok(Some(c))
}
/// Peeks the next character in the stream without consuming it
///
/// # Important
/// This does not increment the line or column counters
fn peek_next_char(&mut self) -> Result<Option<char>, Error> {
let current_pos = self.reader.stream_position()?;
let to_return = if self.reader.read(&mut self.char_buffer)? == 0 {
None
} else {
self.reader.seek(SeekFrom::Start(current_pos))?;
// Safety: The buffer is guaranteed to have 1 value as it is initialized with a size of 1
Some(self.char_buffer[0] as char)
};
Ok(to_return)
}
/// Skips the current line in the stream.
/// Useful for skipping comments or empty lines
///
/// # Important
/// This function will increment the line and column counters
fn skip_line(&mut self) -> Result<(), Error> {
while let Some(next_char) = self.next_char()? {
if next_char == '\n' {
break;
}
}
Ok(())
}
/// Consumes the tokenizer and returns the next token in the stream
/// If there are no more tokens in the stream, this function returns None
pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
while let Some(next_char) = self.next_char()? {
// skip whitespace
if next_char.is_whitespace() {
continue;
}
// skip comments
if next_char == '/' && self.peek_next_char()? == Some('/') {
self.skip_line()?;
continue;
}
match next_char {
// numbers
'0'..='9' => {
return self.tokenize_number(next_char).map(Some);
}
// strings
'"' | '\'' => return self.tokenize_string(next_char).map(Some),
// symbols excluding `"` and `'`
char if !char.is_alphanumeric() && char != '"' && char != '\'' => {
return self.tokenize_symbol(next_char).map(Some);
}
// keywords and identifiers
char if char.is_alphabetic() => {
return self.tokenize_keyword_or_identifier(next_char).map(Some);
}
_ => {
return Err(Error::UnknownSymbolError(next_char, self.line, self.column));
}
}
}
if self.returned_eof {
Ok(None)
} else {
self.returned_eof = true;
Ok(Some(Token::new(TokenType::EOF, self.line, self.column)))
}
}
/// Peeks the next token in the stream without consuming it
/// If there are no more tokens in the stream, this function returns None
pub fn peek_next(&mut self) -> Result<Option<Token>, Error> {
let current_pos = self.reader.stream_position()?;
let column = self.column;
let line = self.line;
let token = self.next_token()?;
self.reader.seek(SeekFrom::Start(current_pos))?;
self.column = column;
self.line = line;
Ok(token)
}
/// Tokenizes a symbol
fn tokenize_symbol(&mut self, first_symbol: char) -> Result<Token, Error> {
/// Helper macro to create a symbol token
macro_rules! symbol {
($symbol:ident) => {
Ok(Token::new(
TokenType::Symbol(Symbol::$symbol),
self.line,
self.column,
))
};
}
match first_symbol {
// single character symbols
'(' => symbol!(LParen),
')' => symbol!(RParen),
'{' => symbol!(LBrace),
'}' => symbol!(RBrace),
'[' => symbol!(LBracket),
']' => symbol!(RBracket),
';' => symbol!(Semicolon),
':' => symbol!(Colon),
',' => symbol!(Comma),
'+' => symbol!(Plus),
'-' => symbol!(Minus),
'/' => symbol!(Slash),
'.' => symbol!(Dot),
'^' => symbol!(Caret),
'%' => symbol!(Percent),
// multi-character symbols
'<' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(LessThanOrEqual)
}
'<' => symbol!(LessThan),
'>' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(GreaterThanOrEqual)
}
'>' => symbol!(GreaterThan),
'=' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(Equal)
}
'=' => symbol!(Assign),
'!' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(NotEqual)
}
'!' => symbol!(LogicalNot),
'*' if self.peek_next_char()? == Some('*') => {
self.next_char()?;
symbol!(Exp)
}
'*' => symbol!(Asterisk),
'&' if self.peek_next_char()? == Some('&') => {
self.next_char()?;
symbol!(LogicalAnd)
}
'|' if self.peek_next_char()? == Some('|') => {
self.next_char()?;
symbol!(LogicalOr)
}
_ => Err(Error::UnknownSymbolError(
first_symbol,
self.line,
self.column,
)),
}
}
/// Tokenizes a number literal. Also handles temperatures with a suffix of `c`, `f`, or `k`.
fn tokenize_number(&mut self, first_char: char) -> Result<Token, Error> {
let mut primary = String::with_capacity(16);
let mut decimal: Option<String> = None;
let mut reading_decimal = false;
let column = self.column;
let line = self.line;
primary.push(first_char);
while let Some(next_char) = self.peek_next_char()? {
if next_char.is_whitespace() {
break;
}
if next_char == '.' {
reading_decimal = true;
self.next_char()?;
continue;
}
// support underscores in numbers for readability
if next_char == '_' {
self.next_char()?;
continue;
}
// This is for the times when we have a number followed by a symbol (like a semicolon or =)
if !next_char.is_numeric() {
break;
}
if reading_decimal {
decimal.get_or_insert_with(String::new).push(next_char);
} else {
primary.push(next_char);
}
self.next_char()?;
}
let number: Number = if let Some(decimal) = decimal {
let decimal_scale = decimal.len() as u32;
let number = format!("{}{}", primary, decimal)
.parse::<i128>()
.map_err(|e| Error::NumberParseError(e, self.line, self.column))?;
Number::Decimal(
Decimal::try_from_i128_with_scale(number, decimal_scale)
.map_err(|e| Error::DecimalParseError(e, line, column))?,
)
} else {
Number::Integer(
primary
.parse()
.map_err(|e| Error::NumberParseError(e, line, column))?,
)
};
// check if the next char is a temperature suffix
if let Some(next_char) = self.peek_next_char()? {
let temperature = match next_char {
'c' => Temperature::Celsius(number),
'f' => Temperature::Fahrenheit(number),
'k' => Temperature::Kelvin(number),
_ => return Ok(Token::new(TokenType::Number(number), line, column)),
}
.to_kelvin();
self.next_char()?;
Ok(Token::new(TokenType::Number(temperature), line, column))
} else {
Ok(Token::new(TokenType::Number(number), line, column))
}
}
/// Tokenizes a string literal
fn tokenize_string(&mut self, beginning_quote: char) -> Result<Token, Error> {
let mut buffer = String::with_capacity(16);
let column = self.column;
let line = self.line;
while let Some(next_char) = self.next_char()? {
if next_char == beginning_quote {
break;
}
buffer.push(next_char);
}
Ok(Token::new(TokenType::String(buffer), line, column))
}
/// Tokenizes a keyword or an identifier. Also handles boolean literals
fn tokenize_keyword_or_identifier(&mut self, first_char: char) -> Result<Token, Error> {
macro_rules! keyword {
($keyword:ident) => {{
return Ok(Token::new(
TokenType::Keyword(Keyword::$keyword),
self.line,
self.column,
));
}};
}
/// Helper macro to check if the next character is whitespace or not alphanumeric
macro_rules! next_ws {
() => {
matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || !x.is_alphanumeric()) || self.peek_next_char()?.is_none()
};
}
let mut buffer = String::with_capacity(16);
let line = self.line;
let column = self.column;
let mut looped_char = Some(first_char);
while let Some(next_char) = looped_char {
if next_char.is_whitespace() {
break;
}
if !next_char.is_alphanumeric() {
break;
}
buffer.push(next_char);
match buffer.as_str() {
"let" if next_ws!() => keyword!(Let),
"fn" if next_ws!() => keyword!(Fn),
"if" if next_ws!() => keyword!(If),
"else" if next_ws!() => keyword!(Else),
"return" if next_ws!() => keyword!(Return),
"enum" if next_ws!() => keyword!(Enum),
"device" if next_ws!() => keyword!(Device),
"loop" if next_ws!() => keyword!(Loop),
"break" if next_ws!() => keyword!(Break),
"while" if next_ws!() => keyword!(While),
"continue" if next_ws!() => keyword!(Continue),
// boolean literals
"true" if next_ws!() => {
return Ok(Token::new(TokenType::Boolean(true), self.line, self.column));
}
"false" if next_ws!() => {
return Ok(Token::new(
TokenType::Boolean(false),
self.line,
self.column,
));
}
// if the next character is whitespace or not alphanumeric, then we have an identifier
// this is because keywords are checked first
val if next_ws!() => {
return Ok(Token::new(
TokenType::Identifier(val.to_string()),
line,
column,
));
}
_ => {}
}
looped_char = self.next_char()?;
}
Err(Error::UnknownKeywordOrIdentifierError(buffer, line, column))
}
}
pub struct TokenizerBuffer {
tokenizer: Tokenizer,
buffer: VecDeque<Token>,
history: VecDeque<Token>,
}
impl TokenizerBuffer {
pub fn new(tokenizer: Tokenizer) -> Self {
Self {
tokenizer,
buffer: VecDeque::new(),
history: VecDeque::with_capacity(128),
}
}
/// Reads the next token from the tokenizer, pushing the value to the back of the history
/// and returning the token
pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
if let Some(token) = self.buffer.pop_front() {
self.history.push_back(token.clone());
return Ok(Some(token));
}
let token = self.tokenizer.next_token()?;
if let Some(ref token) = token {
self.history.push_back(token.clone());
}
Ok(token)
}
/// Peeks the next token in the stream without adding to the history stack
pub fn peek(&mut self) -> Result<Option<Token>, Error> {
if let Some(token) = self.buffer.front() {
return Ok(Some(token.clone()));
}
let token = self.tokenizer.peek_next()?;
Ok(token)
}
fn seek_from_current(&mut self, seek_to: i64) -> Result<(), Error> {
use Ordering::*;
// if seek_to > 0 then we need to check if the buffer has enough tokens to pop, otherwise we need to read from the tokenizer
// if seek_to < 0 then we need to pop from the history and push to the front of the buffer. If not enough, then we throw (we reached the front of the history)
// if seek_to == 0 then we don't need to do anything
match seek_to.cmp(&0) {
Greater => {
let mut tokens = Vec::with_capacity(seek_to as usize);
for _ in 0..seek_to {
if let Some(token) = self.tokenizer.next_token()? {
tokens.push(token);
} else {
return Err(Error::IOError(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"Unexpected EOF",
)));
}
}
self.history.extend(tokens);
}
Less => {
let seek_to = seek_to.unsigned_abs() as usize;
let mut tokens = Vec::with_capacity(seek_to);
for _ in 0..seek_to {
if let Some(token) = self.history.pop_back() {
tokens.push(token);
} else {
return Err(Error::IOError(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"Unexpected EOF",
)));
}
}
self.buffer.extend(tokens.into_iter().rev());
}
_ => {}
}
Ok(())
}
/// Adds to or removes from the History stack, allowing the user to move back and forth in the stream
pub fn seek(&mut self, from: SeekFrom) -> Result<(), Error> {
match from {
SeekFrom::Current(seek_to) => self.seek_from_current(seek_to)?,
SeekFrom::End(_) => unimplemented!("SeekFrom::End will not be implemented"),
SeekFrom::Start(_) => unimplemented!("SeekFrom::Start will not be implemented"),
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
use rust_decimal::Decimal;
const TEST_FILE: &str = "tests/file.stlg";
const TEST_STRING: &str = r#"
fn test() {
let x = 10;
return x + 2;
}
"#;
#[test]
fn test_seek_from_current() -> Result<()> {
let tokenizer = Tokenizer::from(TEST_STRING.to_owned());
let mut buffer = TokenizerBuffer::new(tokenizer);
let token = buffer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn));
buffer.seek(SeekFrom::Current(1))?;
let token = buffer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Symbol(Symbol::LParen));
Ok(())
}
#[test]
fn test_tokenizer_from_path_ok() {
let tokenizer = Tokenizer::from_path(TEST_FILE);
assert!(tokenizer.is_ok());
}
#[test]
fn test_tokenizer_from_path_err() {
let tokenizer = Tokenizer::from_path("non_existent_file.stlg");
assert!(tokenizer.is_err());
}
#[test]
fn test_next_char() -> Result<()> {
let mut tokenizer = Tokenizer::from(TEST_STRING.to_owned());
let char = tokenizer.next_char()?;
assert_eq!(char, Some('\n'));
assert_eq!(tokenizer.line, 2);
assert_eq!(tokenizer.column, 1);
let mut tokenizer = Tokenizer::from(String::from("fn"));
let char = tokenizer.next_char()?;
assert_eq!(char, Some('f'));
assert_eq!(tokenizer.line, 1);
assert_eq!(tokenizer.column, 2);
Ok(())
}
#[test]
fn test_peek_next_char() -> Result<()> {
let mut tokenizer = Tokenizer::from(TEST_STRING.to_owned());
let char = tokenizer.peek_next_char()?;
assert_eq!(char, Some('\n'));
assert_eq!(tokenizer.line, 1);
assert_eq!(tokenizer.column, 1);
let char = tokenizer.next_char()?;
assert_eq!(char, Some('\n'));
assert_eq!(tokenizer.line, 2);
assert_eq!(tokenizer.column, 1);
let char = tokenizer.peek_next_char()?;
assert_eq!(char, Some(' '));
assert_eq!(tokenizer.line, 2);
assert_eq!(tokenizer.column, 1);
Ok(())
}
#[test]
fn test_temperature_unit() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("10c 14f 10k"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Number(Number::Decimal(Decimal::new(28315, 2)))
);
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Number(Number::Decimal(Decimal::new(26315, 2)))
);
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Number(Number::Integer(10)));
Ok(())
}
#[test]
fn test_parse_integer() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("10"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Number(Number::Integer(10)));
Ok(())
}
#[test]
fn test_parse_integer_with_underscore() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("1_000"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Number(Number::Integer(1000)));
Ok(())
}
#[test]
fn test_parse_decimal() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("10.5"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Number(Number::Decimal(Decimal::new(105, 1))) // 10.5
);
Ok(())
}
#[test]
fn test_parse_decimal_with_underscore() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("1_000.000_6"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Number(Number::Decimal(Decimal::new(10000006, 4))) // 1000.0006
);
Ok(())
}
#[test]
fn test_parse_number_with_symbol() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("10;"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Number(Number::Integer(10)));
let next_char = tokenizer.next_char()?;
assert_eq!(next_char, Some(';'));
Ok(())
}
#[test]
fn test_string_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from(r#""Hello, World!""#));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::String(String::from("Hello, World!"))
);
let mut tokenizer = Tokenizer::from(String::from(r#"'Hello, World!'"#));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::String(String::from("Hello, World!"))
);
Ok(())
}
#[test]
fn test_symbol_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from(
"^ ! () [] {} , . ; : + - * / < > = != && || >= <=**%",
));
let expected_tokens = vec![
TokenType::Symbol(Symbol::Caret),
TokenType::Symbol(Symbol::LogicalNot),
TokenType::Symbol(Symbol::LParen),
TokenType::Symbol(Symbol::RParen),
TokenType::Symbol(Symbol::LBracket),
TokenType::Symbol(Symbol::RBracket),
TokenType::Symbol(Symbol::LBrace),
TokenType::Symbol(Symbol::RBrace),
TokenType::Symbol(Symbol::Comma),
TokenType::Symbol(Symbol::Dot),
TokenType::Symbol(Symbol::Semicolon),
TokenType::Symbol(Symbol::Colon),
TokenType::Symbol(Symbol::Plus),
TokenType::Symbol(Symbol::Minus),
TokenType::Symbol(Symbol::Asterisk),
TokenType::Symbol(Symbol::Slash),
TokenType::Symbol(Symbol::LessThan),
TokenType::Symbol(Symbol::GreaterThan),
TokenType::Symbol(Symbol::Assign),
TokenType::Symbol(Symbol::NotEqual),
TokenType::Symbol(Symbol::LogicalAnd),
TokenType::Symbol(Symbol::LogicalOr),
TokenType::Symbol(Symbol::GreaterThanOrEqual),
TokenType::Symbol(Symbol::LessThanOrEqual),
TokenType::Symbol(Symbol::Exp),
TokenType::Symbol(Symbol::Percent),
];
for expected_token in expected_tokens {
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, expected_token);
}
Ok(())
}
#[test]
fn test_keyword_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("let fn if else return enum"));
let expected_tokens = vec![
TokenType::Keyword(Keyword::Let),
TokenType::Keyword(Keyword::Fn),
TokenType::Keyword(Keyword::If),
TokenType::Keyword(Keyword::Else),
TokenType::Keyword(Keyword::Return),
TokenType::Keyword(Keyword::Enum),
];
for expected_token in expected_tokens {
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, expected_token);
}
Ok(())
}
#[test]
fn test_identifier_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("fn test"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Identifier(String::from("test"))
);
Ok(())
}
#[test]
fn test_boolean_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("true false"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Boolean(true));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Boolean(false));
Ok(())
}
#[test]
fn test_full_source() -> Result<()> {
let mut tokenizer = Tokenizer::from(TEST_STRING.to_owned());
let expected_tokens = vec![
TokenType::Keyword(Keyword::Fn),
TokenType::Identifier(String::from("test")),
TokenType::Symbol(Symbol::LParen),
TokenType::Symbol(Symbol::RParen),
TokenType::Symbol(Symbol::LBrace),
TokenType::Keyword(Keyword::Let),
TokenType::Identifier(String::from("x")),
TokenType::Symbol(Symbol::Assign),
TokenType::Number(Number::Integer(10)),
TokenType::Symbol(Symbol::Semicolon),
TokenType::Keyword(Keyword::Return),
TokenType::Identifier(String::from("x")),
TokenType::Symbol(Symbol::Plus),
TokenType::Number(Number::Integer(2)),
TokenType::Symbol(Symbol::Semicolon),
TokenType::Symbol(Symbol::RBrace),
];
for expected_token in expected_tokens {
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, expected_token);
}
Ok(())
}
#[test]
fn test_peek_next() -> Result<()> {
let mut tokenizer = Tokenizer::from(TEST_STRING.to_owned());
let column = tokenizer.column;
let line = tokenizer.line;
let peeked_token = tokenizer.peek_next()?;
assert_eq!(
peeked_token.unwrap().token_type,
TokenType::Keyword(Keyword::Fn)
);
assert_eq!(tokenizer.column, column);
assert_eq!(tokenizer.line, line);
let next_token = tokenizer.next_token()?;
assert_eq!(
next_token.unwrap().token_type,
TokenType::Keyword(Keyword::Fn)
);
assert_ne!(tokenizer.column, column);
assert_ne!(tokenizer.line, line);
Ok(())
}
#[test]
fn test_compact_syntax() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("if(true) while(false)"));
// if(true)
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Keyword(Keyword::If)
);
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Symbol(Symbol::LParen)
);
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Boolean(true)
);
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Symbol(Symbol::RParen)
);
// while(false)
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Keyword(Keyword::While)
);
assert_eq!(
tokenizer.next_token()?.unwrap().token_type,
TokenType::Symbol(Symbol::LParen)
);
Ok(())
}
}

235
rust_compiler/libs/tokenizer/src/token.rs Normal file
View File

@@ -0,0 +1,235 @@
use rust_decimal::Decimal;
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token {
/// The type of the token
pub token_type: TokenType,
/// The line where the token was found
pub line: usize,
/// The column where the token was found
pub column: usize,
}
impl Token {
pub fn new(token_type: TokenType, line: usize, column: usize) -> Self {
Self {
token_type,
line,
column,
}
}
}
#[derive(Debug, PartialEq, Hash, Eq, Clone)]
pub enum Temperature {
Celsius(Number),
Fahrenheit(Number),
Kelvin(Number),
}
impl std::fmt::Display for Temperature {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Temperature::Celsius(n) => write!(f, "{}°C", n),
Temperature::Fahrenheit(n) => write!(f, "{}°F", n),
Temperature::Kelvin(n) => write!(f, "{}K", n),
}
}
}
impl Temperature {
pub fn to_kelvin(self) -> Number {
match self {
Temperature::Celsius(n) => {
let n = match n {
Number::Integer(i) => Decimal::new(i as i64, 0),
Number::Decimal(d) => d,
};
Number::Decimal(n + Decimal::new(27315, 2))
}
Temperature::Fahrenheit(n) => {
let n = match n {
Number::Integer(i) => Decimal::new(i as i64, 0),
Number::Decimal(d) => d,
};
let a = n - Decimal::new(32, 0);
let b = Decimal::new(5, 0) / Decimal::new(9, 0);
Number::Decimal(a * b + Decimal::new(27315, 2))
}
Temperature::Kelvin(n) => n,
}
}
}
#[derive(Debug, PartialEq, Hash, Eq, Clone)]
pub enum TokenType {
/// Represents a string token
String(String),
/// Represents a number token
Number(Number),
/// Represents a boolean token
Boolean(bool),
/// Represents a keyword token
Keyword(Keyword),
/// Represents an identifier token
Identifier(String),
/// Represents a symbol token
Symbol(Symbol),
/// Represents an end of file token
EOF,
}
impl std::fmt::Display for TokenType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TokenType::String(s) => write!(f, "{}", s),
TokenType::Number(n) => write!(f, "{}", n),
TokenType::Boolean(b) => write!(f, "{}", b),
TokenType::Keyword(k) => write!(f, "{:?}", k),
TokenType::Identifier(i) => write!(f, "{}", i),
TokenType::Symbol(s) => write!(f, "{:?}", s),
TokenType::EOF => write!(f, "EOF"),
}
}
}
#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)]
pub enum Number {
/// Represents an integer number
Integer(u128),
/// Represents a decimal type number with a precision of 64 bits
Decimal(Decimal),
}
impl std::fmt::Display for Number {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Number::Integer(i) => write!(f, "{}", i),
Number::Decimal(d) => write!(f, "{}", d),
}
}
}
impl std::convert::From<Number> for String {
fn from(value: Number) -> Self {
value.to_string()
}
}
#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)]
pub enum Symbol {
// Single Character Symbols
/// Represents the `(` symbol
LParen,
/// Represents the `)` symbol
RParen,
/// Represents the `{` symbol
LBrace,
/// Represents the `}` symbol
RBrace,
/// Represents the `[` symbol
LBracket,
/// Represents the `]` symbol
RBracket,
/// Represents the `;` symbol
Semicolon,
/// Represents the `:` symbol
Colon,
/// Represents the `,` symbol
Comma,
/// Represents the `+` symbol
Plus,
/// Represents the `-` symbol
Minus,
/// Represents the `*` symbol
Asterisk,
/// Represents the `/` symbol
Slash,
/// Represents the `<` symbol
LessThan,
/// Represents the `>` symbol
GreaterThan,
/// Represents the `=` symbol
Assign,
/// Represents the `!` symbol
LogicalNot,
/// Represents the `.` symbol
Dot,
/// Represents the `^` symbol
Caret,
/// Represents the `%` symbol
Percent,
// Double Character Symbols
/// Represents the `==` symbol
Equal,
/// Represents the `!=` symbol
NotEqual,
/// Represents the `&&` Symbol
LogicalAnd,
// Represents the `||` Symbol
LogicalOr,
/// Represents the `<=` symbol
LessThanOrEqual,
/// Represents the `>=` symbol
GreaterThanOrEqual,
/// Represents the `**` symbol
Exp,
}
impl Symbol {
pub fn is_operator(&self) -> bool {
matches!(
self,
Symbol::Plus
| Symbol::Minus
| Symbol::Asterisk
| Symbol::Slash
| Symbol::Exp
| Symbol::Percent
)
}
pub fn is_comparison(&self) -> bool {
matches!(
self,
Symbol::LessThan
| Symbol::GreaterThan
| Symbol::Equal
| Symbol::NotEqual
| Symbol::LessThanOrEqual
| Symbol::GreaterThanOrEqual,
)
}
pub fn is_logical(&self) -> bool {
matches!(self, Symbol::LogicalAnd | Symbol::LogicalOr)
}
}
#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)]
pub enum Keyword {
/// Represents the `continue` keyword
Continue,
/// Represents the `let` keyword
Let,
/// Represents the `fn` keyword
Fn,
/// Represents the `if` keyword
If,
/// Represents the `device` keyword. Useful for defining a device at a specific address (ex. d0, d1, d2, etc.)
Device,
/// Represents the `else` keyword
Else,
/// Represents the `return` keyword
Return,
/// Represents the `enum` keyword
Enum,
/// Represents the `loop` keyword
Loop,
/// Represents the `break` keyword
Break,
/// Represents the `while` keyword
While,
}

9
rust_compiler/libs/tokenizer/tests/file.stlg Normal file
View File

@@ -0,0 +1,9 @@
device self = "db";
device airConditioner = "d1";
device roomTemperatureSensor = "d2";
let roomTemperatureMin = 20c;
let roomTemperatureMax = 30c;
let averageTemperature = (roomTemperatureMax + roomTemperatureMin) / 2;