From 836fd3bf99aa9c09e37348da342b04b3e17dd73f Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 18:46:59 -0700 Subject: [PATCH 01/19] wip -- dot notation --- rust_compiler/libs/compiler/src/v1.rs | 160 +++++++++- rust_compiler/libs/parser/src/lib.rs | 327 +++++++++++++-------- rust_compiler/libs/parser/src/tree_node.rs | 44 ++- 3 files changed, 407 insertions(+), 124 deletions(-) diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index e4ef716..1eb455d 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -6,7 +6,8 @@ use parser::{ tree_node::{ AssignmentExpression, BinaryExpression, BlockExpression, DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression, InvocationExpression, Literal, - LiteralOrVariable, LogicalExpression, LoopExpression, Span, Spanned, WhileExpression, + LiteralOrVariable, LogicalExpression, LoopExpression, MethodCallExpression, Span, Spanned, + WhileExpression, }, }; use quick_error::quick_error; @@ -351,6 +352,9 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { temp_name: Some(result_name), })) } + Expression::MethodCall(method_expr) => { + self.expression_method_call(method_expr.node, scope, method_expr.span) + } _ => Err(Error::Unknown( format!( "Expression type not yet supported in general expression context: {:?}", @@ -540,6 +544,28 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope, ); } + Expression::MethodCall(method_expr) => { + if let Some(result) = + self.expression_method_call(method_expr.node, scope, method_expr.span)? + { + let var_loc = scope.add_variable(&name_str, LocationRequest::Persist)?; + + // Move result from temp to new persistent variable + let result_reg = self.resolve_register(&result.location)?; + self.emit_variable_assignment(&name_str, &var_loc, result_reg)?; + + // Free the temp result + if let Some(name) = result.temp_name { + scope.free_temp(name)?; + } + (var_loc, None) + } else { + return Err(Error::Unknown( + "Method call did not return a value".into(), + Some(method_expr.span), + )); + } + } _ => { return Err(Error::Unknown( format!("`{name_str}` declaration of this type is not supported/implemented."), @@ -1252,6 +1278,25 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope.free_temp(name)?; } } + Expression::Invocation(invoke_expr) => { + self.expression_function_invocation(invoke_expr, scope)?; + // The result is already in RETURN_REGISTER from the call + } + Expression::MethodCall(method_expr) => { + if let Some(result) = + self.expression_method_call(method_expr.node, scope, method_expr.span)? + { + let result_reg = self.resolve_register(&result.location)?; + self.write_output(format!( + "move r{} {}", + VariableScope::RETURN_REGISTER, + result_reg + ))?; + if let Some(name) = result.temp_name { + scope.free_temp(name)?; + } + } + } _ => { return Err(Error::Unknown( format!("Unsupported `return` statement: {:?}", expr), @@ -1546,4 +1591,117 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { self.write_output("j ra")?; Ok(()) } + + fn expression_method_call<'v>( + &mut self, + expr: MethodCallExpression, + scope: &mut VariableScope<'v>, + span: Span, + ) -> Result, Error> { + let object_name = expr.object.node; + + if let Some(device_val) = self.devices.get(&object_name).cloned() { + match expr.method.node.as_str() { + "load" => { + if expr.arguments.len() != 1 { + return Err(Error::AgrumentMismatch( + "load expects 1 argument".into(), + span, + )); + } + let arg = expr.arguments.into_iter().next().unwrap(); + let logic_type = match arg.node { + Expression::Literal(l) => match l.node { + Literal::String(s) => s, + _ => { + return Err(Error::AgrumentMismatch( + "load argument must be a string literal".into(), + arg.span, + )); + } + }, + _ => { + return Err(Error::AgrumentMismatch( + "load argument must be a string literal".into(), + arg.span, + )); + } + }; + + self.write_output(format!( + "l r{} {} {}", + VariableScope::RETURN_REGISTER, + device_val, + logic_type + ))?; + + // Move result to a temp register to be safe for use in expressions + let temp_name = self.next_temp_name(); + let temp_loc = scope.add_variable(&temp_name, LocationRequest::Temp)?; + self.emit_variable_assignment( + &temp_name, + &temp_loc, + format!("r{}", VariableScope::RETURN_REGISTER), + )?; + + return Ok(Some(CompilationResult { + location: temp_loc, + temp_name: Some(temp_name), + })); + } + "set" => { + if expr.arguments.len() != 2 { + return Err(Error::AgrumentMismatch( + "set expects 2 arguments".into(), + span, + )); + } + + let mut args_iter = expr.arguments.into_iter(); + let logic_type_arg = args_iter.next().unwrap(); + let value_arg = args_iter.next().unwrap(); + + let logic_type = match logic_type_arg.node { + Expression::Literal(l) => match l.node { + Literal::String(s) => s, + _ => { + return Err(Error::AgrumentMismatch( + "set expects a string literal as first argument".into(), + logic_type_arg.span, + )); + } + }, + _ => { + return Err(Error::AgrumentMismatch( + "set expects a string literal as first argument".into(), + logic_type_arg.span, + )); + } + }; + + let (val_str, cleanup) = self.compile_operand(value_arg, scope)?; + + self.write_output(format!("s {} {} {}", device_val, logic_type, val_str))?; + + if let Some(name) = cleanup { + scope.free_temp(name)?; + } + + return Ok(None); + } + _ => { + return Err(Error::Unknown( + format!( + "Unknown method '{}' on device '{}'", + expr.method.node, object_name + ), + Some(span), + )); + } + } + } + + Err(Error::UnknownIdentifier(object_name, expr.object.span)) + } } + diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index bf5217e..994e71f 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -121,7 +121,6 @@ impl<'a> Parser<'a> { } /// Calculates a Span from a given Token reference. - /// This is a static helper to avoid borrowing `self` when we already have a token ref. fn token_to_span(t: &Token) -> Span { let len = t.original_string.as_ref().map(|s| s.len()).unwrap_or(0); Span { @@ -149,7 +148,6 @@ impl<'a> Parser<'a> { where F: FnOnce(&mut Self) -> Result, { - // Peek at the start token. If no current token (parsing hasn't started), peek the buffer. let start_token = if self.current_token.is_some() { self.current_token.clone() } else { @@ -163,7 +161,6 @@ impl<'a> Parser<'a> { let node = parser(self)?; - // The end token is the current_token after parsing. let end_token = self.current_token.as_ref(); let (end_line, end_col) = end_token @@ -184,26 +181,15 @@ impl<'a> Parser<'a> { }) } - /// Skips tokens until a statement boundary is found to recover from errors. fn synchronize(&mut self) -> Result<(), Error> { - // We advance once to consume the error-causing token if we haven't already - // But often the error happens after we consumed something. - // Safe bet: consume current, then look. - - // If we assign next, we might be skipping the very token we want to sync on if the error didn't consume it? - // Usually, in recursive descent, the error is raised when `current` is unexpected. - // We want to discard `current` and move on. self.assign_next()?; while let Some(token) = &self.current_token { if token.token_type == TokenType::Symbol(Symbol::Semicolon) { - // Consuming the semicolon is a good place to stop and resume parsing next statement self.assign_next()?; return Ok(()); } - // Check if the token looks like the start of a statement. - // If so, we don't consume it; we return so the loop in parse_all can try to parse it. match token.token_type { TokenType::Keyword(Keyword::Fn) | TokenType::Keyword(Keyword::Let) @@ -231,7 +217,6 @@ impl<'a> Parser<'a> { let mut expressions = Vec::>::new(); loop { - // Check EOF without unwrapping error match self.tokenizer.peek() { Ok(None) => break, Err(e) => { @@ -248,19 +233,13 @@ impl<'a> Parser<'a> { Ok(None) => break, Err(e) => { self.errors.push(e); - // Recover if self.synchronize().is_err() { - // If sync failed (e.g. EOF during sync), break break; } } } } - // Even if we had errors, we return whatever partial AST we managed to build. - // If expressions is empty and we had errors, it's a failed parse, but we return a block. - - // Use the last token position for end span, or start if nothing parsed let end_token_opt = self.tokenizer.peek().unwrap_or(None); let (end_line, end_col) = end_token_opt .map(|tok| { @@ -285,7 +264,6 @@ impl<'a> Parser<'a> { pub fn parse(&mut self) -> Result>, Error> { self.assign_next()?; - // If assign_next hit EOF or error? if self.current_token.is_none() { return Ok(None); } @@ -317,15 +295,18 @@ impl<'a> Parser<'a> { return Ok(None); }; - // check if the next or current token is an operator, comparison, or logical symbol + // Handle Postfix operators (Member Access, Method Call) immediately after unary + let lhs = self.parse_postfix(lhs)?; + + // Handle Infix operators (Binary, Logical, Assignment) if self_matches_peek!( self, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign) ) { return Ok(Some(self.infix(lhs)?)); } else if self_matches_current!( self, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign) ) { self.tokenizer.seek(SeekFrom::Current(-1))?; return Ok(Some(self.infix(lhs)?)); @@ -334,6 +315,116 @@ impl<'a> Parser<'a> { Ok(Some(lhs)) } + /// Handles dot notation chains: x.y.z() + fn parse_postfix( + &mut self, + mut lhs: Spanned, + ) -> Result, Error> { + loop { + if self_matches_peek!(self, TokenType::Symbol(Symbol::Dot)) { + self.assign_next()?; // consume Dot + + let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_span = Self::token_to_span(identifier_token); + let identifier = match identifier_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => { + return Err(Error::UnexpectedToken( + identifier_span, + identifier_token.clone(), + )); + } + }; + + // Check for Method Call '()' + if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) { + // Method Call + self.assign_next()?; // consume '(' + let mut arguments = Vec::>::new(); + + while !token_matches!( + self.get_next()?.ok_or(Error::UnexpectedEOF)?, + TokenType::Symbol(Symbol::RParen) + ) { + let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + + // Block expressions not allowed in args + if let Expression::Block(_) = expression.node { + return Err(Error::InvalidSyntax( + self.current_span(), + String::from("Block expressions are not allowed in method calls"), + )); + } + arguments.push(expression); + + if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) + && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) + { + let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + return Err(Error::UnexpectedToken( + Self::token_to_span(next_token), + next_token.clone(), + )); + } + + if !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { + self.assign_next()?; + } + } + + // End span is the ')' + let end_span = self.current_span(); + let combined_span = Span { + start_line: lhs.span.start_line, + start_col: lhs.span.start_col, + end_line: end_span.end_line, + end_col: end_span.end_col, + }; + + lhs = Spanned { + span: combined_span, + node: Expression::MethodCall(Spanned { + span: combined_span, + node: MethodCallExpression { + object: boxed!(lhs), + method: Spanned { + span: identifier_span, + node: identifier, + }, + arguments, + }, + }), + }; + } else { + // Member Access + let combined_span = Span { + start_line: lhs.span.start_line, + start_col: lhs.span.start_col, + end_line: identifier_span.end_line, + end_col: identifier_span.end_col, + }; + + lhs = Spanned { + span: combined_span, + node: Expression::MemberAccess(Spanned { + span: combined_span, + node: MemberAccessExpression { + object: boxed!(lhs), + member: Spanned { + span: identifier_span, + node: identifier, + }, + }, + }), + }; + } + } else { + break; + } + } + Ok(lhs) + } + fn unary(&mut self) -> Result>, Error> { macro_rules! matches_keyword { ($keyword:expr, $($pattern:pat),+) => { @@ -357,10 +448,7 @@ impl<'a> Parser<'a> { )); } - TokenType::Keyword(Keyword::Let) => { - // declaration is wrapped in spanned inside the function, but expects 'let' to be current - Some(self.spanned(|p| p.declaration())?) - } + TokenType::Keyword(Keyword::Let) => Some(self.spanned(|p| p.declaration())?), TokenType::Keyword(Keyword::Device) => { let spanned_dev = self.spanned(|p| p.device())?; @@ -404,7 +492,6 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Break) => { let span = self.current_span(); - // make sure the next token is a semi-colon let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { return Err(Error::UnexpectedToken( @@ -451,16 +538,6 @@ impl<'a> Parser<'a> { }) } - TokenType::Identifier(_) - if self_matches_peek!(self, TokenType::Symbol(Symbol::Assign)) => - { - let spanned_assign = self.spanned(|p| p.assignment())?; - Some(Spanned { - span: spanned_assign.span, - node: Expression::Assignment(spanned_assign), - }) - } - TokenType::Identifier(ref id) => { let span = self.current_span(); Some(Spanned { @@ -489,24 +566,36 @@ impl<'a> Parser<'a> { } TokenType::Symbol(Symbol::LParen) => { - // Priority handles its own spanning self.spanned(|p| p.priority())?.node.map(|node| *node) } TokenType::Symbol(Symbol::Minus) => { - // Need to handle span manually because unary call is next let start_span = self.current_span(); self.assign_next()?; let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + // NOTE: Unary negation can also have postfix applied to the inner expression + // But generally -a.b parses as -(a.b), which is what parse_postfix ensures if called here. + // However, we call parse_postfix on the RESULT of unary in expression(), so + // `expression` sees `Negation`. `parse_postfix` doesn't apply to Negation node unless we allow it? + // Actually, `x.y` binds tighter than `-`. `postfix` logic belongs inside `unary` logic or + // `expression` logic. + // If I have `-x.y`, standard precedence says `-(x.y)`. + // `unary` returns `Negation(x)`. Then `expression` calls `postfix` on `Negation(x)`. + // `postfix` loop runs on `Negation`. This implies `(-x).y`. This is usually WRONG. + // `.` binds tighter than `-`. + // So `unary` must call `postfix` on the *operand* of the negation. + + let inner_with_postfix = self.parse_postfix(inner_expr)?; + let combined_span = Span { start_line: start_span.start_line, start_col: start_span.start_col, - end_line: inner_expr.span.end_line, - end_col: inner_expr.span.end_col, + end_line: inner_with_postfix.span.end_line, + end_col: inner_with_postfix.span.end_col, }; Some(Spanned { span: combined_span, - node: Expression::Negation(boxed!(inner_expr)), + node: Expression::Negation(boxed!(inner_with_postfix)), }) } @@ -514,17 +603,18 @@ impl<'a> Parser<'a> { let start_span = self.current_span(); self.assign_next()?; let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_with_postfix = self.parse_postfix(inner_expr)?; let combined_span = Span { start_line: start_span.start_line, start_col: start_span.start_col, - end_line: inner_expr.span.end_line, - end_col: inner_expr.span.end_col, + end_line: inner_with_postfix.span.end_line, + end_col: inner_with_postfix.span.end_col, }; Some(Spanned { span: combined_span, node: Expression::Logical(Spanned { span: combined_span, - node: LogicalExpression::Not(boxed!(inner_expr)), + node: LogicalExpression::Not(boxed!(inner_with_postfix)), }), }) } @@ -543,41 +633,44 @@ impl<'a> Parser<'a> { fn get_infix_child_node(&mut self) -> Result, Error> { let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; - match current_token.token_type { + let start_span = self.current_span(); + + let expr = match current_token.token_type { TokenType::Number(_) | TokenType::Boolean(_) => { let lit = self.spanned(|p| p.literal())?; - Ok(Spanned { + Spanned { span: lit.span, node: Expression::Literal(lit), - }) + } } TokenType::Identifier(ref ident) if !self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) => { + // This is a Variable. We need to check for Postfix operations on it. let span = self.current_span(); - Ok(Spanned { + Spanned { span, node: Expression::Variable(Spanned { span, node: ident.clone(), }), - }) + } } - TokenType::Symbol(Symbol::LParen) => Ok(*self + TokenType::Symbol(Symbol::LParen) => *self .spanned(|p| p.priority())? .node - .ok_or(Error::UnexpectedEOF)?), + .ok_or(Error::UnexpectedEOF)?, + TokenType::Identifier(_) if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) => { let inv = self.spanned(|p| p.invocation())?; - Ok(Spanned { + Spanned { span: inv.span, node: Expression::Invocation(inv), - }) + } } TokenType::Symbol(Symbol::Minus) => { - let start_span = self.current_span(); self.assign_next()?; let inner = self.get_infix_child_node()?; let span = Span { @@ -586,13 +679,12 @@ impl<'a> Parser<'a> { end_line: inner.span.end_line, end_col: inner.span.end_col, }; - Ok(Spanned { + Spanned { span, node: Expression::Negation(boxed!(inner)), - }) + } } TokenType::Symbol(Symbol::LogicalNot) => { - let start_span = self.current_span(); self.assign_next()?; let inner = self.get_infix_child_node()?; let span = Span { @@ -601,19 +693,25 @@ impl<'a> Parser<'a> { end_line: inner.span.end_line, end_col: inner.span.end_col, }; - Ok(Spanned { + Spanned { span, node: Expression::Logical(Spanned { span, node: LogicalExpression::Not(boxed!(inner)), }), - }) + } } - _ => Err(Error::UnexpectedToken( - self.current_span(), - current_token.clone(), - )), - } + _ => { + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); + } + }; + + // Important: We must check for postfix operations here too + // e.g. a + b.c + self.parse_postfix(expr) } fn device(&mut self) -> Result { @@ -665,39 +763,6 @@ impl<'a> Parser<'a> { }) } - fn assignment(&mut self) -> Result { - let identifier_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; - let identifier_span = Self::token_to_span(identifier_token); - let identifier = match identifier_token.token_type { - TokenType::Identifier(ref id) => id.clone(), - _ => { - return Err(Error::UnexpectedToken( - self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, - )); - } - }; - - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); - if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { - return Err(Error::UnexpectedToken( - Self::token_to_span(¤t_token), - current_token.clone(), - )); - } - self.assign_next()?; - - let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; - - Ok(AssignmentExpression { - identifier: Spanned { - span: identifier_span, - node: identifier, - }, - expression: boxed!(expression), - }) - } - fn infix(&mut self, previous: Spanned) -> Result, Error> { let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); @@ -708,7 +773,9 @@ impl<'a> Parser<'a> { | Expression::Priority(_) | Expression::Literal(_) | Expression::Variable(_) - | Expression::Negation(_) => {} + | Expression::Negation(_) + | Expression::MemberAccess(_) + | Expression::MethodCall(_) => {} _ => { return Err(Error::InvalidSyntax( self.current_span(), @@ -722,9 +789,10 @@ impl<'a> Parser<'a> { let mut temp_token = current_token.clone(); + // Include Assign in the operator loop while token_matches!( temp_token, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign) ) { let operator = match temp_token.token_type { TokenType::Symbol(s) => s, @@ -955,6 +1023,37 @@ impl<'a> Parser<'a> { } operators.retain(|symbol| !matches!(symbol, Symbol::LogicalOr)); + // --- PRECEDENCE LEVEL 8: Assignment (=) --- + // Assignment is Right Associative: a = b = c => a = (b = c) + // We iterate Right to Left + for (i, operator) in operators.iter().enumerate().rev() { + if matches!(operator, Symbol::Assign) { + let right = expressions.remove(i + 1); + let left = expressions.remove(i); + let span = Span { + start_line: left.span.start_line, + start_col: left.span.start_col, + end_line: right.span.end_line, + end_col: right.span.end_col, + }; + + expressions.insert( + i, + Spanned { + span, + node: Expression::Assignment(Spanned { + span, + node: AssignmentExpression { + assignee: boxed!(left), + expression: boxed!(right), + }, + }), + }, + ); + } + } + operators.retain(|symbol| !matches!(symbol, Symbol::Assign)); + if expressions.len() != 1 || !operators.is_empty() { return Err(Error::InvalidSyntax( self.current_span(), @@ -1506,7 +1605,6 @@ impl<'a> Parser<'a> { Literal::String(variable), ))) } - // ... (implementing other syscalls similarly using patterns above) "setOnDevice" => { check_length(self, &invocation.arguments, 3)?; let mut args = invocation.arguments.into_iter(); @@ -1531,23 +1629,10 @@ impl<'a> Parser<'a> { boxed!(variable), ))) } - _ => { - // For Math functions or unknown functions - if SysCall::is_syscall(&invocation.name.node) { - // Attempt to parse as math if applicable, or error if strict - // Here we are falling back to simple handling or error. - // Since Math isn't fully expanded in this snippet, we return Unsupported. - Err(Error::UnsupportedKeyword( - self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, - )) - } else { - Err(Error::UnsupportedKeyword( - self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, - )) - } - } + _ => Err(Error::UnsupportedKeyword( + self.current_span(), + self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + )), } } } diff --git a/rust_compiler/libs/parser/src/tree_node.rs b/rust_compiler/libs/parser/src/tree_node.rs index a968ed4..1d39d64 100644 --- a/rust_compiler/libs/parser/src/tree_node.rs +++ b/rust_compiler/libs/parser/src/tree_node.rs @@ -74,13 +74,13 @@ impl std::fmt::Display for LogicalExpression { #[derive(Debug, PartialEq, Eq)] pub struct AssignmentExpression { - pub identifier: Spanned, + pub assignee: Box>, pub expression: Box>, } impl std::fmt::Display for AssignmentExpression { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "({} = {})", self.identifier, self.expression) + write!(f, "({} = {})", self.assignee, self.expression) } } @@ -145,6 +145,41 @@ impl std::fmt::Display for InvocationExpression { } } +#[derive(Debug, PartialEq, Eq)] +pub struct MemberAccessExpression { + pub object: Box>, + pub member: Spanned, +} + +impl std::fmt::Display for MemberAccessExpression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.object, self.member) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct MethodCallExpression { + pub object: Box>, + pub method: Spanned, + pub arguments: Vec>, +} + +impl std::fmt::Display for MethodCallExpression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}.{}({})", + self.object, + self.method, + self.arguments + .iter() + .map(|e| e.to_string()) + .collect::>() + .join(", ") + ) + } +} + #[derive(Debug, PartialEq, Eq)] pub enum LiteralOrVariable { Literal(Literal), @@ -290,6 +325,8 @@ pub enum Expression { Literal(Spanned), Logical(Spanned), Loop(Spanned), + MemberAccess(Spanned), + MethodCall(Spanned), Negation(Box>), Priority(Box>), Return(Box>), @@ -314,6 +351,8 @@ impl std::fmt::Display for Expression { Expression::Literal(l) => write!(f, "{}", l), Expression::Logical(e) => write!(f, "{}", e), Expression::Loop(e) => write!(f, "{}", e), + Expression::MemberAccess(e) => write!(f, "{}", e), + Expression::MethodCall(e) => write!(f, "{}", e), Expression::Negation(e) => write!(f, "(-{})", e), Expression::Priority(e) => write!(f, "({})", e), Expression::Return(e) => write!(f, "(return {})", e), @@ -323,3 +362,4 @@ impl std::fmt::Display for Expression { } } } + From 48049b79ec09ade303acee0fba1e554bb04090aa Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 18:48:37 -0700 Subject: [PATCH 02/19] revert compiler --- rust_compiler/libs/compiler/src/v1.rs | 159 +------------------------- 1 file changed, 1 insertion(+), 158 deletions(-) diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index 1eb455d..2125d97 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -6,8 +6,7 @@ use parser::{ tree_node::{ AssignmentExpression, BinaryExpression, BlockExpression, DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression, InvocationExpression, Literal, - LiteralOrVariable, LogicalExpression, LoopExpression, MethodCallExpression, Span, Spanned, - WhileExpression, + LiteralOrVariable, LogicalExpression, LoopExpression, Span, Spanned, WhileExpression, }, }; use quick_error::quick_error; @@ -352,9 +351,6 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { temp_name: Some(result_name), })) } - Expression::MethodCall(method_expr) => { - self.expression_method_call(method_expr.node, scope, method_expr.span) - } _ => Err(Error::Unknown( format!( "Expression type not yet supported in general expression context: {:?}", @@ -544,28 +540,6 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope, ); } - Expression::MethodCall(method_expr) => { - if let Some(result) = - self.expression_method_call(method_expr.node, scope, method_expr.span)? - { - let var_loc = scope.add_variable(&name_str, LocationRequest::Persist)?; - - // Move result from temp to new persistent variable - let result_reg = self.resolve_register(&result.location)?; - self.emit_variable_assignment(&name_str, &var_loc, result_reg)?; - - // Free the temp result - if let Some(name) = result.temp_name { - scope.free_temp(name)?; - } - (var_loc, None) - } else { - return Err(Error::Unknown( - "Method call did not return a value".into(), - Some(method_expr.span), - )); - } - } _ => { return Err(Error::Unknown( format!("`{name_str}` declaration of this type is not supported/implemented."), @@ -1278,25 +1252,6 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope.free_temp(name)?; } } - Expression::Invocation(invoke_expr) => { - self.expression_function_invocation(invoke_expr, scope)?; - // The result is already in RETURN_REGISTER from the call - } - Expression::MethodCall(method_expr) => { - if let Some(result) = - self.expression_method_call(method_expr.node, scope, method_expr.span)? - { - let result_reg = self.resolve_register(&result.location)?; - self.write_output(format!( - "move r{} {}", - VariableScope::RETURN_REGISTER, - result_reg - ))?; - if let Some(name) = result.temp_name { - scope.free_temp(name)?; - } - } - } _ => { return Err(Error::Unknown( format!("Unsupported `return` statement: {:?}", expr), @@ -1591,117 +1546,5 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { self.write_output("j ra")?; Ok(()) } - - fn expression_method_call<'v>( - &mut self, - expr: MethodCallExpression, - scope: &mut VariableScope<'v>, - span: Span, - ) -> Result, Error> { - let object_name = expr.object.node; - - if let Some(device_val) = self.devices.get(&object_name).cloned() { - match expr.method.node.as_str() { - "load" => { - if expr.arguments.len() != 1 { - return Err(Error::AgrumentMismatch( - "load expects 1 argument".into(), - span, - )); - } - let arg = expr.arguments.into_iter().next().unwrap(); - let logic_type = match arg.node { - Expression::Literal(l) => match l.node { - Literal::String(s) => s, - _ => { - return Err(Error::AgrumentMismatch( - "load argument must be a string literal".into(), - arg.span, - )); - } - }, - _ => { - return Err(Error::AgrumentMismatch( - "load argument must be a string literal".into(), - arg.span, - )); - } - }; - - self.write_output(format!( - "l r{} {} {}", - VariableScope::RETURN_REGISTER, - device_val, - logic_type - ))?; - - // Move result to a temp register to be safe for use in expressions - let temp_name = self.next_temp_name(); - let temp_loc = scope.add_variable(&temp_name, LocationRequest::Temp)?; - self.emit_variable_assignment( - &temp_name, - &temp_loc, - format!("r{}", VariableScope::RETURN_REGISTER), - )?; - - return Ok(Some(CompilationResult { - location: temp_loc, - temp_name: Some(temp_name), - })); - } - "set" => { - if expr.arguments.len() != 2 { - return Err(Error::AgrumentMismatch( - "set expects 2 arguments".into(), - span, - )); - } - - let mut args_iter = expr.arguments.into_iter(); - let logic_type_arg = args_iter.next().unwrap(); - let value_arg = args_iter.next().unwrap(); - - let logic_type = match logic_type_arg.node { - Expression::Literal(l) => match l.node { - Literal::String(s) => s, - _ => { - return Err(Error::AgrumentMismatch( - "set expects a string literal as first argument".into(), - logic_type_arg.span, - )); - } - }, - _ => { - return Err(Error::AgrumentMismatch( - "set expects a string literal as first argument".into(), - logic_type_arg.span, - )); - } - }; - - let (val_str, cleanup) = self.compile_operand(value_arg, scope)?; - - self.write_output(format!("s {} {} {}", device_val, logic_type, val_str))?; - - if let Some(name) = cleanup { - scope.free_temp(name)?; - } - - return Ok(None); - } - _ => { - return Err(Error::Unknown( - format!( - "Unknown method '{}' on device '{}'", - expr.method.node, object_name - ), - Some(span), - )); - } - } - } - - Err(Error::UnknownIdentifier(object_name, expr.object.span)) - } } From 5de614cc38ebd095cc0a2ec7b5709628a1d3958c Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 19:01:32 -0700 Subject: [PATCH 03/19] compiler support for dot notation --- rust_compiler/libs/compiler/src/v1.rs | 231 +++++++++++++++++++++----- 1 file changed, 192 insertions(+), 39 deletions(-) diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index 2125d97..de7ad3a 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -6,7 +6,8 @@ use parser::{ tree_node::{ AssignmentExpression, BinaryExpression, BlockExpression, DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression, InvocationExpression, Literal, - LiteralOrVariable, LogicalExpression, LoopExpression, Span, Spanned, WhileExpression, + LiteralOrVariable, LogicalExpression, LoopExpression, MemberAccessExpression, Span, + Spanned, WhileExpression, }, }; use quick_error::quick_error; @@ -332,6 +333,42 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { } } } + Expression::MemberAccess(access) => { + // "load" behavior (e.g. `let x = d0.On`) + let MemberAccessExpression { object, member } = access.node; + + // 1. Resolve the object to a device string (e.g., "d0" or "rX") + let (device_str, cleanup) = self.resolve_device(*object, scope)?; + + // 2. Allocate a temp register for the result + let result_name = self.next_temp_name(); + let loc = scope.add_variable(&result_name, LocationRequest::Temp)?; + let reg = self.resolve_register(&loc)?; + + // 3. Emit load instruction: l rX device member + self.write_output(format!("l {} {} {}", reg, device_str, member.node))?; + + // 4. Cleanup + if let Some(c) = cleanup { + scope.free_temp(c)?; + } + + Ok(Some(CompilationResult { + location: loc, + temp_name: Some(result_name), + })) + } + Expression::MethodCall(call) => { + // Methods are not yet fully supported (e.g. `d0.SomeFunc()`). + // This would likely map to specialized syscalls or batch instructions. + Err(Error::Unknown( + format!( + "Method calls are not yet supported: {}", + call.node.method.node + ), + Some(call.span), + )) + } Expression::Priority(inner_expr) => self.expression(*inner_expr, scope), Expression::Negation(inner_expr) => { // Compile negation as 0 - inner @@ -361,6 +398,24 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { } } + /// Resolves an expression to a device identifier string for use in instructions like `s` or `l`. + /// Returns (device_string, optional_cleanup_temp_name). + fn resolve_device<'v>( + &mut self, + expr: Spanned, + scope: &mut VariableScope<'v>, + ) -> Result<(String, Option), Error> { + // If it's a direct variable reference, check if it's a known device alias first + if let Expression::Variable(ref name) = expr.node + && let Some(device_id) = self.devices.get(&name.node) + { + return Ok((device_id.clone(), None)); + } + + // Otherwise, compile it as an operand (e.g. it might be a register holding a device hash/id) + self.compile_operand(expr, scope) + } + fn emit_variable_assignment( &mut self, var_name: &str, @@ -540,6 +595,35 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope, ); } + Expression::MemberAccess(access) => { + // Compile the member access (load instruction) + let result = self.expression( + Spanned { + node: Expression::MemberAccess(access), + span: name_span, // Use declaration span roughly + }, + scope, + )?; + + // Result is in a temp register + let Some(comp_res) = result else { + return Err(Error::Unknown( + "Member access did not return a value".into(), + Some(name_span), + )); + }; + + let var_loc = scope.add_variable(&name_str, LocationRequest::Persist)?; + let result_reg = self.resolve_register(&comp_res.location)?; + + self.emit_variable_assignment(&name_str, &var_loc, result_reg)?; + + if let Some(temp) = comp_res.temp_name { + scope.free_temp(temp)?; + } + + (var_loc, None) + } _ => { return Err(Error::Unknown( format!("`{name_str}` declaration of this type is not supported/implemented."), @@ -560,49 +644,76 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope: &mut VariableScope<'v>, ) -> Result<(), Error> { let AssignmentExpression { - identifier, + assignee, expression, } = expr; - let location = match scope.get_location_of(&identifier.node) { - Ok(l) => l, - Err(_) => { - self.errors.push(Error::UnknownIdentifier( - identifier.node.clone(), - identifier.span, + match assignee.node { + Expression::Variable(identifier) => { + let location = match scope.get_location_of(&identifier.node) { + Ok(l) => l, + Err(_) => { + self.errors.push(Error::UnknownIdentifier( + identifier.node.clone(), + identifier.span, + )); + VariableLocation::Temporary(0) + } + }; + + let (val_str, cleanup) = self.compile_operand(*expression, scope)?; + + let debug_tag = if self.config.debug { + format!(" #{}", identifier.node) + } else { + String::new() + }; + + match location { + VariableLocation::Temporary(reg) | VariableLocation::Persistant(reg) => { + self.write_output(format!("move r{reg} {val_str}{debug_tag}"))?; + } + VariableLocation::Stack(offset) => { + // Calculate address: sp - offset + self.write_output(format!( + "sub r{0} sp {offset}", + VariableScope::TEMP_STACK_REGISTER + ))?; + // Store value to stack/db at address + self.write_output(format!( + "put db r{0} {val_str}{debug_tag}", + VariableScope::TEMP_STACK_REGISTER + ))?; + } + } + + if let Some(name) = cleanup { + scope.free_temp(name)?; + } + } + Expression::MemberAccess(access) => { + // Set instruction: s device member value + let MemberAccessExpression { object, member } = access.node; + + let (device_str, dev_cleanup) = self.resolve_device(*object, scope)?; + let (val_str, val_cleanup) = self.compile_operand(*expression, scope)?; + + self.write_output(format!("s {} {} {}", device_str, member.node, val_str))?; + + if let Some(c) = dev_cleanup { + scope.free_temp(c)?; + } + if let Some(c) = val_cleanup { + scope.free_temp(c)?; + } + } + _ => { + return Err(Error::Unknown( + "Invalid assignment target. Only variables and member access are supported." + .into(), + Some(assignee.span), )); - VariableLocation::Temporary(0) } - }; - - let (val_str, cleanup) = self.compile_operand(*expression, scope)?; - - let debug_tag = if self.config.debug { - format!(" #{}", identifier.node) - } else { - String::new() - }; - - match location { - VariableLocation::Temporary(reg) | VariableLocation::Persistant(reg) => { - self.write_output(format!("move r{reg} {val_str}{debug_tag}"))?; - } - VariableLocation::Stack(offset) => { - // Calculate address: sp - offset - self.write_output(format!( - "sub r{0} sp {offset}", - VariableScope::TEMP_STACK_REGISTER - ))?; - // Store value to stack/db at address - self.write_output(format!( - "put db r{0} {val_str}{debug_tag}", - VariableScope::TEMP_STACK_REGISTER - ))?; - } - } - - if let Some(name) = cleanup { - scope.free_temp(name)?; } Ok(()) @@ -705,6 +816,31 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { stack.free_temp(name)?; } } + Expression::MemberAccess(access) => { + // Compile member access to temp and push + let result_opt = self.expression( + Spanned { + node: Expression::MemberAccess(access), + span: Span { + start_col: 0, + end_col: 0, + start_line: 0, + end_line: 0, + }, // Dummy span + }, + stack, + )?; + + if let Some(result) = result_opt { + let reg_str = self.resolve_register(&result.location)?; + self.write_output(format!("push {reg_str}"))?; + if let Some(name) = result.temp_name { + stack.free_temp(name)?; + } + } else { + self.write_output("push 0")?; // Should fail ideally + } + } _ => { return Err(Error::Unknown( format!( @@ -1252,6 +1388,23 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { scope.free_temp(name)?; } } + Expression::MemberAccess(access) => { + // Return result of member access + let res_opt = self.expression( + Spanned { + node: Expression::MemberAccess(access), + span: expr.span, + }, + scope, + )?; + if let Some(res) = res_opt { + let reg = self.resolve_register(&res.location)?; + self.write_output(format!("move r{} {}", VariableScope::RETURN_REGISTER, reg))?; + if let Some(temp) = res.temp_name { + scope.free_temp(temp)?; + } + } + } _ => { return Err(Error::Unknown( format!("Unsupported `return` statement: {:?}", expr), From b3c732bbb71401cad667b7067269f88c45236f1d Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 23:08:56 -0700 Subject: [PATCH 04/19] Start generating documentation for built-in types and functions --- rust_compiler/Cargo.lock | 1 + rust_compiler/libs/parser/Cargo.toml | 1 + rust_compiler/libs/parser/src/lib.rs | 5 + rust_compiler/libs/parser/src/macros.rs | 85 ++++++ rust_compiler/libs/parser/src/sys_call.rs | 303 ++++++++++++--------- rust_compiler/libs/parser/src/test/docs.rs | 12 + rust_compiler/libs/parser/src/test/mod.rs | 10 +- 7 files changed, 279 insertions(+), 138 deletions(-) create mode 100644 rust_compiler/libs/parser/src/macros.rs create mode 100644 rust_compiler/libs/parser/src/test/docs.rs diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index b42d3c4..0c3ea01 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -496,6 +496,7 @@ version = "0.1.0" dependencies = [ "anyhow", "lsp-types", + "pretty_assertions", "quick-error", "tokenizer", ] diff --git a/rust_compiler/libs/parser/Cargo.toml b/rust_compiler/libs/parser/Cargo.toml index 5ff0cd5..504d535 100644 --- a/rust_compiler/libs/parser/Cargo.toml +++ b/rust_compiler/libs/parser/Cargo.toml @@ -11,3 +11,4 @@ lsp-types = { workspace = true } [dev-dependencies] anyhow = { version = "1" } +pretty_assertions = "1.4" diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index 994e71f..f58560f 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -1,3 +1,4 @@ +mod macros; #[cfg(test)] mod test; @@ -14,6 +15,10 @@ use tokenizer::{ }; use tree_node::*; +pub trait Documentation { + fn docs(&self) -> String; +} + #[macro_export] /// A macro to create a boxed value. macro_rules! boxed { diff --git a/rust_compiler/libs/parser/src/macros.rs b/rust_compiler/libs/parser/src/macros.rs new file mode 100644 index 0000000..6eefb02 --- /dev/null +++ b/rust_compiler/libs/parser/src/macros.rs @@ -0,0 +1,85 @@ +#[macro_export] +macro_rules! documented { + // ------------------------------------------------------------------------- + // Internal Helper: Filter doc comments + // ------------------------------------------------------------------------- + + // Case 1: Doc comment. Return Some("string"). + // We match the specific structure of a doc attribute. + (@doc_filter #[doc = $doc:expr]) => { + Some($doc) + }; + + // Case 2: Other attributes (derives, etc.). Return None. + // We catch any other token sequence inside the brackets. + (@doc_filter #[$($attr:tt)*]) => { + None + }; + + // ------------------------------------------------------------------------- + // Internal Helper: Match patterns for `match self` + // ------------------------------------------------------------------------- + (@arm $name:ident $variant:ident) => { + $name::$variant + }; + (@arm $name:ident $variant:ident ( $($tuple:tt)* )) => { + $name::$variant(..) + }; + (@arm $name:ident $variant:ident { $($structure:tt)* }) => { + $name::$variant{..} + }; + + // ------------------------------------------------------------------------- + // Main Macro Entry Point + // ------------------------------------------------------------------------- + ( + $(#[$enum_attr:meta])* $vis:vis enum $name:ident { + $( + // Capture attributes as a sequence of token trees inside brackets + // to avoid "local ambiguity" and handle multi-token attributes (like doc="..."). + $(#[ $($variant_attr:tt)* ])* + $variant:ident + $( ($($tuple:tt)*) )? + $( {$($structure:tt)*} )? + ),* $(,)? + } + ) => { + // 1. Generate the actual Enum definition + $(#[$enum_attr])* + $vis enum $name { + $( + $(#[ $($variant_attr)* ])* + $variant + $( ($($tuple)*) )? + $( {$($structure)*} )?, + )* + } + + // 2. Implement the Trait + impl Documentation for $name { + fn docs(&self) -> String { + match self { + $( + documented!(@arm $name $variant $( ($($tuple)*) )? $( {$($structure)*} )? ) => { + // Create a temporary array of Option<&str> for all attributes + let doc_lines: &[Option<&str>] = &[ + $( + documented!(@doc_filter #[ $($variant_attr)* ]) + ),* + ]; + + // Filter out the Nones (non-doc attributes), join, and return + doc_lines.iter() + .filter_map(|&d| d) + .collect::>() + .join("\n") + .trim() + .to_string() + } + )* + } + } + } + }; +} + diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index 494fac5..ed9d2c9 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -1,73 +1,107 @@ -use crate::tree_node::{Expression, Literal, Spanned}; - use super::LiteralOrVariable; +use crate::tree_node::{Expression, Literal, Spanned}; +use crate::{Documentation, documented}; -#[derive(Debug, PartialEq, Eq)] -pub enum Math { - /// Returns the angle in radians whose cosine is the specified number. - /// ## In Game - /// `acos r? a(r?|num)` - Acos(LiteralOrVariable), - /// Returns the angle in radians whose sine is the specified number. - /// ## In Game - /// `asin r? a(r?|num)` - Asin(LiteralOrVariable), - /// Returns the angle in radians whose tangent is the specified number. - /// ## In Game - /// `atan r? a(r?|num)` - Atan(LiteralOrVariable), - /// Returns the angle in radians whose tangent is the quotient of the specified numbers. - /// ## In Game - /// `atan2 r? a(r?|num) b(r?|num)` - Atan2(LiteralOrVariable, LiteralOrVariable), - /// Gets the absolute value of a number. - /// ## In Game - /// `abs r? a(r?|num)` - Abs(LiteralOrVariable), - /// Rounds a number up to the nearest whole number. - /// ## In Game - /// `ceil r? a(r?|num)` - Ceil(LiteralOrVariable), - /// Returns the cosine of the specified angle in radians. - /// ## In Game - /// cos r? a(r?|num) - Cos(LiteralOrVariable), - /// Rounds a number down to the nearest whole number. - /// ## In Game - /// `floor r? a(r?|num)` - Floor(LiteralOrVariable), - /// Computes the natural logarithm of a number. - /// ## In Game - /// `log r? a(r?|num)` - Log(LiteralOrVariable), - /// Computes the maximum of two numbers. - /// ## In Game - /// `max r? a(r?|num) b(r?|num)` - Max(LiteralOrVariable, LiteralOrVariable), - /// Computes the minimum of two numbers. - /// ## In Game - /// `min r? a(r?|num) b(r?|num)` - Min(LiteralOrVariable, LiteralOrVariable), - /// Gets a random number between 0 and 1. - /// ## In Game - /// `rand r?` - Rand, - /// Returns the sine of the specified angle in radians. - /// ## In Game - /// `sin r? a(r?|num)` - Sin(LiteralOrVariable), - /// Computes the square root of a number. - /// ## In Game - /// `sqrt r? a(r?|num)` - Sqrt(LiteralOrVariable), - /// Returns the tangent of the specified angle in radians. - /// ## In Game - /// `tan r? a(r?|num)` - Tan(LiteralOrVariable), - /// Truncates a number by removing the decimal portion. - /// ## In Game - /// `trunc r? a(r?|num)` - Trunc(LiteralOrVariable), +documented! { + #[derive(Debug, PartialEq, Eq)] + pub enum Math { + /// Returns the angle in radians whose cosine is the specified number. + /// ## IC10 + /// `acos r? a(r?|num)` + /// ## Slang + /// `(number|var).acos();` + Acos(LiteralOrVariable), + /// Returns the angle in radians whose sine is the specified number. + /// ## IC10 + /// `asin r? a(r?|num)` + /// ## Slang + /// `(number|var).asin();` + Asin(LiteralOrVariable), + /// Returns the angle in radians whose tangent is the specified number. + /// ## IC10 + /// `atan r? a(r?|num)` + /// ## Slang + /// `(number|var).atan();` + Atan(LiteralOrVariable), + /// Returns the angle in radians whose tangent is the quotient of the specified numbers. + /// ## IC10 + /// `atan2 r? a(r?|num) b(r?|num)` + /// ## Slang + /// `(number|var).atan2((number|var));` + Atan2(LiteralOrVariable, LiteralOrVariable), + /// Gets the absolute value of a number. + /// ## IC10 + /// `abs r? a(r?|num)` + /// ## Slang + /// `(number|var).abs();` + Abs(LiteralOrVariable), + /// Rounds a number up to the nearest whole number. + /// ## IC10 + /// `ceil r? a(r?|num)` + /// ## Slang + /// `(number|var).ceil();` + Ceil(LiteralOrVariable), + /// Returns the cosine of the specified angle in radians. + /// ## IC10 + /// `cos r? a(r?|num)` + /// ## Slang + /// `(number|var).cos();` + Cos(LiteralOrVariable), + /// Rounds a number down to the nearest whole number. + /// ## In Game + /// `floor r? a(r?|num)` + /// ## Slang + /// `(number|var).floor();` + Floor(LiteralOrVariable), + /// Computes the natural logarithm of a number. + /// ## IC10 + /// `log r? a(r?|num)` + /// ## Slang + /// `(number|var).log();` + Log(LiteralOrVariable), + /// Computes the maximum of two numbers. + /// ## IC10 + /// `max r? a(r?|num) b(r?|num)` + /// ## Slang + /// `(number|var).max((number|var));` + Max(LiteralOrVariable, LiteralOrVariable), + /// Computes the minimum of two numbers. + /// ## IC10 + /// `min r? a(r?|num) b(r?|num)` + /// ## Slang + /// `(number|var).min((number|var));` + Min(LiteralOrVariable, LiteralOrVariable), + /// Gets a random number between 0 and 1. + /// ## IC10 + /// `rand r?` + /// ## Slang + /// `rand();` + Rand, + /// Returns the sine of the specified angle in radians. + /// ## IC10 + /// `sin r? a(r?|num)` + /// ## Slang + /// `(number|var).sin();` + Sin(LiteralOrVariable), + /// Computes the square root of a number. + /// ## IC10 + /// `sqrt r? a(r?|num)` + /// ## Slang + /// `(number|var).sqrt();` + Sqrt(LiteralOrVariable), + /// Returns the tangent of the specified angle in radians. + /// ## IC10 + /// `tan r? a(r?|num)` + /// ## Slang + /// `(number|var).tan();` + Tan(LiteralOrVariable), + /// Truncates a number by removing the decimal portion. + /// ## IC10 + /// `trunc r? a(r?|num)` + /// ## Slang + /// `(number|var).trunc();` + Trunc(LiteralOrVariable), + } } impl std::fmt::Display for Math { @@ -93,71 +127,73 @@ impl std::fmt::Display for Math { } } -#[derive(Debug, PartialEq, Eq)] -pub enum System { - /// Pauses execution for exactly 1 tick and then resumes. - /// ## In Game - /// yield - Yield, - /// Represents a function that can be called to sleep for a certain amount of time. - /// ## In Game - /// `sleep a(r?|num)` - Sleep(Box>), - /// Gets the in-game hash for a specific prefab name. - /// ## In Game - /// `HASH("prefabName")` - Hash(Literal), - /// Represents a function which loads a device variable into a register. - /// ## In Game - /// `l r? d? var` - /// ## Examples - /// `l r0 d0 Setting` - /// `l r1 d5 Pressure` - LoadFromDevice(LiteralOrVariable, Literal), - /// Function which gets a LogicType from all connected network devices that match - /// the provided device hash and name, aggregating them via a batchMode - /// ## In Game - /// lbn r? deviceHash nameHash logicType batchMode - /// ## Examples - /// lbn r0 HASH("StructureWallLight") HASH("wallLight") On Minimum - LoadBatchNamed( - LiteralOrVariable, - Box>, - Literal, - Literal, - ), - /// Loads a LogicType from all connected network devices, aggregating them via a - /// batchMode - /// ## In Game - /// lb r? deviceHash logicType batchMode - /// ## Examples - /// lb r0 HASH("StructureWallLight") On Minimum - LoadBatch(LiteralOrVariable, Literal, Literal), - /// Represents a function which stores a setting into a specific device. - /// ## In Game - /// `s d? logicType r?` - /// ## Example - /// `s d0 Setting r0` - SetOnDevice(LiteralOrVariable, Literal, Box>), - /// Represents a function which stores a setting to all devices that match - /// the given deviceHash - /// ## In Game - /// `sb deviceHash logicType r?` - /// ## Example - /// `sb HASH("Doors") Lock 1` - SetOnDeviceBatched(LiteralOrVariable, Literal, Box>), - /// Represents a function which stores a setting to all devices that match - /// both the given deviceHash AND the given nameHash - /// ## In Game - /// `sbn deviceHash nameHash logicType r?` - /// ## Example - /// `sbn HASH("Doors") HASH("Exterior") Lock 1` - SetOnDeviceBatchedNamed( - LiteralOrVariable, - LiteralOrVariable, - Literal, - Box>, - ), +documented! { + #[derive(Debug, PartialEq, Eq)] + pub enum System { + /// Pauses execution for exactly 1 tick and then resumes. + /// ## In Game + /// yield + Yield, + /// Represents a function that can be called to sleep for a certain amount of time. + /// ## In Game + /// `sleep a(r?|num)` + Sleep(Box>), + /// Gets the in-game hash for a specific prefab name. + /// ## In Game + /// `HASH("prefabName")` + Hash(Literal), + /// Represents a function which loads a device variable into a register. + /// ## In Game + /// `l r? d? var` + /// ## Examples + /// `l r0 d0 Setting` + /// `l r1 d5 Pressure` + LoadFromDevice(LiteralOrVariable, Literal), + /// Function which gets a LogicType from all connected network devices that match + /// the provided device hash and name, aggregating them via a batchMode + /// ## In Game + /// lbn r? deviceHash nameHash logicType batchMode + /// ## Examples + /// lbn r0 HASH("StructureWallLight") HASH("wallLight") On Minimum + LoadBatchNamed( + LiteralOrVariable, + Box>, + Literal, + Literal, + ), + /// Loads a LogicType from all connected network devices, aggregating them via a + /// batchMode + /// ## In Game + /// lb r? deviceHash logicType batchMode + /// ## Examples + /// lb r0 HASH("StructureWallLight") On Minimum + LoadBatch(LiteralOrVariable, Literal, Literal), + /// Represents a function which stores a setting into a specific device. + /// ## In Game + /// `s d? logicType r?` + /// ## Example + /// `s d0 Setting r0` + SetOnDevice(LiteralOrVariable, Literal, Box>), + /// Represents a function which stores a setting to all devices that match + /// the given deviceHash + /// ## In Game + /// `sb deviceHash logicType r?` + /// ## Example + /// `sb HASH("Doors") Lock 1` + SetOnDeviceBatched(LiteralOrVariable, Literal, Box>), + /// Represents a function which stores a setting to all devices that match + /// both the given deviceHash AND the given nameHash + /// ## In Game + /// `sbn deviceHash nameHash logicType r?` + /// ## Example + /// `sbn HASH("Doors") HASH("Exterior") Lock 1` + SetOnDeviceBatchedNamed( + LiteralOrVariable, + LiteralOrVariable, + Literal, + Box>, + ), + } } impl std::fmt::Display for System { @@ -229,4 +265,3 @@ impl SysCall { ) } } - diff --git a/rust_compiler/libs/parser/src/test/docs.rs b/rust_compiler/libs/parser/src/test/docs.rs new file mode 100644 index 0000000..9a442f9 --- /dev/null +++ b/rust_compiler/libs/parser/src/test/docs.rs @@ -0,0 +1,12 @@ +use crate::Documentation; +use crate::sys_call; +use pretty_assertions::assert_eq; + +#[test] +fn test_token_tree_docs() -> anyhow::Result<()> { + let syscall = sys_call::System::Yield; + + assert_eq!(syscall.docs(), ""); + + Ok(()) +} diff --git a/rust_compiler/libs/parser/src/test/mod.rs b/rust_compiler/libs/parser/src/test/mod.rs index c23869a..b7a7177 100644 --- a/rust_compiler/libs/parser/src/test/mod.rs +++ b/rust_compiler/libs/parser/src/test/mod.rs @@ -6,9 +6,11 @@ macro_rules! parser { } mod blocks; +mod docs; use super::Parser; use super::Tokenizer; use anyhow::Result; +use pretty_assertions::assert_eq; #[test] fn test_unsupported_keywords() -> Result<()> { @@ -99,16 +101,16 @@ fn test_priority_expression() -> Result<()> { #[test] fn test_binary_expression() -> Result<()> { - let expr = parser!("4 ** 2 + 5 ** 2").parse()?.unwrap(); + let expr = parser!("4 ** 2 + 5 ** 2;").parse()?.unwrap(); assert_eq!("((4 ** 2) + (5 ** 2))", expr.to_string()); - let expr = parser!("2 ** 3 ** 4").parse()?.unwrap(); + let expr = parser!("2 ** 3 ** 4;").parse()?.unwrap(); assert_eq!("(2 ** (3 ** 4))", expr.to_string()); - let expr = parser!("45 * 2 - 15 / 5 + 5 ** 2").parse()?.unwrap(); + let expr = parser!("45 * 2 - 15 / 5 + 5 ** 2;").parse()?.unwrap(); assert_eq!("(((45 * 2) - (15 / 5)) + (5 ** 2))", expr.to_string()); - let expr = parser!("(5 - 2) * 10").parse()?.unwrap(); + let expr = parser!("(5 - 2) * 10;").parse()?.unwrap(); assert_eq!("((5 - 2) * 10)", expr.to_string()); Ok(()) From 8aadb95f3632a8b81c3e398a57686b1246ccd325 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 23:43:40 -0700 Subject: [PATCH 05/19] More documentation --- rust_compiler/Cargo.lock | 7 + rust_compiler/Cargo.toml | 1 + rust_compiler/libs/helpers/Cargo.toml | 6 + rust_compiler/libs/helpers/src/lib.rs | 12 ++ .../libs/{parser => helpers}/src/macros.rs | 1 - rust_compiler/libs/parser/Cargo.toml | 1 + rust_compiler/libs/parser/src/lib.rs | 1 - rust_compiler/libs/parser/src/sys_call.rs | 66 ++++---- rust_compiler/libs/parser/src/test/docs.rs | 2 +- rust_compiler/libs/tokenizer/Cargo.toml | 1 + rust_compiler/libs/tokenizer/src/token.rs | 151 +++++++++++++++--- 11 files changed, 195 insertions(+), 54 deletions(-) create mode 100644 rust_compiler/libs/helpers/Cargo.toml create mode 100644 rust_compiler/libs/helpers/src/lib.rs rename rust_compiler/libs/{parser => helpers}/src/macros.rs (99%) diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 0c3ea01..7a09c6c 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -360,6 +360,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "helpers" +version = "0.1.0" + [[package]] name = "indexmap" version = "2.12.1" @@ -495,6 +499,7 @@ name = "parser" version = "0.1.0" dependencies = [ "anyhow", + "helpers", "lsp-types", "pretty_assertions", "quick-error", @@ -829,6 +834,7 @@ dependencies = [ "anyhow", "clap", "compiler", + "helpers", "lsp-types", "parser", "quick-error", @@ -926,6 +932,7 @@ name = "tokenizer" version = "0.1.0" dependencies = [ "anyhow", + "helpers", "lsp-types", "quick-error", "rust_decimal", diff --git a/rust_compiler/Cargo.toml b/rust_compiler/Cargo.toml index 8286678..c0b0744 100644 --- a/rust_compiler/Cargo.toml +++ b/rust_compiler/Cargo.toml @@ -40,6 +40,7 @@ rust_decimal = { workspace = true } tokenizer = { path = "libs/tokenizer" } parser = { path = "libs/parser" } compiler = { path = "libs/compiler" } +helpers = { path = "libs/helpers" } safer-ffi = { workspace = true } [dev-dependencies] diff --git a/rust_compiler/libs/helpers/Cargo.toml b/rust_compiler/libs/helpers/Cargo.toml new file mode 100644 index 0000000..cb9df2c --- /dev/null +++ b/rust_compiler/libs/helpers/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "helpers" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/rust_compiler/libs/helpers/src/lib.rs b/rust_compiler/libs/helpers/src/lib.rs new file mode 100644 index 0000000..3060c3e --- /dev/null +++ b/rust_compiler/libs/helpers/src/lib.rs @@ -0,0 +1,12 @@ +mod macros; + +/// This trait will allow the LSP to emit documentation for various tokens and expressions. +/// You can easily create documentation for large enums with the `documented!` macro. +pub trait Documentation { + /// Retreive documentation for this specific item. + fn docs(&self) -> String; +} + +pub mod prelude { + pub use super::{Documentation, documented}; +} diff --git a/rust_compiler/libs/parser/src/macros.rs b/rust_compiler/libs/helpers/src/macros.rs similarity index 99% rename from rust_compiler/libs/parser/src/macros.rs rename to rust_compiler/libs/helpers/src/macros.rs index 6eefb02..822096b 100644 --- a/rust_compiler/libs/parser/src/macros.rs +++ b/rust_compiler/libs/helpers/src/macros.rs @@ -82,4 +82,3 @@ macro_rules! documented { } }; } - diff --git a/rust_compiler/libs/parser/Cargo.toml b/rust_compiler/libs/parser/Cargo.toml index 504d535..336b498 100644 --- a/rust_compiler/libs/parser/Cargo.toml +++ b/rust_compiler/libs/parser/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] quick-error = { workspace = true } tokenizer = { path = "../tokenizer" } +helpers = { path = "../helpers" } lsp-types = { workspace = true } diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index f58560f..f293a4d 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -1,4 +1,3 @@ -mod macros; #[cfg(test)] mod test; diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index ed9d2c9..9e217fc 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -1,6 +1,6 @@ use super::LiteralOrVariable; use crate::tree_node::{Expression, Literal, Spanned}; -use crate::{Documentation, documented}; +use helpers::prelude::*; documented! { #[derive(Debug, PartialEq, Eq)] @@ -48,7 +48,7 @@ documented! { /// `(number|var).cos();` Cos(LiteralOrVariable), /// Rounds a number down to the nearest whole number. - /// ## In Game + /// ## IC10 /// `floor r? a(r?|num)` /// ## Slang /// `(number|var).floor();` @@ -131,30 +131,35 @@ documented! { #[derive(Debug, PartialEq, Eq)] pub enum System { /// Pauses execution for exactly 1 tick and then resumes. - /// ## In Game - /// yield + /// ## IC10 + /// `yield` + /// ## Slang + /// `yield();` Yield, /// Represents a function that can be called to sleep for a certain amount of time. - /// ## In Game + /// ## IC10 /// `sleep a(r?|num)` + /// ## Slang + /// `sleep(number|var);` Sleep(Box>), /// Gets the in-game hash for a specific prefab name. - /// ## In Game + /// ## IC10 /// `HASH("prefabName")` + /// ## Slang + /// `HASH("prefabName");` Hash(Literal), /// Represents a function which loads a device variable into a register. - /// ## In Game + /// ## IC10 /// `l r? d? var` - /// ## Examples - /// `l r0 d0 Setting` - /// `l r1 d5 Pressure` + /// ## Slang + /// `loadFromDevice(deviceType, "LogicType");` LoadFromDevice(LiteralOrVariable, Literal), /// Function which gets a LogicType from all connected network devices that match /// the provided device hash and name, aggregating them via a batchMode - /// ## In Game - /// lbn r? deviceHash nameHash logicType batchMode - /// ## Examples - /// lbn r0 HASH("StructureWallLight") HASH("wallLight") On Minimum + /// ## IC10 + /// `lbn r? deviceHash nameHash logicType batchMode` + /// ## Slang + /// `loadFromDeviceBatchedNamed(deviceHash, deviceName, "LogicType", "BatchMode");` LoadBatchNamed( LiteralOrVariable, Box>, @@ -163,30 +168,28 @@ documented! { ), /// Loads a LogicType from all connected network devices, aggregating them via a /// batchMode - /// ## In Game - /// lb r? deviceHash logicType batchMode - /// ## Examples - /// lb r0 HASH("StructureWallLight") On Minimum + /// ## IC10 + /// `lb r? deviceHash logicType batchMode` + /// ## Slang + /// `loadFromDeviceBatched(deviceHash, "Variable", "LogicType");` LoadBatch(LiteralOrVariable, Literal, Literal), /// Represents a function which stores a setting into a specific device. - /// ## In Game + /// ## IC10 /// `s d? logicType r?` - /// ## Example - /// `s d0 Setting r0` + /// ## Slang + /// `setOnDevice(deviceType, "Variable", (number|var));` SetOnDevice(LiteralOrVariable, Literal, Box>), /// Represents a function which stores a setting to all devices that match /// the given deviceHash - /// ## In Game + /// ## IC10 /// `sb deviceHash logicType r?` - /// ## Example - /// `sb HASH("Doors") Lock 1` SetOnDeviceBatched(LiteralOrVariable, Literal, Box>), /// Represents a function which stores a setting to all devices that match /// both the given deviceHash AND the given nameHash - /// ## In Game + /// ## IC10 /// `sbn deviceHash nameHash logicType r?` - /// ## Example - /// `sbn HASH("Doors") HASH("Exterior") Lock 1` + /// ## Slang + /// `setOnDeviceBatchedNamed(deviceType, nameHash, "LogicType", (number|var))` SetOnDeviceBatchedNamed( LiteralOrVariable, LiteralOrVariable, @@ -226,6 +229,15 @@ pub enum SysCall { Math(Math), } +impl Documentation for SysCall { + fn docs(&self) -> String { + match self { + Self::System(s) => s.docs(), + Self::Math(m) => m.docs(), + } + } +} + impl std::fmt::Display for SysCall { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/rust_compiler/libs/parser/src/test/docs.rs b/rust_compiler/libs/parser/src/test/docs.rs index 9a442f9..28aae6b 100644 --- a/rust_compiler/libs/parser/src/test/docs.rs +++ b/rust_compiler/libs/parser/src/test/docs.rs @@ -1,5 +1,5 @@ -use crate::Documentation; use crate::sys_call; +use helpers::Documentation; use pretty_assertions::assert_eq; #[test] diff --git a/rust_compiler/libs/tokenizer/Cargo.toml b/rust_compiler/libs/tokenizer/Cargo.toml index 38032f4..7433cab 100644 --- a/rust_compiler/libs/tokenizer/Cargo.toml +++ b/rust_compiler/libs/tokenizer/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" rust_decimal = { workspace = true } quick-error = { workspace = true } lsp-types = { workspace = true } +helpers = { path = "../helpers" } [dev-dependencies] anyhow = { version = "^1" } diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index b471d6d..5938ddc 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -1,3 +1,4 @@ +use helpers::prelude::*; use rust_decimal::Decimal; #[derive(Debug, PartialEq, Eq, Clone)] @@ -264,28 +265,130 @@ impl std::fmt::Display for Symbol { } } -#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)] -pub enum Keyword { - /// Represents the `continue` keyword - Continue, - /// Represents the `let` keyword - Let, - /// Represents the `fn` keyword - Fn, - /// Represents the `if` keyword - If, - /// Represents the `device` keyword. Useful for defining a device at a specific address (ex. d0, d1, d2, etc.) - Device, - /// Represents the `else` keyword - Else, - /// Represents the `return` keyword - Return, - /// Represents the `enum` keyword - Enum, - /// Represents the `loop` keyword - Loop, - /// Represents the `break` keyword - Break, - /// Represents the `while` keyword - While, +documented! { + #[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)] + pub enum Keyword { + /// Represents the `continue` keyword. This will allow you to bypass the current iteration in a loop and start the next one. + /// ## Example + /// ``` + /// let item = 0; + /// loop { + /// if (item % 2 == 0) { + /// // This will NOT increment `item` and will continue with the next iteration of the + /// // loop + /// continue; + /// } + /// item = item + 1; + /// } + /// ``` + Continue, + /// Represents the `let` keyword, used to declare variables within Slang. + /// ## Example + /// ``` + /// // This variable now exists either in a register or the stack depending on how many + /// // free registers were available when declaring it. + /// let item = 0; + /// ``` + Let, + /// Represents the `fn` keyword, used to declare functions within Slang. + /// ## Example + /// ``` + /// // This allows you to now call `doSomething` with specific arguments. + /// fn doSomething(arg1, arg2) { + /// + /// } + /// ``` + Fn, + /// Represents the `if` keyword, allowing you to create branched logic. + /// ## Example + /// ``` + /// let i = 0; + /// if (i == 0) { + /// i = 1; + /// } + /// // At this line, `i` is now `1` + /// ``` + If, + /// Represents the `device` keyword. Useful for defining a device at a specific address + /// (ex. d0, d1, d2, etc.). This also allows you to perform direct operations ON a device. + /// ## Example + /// ``` + /// device self = "db"; + /// + /// // This is the same as `s db Setting 123` + /// self.Setting = 123; + /// ``` + Device, + /// Represents the `else` keyword. Useful if you want to check a condition but run run + /// seperate logic in case that condition fails. + /// ## Example + /// ``` + /// device self = "db"; + /// let i = 0; + /// if (i < 0) { + /// self.Setting = 0; + /// } else { + /// self.Setting = 1; + /// } + /// // Here, the `Setting` on the current housing is `1` because i was NOT less than 0 + /// ``` + Else, + /// Represents the `return` keyword. Allows you to pass values from a function back to + /// the caller. + /// ## Example + /// ``` + /// fn doSomething() { + /// return 1 + 2; + /// } + /// + /// // `returnedValue` now holds the value `3` + /// let returnedValue = doSomething(); + /// ``` + Return, + /// Represents the `enum` keyword. This is currently not supported, but is kept as a + /// reserved keyword in the future case that this is implemented. + Enum, + /// Represents the `loop` keyword. This allows you to create an infinate loop, but can be + /// broken with the `break` keyword. + /// ## Example + /// ``` + /// device self = "db"; + /// let i = 0; + /// loop { + /// i = i + 1; + /// // The current housing will infinately increment it's `Setting` value. + /// self.Setting = i; + /// } + /// ``` + Loop, + /// Represents the `break` keyword. This allows you to "break out of" a loop prematurely, + /// such as when an if() conditon is true, etc. + /// ## Example + /// ``` + /// let i = 0; + /// // This loop will run until the value of `i` is greater than 10,000, + /// // which will then trigger the `break` keyword and it will stop looping + /// loop { + /// if (i > 10_000) { + /// break; + /// } + /// i = i + 1; + /// } + /// ``` + Break, + /// Represents the `while` keyword. This is similar to the `loop` keyword but different in + /// that you don't need an `if` statement to break out of a loop, that is handled + /// automatically when invoking `while` + /// ## Example + /// ``` + /// let i = 0; + /// // This loop will run until the value of `i` is greater than 10,000, in which case the + /// // while loop will automatically stop running and code will continue AFTER the last + /// // bracket. + /// while (i < 10_000) { + /// i = i + 1; + /// } + /// ``` + While, + } } From 0f1613d521337fa06edb3c9a616c1a0baad55c36 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Tue, 2 Dec 2025 00:00:42 -0700 Subject: [PATCH 06/19] Support tokenization tooltips in the C# mod --- csharp_mod/Extensions.cs | 4 +++- rust_compiler/libs/tokenizer/src/token.rs | 9 +++++++++ rust_compiler/src/ffi/mod.rs | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index 35416fb..6367094 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -63,7 +63,9 @@ public static unsafe class SlangExtensions colIndex, token.length, color, - token.token_kind + token.token_kind, + 0, + token.tooltip.AsString() ); string errMsg = token.error.AsString(); diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index 5938ddc..59f2c7f 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -88,6 +88,15 @@ pub enum TokenType { EOF, } +impl Documentation for TokenType { + fn docs(&self) -> String { + match self { + Self::Keyword(k) => k.docs(), + _ => "".into(), + } + } +} + impl From for u32 { fn from(value: TokenType) -> Self { use TokenType::*; diff --git a/rust_compiler/src/ffi/mod.rs b/rust_compiler/src/ffi/mod.rs index 5c0750e..938a5a7 100644 --- a/rust_compiler/src/ffi/mod.rs +++ b/rust_compiler/src/ffi/mod.rs @@ -1,4 +1,5 @@ use compiler::Compiler; +use helpers::Documentation; use parser::Parser; use safer_ffi::prelude::*; use std::io::BufWriter; @@ -151,8 +152,8 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec Date: Tue, 2 Dec 2025 14:27:06 -0700 Subject: [PATCH 07/19] Implement changes from latest version of IC10Editor.dll --- csharp_mod/Formatter.cs | 110 ++++++++++++++++++----------- csharp_mod/Patches.cs | 1 - csharp_mod/stationeersSlang.csproj | 4 ++ 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs index a3dc844..2be094a 100644 --- a/csharp_mod/Formatter.cs +++ b/csharp_mod/Formatter.cs @@ -4,15 +4,14 @@ using System; using System.Collections.Generic; using System.Linq; using System.Threading; -using System.Threading.Tasks; using System.Timers; +using Cysharp.Threading.Tasks; using StationeersIC10Editor; public class SlangFormatter : ICodeFormatter { private CancellationTokenSource? _lspCancellationToken; - private readonly SynchronizationContext? _mainThreadContext; - private volatile bool IsDiagnosing = false; + private object _tokenLock = new(); public static readonly uint ColorInstruction = ColorFromHTML("#ffff00"); public static readonly uint ColorString = ColorFromHTML("#ce9178"); @@ -20,10 +19,39 @@ public class SlangFormatter : ICodeFormatter private HashSet _linesWithErrors = new(); public SlangFormatter() + : base() { - // 1. Capture the Main Thread context. - // This works because the Editor instantiates this class on the main thread. - _mainThreadContext = SynchronizationContext.Current; + OnCodeChanged += HandleCodeChanged; + } + + public static double MatchingScore(string input) + { + // Empty input is not valid Slang + if (string.IsNullOrWhiteSpace(input)) + return 0d; + + // Run the compiler to get diagnostics + var diagnostics = Marshal.DiagnoseSource(input); + + // Count the number of actual Errors (Severity 1). + // We ignore Warnings (2), Info (3), etc. + double errorCount = diagnostics.Count(d => d.Severity == 1); + + // Get the total line count to calculate error density + double lineCount = input.Split('\n').Length; + + // Prevent division by zero + if (lineCount == 0) + return 0d; + + // Calculate score: Start at 1.0 (100%) and subtract the ratio of errors per line. + // Example: 10 lines with 0 errors = 1.0 + // Example: 10 lines with 2 errors = 0.8 + // Example: 10 lines with 10+ errors = 0.0 + double score = 1.0d - (errorCount / lineCount); + + // Clamp the result between 0 and 1 + return Math.Max(0d, Math.Min(1d, score)); } public override string Compile() @@ -33,65 +61,65 @@ public class SlangFormatter : ICodeFormatter public override Line ParseLine(string line) { - HandleCodeChanged(); return Marshal.TokenizeLine(line); } private void HandleCodeChanged() { - if (IsDiagnosing) - return; + CancellationToken token; + string inputSrc; + lock (_tokenLock) + { + _lspCancellationToken?.Cancel(); + _lspCancellationToken = new CancellationTokenSource(); + token = _lspCancellationToken.Token; + inputSrc = this.RawText; + } - _lspCancellationToken?.Cancel(); - _lspCancellationToken?.Dispose(); - - _lspCancellationToken = new CancellationTokenSource(); - - _ = Task.Run(() => HandleLsp(_lspCancellationToken.Token), _lspCancellationToken.Token); + HandleLsp(inputSrc, token).Forget(); } private void OnTimerElapsed(object sender, ElapsedEventArgs e) { } - private async Task HandleLsp(CancellationToken cancellationToken) + private async UniTaskVoid HandleLsp(string inputSrc, CancellationToken cancellationToken) { try { - await Task.Delay(200, cancellationToken); + await UniTask.SwitchToThreadPool(); if (cancellationToken.IsCancellationRequested) - { return; - } - // 3. Dispatch the UI update to the Main Thread - if (_mainThreadContext != null) - { - // Post ensures ApplyDiagnostics runs on the captured thread (Main Thread) - _mainThreadContext.Post(_ => ApplyDiagnostics(), null); - } - else - { - // Fallback: If context is null (rare in Unity), try running directly - // but warn, as this might crash if not thread-safe. - L.Warning("SynchronizationContext was null. Attempting direct update (risky)."); - ApplyDiagnostics(); - } + await System.Threading.Tasks.Task.Delay(200, cancellationToken: cancellationToken); + + if (cancellationToken.IsCancellationRequested) + return; + + var dict = Marshal + .DiagnoseSource(inputSrc) + .GroupBy(d => d.Range.StartLine) + .ToDictionary(g => g.Key); + + if (cancellationToken.IsCancellationRequested) + return; + + await UniTask.Yield(PlayerLoopTiming.Update, cancellationToken); + + ApplyDiagnostics(dict); + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + L.Error(ex.Message); } - finally { } } // This runs on the Main Thread - private void ApplyDiagnostics() + private void ApplyDiagnostics(Dictionary> dict) { - List diagnosis = Marshal.DiagnoseSource(this.RawText); - - var dict = diagnosis.GroupBy(d => d.Range.StartLine).ToDictionary(g => g.Key); - var linesToRefresh = new HashSet(dict.Keys); linesToRefresh.UnionWith(_linesWithErrors); - IsDiagnosing = true; - foreach (var lineIndex in linesToRefresh) { // safety check for out of bounds (in case lines were deleted) @@ -134,7 +162,5 @@ public class SlangFormatter : ICodeFormatter } _linesWithErrors = new HashSet(dict.Keys); - - IsDiagnosing = false; } } diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs index 22a4dd0..0222b50 100644 --- a/csharp_mod/Patches.cs +++ b/csharp_mod/Patches.cs @@ -214,7 +214,6 @@ public static class SlangPatches [HarmonyPrefix] public static void isc_ButtonInputCancel() { - L.Info("ButtonInputCancel called on the InputSourceCode static instance."); if (_currentlyEditingMotherboard is null || _motherboardCachedCode is null) { return; diff --git a/csharp_mod/stationeersSlang.csproj b/csharp_mod/stationeersSlang.csproj index d613b2e..e464914 100644 --- a/csharp_mod/stationeersSlang.csproj +++ b/csharp_mod/stationeersSlang.csproj @@ -41,6 +41,10 @@ $(ManagedDir)/Assembly-CSharp.dll False + + $(ManagedDir)/UniTask.dll + False + ./ref/IC10Editor.dll From bf1daf12cc70d1e938d44dee67a55c4f3d175d56 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Tue, 2 Dec 2025 14:40:02 -0700 Subject: [PATCH 08/19] Remove unnecessary cancellation check after the Marshal call --- csharp_mod/Formatter.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs index 2be094a..cdb09c7 100644 --- a/csharp_mod/Formatter.cs +++ b/csharp_mod/Formatter.cs @@ -100,9 +100,6 @@ public class SlangFormatter : ICodeFormatter .GroupBy(d => d.Range.StartLine) .ToDictionary(g => g.Key); - if (cancellationToken.IsCancellationRequested) - return; - await UniTask.Yield(PlayerLoopTiming.Update, cancellationToken); ApplyDiagnostics(dict); From 75f1c5c44a45785bb87c88a39612cadf9ef85dfd Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Tue, 2 Dec 2025 17:59:40 -0700 Subject: [PATCH 09/19] Stationpedia docs --- csharp_mod/Extensions.cs | 31 +++++++++++++ csharp_mod/FfiGlue.cs | 30 +++++++++++++ csharp_mod/Marshal.cs | 9 ++++ csharp_mod/Patches.cs | 10 +++++ csharp_mod/StationpediaDocumentation.cs | 22 --------- csharp_mod/TmpFormatter.cs | 54 +++++++++++++++++++++++ rust_compiler/libs/helpers/src/lib.rs | 2 + rust_compiler/libs/helpers/src/macros.rs | 29 +++++++++++- rust_compiler/libs/parser/src/sys_call.rs | 7 +++ rust_compiler/libs/tokenizer/src/token.rs | 4 ++ rust_compiler/src/ffi/mod.rs | 37 +++++++++++++++- 11 files changed, 211 insertions(+), 24 deletions(-) delete mode 100644 csharp_mod/StationpediaDocumentation.cs create mode 100644 csharp_mod/TmpFormatter.cs diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index 6367094..648062b 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -3,6 +3,7 @@ namespace Slang; using System; using System.Collections.Generic; using System.Text; +using Assets.Scripts.UI; using StationeersIC10Editor; public static unsafe class SlangExtensions @@ -133,4 +134,34 @@ public static unsafe class SlangExtensions return SlangFormatter.ColorDefault; } } + + public static unsafe List ToList(this Vec_FfiDocumentedItem_t vec) + { + var toReturn = new List((int)vec.len); + + var currentPtr = vec.ptr; + + for (int i = 0; i < (int)vec.len; i++) + { + var doc = currentPtr[i]; + var docItemName = doc.item_name.AsString(); + + var formattedText = TextMeshProFormatter.FromMarkdown(doc.docs.AsString()); + + var pediaPage = new StationpediaPage( + $"slang-item-{docItemName}", + docItemName, + formattedText + ); + + pediaPage.Text = formattedText; + pediaPage.Description = formattedText; + pediaPage.ParsePage(); + + toReturn.Add(pediaPage); + } + + Ffi.free_docs_vec(vec); + return toReturn; + } } diff --git a/csharp_mod/FfiGlue.cs b/csharp_mod/FfiGlue.cs index 5c49ef5..3489eb4 100644 --- a/csharp_mod/FfiGlue.cs +++ b/csharp_mod/FfiGlue.cs @@ -121,6 +121,31 @@ public unsafe partial class Ffi { slice_ref_uint16_t input); } +[StructLayout(LayoutKind.Sequential, Size = 48)] +public unsafe struct FfiDocumentedItem_t { + public Vec_uint8_t item_name; + + public Vec_uint8_t docs; +} + +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_FfiDocumentedItem_t { + public FfiDocumentedItem_t * ptr; + + public UIntPtr len; + + public UIntPtr cap; +} + +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_docs_vec ( + Vec_FfiDocumentedItem_t v); +} + public unsafe partial class Ffi { [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern void free_ffi_diagnostic_vec ( @@ -164,6 +189,11 @@ public unsafe partial class Ffi { Vec_uint8_t s); } +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_FfiDocumentedItem_t get_docs (); +} + public unsafe partial class Ffi { [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern Vec_FfiToken_t tokenize_line ( diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs index c058e6b..3a7b385 100644 --- a/csharp_mod/Marshal.cs +++ b/csharp_mod/Marshal.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Reflection; using System.Runtime.InteropServices; +using Assets.Scripts.UI; using StationeersIC10Editor; public struct Range @@ -151,6 +152,14 @@ public static class Marshal } } + /// + /// Gets the currently documented items from the Slang compiler and returns new StationpediaPages with correct formatting. + /// + public static unsafe List GetSlangDocs() + { + return Ffi.get_docs().ToList(); + } + private static string ExtractNativeLibrary(string libName) { string destinationPath = Path.Combine(Path.GetTempPath(), libName); diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs index 0222b50..a96809e 100644 --- a/csharp_mod/Patches.cs +++ b/csharp_mod/Patches.cs @@ -224,4 +224,14 @@ public static class SlangPatches _currentlyEditingMotherboard = null; _motherboardCachedCode = null; } + + [HarmonyPatch(typeof(Stationpedia), nameof(Stationpedia.Regenerate))] + [HarmonyPostfix] + public static void Stationpedia_Regenerate() + { + foreach (var page in Marshal.GetSlangDocs()) + { + Stationpedia.Register(page); + } + } } diff --git a/csharp_mod/StationpediaDocumentation.cs b/csharp_mod/StationpediaDocumentation.cs deleted file mode 100644 index e03814a..0000000 --- a/csharp_mod/StationpediaDocumentation.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Assets.Scripts.UI; - -namespace Slang -{ - public static class SlangDocs - { - public static StationpediaPage[] Pages - { - get - { - return - [ - new StationpediaPage( - "slang-init", - "Slang", - "Slang is a new high level language built specifically for Stationeers" - ), - ]; - } - } - } -} diff --git a/csharp_mod/TmpFormatter.cs b/csharp_mod/TmpFormatter.cs new file mode 100644 index 0000000..aab5802 --- /dev/null +++ b/csharp_mod/TmpFormatter.cs @@ -0,0 +1,54 @@ +using System.Text.RegularExpressions; + +namespace Slang; + +public static class TextMeshProFormatter +{ + private const string CODE_COLOR = "#FFD700"; + + public static string FromMarkdown(string markdown) + { + if (string.IsNullOrEmpty(markdown)) + return ""; + + // 1. Normalize Line Endings + string text = markdown.Replace("\r\n", "\n"); + + // 2. Handle Code Blocks (```) + text = Regex.Replace( + text, + @"```\s*(.*?)\s*```", + match => + { + var codeContent = match.Groups[1].Value; + return $"{codeContent}"; // Gold color for code + }, + RegexOptions.Singleline + ); + + // 3. Handle Headers (## Header) + // Convert ## Header to large bold text + text = Regex.Replace( + text, + @"^##(\s+)?(.+)$", + "$1", + RegexOptions.Multiline + ); + + // 4. Handle Inline Code (`code`) + text = Regex.Replace(text, @"`([^`]+)`", $"$1"); + + // 5. Handle Bold (**text**) + text = Regex.Replace(text, @"\*\*(.+?)\*\*", "$1"); + + // 6. Handle Italics (*text*) + text = Regex.Replace(text, @"\*(.+?)\*", "$1"); + + // 7. Convert Newlines to TMP Line Breaks + // Stationpedia needs
or explicit newlines. + // Often just ensuring \n is preserved is enough, but
is safer for HTML-like parsers. + text = text.Replace("\n", "
"); + + return text; + } +} diff --git a/rust_compiler/libs/helpers/src/lib.rs b/rust_compiler/libs/helpers/src/lib.rs index 3060c3e..18359cf 100644 --- a/rust_compiler/libs/helpers/src/lib.rs +++ b/rust_compiler/libs/helpers/src/lib.rs @@ -5,6 +5,8 @@ mod macros; pub trait Documentation { /// Retreive documentation for this specific item. fn docs(&self) -> String; + + fn get_all_documentation() -> Vec<(&'static str, String)>; } pub mod prelude { diff --git a/rust_compiler/libs/helpers/src/macros.rs b/rust_compiler/libs/helpers/src/macros.rs index 822096b..9c51e46 100644 --- a/rust_compiler/libs/helpers/src/macros.rs +++ b/rust_compiler/libs/helpers/src/macros.rs @@ -55,7 +55,7 @@ macro_rules! documented { )* } - // 2. Implement the Trait + // 2. Implement the Documentation Trait impl Documentation for $name { fn docs(&self) -> String { match self { @@ -79,6 +79,33 @@ macro_rules! documented { )* } } + + // 3. Implement Static Documentation Provider + #[allow(dead_code)] + fn get_all_documentation() -> Vec<(&'static str, String)> { + vec![ + $( + ( + stringify!($variant), + { + // Re-use the same extraction logic + let doc_lines: &[Option<&str>] = &[ + $( + documented!(@doc_filter #[ $($variant_attr)* ]) + ),* + ]; + doc_lines.iter() + .filter_map(|&d| d) + .collect::>() + .join("\n") + .trim() + .to_string() + } + ) + ),* + ] + } } }; } + diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index 9e217fc..96f5a26 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -236,6 +236,13 @@ impl Documentation for SysCall { Self::Math(m) => m.docs(), } } + + fn get_all_documentation() -> Vec<(&'static str, String)> { + let mut all_docs = System::get_all_documentation(); + all_docs.extend(Math::get_all_documentation()); + + all_docs + } } impl std::fmt::Display for SysCall { diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index 59f2c7f..ca61cee 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -95,6 +95,10 @@ impl Documentation for TokenType { _ => "".into(), } } + + fn get_all_documentation() -> Vec<(&'static str, String)> { + Keyword::get_all_documentation() + } } impl From for u32 { diff --git a/rust_compiler/src/ffi/mod.rs b/rust_compiler/src/ffi/mod.rs index 938a5a7..ee31887 100644 --- a/rust_compiler/src/ffi/mod.rs +++ b/rust_compiler/src/ffi/mod.rs @@ -1,6 +1,6 @@ use compiler::Compiler; use helpers::Documentation; -use parser::Parser; +use parser::{sys_call::SysCall, Parser}; use safer_ffi::prelude::*; use std::io::BufWriter; use tokenizer::{ @@ -27,6 +27,13 @@ pub struct FfiRange { end_line: u32, } +#[derive_ReprC] +#[repr(C)] +pub struct FfiDocumentedItem { + item_name: safer_ffi::String, + docs: safer_ffi::String, +} + impl From for FfiRange { fn from(value: lsp_types::Range) -> Self { Self { @@ -77,6 +84,11 @@ pub fn free_string(s: safer_ffi::String) { drop(s) } +#[ffi_export] +pub fn free_docs_vec(v: safer_ffi::Vec) { + drop(v) +} + /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. /// This should result in the ability to compile many times without triggering frame drops @@ -184,3 +196,26 @@ pub fn diagnose_source(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec< res.unwrap_or(vec![].into()) } + +#[ffi_export] +pub fn get_docs() -> safer_ffi::Vec { + let res = std::panic::catch_unwind(|| { + let mut docs = SysCall::get_all_documentation(); + docs.extend(TokenType::get_all_documentation()); + + docs + }); + + let Ok(result) = res else { + return vec![].into(); + }; + + result + .into_iter() + .map(|(key, doc)| FfiDocumentedItem { + item_name: key.into(), + docs: doc.into(), + }) + .collect::>() + .into() +} From baab8b9d0bc6d6e49c346d228863ffe662df02e5 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Tue, 2 Dec 2025 18:05:21 -0700 Subject: [PATCH 10/19] update dependencies --- rust_compiler/Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 7a09c6c..f12b00b 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "lsp-types" @@ -997,9 +997,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "js-sys", "wasm-bindgen", From e05f130040476320de73ffd39d0bf78442930675 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Tue, 2 Dec 2025 23:37:13 -0700 Subject: [PATCH 11/19] Fixed stack overflow error caused by improper cleanup of block scoped variables --- rust_compiler/libs/compiler/src/v1.rs | 15 ++++--- .../libs/compiler/src/variable_manager.rs | 40 ++++++++++++------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index de7ad3a..f97e99b 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -1252,7 +1252,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { fn expression_block<'v>( &mut self, mut expr: BlockExpression, - scope: &mut VariableScope<'v>, + parent_scope: &mut VariableScope<'v>, ) -> Result<(), Error> { // First, sort the expressions to ensure functions are hoisted expr.0.sort_by(|a, b| { @@ -1267,10 +1267,12 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { } }); + let mut scope = VariableScope::scoped(parent_scope); + for expr in expr.0 { if !self.declared_main && !matches!(expr.node, Expression::Function(_)) - && !scope.has_parent() + && !parent_scope.has_parent() { self.write_output("main:")?; self.declared_main = true; @@ -1278,11 +1280,11 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { match expr.node { Expression::Return(ret_expr) => { - self.expression_return(*ret_expr, scope)?; + self.expression_return(*ret_expr, &mut scope)?; } _ => { // Swallow errors within expressions so block can continue - if let Err(e) = self.expression(expr, scope).and_then(|result| { + if let Err(e) = self.expression(expr, &mut scope).and_then(|result| { // If the expression was a statement that returned a temp result (e.g. `1 + 2;` line), // we must free it to avoid leaking registers. if let Some(comp_res) = result @@ -1298,6 +1300,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { } } + if scope.stack_offset() > 0 { + self.write_output(format!("sub sp sp {}", scope.stack_offset()))?; + } + Ok(()) } @@ -1700,4 +1706,3 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { Ok(()) } } - diff --git a/rust_compiler/libs/compiler/src/variable_manager.rs b/rust_compiler/libs/compiler/src/variable_manager.rs index ca47f1c..c22a39f 100644 --- a/rust_compiler/libs/compiler/src/variable_manager.rs +++ b/rust_compiler/libs/compiler/src/variable_manager.rs @@ -91,6 +91,8 @@ impl<'a> VariableScope<'a> { pub fn scoped(parent: &'a VariableScope<'a>) -> Self { Self { parent: Option::Some(parent), + temporary_vars: parent.temporary_vars.clone(), + persistant_vars: parent.persistant_vars.clone(), ..Default::default() } } @@ -140,24 +142,32 @@ impl<'a> VariableScope<'a> { Ok(var_location) } - pub fn get_location_of( - &mut self, - var_name: impl Into, - ) -> Result { + pub fn get_location_of(&self, var_name: impl Into) -> Result { let var_name = var_name.into(); - let var = self - .var_lookup_table - .get(var_name.as_str()) - .cloned() - .ok_or(Error::UnknownVariable(var_name))?; - if let VariableLocation::Stack(inserted_at_offset) = var { - Ok(VariableLocation::Stack( - self.stack_offset - inserted_at_offset, - )) - } else { - Ok(var) + // 1. Check this scope + if let Some(var) = self.var_lookup_table.get(var_name.as_str()) { + if let VariableLocation::Stack(inserted_at_offset) = var { + // Return offset relative to CURRENT sp + return Ok(VariableLocation::Stack( + self.stack_offset - inserted_at_offset, + )); + } else { + return Ok(var.clone()); + } } + + // 2. Recursively check parent + if let Some(parent) = self.parent { + let loc = parent.get_location_of(var_name)?; + + if let VariableLocation::Stack(parent_offset) = loc { + return Ok(VariableLocation::Stack(parent_offset + self.stack_offset)); + } + return Ok(loc); + } + + Err(Error::UnknownVariable(var_name)) } pub fn has_parent(&self) -> bool { From 4b6c7eb63c39cec00feb61b3cc4b165521d35888 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Thu, 4 Dec 2025 18:01:16 -0700 Subject: [PATCH 12/19] Implement AST for 'const' expressions. TODO -- add const expressions to compiler --- .../libs/compiler/src/test/branching.rs | 1 + .../test/declaration_function_invocation.rs | 5 ++ .../compiler/src/test/declaration_literal.rs | 1 + rust_compiler/libs/parser/src/lib.rs | 49 +++++++++++++++++++ rust_compiler/libs/parser/src/test/docs.rs | 12 ----- rust_compiler/libs/parser/src/test/mod.rs | 41 +++++++++++++--- rust_compiler/libs/parser/src/tree_node.rs | 15 +++++- rust_compiler/libs/tokenizer/src/lib.rs | 1 + rust_compiler/libs/tokenizer/src/token.rs | 17 +++++++ 9 files changed, 123 insertions(+), 19 deletions(-) delete mode 100644 rust_compiler/libs/parser/src/test/docs.rs diff --git a/rust_compiler/libs/compiler/src/test/branching.rs b/rust_compiler/libs/compiler/src/test/branching.rs index d23d880..1a852da 100644 --- a/rust_compiler/libs/compiler/src/test/branching.rs +++ b/rust_compiler/libs/compiler/src/test/branching.rs @@ -149,6 +149,7 @@ fn test_spilled_variable_update_in_branch() -> anyhow::Result<()> { sub r0 sp 1 put db r0 99 #h L1: + sub sp sp 1 " } ); diff --git a/rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs b/rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs index 2e0c3c2..43e3131 100644 --- a/rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs +++ b/rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs @@ -34,6 +34,8 @@ fn no_arguments() -> anyhow::Result<()> { #[test] fn let_var_args() -> anyhow::Result<()> { + // !IMPORTANT this needs to be stabilized as it currently incorrectly calculates sp offset at + // both ends of the cleanup lifecycle let compiled = compile! { debug " @@ -64,6 +66,7 @@ fn let_var_args() -> anyhow::Result<()> { get r8 db r0 sub sp sp 1 move r9 r15 #i + sub sp sp 1 " } ); @@ -123,6 +126,7 @@ fn inline_literal_args() -> anyhow::Result<()> { get r8 db r0 sub sp sp 1 move r9 r15 #returnedValue + sub sp sp 1 " } ); @@ -164,6 +168,7 @@ fn mixed_args() -> anyhow::Result<()> { get r8 db r0 sub sp sp 1 move r9 r15 #returnValue + sub sp sp 1 " } ); diff --git a/rust_compiler/libs/compiler/src/test/declaration_literal.rs b/rust_compiler/libs/compiler/src/test/declaration_literal.rs index 9316e54..c42624c 100644 --- a/rust_compiler/libs/compiler/src/test/declaration_literal.rs +++ b/rust_compiler/libs/compiler/src/test/declaration_literal.rs @@ -56,6 +56,7 @@ fn variable_declaration_numeric_literal_stack_spillover() -> anyhow::Result<()> push 7 #h push 8 #i push 9 #j + sub sp sp 3 " } ); diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index f293a4d..dd9f582 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -462,6 +462,15 @@ impl<'a> Parser<'a> { }) } + TokenType::Keyword(Keyword::Const) => { + let spanned_const = self.spanned(|p| p.const_declaration())?; + + Some(Spanned { + span: spanned_const.span, + node: Expression::ConstDeclaration(spanned_const), + }) + } + TokenType::Keyword(Keyword::Fn) => { let spanned_fn = self.spanned(|p| p.function())?; Some(Spanned { @@ -1220,6 +1229,46 @@ impl<'a> Parser<'a> { Ok(BlockExpression(expressions)) } + fn const_declaration(&mut self) -> Result { + // const + let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + if !self_matches_current!(self, TokenType::Keyword(Keyword::Const)) { + return Err(Error::UnexpectedToken( + self.current_span(), + current_token.clone(), + )); + } + + // variable_name + let ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let ident_span = Self::token_to_span(ident_token); + let ident = match ident_token.token_type { + TokenType::Identifier(ref id) => id.clone(), + _ => return Err(Error::UnexpectedToken(ident_span, ident_token.clone())), + }; + + // `=` + let assign_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + if !token_matches!(assign_token, TokenType::Symbol(Symbol::Assign)) { + return Err(Error::UnexpectedToken( + Self::token_to_span(&assign_token), + assign_token, + )); + } + + // literal value + self.assign_next()?; + let lit = self.spanned(|p| p.literal())?; + + Ok(ConstDeclarationExpression { + name: Spanned { + span: ident_span, + node: ident, + }, + value: lit, + }) + } + fn declaration(&mut self) -> Result { let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) { diff --git a/rust_compiler/libs/parser/src/test/docs.rs b/rust_compiler/libs/parser/src/test/docs.rs deleted file mode 100644 index 28aae6b..0000000 --- a/rust_compiler/libs/parser/src/test/docs.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::sys_call; -use helpers::Documentation; -use pretty_assertions::assert_eq; - -#[test] -fn test_token_tree_docs() -> anyhow::Result<()> { - let syscall = sys_call::System::Yield; - - assert_eq!(syscall.docs(), ""); - - Ok(()) -} diff --git a/rust_compiler/libs/parser/src/test/mod.rs b/rust_compiler/libs/parser/src/test/mod.rs index b7a7177..beb874c 100644 --- a/rust_compiler/libs/parser/src/test/mod.rs +++ b/rust_compiler/libs/parser/src/test/mod.rs @@ -1,12 +1,11 @@ #[macro_export] macro_rules! parser { ($input:expr) => { - Parser::new(Tokenizer::from($input.to_owned())) + Parser::new(Tokenizer::from($input)) }; } mod blocks; -mod docs; use super::Parser; use super::Tokenizer; use anyhow::Result; @@ -33,7 +32,7 @@ fn test_declarations() -> Result<()> { // The below line should fail let y = 234 "#; - let tokenizer = Tokenizer::from(input.to_owned()); + let tokenizer = Tokenizer::from(input); let mut parser = Parser::new(tokenizer); let expression = parser.parse()?.unwrap(); @@ -45,6 +44,36 @@ fn test_declarations() -> Result<()> { Ok(()) } +#[test] +fn test_const_declaration() -> Result<()> { + let input = r#" + const item = 20c; + const decimal = 200.15; + const nameConst = "str_lit"; + "#; + let tokenizer = Tokenizer::from(input); + let mut parser = Parser::new(tokenizer); + + assert_eq!( + "(const item = 293.15)", + parser.parse()?.unwrap().to_string() + ); + + assert_eq!( + "(const decimal = 200.15)", + parser.parse()?.unwrap().to_string() + ); + + assert_eq!( + r#"(const nameConst = "str_lit")"#, + parser.parse()?.unwrap().to_string() + ); + + assert_eq!(None, parser.parse()?); + + Ok(()) +} + #[test] fn test_function_expression() -> Result<()> { let input = r#" @@ -54,7 +83,7 @@ fn test_function_expression() -> Result<()> { } "#; - let tokenizer = Tokenizer::from(input.to_owned()); + let tokenizer = Tokenizer::from(input); let mut parser = Parser::new(tokenizer); let expression = parser.parse()?.unwrap(); @@ -73,7 +102,7 @@ fn test_function_invocation() -> Result<()> { add(); "#; - let tokenizer = Tokenizer::from(input.to_owned()); + let tokenizer = Tokenizer::from(input); let mut parser = Parser::new(tokenizer); let expression = parser.parse()?.unwrap(); @@ -89,7 +118,7 @@ fn test_priority_expression() -> Result<()> { let x = (4); "#; - let tokenizer = Tokenizer::from(input.to_owned()); + let tokenizer = Tokenizer::from(input); let mut parser = Parser::new(tokenizer); let expression = parser.parse()?.unwrap(); diff --git a/rust_compiler/libs/parser/src/tree_node.rs b/rust_compiler/libs/parser/src/tree_node.rs index 1d39d64..daca733 100644 --- a/rust_compiler/libs/parser/src/tree_node.rs +++ b/rust_compiler/libs/parser/src/tree_node.rs @@ -195,6 +195,18 @@ impl std::fmt::Display for LiteralOrVariable { } } +#[derive(Debug, PartialEq, Eq)] +pub struct ConstDeclarationExpression { + pub name: Spanned, + pub value: Spanned, +} + +impl std::fmt::Display for ConstDeclarationExpression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "(const {} = {})", self.name, self.value) + } +} + #[derive(Debug, PartialEq, Eq)] pub struct DeviceDeclarationExpression { /// any variable-like name @@ -316,6 +328,7 @@ pub enum Expression { Binary(Spanned), Block(Spanned), Break(Span), + ConstDeclaration(Spanned), Continue(Span), Declaration(Spanned, Box>), DeviceDeclaration(Spanned), @@ -342,6 +355,7 @@ impl std::fmt::Display for Expression { Expression::Binary(e) => write!(f, "{}", e), Expression::Block(e) => write!(f, "{}", e), Expression::Break(_) => write!(f, "break"), + Expression::ConstDeclaration(e) => write!(f, "{}", e), Expression::Continue(_) => write!(f, "continue"), Expression::Declaration(id, e) => write!(f, "(let {} = {})", id, e), Expression::DeviceDeclaration(e) => write!(f, "{}", e), @@ -362,4 +376,3 @@ impl std::fmt::Display for Expression { } } } - diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index c6a5fba..ed5619d 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -463,6 +463,7 @@ impl<'a> Tokenizer<'a> { "break" if next_ws!() => keyword!(Break), "while" if next_ws!() => keyword!(While), "continue" if next_ws!() => keyword!(Continue), + "const" if next_ws!() => keyword!(Const), "true" if next_ws!() => { return Ok(Token::new( TokenType::Boolean(true), diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index ca61cee..13af954 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -295,6 +295,20 @@ documented! { /// } /// ``` Continue, + /// Prepresents the `const` keyword. This allows you to define a variable that will never + /// change throughout the lifetime of the program, similar to `define` in IC10. If you are + /// not planning on mutating the variable (changing it), it is recommend you store it as a + /// const, as the compiler will not assign it to a register or stack variable. + /// + /// ## Example + /// ``` + /// const targetTemp = 20c; + /// device gasSensor = "d0"; + /// device airCon = "d1"; + /// + /// airCon.On = gasSensor.Temperature > targetTemp; + /// ``` + Const, /// Represents the `let` keyword, used to declare variables within Slang. /// ## Example /// ``` @@ -304,6 +318,9 @@ documented! { /// ``` Let, /// Represents the `fn` keyword, used to declare functions within Slang. + /// # WARNING + /// Functions are currently unstable and are subject to change until stabilized. Use at + /// your own risk! (They are also heavily not optimized and produce a LOT of code bloat) /// ## Example /// ``` /// // This allows you to now call `doSomething` with specific arguments. From d34ce32d978861b9aaa8ec6f113f1045552092c1 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Thu, 4 Dec 2025 21:18:57 -0700 Subject: [PATCH 13/19] v1 constant expressions --- rust_compiler/libs/compiler/src/v1.rs | 93 +++++++++++++++++-- .../libs/compiler/src/variable_manager.rs | 22 ++++- 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index f97e99b..6633554 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -4,10 +4,10 @@ use parser::{ Parser as ASTParser, sys_call::{SysCall, System}, tree_node::{ - AssignmentExpression, BinaryExpression, BlockExpression, DeviceDeclarationExpression, - Expression, FunctionExpression, IfExpression, InvocationExpression, Literal, - LiteralOrVariable, LogicalExpression, LoopExpression, MemberAccessExpression, Span, - Spanned, WhileExpression, + AssignmentExpression, BinaryExpression, BlockExpression, ConstDeclarationExpression, + DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression, + InvocationExpression, Literal, LiteralOrVariable, LogicalExpression, LoopExpression, + MemberAccessExpression, Span, Spanned, WhileExpression, }, }; use quick_error::quick_error; @@ -34,6 +34,20 @@ macro_rules! debug { }; } +fn extract_literal(literal: Literal, allow_strings: bool) -> Result { + if !allow_strings && matches!(literal, Literal::String(_)) { + return Err(Error::Unknown( + "Literal strings are not allowed in this context".to_string(), + None, + )); + } + Ok(match literal { + Literal::String(s) => s, + Literal::Number(n) => n.to_string(), + Literal::Boolean(b) => if b { "1" } else { "0" }.into(), + }) +} + quick_error! { #[derive(Debug)] pub enum Error { @@ -58,6 +72,9 @@ quick_error! { AgrumentMismatch(func_name: String, span: Span) { display("Incorrect number of arguments passed into `{func_name}`") } + ConstAssignment(ident: String, span: Span) { + display("Attempted to re-assign a value to const variable `{ident}`") + } Unknown(reason: String, span: Option) { display("{reason}") } @@ -84,6 +101,7 @@ impl From for lsp_types::Diagnostic { DuplicateIdentifier(_, span) | UnknownIdentifier(_, span) | InvalidDevice(_, span) + | ConstAssignment(_, span) | AgrumentMismatch(_, span) => Diagnostic { range: span.into(), message: value.to_string(), @@ -267,6 +285,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { // decl_expr is Box> self.expression_declaration(var_name, *decl_expr, scope) } + Expression::ConstDeclaration(const_decl_expr) => { + self.expression_const_declaration(const_decl_expr.node, scope)?; + Ok(None) + } Expression::Assignment(assign_expr) => { self.expression_assignment(assign_expr.node, scope)?; Ok(None) @@ -435,6 +457,14 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { VariableLocation::Stack(_) => { self.write_output(format!("push {}{debug_tag}", source_value.into()))?; } + VariableLocation::Constant(_) => { + return Err(Error::Unknown( + r#"Attempted to emit a variable assignent for a constant value. + This is a Compiler bug and should be reported to the developer."# + .into(), + None, + )); + } } Ok(()) @@ -581,6 +611,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { ))?; format!("r{}", VariableScope::TEMP_STACK_REGISTER) } + VariableLocation::Constant(_) => unreachable!(), }; self.emit_variable_assignment(&name_str, &var_loc, src_str)?; (var_loc, None) @@ -638,6 +669,22 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { })) } + fn expression_const_declaration<'v>( + &mut self, + expr: ConstDeclarationExpression, + scope: &mut VariableScope<'v>, + ) -> Result { + let ConstDeclarationExpression { + name: const_name, + value: const_value, + } = expr; + + Ok(CompilationResult { + location: scope.define_const(const_name.node, const_value.node)?, + temp_name: None, + }) + } + fn expression_assignment<'v>( &mut self, expr: AssignmentExpression, @@ -685,6 +732,9 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { VariableScope::TEMP_STACK_REGISTER ))?; } + VariableLocation::Constant(_) => { + return Err(Error::ConstAssignment(identifier.node, identifier.span)); + } } if let Some(name) = cleanup { @@ -782,6 +832,9 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { VariableLocation::Persistant(reg) | VariableLocation::Temporary(reg) => { self.write_output(format!("push r{reg}"))?; } + VariableLocation::Constant(lit) => { + self.write_output(format!("push {}", extract_literal(lit, false)?))?; + } VariableLocation::Stack(stack_offset) => { self.write_output(format!( "sub r{0} sp {stack_offset}", @@ -1041,6 +1094,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { fn resolve_register(&self, loc: &VariableLocation) -> Result { match loc { VariableLocation::Temporary(r) | VariableLocation::Persistant(r) => Ok(format!("r{r}")), + VariableLocation::Constant(_) => Err(Error::Unknown( + "Cannot resolve a constant value to register".into(), + None, + )), VariableLocation::Stack(_) => Err(Error::Unknown( "Cannot resolve Stack location directly to register string without context".into(), None, @@ -1090,6 +1147,11 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { VariableLocation::Temporary(r) | VariableLocation::Persistant(r) => { Ok((format!("r{r}"), result.temp_name)) } + VariableLocation::Constant(lit) => match lit { + Literal::Number(n) => Ok((n.to_string(), None)), + Literal::Boolean(b) => Ok((if b { "1" } else { "0" }.to_string(), None)), + Literal::String(s) => Ok((s, None)), + }, VariableLocation::Stack(offset) => { // If it's on the stack, we must load it into a temp to use it as an operand let temp_name = self.next_temp_name(); @@ -1256,11 +1318,18 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { ) -> Result<(), Error> { // First, sort the expressions to ensure functions are hoisted expr.0.sort_by(|a, b| { - if matches!(b.node, Expression::Function(_)) - && matches!(a.node, Expression::Function(_)) - { + if matches!( + b.node, + Expression::Function(_) | Expression::ConstDeclaration(_) + ) && matches!( + a.node, + Expression::Function(_) | Expression::ConstDeclaration(_) + ) { std::cmp::Ordering::Equal - } else if matches!(a.node, Expression::Function(_)) { + } else if matches!( + a.node, + Expression::Function(_) | Expression::ConstDeclaration(_) + ) { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater @@ -1332,6 +1401,14 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { debug!(self, "#returnValue") ))?; } + VariableLocation::Constant(lit) => { + let str = extract_literal(lit, false)?; + self.write_output(format!( + "move r{} {str} {}", + VariableScope::RETURN_REGISTER, + debug!(self, "#returnValue") + ))? + } VariableLocation::Stack(offset) => { self.write_output(format!( "sub r{} sp {offset}", diff --git a/rust_compiler/libs/compiler/src/variable_manager.rs b/rust_compiler/libs/compiler/src/variable_manager.rs index c22a39f..15f0aea 100644 --- a/rust_compiler/libs/compiler/src/variable_manager.rs +++ b/rust_compiler/libs/compiler/src/variable_manager.rs @@ -3,6 +3,7 @@ // r1 - r7 : Temporary Variables // r8 - r14 : Persistant Variables +use parser::tree_node::Literal; use quick_error::quick_error; use std::collections::{HashMap, VecDeque}; @@ -43,11 +44,14 @@ pub enum VariableLocation { Persistant(u8), /// Represents a a stack offset (current stack - offset = variable loc) Stack(u16), + /// Represents a constant value and should be directly substituted as such. + Constant(Literal), } pub struct VariableScope<'a> { temporary_vars: VecDeque, persistant_vars: VecDeque, + constant_vars: HashMap, var_lookup_table: HashMap, stack_offset: u16, parent: Option<&'a VariableScope<'a>>, @@ -61,6 +65,7 @@ impl<'a> Default for VariableScope<'a> { persistant_vars: PERSIST.to_vec().into(), temporary_vars: TEMP.to_vec().into(), var_lookup_table: HashMap::new(), + constant_vars: HashMap::new(), } } } @@ -93,6 +98,7 @@ impl<'a> VariableScope<'a> { parent: Option::Some(parent), temporary_vars: parent.temporary_vars.clone(), persistant_vars: parent.persistant_vars.clone(), + constant_vars: parent.constant_vars.clone(), ..Default::default() } } @@ -142,6 +148,20 @@ impl<'a> VariableScope<'a> { Ok(var_location) } + pub fn define_const( + &mut self, + var_name: impl Into, + value: Literal, + ) -> Result { + let var_name = var_name.into(); + if self.constant_vars.contains_key(&var_name) { + return Err(Error::DuplicateVariable(var_name)); + } + + self.constant_vars.insert(var_name, value.clone()); + Ok(VariableLocation::Constant(value)) + } + pub fn get_location_of(&self, var_name: impl Into) -> Result { let var_name = var_name.into(); @@ -190,7 +210,7 @@ impl<'a> VariableScope<'a> { "Attempted to free a `let` variable.", ))); } - VariableLocation::Stack(_) => {} + _ => {} }; Ok(()) From 759f798fda1720d3fc6946c48be69e859c8993e5 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Thu, 4 Dec 2025 21:32:00 -0700 Subject: [PATCH 14/19] Fixed VariableScope so it stores const values in the var_lookup_table --- rust_compiler/libs/compiler/src/variable_manager.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust_compiler/libs/compiler/src/variable_manager.rs b/rust_compiler/libs/compiler/src/variable_manager.rs index 15f0aea..61dde65 100644 --- a/rust_compiler/libs/compiler/src/variable_manager.rs +++ b/rust_compiler/libs/compiler/src/variable_manager.rs @@ -51,7 +51,6 @@ pub enum VariableLocation { pub struct VariableScope<'a> { temporary_vars: VecDeque, persistant_vars: VecDeque, - constant_vars: HashMap, var_lookup_table: HashMap, stack_offset: u16, parent: Option<&'a VariableScope<'a>>, @@ -65,7 +64,6 @@ impl<'a> Default for VariableScope<'a> { persistant_vars: PERSIST.to_vec().into(), temporary_vars: TEMP.to_vec().into(), var_lookup_table: HashMap::new(), - constant_vars: HashMap::new(), } } } @@ -98,7 +96,6 @@ impl<'a> VariableScope<'a> { parent: Option::Some(parent), temporary_vars: parent.temporary_vars.clone(), persistant_vars: parent.persistant_vars.clone(), - constant_vars: parent.constant_vars.clone(), ..Default::default() } } @@ -154,12 +151,14 @@ impl<'a> VariableScope<'a> { value: Literal, ) -> Result { let var_name = var_name.into(); - if self.constant_vars.contains_key(&var_name) { + if self.var_lookup_table.contains_key(&var_name) { return Err(Error::DuplicateVariable(var_name)); } - self.constant_vars.insert(var_name, value.clone()); - Ok(VariableLocation::Constant(value)) + let new_value = VariableLocation::Constant(value); + + self.var_lookup_table.insert(var_name, new_value.clone()); + Ok(new_value) } pub fn get_location_of(&self, var_name: impl Into) -> Result { From b06ad778d9c623f192ef5890b3ac963fb3990c74 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Thu, 4 Dec 2025 22:44:47 -0700 Subject: [PATCH 15/19] Allow underscores in identifiers --- rust_compiler/libs/tokenizer/src/lib.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index ed5619d..b73c514 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -199,7 +199,7 @@ impl<'a> Tokenizer<'a> { .tokenize_symbol(next_char, start_line, start_col) .map(Some); } - char if char.is_alphabetic() => { + char if char.is_alphabetic() || char == '_' => { return self .tokenize_keyword_or_identifier(next_char, start_line, start_col) .map(Some); @@ -439,14 +439,15 @@ impl<'a> Tokenizer<'a> { }}; } macro_rules! next_ws { - () => { matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || !x.is_alphanumeric()) || self.peek_next_char()?.is_none() }; + () => { matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || (!x.is_alphanumeric()) && x != '_') || self.peek_next_char()?.is_none() }; } let mut buffer = String::with_capacity(16); let mut looped_char = Some(first_char); while let Some(next_char) = looped_char { - if next_char.is_whitespace() || !next_char.is_alphanumeric() { + // allow UNDERSCORE_IDENTS + if next_char.is_whitespace() || (!next_char.is_alphanumeric() && next_char != '_') { break; } buffer.push(next_char); @@ -838,7 +839,9 @@ mod tests { #[test] fn test_keyword_parse() -> Result<()> { - let mut tokenizer = Tokenizer::from(String::from("let fn if else return enum")); + let mut tokenizer = Tokenizer::from(String::from( + "let fn if else return enum continue break const", + )); let expected_tokens = vec![ TokenType::Keyword(Keyword::Let), @@ -847,6 +850,9 @@ mod tests { TokenType::Keyword(Keyword::Else), TokenType::Keyword(Keyword::Return), TokenType::Keyword(Keyword::Enum), + TokenType::Keyword(Keyword::Continue), + TokenType::Keyword(Keyword::Break), + TokenType::Keyword(Keyword::Const), ]; for expected_token in expected_tokens { @@ -860,7 +866,7 @@ mod tests { #[test] fn test_identifier_parse() -> Result<()> { - let mut tokenizer = Tokenizer::from(String::from("fn test")); + let mut tokenizer = Tokenizer::from(String::from("fn test fn test_underscores")); let token = tokenizer.next_token()?.unwrap(); assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn)); @@ -869,6 +875,13 @@ mod tests { token.token_type, TokenType::Identifier(String::from("test")) ); + let token = tokenizer.next_token()?.unwrap(); + assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn)); + let token = tokenizer.next_token()?.unwrap(); + assert_eq!( + token.token_type, + TokenType::Identifier(String::from("test_underscores")) + ); Ok(()) } From f2aedb96df54aea2efd2a7a69bb540daed334e1c Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 5 Dec 2025 00:25:24 -0700 Subject: [PATCH 16/19] update syntax highlighting to use vscode dark mode theme --- csharp_mod/Extensions.cs | 37 ++++++++++------ csharp_mod/Formatter.cs | 15 ++++++- rust_compiler/libs/helpers/src/lib.rs | 3 +- rust_compiler/libs/helpers/src/syscall.rs | 32 ++++++++++++++ rust_compiler/libs/parser/src/sys_call.rs | 27 +----------- rust_compiler/libs/tokenizer/src/token.rs | 51 +++++++++++++++++++---- 6 files changed, 116 insertions(+), 49 deletions(-) create mode 100644 rust_compiler/libs/helpers/src/syscall.rs diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index 648062b..71499a6 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -118,18 +118,31 @@ public static unsafe class SlangExtensions { switch (kind) { - case 1: - return SlangFormatter.ColorString; // String - case 2: - return SlangFormatter.ColorString; // Number - case 3: - return SlangFormatter.ColorInstruction; // Boolean - case 4: - return SlangFormatter.ColorSelection; // Keyword - case 5: - return SlangFormatter.ColorLineNumber; // Identifier - case 6: - return SlangFormatter.ColorDefault; // Symbol + case 1: // Strings + return SlangFormatter.ColorString; + case 2: // Numbers + return SlangFormatter.ColorNumber; + case 3: // Booleans + return SlangFormatter.ColorBoolean; + + case 4: // (if, else, loop) + return SlangFormatter.ColorControl; + case 5: // (let, const, device) + return SlangFormatter.ColorDeclaration; + + case 6: // (variables) + return SlangFormatter.ColorIdentifier; + case 7: // (punctuation) + return SlangFormatter.ColorDefault; + + case 10: // (syscalls) + return SlangFormatter.ColorFunction; + + case 11: // Comparisons + case 12: // Math + case 13: // Logic + return SlangFormatter.ColorOperator; + default: return SlangFormatter.ColorDefault; } diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs index cdb09c7..13eaf16 100644 --- a/csharp_mod/Formatter.cs +++ b/csharp_mod/Formatter.cs @@ -13,8 +13,19 @@ public class SlangFormatter : ICodeFormatter private CancellationTokenSource? _lspCancellationToken; private object _tokenLock = new(); - public static readonly uint ColorInstruction = ColorFromHTML("#ffff00"); - public static readonly uint ColorString = ColorFromHTML("#ce9178"); + // VS Code Dark Theme Palette + public static readonly uint ColorControl = ColorFromHTML("#C586C0"); // Pink (if, return, loop) + public static readonly uint ColorDeclaration = ColorFromHTML("#569CD6"); // Blue (let, device, fn) + public static readonly uint ColorFunction = ColorFromHTML("#DCDCAA"); // Yellow (syscalls) + public static readonly uint ColorString = ColorFromHTML("#CE9178"); // Orange + public static new readonly uint ColorNumber = ColorFromHTML("#B5CEA8"); // Light Green + public static readonly uint ColorBoolean = ColorFromHTML("#569CD6"); // Blue (true/false) + public static readonly uint ColorIdentifier = ColorFromHTML("#9CDCFE"); // Light Blue (variables) + public static new readonly uint ColorDefault = ColorFromHTML("#D4D4D4"); // White (punctuation ; { } ) + + // Operators are often the same color as default text in VS Code Dark, + // but having a separate definition lets you tweak it (e.g. make them slightly darker or distinct) + public static readonly uint ColorOperator = ColorFromHTML("#D4D4D4"); private HashSet _linesWithErrors = new(); diff --git a/rust_compiler/libs/helpers/src/lib.rs b/rust_compiler/libs/helpers/src/lib.rs index 18359cf..40b9b5a 100644 --- a/rust_compiler/libs/helpers/src/lib.rs +++ b/rust_compiler/libs/helpers/src/lib.rs @@ -1,4 +1,5 @@ mod macros; +mod syscall; /// This trait will allow the LSP to emit documentation for various tokens and expressions. /// You can easily create documentation for large enums with the `documented!` macro. @@ -10,5 +11,5 @@ pub trait Documentation { } pub mod prelude { - pub use super::{Documentation, documented}; + pub use super::{Documentation, documented, with_syscalls}; } diff --git a/rust_compiler/libs/helpers/src/syscall.rs b/rust_compiler/libs/helpers/src/syscall.rs new file mode 100644 index 0000000..37daaa9 --- /dev/null +++ b/rust_compiler/libs/helpers/src/syscall.rs @@ -0,0 +1,32 @@ +#[macro_export] +macro_rules! with_syscalls { + ($matcher:ident) => { + $matcher!( + "yield", + "sleep", + "hash", + "loadFromDevice", + "loadBatchNamed", + "loadBatch", + "setOnDevice", + "setOnDeviceBatched", + "setOnDeviceBatchedNamed", + "acos", + "asin", + "atan", + "atan2", + "abs", + "ceil", + "cos", + "floor", + "log", + "max", + "min", + "rand", + "sin", + "sqrt", + "tan", + "trunc" + ); + }; +} diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index 96f5a26..6a4aa88 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -256,31 +256,6 @@ impl std::fmt::Display for SysCall { impl SysCall { pub fn is_syscall(identifier: &str) -> bool { - matches!( - identifier, - "yield" - | "sleep" - | "hash" - | "loadFromDevice" - | "setOnDevice" - | "setOnDeviceBatched" - | "setOnDeviceBatchedNamed" - | "acos" - | "asin" - | "atan" - | "atan2" - | "abs" - | "ceil" - | "cos" - | "floor" - | "log" - | "max" - | "min" - | "rand" - | "sin" - | "sqrt" - | "tan" - | "trunc" - ) + tokenizer::token::is_syscall(identifier) } } diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index 13af954..3befb9f 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -1,6 +1,15 @@ use helpers::prelude::*; use rust_decimal::Decimal; +// Define a local macro to consume the list +macro_rules! generate_check { + ($($name:literal),*) => { + pub fn is_syscall(s: &str) -> bool { + matches!(s, $($name)|*) + } + } +} + #[derive(Debug, PartialEq, Eq, Clone)] pub struct Token { /// The type of the token @@ -101,17 +110,43 @@ impl Documentation for TokenType { } } +helpers::with_syscalls!(generate_check); + impl From for u32 { fn from(value: TokenType) -> Self { - use TokenType::*; match value { - String(_) => 1, - Number(_) => 2, - Boolean(_) => 3, - Keyword(_) => 4, - Identifier(_) => 5, - Symbol(_) => 6, - EOF => 0, + TokenType::String(_) => 1, + TokenType::Number(_) => 2, + TokenType::Boolean(_) => 3, + TokenType::Keyword(k) => match k { + Keyword::If + | Keyword::Else + | Keyword::Loop + | Keyword::While + | Keyword::Break + | Keyword::Continue + | Keyword::Return => 4, + _ => 5, + }, + TokenType::Identifier(s) => { + if is_syscall(&s) { + 10 + } else { + 6 + } + } + TokenType::Symbol(s) => { + if s.is_comparison() { + 11 + } else if s.is_operator() { + 12 + } else if s.is_logical() { + 13 + } else { + 7 + } + } + TokenType::EOF => 0, } } } From 320d288a6b8aa1903126987703d18db60b6e9ac8 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 5 Dec 2025 02:05:20 -0700 Subject: [PATCH 17/19] Start setting up github actions --- .github/workflows/build.yml | 67 ++++++++++++++++++++++++++++++ Dockerfile.build | 19 +++++++++ build.sh | 2 - csharp_mod/Plugin.cs | 5 ++- csharp_mod/stationeersSlang.csproj | 20 ++++----- 5 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 Dockerfile.build diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..71eabaf --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,67 @@ +name: CI/CD Pipeline + +# Trigger conditions +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +jobs: + # JOB 1: RUN TESTS + # This runs on every PR and every push to master. + # It validates that the Rust code is correct. + test: + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + + # 1. Build the Docker Image (Cached) + - name: Build Docker Image + run: docker build -t slang-builder -f Dockerfile.build . + + # 2. Run Rust Tests + # --manifest-path: Point to the nested Cargo.toml + # --workspace: Test all crates (compiler, parser, tokenizer, helpers) + # --all-targets: Test lib, bin, and tests folder (skips doc-tests for speed) + - name: Run Rust Tests + run: | + docker run --rm \ + -v "$PWD":/app \ + slang-builder \ + cargo test --manifest-path rust_compiler/Cargo.toml --workspace --all-targets + + build: + needs: test + runs-on: self-hosted + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v4 + + # 1. Build Image (Fast, uses cache from previous job if available/local) + - name: Build Docker Image + run: docker build -t slang-builder -f Dockerfile.build . + + # 2. Run the Build Script + # Mounts the references from the server's secure storage + - name: Build Release Artifacts + run: | + docker run --rm \ + -v "$PWD":/app \ + -v "/home/github-runner/permanent-refs":/app/csharp_mod/refs \ + slang-builder \ + ./build.sh + + # 3. Fix Permissions + # Docker writes files as root. We need to own them to upload them. + - name: Fix Permissions + if: always() + run: sudo chown -R $USER:$USER release/ + + # 4. Upload to GitHub + - name: Upload Release Artifacts + uses: actions/upload-artifact@v4 + with: + name: StationeersSlang-Release + path: release/ diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000..cbb06c5 --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,19 @@ +FROM rust:latest + +RUN apt-get update && apt-get install -y \ + mingw-w64 \ + wget \ + apt-transport-https \ + && rm -rf /var/lib/apt/lists/* + +RUN wget --progress=dot:giga https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ + && chmod +x ./dotnet-install.sh \ + && ./dotnet-install.sh --channel 8.0 --install-dir /usr/share/dotnet \ + && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet + +RUN rustup target add x86_64-pc-windows-gnu && rustup target add x86_64-unknown-linux-gnu + +WORKDIR /app + +# The command will be provided at runtime +CMD ["./build.sh"] diff --git a/build.sh b/build.sh index 0bbc5a5..2164aba 100755 --- a/build.sh +++ b/build.sh @@ -39,7 +39,6 @@ echo "--------------------" RUST_WIN_EXE="$RUST_DIR/target/x86_64-pc-windows-gnu/release/slang.exe" RUST_LINUX_BIN="$RUST_DIR/target/x86_64-unknown-linux-gnu/release/slang" CHARP_DLL="$CSHARP_DIR/bin/Release/net48/StationeersSlang.dll" -CHARP_PDB="$CSHARP_DIR/bin/Release/net48/StationeersSlang.pdb" # Check if the release dir exists, if not: create it. if [[ ! -d "$RELEASE_DIR" ]]; then @@ -49,4 +48,3 @@ fi cp "$RUST_WIN_EXE" "$RELEASE_DIR/slang.exe" cp "$RUST_LINUX_BIN" "$RELEASE_DIR/slang" cp "$CHARP_DLL" "$RELEASE_DIR/StationeersSlang.dll" -cp "$CHARP_PDB" "$RELEASE_DIR/StationeersSlang.pdb" diff --git a/csharp_mod/Plugin.cs b/csharp_mod/Plugin.cs index d4a8740..94d177b 100644 --- a/csharp_mod/Plugin.cs +++ b/csharp_mod/Plugin.cs @@ -35,14 +35,15 @@ namespace Slang } } - [BepInPlugin(PluginGuid, PluginName, "0.1.0")] + [BepInPlugin(PluginGuid, PluginName, PluginVersion)] [BepInDependency(StationeersIC10Editor.IC10EditorPlugin.PluginGuid)] public class SlangPlugin : BaseUnityPlugin { public const string PluginGuid = "com.biddydev.slang"; public const string PluginName = "Slang"; + public const string PluginVersion = "0.1.0"; - public static Mod MOD = new Mod(PluginName, "0.1.0"); + public static Mod MOD = new Mod(PluginName, PluginVersion); private Harmony? _harmony; diff --git a/csharp_mod/stationeersSlang.csproj b/csharp_mod/stationeersSlang.csproj index e464914..0880395 100644 --- a/csharp_mod/stationeersSlang.csproj +++ b/csharp_mod/stationeersSlang.csproj @@ -11,38 +11,36 @@ - $(STATIONEERS_DIR)/rocketstation_Data/Managed - $(STATIONEERS_DIR)/BepInEx/core - $(ManagedDir)/netstandard.dll + ./ref/netstandard.dll False - $(BepInExDir)/BepInEx.dll + ./ref/BepInEx.dll False - $(BepInExDir)/0Harmony.dll + ./ref/0Harmony.dll False - $(ManagedDir)/UnityEngine.dll + ./ref/UnityEngine.dll False - $(ManagedDir)/UnityEngine.CoreModule.dll + ./ref/UnityEngine.CoreModule.dll False - $(ManagedDir)/Assembly-CSharp.dll + ./ref/Assembly-CSharp.dll False - $(ManagedDir)/UniTask.dll + ./ref/UniTask.dll False @@ -51,11 +49,11 @@ False - $(STATIONEERS_DIR)/BepInEx/plugins/StationeersLaunchPad/LaunchPadBooster.dll + ./ref/LaunchPadBooster.dll False - $(STATIONEERS_DIR)/BepInEx/plugins/StationeersLaunchPad/StationeersMods.Interface.dll + ./ref/StationeersMods.Interface.dll False From dc02133f7e875837b4f69f7f04b3d00f48e03226 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 5 Dec 2025 02:30:37 -0700 Subject: [PATCH 18/19] Update build.yml comment about skipping rust doc tests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 71eabaf..2e690f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: # 2. Run Rust Tests # --manifest-path: Point to the nested Cargo.toml # --workspace: Test all crates (compiler, parser, tokenizer, helpers) - # --all-targets: Test lib, bin, and tests folder (skips doc-tests for speed) + # --all-targets: Test lib, bin, and tests folder (skips doc-tests because they are not valid Rust) - name: Run Rust Tests run: | docker run --rm \ From 4357bf5263c81e54f60b6345c57cf2ef7de8b2fe Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 5 Dec 2025 02:45:54 -0700 Subject: [PATCH 19/19] Attempt to run docker in github actions with correct user and group permissions --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e690f6..6bdd948 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,7 @@ jobs: - name: Run Rust Tests run: | docker run --rm \ + -u $(id -u):$(id -g) \ -v "$PWD":/app \ slang-builder \ cargo test --manifest-path rust_compiler/Cargo.toml --workspace --all-targets @@ -48,6 +49,7 @@ jobs: - name: Build Release Artifacts run: | docker run --rm \ + -u $(id -u):$(id -g) \ -v "$PWD":/app \ -v "/home/github-runner/permanent-refs":/app/csharp_mod/refs \ slang-builder \