diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index dfab846..722dbc9 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -207,4 +207,34 @@ public static unsafe class SlangExtensions Ffi.free_docs_vec(vec); return toReturn; } + + public static unsafe List ToList(this Vec_FfiSymbolInfo_t vec) + { + var toReturn = new List((int)vec.len); + + var currentPtr = vec.ptr; + + for (int i = 0; i < (int)vec.len; i++) + { + var item = currentPtr[i]; + + toReturn.Add( + new Slang.Symbol + { + Name = item.name.AsString(), + Kind = (SymbolKind)item.kind_data.kind, + Span = new Slang.Range + { + StartLine = item.span.start_line, + StartCol = item.span.start_col, + EndLine = item.span.end_line, + EndCol = item.span.end_col, + }, + Description = item.description.AsString(), + } + ); + } + + return toReturn; + } } diff --git a/csharp_mod/FfiGlue.cs b/csharp_mod/FfiGlue.cs index c2eb521..d839f4d 100644 --- a/csharp_mod/FfiGlue.cs +++ b/csharp_mod/FfiGlue.cs @@ -147,6 +147,51 @@ public unsafe partial class Ffi { slice_ref_uint16_t input); } +[StructLayout(LayoutKind.Sequential, Size = 12)] +public unsafe struct FfiSymbolKindData_t { + public UInt32 kind; + + public UInt32 arg_count; + + public UInt32 syscall_type; +} + +[StructLayout(LayoutKind.Sequential, Size = 80)] +public unsafe struct FfiSymbolInfo_t { + public Vec_uint8_t name; + + public FfiSymbolKindData_t kind_data; + + public FfiRange_t span; + + public Vec_uint8_t description; +} + +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_FfiSymbolInfo_t { + public FfiSymbolInfo_t * ptr; + + public UIntPtr len; + + public UIntPtr cap; +} + +[StructLayout(LayoutKind.Sequential, Size = 48)] +public unsafe struct FfiDiagnosticsAndSymbols_t { + public Vec_FfiDiagnostic_t diagnostics; + + public Vec_FfiSymbolInfo_t symbols; +} + +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + FfiDiagnosticsAndSymbols_t diagnose_source_with_symbols ( + slice_ref_uint16_t input); +} + [StructLayout(LayoutKind.Sequential, Size = 48)] public unsafe struct FfiDocumentedItem_t { public Vec_uint8_t item_name; @@ -184,6 +229,12 @@ public unsafe partial class Ffi { Vec_FfiDiagnostic_t v); } +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_ffi_diagnostics_and_symbols ( + FfiDiagnosticsAndSymbols_t v); +} + [StructLayout(LayoutKind.Sequential, Size = 64)] public unsafe struct FfiToken_t { public Vec_uint8_t tooltip; diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs index c4eb4e1..414377d 100644 --- a/csharp_mod/Formatter.cs +++ b/csharp_mod/Formatter.cs @@ -171,18 +171,17 @@ public class SlangFormatter : ICodeFormatter return; // Running this potentially CPU intensive work on a background thread. - var dict = await Task.Run( + var (diagnostics, symbols) = await Task.Run( () => { - return Marshal - .DiagnoseSource(inputSrc) - .GroupBy(d => d.Range.StartLine) - .ToDictionary(g => g.Key); + return Marshal.DiagnoseSourceWithSymbols(inputSrc); }, cancellationToken ); - ApplyDiagnostics(dict); + var dict = diagnostics.GroupBy(d => d.Range.StartLine).ToDictionary(g => g.Key); + + ApplyDiagnosticsAndSymbols(dict, symbols); // If we have valid code, update the IC10 output if (dict.Count > 0) @@ -266,11 +265,11 @@ public class SlangFormatter : ICodeFormatter } /// - /// Takes diagnostics from the Rust FFI compiler and applies it as semantic tokens to the + /// Takes diagnostics and symbols from the Rust FFI compiler and applies them as semantic tokens to the /// source in this editor. /// This runs on the Main Thread /// - private void ApplyDiagnostics(Dictionary> dict) + private void ApplyDiagnosticsAndSymbols(Dictionary> dict, List symbols) { HashSet linesToRefresh; @@ -289,6 +288,12 @@ public class SlangFormatter : ICodeFormatter { linesToRefresh = new HashSet(dict.Keys); linesToRefresh.UnionWith(_linesWithErrors); + + // Also add lines with symbols that may have been modified + foreach (var symbol in symbols) + { + linesToRefresh.Add(symbol.Span.StartLine); + } } _lastLineCount = this.Lines.Count; @@ -328,9 +333,49 @@ public class SlangFormatter : ICodeFormatter } } + // 3. Add symbol tooltips for symbols on this line + foreach (var symbol in symbols) + { + if (symbol.Span.StartLine == lineIndex) + { + var column = (int)symbol.Span.StartCol; + var length = Math.Max(1, (int)(symbol.Span.EndCol - symbol.Span.StartCol)); + + // If there's already a token at this position (from syntax highlighting), use it + // Otherwise, create a new token for the symbol + if (allTokensDict.ContainsKey(column)) + { + // Update existing token with symbol tooltip + var existingToken = allTokensDict[column]; + allTokensDict[column] = new SemanticToken( + line: existingToken.Line, + column: existingToken.Column, + length: existingToken.Length, + type: existingToken.Type, + style: existingToken.Style, + data: symbol.Description, // Use symbol description as tooltip + isError: existingToken.IsError + ); + } + else + { + // Create new token for symbol + allTokensDict[column] = new SemanticToken( + line: (int)lineIndex, + column, + length, + type: 0, + style: ColorIdentifier, + data: symbol.Description, + isError: false + ); + } + } + } + var allTokens = allTokensDict.Values.ToList(); - // 3. Update the line (this clears existing tokens and uses the list we just built) + // 4. Update the line (this clears existing tokens and uses the list we just built) line.Update(allTokens); ReattachMetadata(line, allTokens); @@ -339,6 +384,16 @@ public class SlangFormatter : ICodeFormatter _linesWithErrors = new HashSet(dict.Keys); } + /// + /// Takes diagnostics from the Rust FFI compiler and applies it as semantic tokens to the + /// source in this editor. + /// This runs on the Main Thread + /// + private void ApplyDiagnostics(Dictionary> dict) + { + ApplyDiagnosticsAndSymbols(dict, new List()); + } + // Helper to map SemanticToken data (tooltips/errors) back to the tokens in the line private void ReattachMetadata(StyledLine line, List semanticTokens) { diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs index 672f1ac..1f44090 100644 --- a/csharp_mod/Marshal.cs +++ b/csharp_mod/Marshal.cs @@ -47,6 +47,33 @@ public struct SourceMapEntry } } +public struct Symbol +{ + public string Name; + public Range Span; + public SymbolKind Kind; + public string Description; + + public override string ToString() + { + return $"{Kind}: {Name} at {Span}"; + } +} + +public enum SymbolKind +{ + Function = 0, + Syscall = 1, + Variable = 2, +} + +public struct SymbolData +{ + public uint Kind; + public uint ArgCount; + public uint SyscallType; // 0=System, 1=Math +} + public static class Marshal { private static IntPtr _libraryHandle = IntPtr.Zero; @@ -164,6 +191,59 @@ public static class Marshal } } + public static unsafe (List, List) DiagnoseSourceWithSymbols(string inputString) + { + if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded()) + { + return (new(), new()); + } + + fixed (char* ptrInput = inputString) + { + var input = new slice_ref_uint16_t + { + ptr = (ushort*)ptrInput, + len = (UIntPtr)inputString.Length, + }; + + var result = Ffi.diagnose_source_with_symbols(input); + + // Convert diagnostics + var diagnostics = result.diagnostics.ToList(); + + // Convert symbols + var symbols = new List(); + var symbolPtr = result.symbols.ptr; + var symbolCount = (int)result.symbols.len; + + for (int i = 0; i < symbolCount; i++) + { + var ffiSymbol = symbolPtr[i]; + var kind = (SymbolKind)ffiSymbol.kind_data.kind; + + // Use the actual description from the FFI (includes doc comments and syscall docs) + var description = ffiSymbol.description.AsString(); + + symbols.Add(new Symbol + { + Name = ffiSymbol.name.AsString(), + Kind = kind, + Span = new Range( + ffiSymbol.span.start_line, + ffiSymbol.span.start_col, + ffiSymbol.span.end_line, + ffiSymbol.span.end_col + ), + Description = description, + }); + } + + Ffi.free_ffi_diagnostics_and_symbols(result); + + return (diagnostics, symbols); + } + } + public static unsafe List TokenizeLine(string inputString) { if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded()) diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index f7e2302..2aab08d 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -173,7 +173,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] @@ -224,9 +224,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "clap" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", "clap_derive", @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstream", "anstyle", @@ -253,7 +253,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] @@ -523,9 +523,9 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "js-sys" @@ -571,7 +571,7 @@ dependencies = [ "regex-automata", "regex-syntax", "rustc_version", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] @@ -726,9 +726,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0" dependencies = [ "unicode-ident", ] @@ -909,12 +909,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - [[package]] name = "safer-ffi" version = "0.1.13" @@ -992,20 +986,20 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -1016,7 +1010,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] @@ -1108,9 +1102,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4" dependencies = [ "proc-macro2", "quote", @@ -1153,7 +1147,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] [[package]] @@ -1185,9 +1179,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.4+spec-1.0.0" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3cea6b2aa3b910092f6abd4053ea464fab5f9c170ba5e9a6aead16ec4af2b6" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -1206,9 +1200,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.5+spec-1.0.0" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c03bee5ce3696f31250db0bbaff18bc43301ce0e8db2ed1f07cbb2acf89984c" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ "winnow", ] @@ -1303,7 +1297,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", "wasm-bindgen-shared", ] @@ -1471,5 +1465,11 @@ checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.112", ] + +[[package]] +name = "zmij" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317f17ff091ac4515f17cc7a190d2769a8c9a96d227de5d64b500b01cda8f2cd" diff --git a/rust_compiler/libs/compiler/src/lib.rs b/rust_compiler/libs/compiler/src/lib.rs index 34f0c8b..aa623be 100644 --- a/rust_compiler/libs/compiler/src/lib.rs +++ b/rust_compiler/libs/compiler/src/lib.rs @@ -1,6 +1,8 @@ +pub mod symbols; #[cfg(test)] mod test; mod v1; mod variable_manager; +pub use symbols::{CompilationMetadata, SymbolInfo, SymbolKind, SyscallType}; pub use v1::{CompilationResult, Compiler, CompilerConfig, Error}; diff --git a/rust_compiler/libs/compiler/src/symbols.rs b/rust_compiler/libs/compiler/src/symbols.rs new file mode 100644 index 0000000..5fde9f6 --- /dev/null +++ b/rust_compiler/libs/compiler/src/symbols.rs @@ -0,0 +1,343 @@ +use helpers::Span; +use std::borrow::Cow; + +/// Represents a symbol (function, syscall, variable, etc.) that can be referenced in code. +/// Designed to be LSP-compatible for easy integration with language servers. +#[derive(Debug, Clone)] +pub struct SymbolInfo<'a> { + /// The name of the symbol + pub name: Cow<'a, str>, + /// The kind of symbol and associated metadata + pub kind: SymbolKind<'a>, + /// The source location of this symbol (for IDE features) + pub span: Option, + /// Optional description for tooltips and documentation + pub description: Option>, +} + +impl<'a> SymbolInfo<'a> { + /// Converts to an LSP SymbolInformation for protocol compatibility. + pub fn to_lsp_symbol_information(&self, uri: lsp_types::Uri) -> lsp_types::SymbolInformation { + lsp_types::SymbolInformation { + name: self.name.to_string(), + kind: self.kind.to_lsp_symbol_kind(), + #[allow(deprecated)] + deprecated: None, + location: lsp_types::Location { + uri, + range: self.span.as_ref().map(|s| (*s).into()).unwrap_or_default(), + }, + container_name: None, + tags: None, + } + } + + /// Converts to an LSP CompletionItem for autocomplete. + pub fn to_lsp_completion_item(&self) -> lsp_types::CompletionItem { + lsp_types::CompletionItem { + label: self.name.to_string(), + kind: Some(self.kind.to_lsp_completion_kind()), + documentation: self + .description + .as_ref() + .map(|d| lsp_types::Documentation::String(d.to_string())), + detail: Some(self.kind.detail_string()), + ..Default::default() + } + } +} + +/// Discriminates between different kinds of symbols. +#[derive(Debug, Clone)] +pub enum SymbolKind<'a> { + /// A user-defined function + Function { + /// Names of parameters in order + parameters: Vec>, + /// Type hint for the return type (if applicable) + return_type: Option>, + }, + /// A system or math syscall + Syscall { + /// Whether it's a System or Math syscall + syscall_type: SyscallType, + /// Number of expected arguments + argument_count: usize, + }, + /// A variable declaration + Variable { + /// Type hint for the variable (if applicable) + type_hint: Option>, + }, +} + +impl<'a> SymbolKind<'a> { + /// Converts to LSP SymbolKind for protocol compatibility. + fn to_lsp_symbol_kind(&self) -> lsp_types::SymbolKind { + match self { + SymbolKind::Function { .. } => lsp_types::SymbolKind::FUNCTION, + SymbolKind::Syscall { .. } => lsp_types::SymbolKind::FUNCTION, // Syscalls are function-like + SymbolKind::Variable { .. } => lsp_types::SymbolKind::VARIABLE, + } + } + + /// Converts to LSP CompletionItemKind for autocomplete filtering. + fn to_lsp_completion_kind(&self) -> lsp_types::CompletionItemKind { + match self { + SymbolKind::Function { .. } => lsp_types::CompletionItemKind::FUNCTION, + SymbolKind::Syscall { .. } => lsp_types::CompletionItemKind::FUNCTION, + SymbolKind::Variable { .. } => lsp_types::CompletionItemKind::VARIABLE, + } + } + + /// Returns a human-readable detail string for display in IDEs. + fn detail_string(&self) -> String { + match self { + SymbolKind::Function { + parameters, + return_type, + } => { + let params = parameters + .iter() + .map(|p| p.to_string()) + .collect::>() + .join(", "); + let ret = return_type + .as_ref() + .map(|t| format!(" -> {}", t)) + .unwrap_or_default(); + format!("fn({}){}", params, ret) + } + SymbolKind::Syscall { + syscall_type, + argument_count, + } => { + format!( + "{}(... {} args)", + match syscall_type { + SyscallType::System => "syscall", + SyscallType::Math => "math", + }, + argument_count + ) + } + SymbolKind::Variable { type_hint } => type_hint + .as_ref() + .map(|t| t.to_string()) + .unwrap_or_else(|| "var".to_string()), + } + } +} + +/// Distinguishes between System and Math syscalls. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyscallType { + System, + Math, +} + +/// Metadata collected during compilation, including all referenced symbols. +#[derive(Debug, Default)] +pub struct CompilationMetadata<'a> { + /// All symbols encountered during compilation (functions, syscalls, variables) + pub symbols: Vec>, +} + +impl<'a> CompilationMetadata<'a> { + /// Creates a new empty compilation metadata. + pub fn new() -> Self { + Self { + symbols: Vec::new(), + } + } + + /// Adds a symbol to the metadata. + pub fn add_symbol(&mut self, symbol: SymbolInfo<'a>) { + self.symbols.push(symbol); + } + + /// Adds a function symbol. + pub fn add_function( + &mut self, + name: Cow<'a, str>, + parameters: Vec>, + span: Option, + ) { + self.add_function_with_doc(name, parameters, span, None); + } + + /// Adds a function symbol with optional doc comment. + pub fn add_function_with_doc( + &mut self, + name: Cow<'a, str>, + parameters: Vec>, + span: Option, + description: Option>, + ) { + self.add_symbol(SymbolInfo { + name, + kind: SymbolKind::Function { + parameters, + return_type: None, + }, + span, + description, + }); + } + + /// Adds a syscall symbol. + pub fn add_syscall( + &mut self, + name: Cow<'a, str>, + syscall_type: SyscallType, + argument_count: usize, + span: Option, + ) { + self.add_syscall_with_doc(name, syscall_type, argument_count, span, None); + } + + /// Adds a syscall symbol with optional doc comment. + pub fn add_syscall_with_doc( + &mut self, + name: Cow<'a, str>, + syscall_type: SyscallType, + argument_count: usize, + span: Option, + description: Option>, + ) { + self.add_symbol(SymbolInfo { + name, + kind: SymbolKind::Syscall { + syscall_type, + argument_count, + }, + span, + description, + }); + } + + /// Adds a variable symbol. + pub fn add_variable(&mut self, name: Cow<'a, str>, span: Option) { + self.add_variable_with_doc(name, span, None); + } + + /// Adds a variable symbol with optional doc comment. + pub fn add_variable_with_doc( + &mut self, + name: Cow<'a, str>, + span: Option, + description: Option>, + ) { + self.add_symbol(SymbolInfo { + name, + kind: SymbolKind::Variable { type_hint: None }, + span, + description, + }); + } + + /// Returns all symbols of a specific kind. + pub fn symbols_of_kind(&self, kind: &str) -> Vec<&SymbolInfo<'a>> { + self.symbols + .iter() + .filter(|sym| match (&sym.kind, kind) { + (SymbolKind::Function { .. }, "function") => true, + (SymbolKind::Syscall { .. }, "syscall") => true, + (SymbolKind::Variable { .. }, "variable") => true, + _ => false, + }) + .collect() + } + + /// Converts all symbols to LSP SymbolInformation for protocol compatibility. + pub fn to_lsp_symbols(&self, uri: lsp_types::Uri) -> Vec { + self.symbols + .iter() + .map(|sym| sym.to_lsp_symbol_information(uri.clone())) + .collect() + } + + /// Converts all symbols to LSP CompletionItems for autocomplete. + pub fn to_lsp_completion_items(&self) -> Vec { + self.symbols + .iter() + .map(|sym| sym.to_lsp_completion_item()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_metadata_creation() { + let metadata = CompilationMetadata::new(); + assert!(metadata.symbols.is_empty()); + } + + #[test] + fn test_add_function_symbol() { + let mut metadata = CompilationMetadata::new(); + metadata.add_function("test_func".into(), vec!["x".into(), "y".into()], None); + assert_eq!(metadata.symbols.len(), 1); + assert_eq!(metadata.symbols[0].name, "test_func"); + } + + #[test] + fn test_add_syscall_symbol() { + let mut metadata = CompilationMetadata::new(); + metadata.add_syscall("hash".into(), SyscallType::System, 1, None); + assert_eq!(metadata.symbols.len(), 1); + assert_eq!(metadata.symbols[0].name, "hash"); + } + + #[test] + fn test_symbols_of_kind() { + let mut metadata = CompilationMetadata::new(); + metadata.add_function("func1".into(), vec![], None); + metadata.add_syscall("hash".into(), SyscallType::System, 1, None); + metadata.add_variable("x".into(), None); + + let functions = metadata.symbols_of_kind("function"); + assert_eq!(functions.len(), 1); + + let syscalls = metadata.symbols_of_kind("syscall"); + assert_eq!(syscalls.len(), 1); + + let variables = metadata.symbols_of_kind("variable"); + assert_eq!(variables.len(), 1); + } + + #[test] + fn test_lsp_completion_items() { + let mut metadata = CompilationMetadata::new(); + metadata.add_function("test_func".into(), vec![], None); + metadata.add_syscall("hash".into(), SyscallType::System, 1, None); + metadata.add_variable("x".into(), None); + + let completions = metadata.to_lsp_completion_items(); + assert_eq!(completions.len(), 3); + + // Verify function + assert_eq!(completions[0].label, "test_func"); + assert_eq!( + completions[0].kind, + Some(lsp_types::CompletionItemKind::FUNCTION) + ); + + // Verify syscall + assert_eq!(completions[1].label, "hash"); + assert_eq!( + completions[1].kind, + Some(lsp_types::CompletionItemKind::FUNCTION) + ); + + // Verify variable + assert_eq!(completions[2].label, "x"); + assert_eq!( + completions[2].kind, + Some(lsp_types::CompletionItemKind::VARIABLE) + ); + } +} diff --git a/rust_compiler/libs/compiler/src/test/device_access.rs b/rust_compiler/libs/compiler/src/test/device_access.rs index e4dee6b..a607b36 100644 --- a/rust_compiler/libs/compiler/src/test/device_access.rs +++ b/rust_compiler/libs/compiler/src/test/device_access.rs @@ -272,3 +272,108 @@ fn device_property_with_underscore_name() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn device_index_read() -> anyhow::Result<()> { + let compiled = compile! { + check " + device printer = \"d0\"; + let value = printer[255]; + " + }; + + assert!( + compiled.errors.is_empty(), + "Expected no errors, got: {:?}", + compiled.errors + ); + + assert_eq!( + compiled.output, + indoc! { + " + j main + main: + get r1 d0 255 + move r8 r1 + " + } + ); + + Ok(()) +} + +#[test] +fn device_index_write() -> anyhow::Result<()> { + let compiled = compile! { + check " + device printer = \"d0\"; + printer[255] = 42; + " + }; + + assert!( + compiled.errors.is_empty(), + "Expected no errors, got: {:?}", + compiled.errors + ); + + assert_eq!( + compiled.output, + indoc! { + " + j main + main: + put d0 255 42 + " + } + ); + + Ok(()) +} + +#[test] +fn device_index_db_not_allowed() -> anyhow::Result<()> { + let compiled = compile! { + check " + device stack = \"db\"; + let x = stack[10]; + " + }; + + assert!( + !compiled.errors.is_empty(), + "Expected error for db indexing" + ); + assert!( + compiled.errors[0] + .to_string() + .contains("Direct stack access on 'db' is not yet supported"), + "Expected db restriction error" + ); + + Ok(()) +} + +#[test] +fn device_index_db_write_not_allowed() -> anyhow::Result<()> { + let compiled = compile! { + check " + device stack = \"db\"; + stack[10] = 42; + " + }; + + assert!( + !compiled.errors.is_empty(), + "Expected error for db indexing" + ); + assert!( + compiled.errors[0] + .to_string() + .contains("Direct stack access on 'db' is not yet supported"), + "Expected db restriction error" + ); + + Ok(()) +} diff --git a/rust_compiler/libs/compiler/src/test/mod.rs b/rust_compiler/libs/compiler/src/test/mod.rs index c732971..f34bd34 100644 --- a/rust_compiler/libs/compiler/src/test/mod.rs +++ b/rust_compiler/libs/compiler/src/test/mod.rs @@ -47,6 +47,15 @@ macro_rules! compile { output, } }}; + + (metadata $source:expr) => {{ + let compiler = crate::Compiler::new( + parser::Parser::new(tokenizer::Tokenizer::from($source)), + None, + ); + let res = compiler.compile(); + res.metadata + }}; } mod binary_expression; mod branching; @@ -61,5 +70,6 @@ mod loops; mod math_syscall; mod negation_priority; mod scoping; +mod symbol_documentation; mod syscall; mod tuple_literals; diff --git a/rust_compiler/libs/compiler/src/test/symbol_documentation.rs b/rust_compiler/libs/compiler/src/test/symbol_documentation.rs new file mode 100644 index 0000000..3ba830e --- /dev/null +++ b/rust_compiler/libs/compiler/src/test/symbol_documentation.rs @@ -0,0 +1,120 @@ +#[cfg(test)] +mod test { + use anyhow::Result; + + #[test] + fn test_variable_doc_comment() -> Result<()> { + let metadata = compile!(metadata "/// this is a documented variable\nlet myVar = 42;"); + + let var_symbol = metadata + .symbols + .iter() + .find(|s| s.name == "myVar") + .expect("myVar symbol not found"); + + assert_eq!( + var_symbol.description.as_ref().map(|d| d.as_ref()), + Some("this is a documented variable") + ); + Ok(()) + } + + #[test] + fn test_const_doc_comment() -> Result<()> { + let metadata = compile!(metadata "/// const documentation\nconst myConst = 100;"); + + let const_symbol = metadata + .symbols + .iter() + .find(|s| s.name == "myConst") + .expect("myConst symbol not found"); + + assert_eq!( + const_symbol.description.as_ref().map(|d| d.as_ref()), + Some("const documentation") + ); + Ok(()) + } + + #[test] + fn test_device_doc_comment() -> Result<()> { + let metadata = compile!(metadata "/// device documentation\ndevice myDevice = \"d0\";"); + + let device_symbol = metadata + .symbols + .iter() + .find(|s| s.name == "myDevice") + .expect("myDevice symbol not found"); + + assert_eq!( + device_symbol.description.as_ref().map(|d| d.as_ref()), + Some("device documentation") + ); + Ok(()) + } + + #[test] + fn test_function_doc_comment() -> Result<()> { + let metadata = compile!(metadata "/// function documentation\nfn test() { }"); + + let fn_symbol = metadata + .symbols + .iter() + .find(|s| s.name == "test") + .expect("test symbol not found"); + + assert_eq!( + fn_symbol.description.as_ref().map(|d| d.as_ref()), + Some("function documentation") + ); + Ok(()) + } + + #[test] + fn test_syscall_documentation() -> Result<()> { + let metadata = compile!(metadata "fn test() { clr(d0); }"); + + let clr_symbol = metadata + .symbols + .iter() + .find(|s| s.name == "clr") + .expect("clr syscall not found"); + + // clr should have its built-in documentation + assert!(clr_symbol.description.is_some()); + assert!(!clr_symbol.description.as_ref().unwrap().is_empty()); + Ok(()) + } + + #[test] + fn test_variable_references_have_tooltips() -> Result<()> { + let metadata = compile!(metadata "/// documented variable\nlet myVar = 5;\nlet x = myVar + 2;\nmyVar = 10;"); + + // Count how many times 'myVar' appears in symbols + let myvar_symbols: Vec<_> = metadata + .symbols + .iter() + .filter(|s| s.name == "myVar") + .collect(); + + // We should have at least 2: declaration + 1 reference (in myVar + 2) + // The assignment `myVar = 10` is a write, not a read, so doesn't create a reference + assert!( + myvar_symbols.len() >= 2, + "Expected at least 2 'myVar' symbols (declaration + reference), got {}", + myvar_symbols.len() + ); + + // All should have the same description + let expected_desc = "documented variable"; + for sym in &myvar_symbols { + assert_eq!( + sym.description.as_ref().map(|d| d.as_ref()), + Some(expected_desc), + "Symbol description mismatch at {:?}", + sym.span + ); + } + Ok(()) + } +} diff --git a/rust_compiler/libs/compiler/src/test/syscall.rs b/rust_compiler/libs/compiler/src/test/syscall.rs index 22ce165..f894eb3 100644 --- a/rust_compiler/libs/compiler/src/test/syscall.rs +++ b/rust_compiler/libs/compiler/src/test/syscall.rs @@ -287,3 +287,68 @@ fn test_load_reagent() -> anyhow::Result<()> { Ok(()) } +#[test] +fn test_clr() -> anyhow::Result<()> { + let compiled = compile! { + check + " + device stackDevice = \"d0\"; + clr(stackDevice); + let deviceRef = 5; + clr(deviceRef); + " + }; + + assert!( + compiled.errors.is_empty(), + "Expected no errors, got: {:?}", + compiled.errors + ); + + assert_eq!( + compiled.output, + indoc! { + " + j main + main: + clr d0 + move r8 5 + clr r8 + " + } + ); + + Ok(()) +} +#[test] +fn test_rmap() -> anyhow::Result<()> { + let compiled = compile! { + check + " + device printer = \"d0\"; + let reagentHash = 12345; + let itemHash = rmap(printer, reagentHash); + " + }; + + assert!( + compiled.errors.is_empty(), + "Expected no errors, got: {:?}", + compiled.errors + ); + + assert_eq!( + compiled.output, + indoc! { + " + j main + main: + move r8 12345 + rmap r15 d0 r8 + move r9 r15 + " + } + ); + + Ok(()) +} diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index d5daaa5..66e6958 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -8,8 +8,8 @@ use parser::{ tree_node::{ AssignmentExpression, BinaryExpression, BlockExpression, ConstDeclarationExpression, DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression, - InvocationExpression, Literal, LiteralOr, LiteralOrVariable, LogicalExpression, - LoopExpression, MemberAccessExpression, Spanned, TernaryExpression, + IndexAccessExpression, InvocationExpression, Literal, LiteralOr, LiteralOrVariable, + LogicalExpression, LoopExpression, MemberAccessExpression, Spanned, TernaryExpression, TupleAssignmentExpression, TupleDeclarationExpression, WhileExpression, }, }; @@ -67,6 +67,9 @@ pub enum Error<'a> { #[error("Expected a {0}-tuple, but you're trying to destructure into {1} variables")] TupleSizeMismatch(usize, usize, Span), + #[error("{0}")] + OperationNotSupported(String, Span), + #[error("{0}")] Unknown(String, Option), } @@ -89,7 +92,8 @@ impl<'a> From> for lsp_types::Diagnostic { | ConstAssignment(_, span) | DeviceAssignment(_, span) | AgrumentMismatch(_, span) - | TupleSizeMismatch(_, _, span) => Diagnostic { + | TupleSizeMismatch(_, _, span) + | OperationNotSupported(_, span) => Diagnostic { range: span.into(), message: value.to_string(), severity: Some(DiagnosticSeverity::ERROR), @@ -141,6 +145,7 @@ struct CompileLocation<'a> { pub struct CompilationResult<'a> { pub errors: Vec>, pub instructions: Instructions<'a>, + pub metadata: crate::CompilationMetadata<'a>, } /// Metadata for the currently compiling function @@ -198,6 +203,8 @@ pub struct Compiler<'a> { pub source_map: HashMap>, /// Accumulative errors from the compilation process pub errors: Vec>, + /// Metadata about symbols encountered during compilation + pub metadata: crate::CompilationMetadata<'a>, } impl<'a> Compiler<'a> { @@ -215,6 +222,7 @@ impl<'a> Compiler<'a> { loop_stack: Vec::new(), source_map: HashMap::new(), errors: Vec::new(), + metadata: crate::CompilationMetadata::new(), } } @@ -233,6 +241,7 @@ impl<'a> Compiler<'a> { return CompilationResult { errors: self.errors, instructions: self.instructions, + metadata: self.metadata, }; } Err(e) => { @@ -241,6 +250,7 @@ impl<'a> Compiler<'a> { return CompilationResult { errors: self.errors, instructions: self.instructions, + metadata: self.metadata, }; } }; @@ -266,6 +276,7 @@ impl<'a> Compiler<'a> { return CompilationResult { errors: self.errors, instructions: self.instructions, + metadata: self.metadata, }; } @@ -279,6 +290,7 @@ impl<'a> Compiler<'a> { CompilationResult { errors: self.errors, instructions: self.instructions, + metadata: self.metadata, } } @@ -387,6 +399,26 @@ impl<'a> Compiler<'a> { } Expression::Ternary(tern) => Ok(Some(self.expression_ternary(tern.node, scope)?)), Expression::Invocation(expr_invoke) => { + // Special case: hash() with string literal can be evaluated at compile time + if expr_invoke.node.name.node == "hash" && expr_invoke.node.arguments.len() == 1 { + if let Expression::Literal(Spanned { + node: Literal::String(str_to_hash), + .. + }) = &expr_invoke.node.arguments[0].node + { + // Evaluate hash at compile time + let hash_value = crc_hash_signed(str_to_hash); + return Ok(Some(CompileLocation { + location: VariableLocation::Constant(Literal::Number(Number::Integer( + hash_value, + Unit::None, + ))), + temp_name: None, + })); + } + } + + // Non-constant hash calls or other function calls self.expression_function_invocation(expr_invoke, scope)?; // Invocation returns result in r15 (RETURN_REGISTER). // If used as an expression, we must move it to a temp to avoid overwrite. @@ -433,10 +465,23 @@ impl<'a> Compiler<'a> { }, Expression::Variable(name) => { match scope.get_location_of(&name.node, Some(name.span)) { - Ok(loc) => Ok(Some(CompileLocation { - location: loc, - temp_name: None, // User variable, do not free - })), + Ok(loc) => { + // Track this variable reference in metadata (for tooltips on all usages, not just declaration) + let doc_comment: Option> = self + .parser + .get_declaration_doc(name.node.as_ref()) + .map(|s| Cow::Owned(s) as Cow<'a, str>); + self.metadata.add_variable_with_doc( + name.node.clone(), + Some(name.span), + doc_comment, + ); + + Ok(Some(CompileLocation { + location: loc, + temp_name: None, // User variable, do not free + })) + } Err(_) => { // fallback, check devices if let Some(device) = self.devices.get(&name.node) { @@ -487,6 +532,50 @@ impl<'a> Compiler<'a> { temp_name: Some(result_name), })) } + Expression::IndexAccess(access) => { + // "get" behavior (e.g. `let x = d0[255]`) + let IndexAccessExpression { object, index } = access.node; + + // 1. Resolve the object to a device string + let (device, dev_cleanup) = self.resolve_device(*object, scope)?; + + // Check if device is "db" (not allowed) + if let Operand::Device(ref dev_str) = device { + if dev_str.as_ref() == "db" { + return Err(Error::OperationNotSupported( + "Direct stack access on 'db' is not yet supported".to_string(), + expr.span, + )); + } + } + + // 2. Compile the index expression to get the address + let (addr, addr_cleanup) = self.compile_operand(*index, scope)?; + + // 3. Allocate a temp register for the result + let result_name = self.next_temp_name(); + let loc = scope.add_variable(result_name.clone(), LocationRequest::Temp, None)?; + let reg = self.resolve_register(&loc)?; + + // 4. Emit get instruction: get rX device address + self.write_instruction( + Instruction::Get(Operand::Register(reg), device, addr), + Some(expr.span), + )?; + + // 5. Cleanup + if let Some(c) = dev_cleanup { + scope.free_temp(c, None)?; + } + if let Some(c) = addr_cleanup { + scope.free_temp(c, None)?; + } + + Ok(Some(CompileLocation { + location: loc, + temp_name: Some(result_name), + })) + } Expression::MethodCall(call) => { // Methods are not yet fully supported (e.g. `d0.SomeFunc()`). // This would likely map to specialized syscalls or batch instructions. @@ -525,6 +614,28 @@ impl<'a> Compiler<'a> { temp_name: Some(result_name), })) } + Expression::BitwiseNot(inner_expr) => { + // Compile bitwise NOT using the NOT instruction + let (inner_str, cleanup) = self.compile_operand(*inner_expr, scope)?; + let result_name = self.next_temp_name(); + let result_loc = + scope.add_variable(result_name.clone(), LocationRequest::Temp, None)?; + let result_reg = self.resolve_register(&result_loc)?; + + self.write_instruction( + Instruction::Not(Operand::Register(result_reg), inner_str), + Some(expr.span), + )?; + + if let Some(name) = cleanup { + scope.free_temp(name, None)?; + } + + Ok(Some(CompileLocation { + location: result_loc, + temp_name: Some(result_name), + })) + } Expression::TupleDeclaration(tuple_decl) => { self.expression_tuple_declaration(tuple_decl.node, scope)?; Ok(None) @@ -554,6 +665,14 @@ impl<'a> Compiler<'a> { if let Expression::Variable(ref name) = expr.node && let Some(device_id) = self.devices.get(&name.node) { + // Track this device reference in metadata (for tooltips on all usages, not just declaration) + let doc_comment = self + .parser + .get_declaration_doc(name.node.as_ref()) + .map(Cow::Owned); + self.metadata + .add_variable_with_doc(name.node.clone(), Some(expr.span), doc_comment); + return Ok((Operand::Device(device_id.clone()), None)); } @@ -606,6 +725,14 @@ impl<'a> Compiler<'a> { let name_str = var_name.node; let name_span = var_name.span; + // Track the variable in metadata + let doc_comment = self + .parser + .get_declaration_doc(name_str.as_ref()) + .map(Cow::Owned); + self.metadata + .add_variable_with_doc(name_str.clone(), Some(name_span), doc_comment); + // optimization. Check for a negated numeric literal (including nested negations) // e.g., -5, -(-5), -(-(5)), etc. if let Some(num) = self.try_fold_negation(&expr.node) { @@ -889,6 +1016,58 @@ impl<'a> Compiler<'a> { } (var_loc, None) } + Expression::BitwiseNot(_) => { + // Compile the bitwise NOT expression + let result = self.expression(expr, scope)?; + let var_loc = scope.add_variable( + name_str.clone(), + LocationRequest::Persist, + Some(name_span), + )?; + + if let Some(res) = result { + // Move result from temp to new persistent variable + let result_reg = self.resolve_register(&res.location)?; + self.emit_variable_assignment(&var_loc, Operand::Register(result_reg))?; + + // Free the temp result + if let Some(name) = res.temp_name { + scope.free_temp(name, None)?; + } + } else { + return Err(Error::Unknown( + format!("`{name_str}` bitwise NOT expression did not produce a value"), + Some(name_span), + )); + } + (var_loc, None) + } + Expression::IndexAccess(_) => { + // Compile the index access expression + let result = self.expression(expr, scope)?; + let var_loc = scope.add_variable( + name_str.clone(), + LocationRequest::Persist, + Some(name_span), + )?; + + if let Some(res) = result { + // Move result from temp to new persistent variable + let result_reg = self.resolve_register(&res.location)?; + self.emit_variable_assignment(&var_loc, Operand::Register(result_reg))?; + + // Free the temp result + if let Some(name) = res.temp_name { + scope.free_temp(name, None)?; + } + } else { + return Err(Error::Unknown( + format!("`{name_str}` index access expression did not produce a value"), + Some(name_span), + )); + } + (var_loc, None) + } _ => { return Err(Error::Unknown( format!("`{name_str}` declaration of this type is not supported/implemented."), @@ -913,6 +1092,17 @@ impl<'a> Compiler<'a> { value: const_value, } = expr; + // Track the const variable in metadata + let doc_comment = self + .parser + .get_declaration_doc(const_name.node.as_ref()) + .map(Cow::Owned); + self.metadata.add_variable_with_doc( + const_name.node.clone(), + Some(const_name.span), + doc_comment, + ); + // check for a hash expression or a literal let value = match const_value { LiteralOr::Or(Spanned { @@ -1025,6 +1215,37 @@ impl<'a> Compiler<'a> { scope.free_temp(c, None)?; } } + Expression::IndexAccess(access) => { + // Put instruction: put device address value + let IndexAccessExpression { object, index } = access.node; + + let (device, dev_cleanup) = self.resolve_device(*object, scope)?; + + // Check if device is "db" (not allowed) + if let Operand::Device(ref dev_str) = device { + if dev_str.as_ref() == "db" { + return Err(Error::OperationNotSupported( + "Direct stack access on 'db' is not yet supported".to_string(), + assignee.span, + )); + } + } + + let (addr, addr_cleanup) = self.compile_operand(*index, scope)?; + let (val, val_cleanup) = self.compile_operand(*expression, scope)?; + + self.write_instruction(Instruction::Put(device, addr, val), Some(assignee.span))?; + + if let Some(c) = dev_cleanup { + scope.free_temp(c, None)?; + } + if let Some(c) = addr_cleanup { + scope.free_temp(c, None)?; + } + if let Some(c) = val_cleanup { + scope.free_temp(c, None)?; + } + } _ => { return Err(Error::Unknown( @@ -1305,6 +1526,29 @@ impl<'a> Compiler<'a> { ) -> Result<(), Error<'a>> { let TupleDeclarationExpression { names, value } = tuple_decl; + // Track each variable in the tuple declaration + // Get doc for the first variable + let first_var_name = names + .iter() + .find(|n| n.node.as_ref() != "_") + .map(|n| n.node.to_string()); + let doc_comment = first_var_name + .as_ref() + .and_then(|name| self.parser.get_declaration_doc(name)) + .map(Cow::Owned); + + for (i, name_spanned) in names.iter().enumerate() { + if name_spanned.node.as_ref() != "_" { + // Only attach doc comment to the first variable + let comment = if i == 0 { doc_comment.clone() } else { None }; + self.metadata.add_variable_with_doc( + name_spanned.node.clone(), + Some(name_spanned.span), + comment, + ); + } + } + match value.node { Expression::Invocation(invoke_expr) => { // Execute the function call - tuple values will be on the stack @@ -1743,6 +1987,17 @@ impl<'a> Compiler<'a> { &mut self, expr: DeviceDeclarationExpression<'a>, ) -> Result<(), Error<'a>> { + // Track the device declaration in metadata + let doc_comment = self + .parser + .get_declaration_doc(expr.name.node.as_ref()) + .map(Cow::Owned); + self.metadata.add_variable_with_doc( + expr.name.node.clone(), + Some(expr.name.span), + doc_comment, + ); + if self.devices.contains_key(&expr.name.node) { self.errors.push(Error::DuplicateIdentifier( expr.name.node.clone(), @@ -2113,14 +2368,40 @@ impl<'a> Compiler<'a> { expr: Spanned>, scope: &mut VariableScope<'a, '_>, ) -> Result, Error<'a>> { - fn fold_binary_expression<'a>(expr: &BinaryExpression<'a>) -> Option { + fn fold_binary_expression<'a>( + expr: &BinaryExpression<'a>, + scope: &VariableScope<'a, '_>, + ) -> Option { + fn number_to_i64(n: Number) -> Option { + match n { + Number::Integer(i, _) => i64::try_from(i).ok(), + Number::Decimal(d, _) => { + // Convert decimal to i64 by truncating + let int_part = d.trunc(); + i64::try_from(int_part.mantissa() / 10_i128.pow(int_part.scale())).ok() + } + } + } + + fn i64_to_number(i: i64) -> Number { + Number::Integer(i as i128, Unit::None) + } + let (lhs, rhs) = match &expr { BinaryExpression::Add(l, r) | BinaryExpression::Subtract(l, r) | BinaryExpression::Multiply(l, r) | BinaryExpression::Divide(l, r) | BinaryExpression::Exponent(l, r) - | BinaryExpression::Modulo(l, r) => (fold_expression(l)?, fold_expression(r)?), + | BinaryExpression::Modulo(l, r) + | BinaryExpression::BitwiseAnd(l, r) + | BinaryExpression::BitwiseOr(l, r) + | BinaryExpression::BitwiseXor(l, r) + | BinaryExpression::LeftShift(l, r) + | BinaryExpression::RightShiftArithmetic(l, r) + | BinaryExpression::RightShiftLogical(l, r) => { + (fold_expression(l, scope)?, fold_expression(r, scope)?) + } }; match expr { @@ -2129,11 +2410,44 @@ impl<'a> Compiler<'a> { BinaryExpression::Multiply(..) => Some(lhs * rhs), BinaryExpression::Divide(..) => Some(lhs / rhs), // Watch out for div by zero panics! BinaryExpression::Modulo(..) => Some(lhs % rhs), - _ => None, // Handle Exponent separately or implement pow + BinaryExpression::BitwiseAnd(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int & rhs_int)) + } + BinaryExpression::BitwiseOr(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int | rhs_int)) + } + BinaryExpression::BitwiseXor(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int ^ rhs_int)) + } + BinaryExpression::LeftShift(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int << rhs_int)) + } + BinaryExpression::RightShiftArithmetic(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int >> rhs_int)) + } + BinaryExpression::RightShiftLogical(..) => { + let lhs_int = number_to_i64(lhs)?; + let rhs_int = number_to_i64(rhs)?; + Some(i64_to_number(lhs_int >> rhs_int)) + } + _ => None, // Exponent not handled in compile-time folding } } - fn fold_expression<'a>(expr: &Expression<'a>) -> Option { + fn fold_expression<'a>( + expr: &Expression<'a>, + scope: &VariableScope<'a, '_>, + ) -> Option { match expr { // 1. Base Case: It's already a number Expression::Literal(lit) => match lit.node { @@ -2142,23 +2456,60 @@ impl<'a> Compiler<'a> { }, // 2. Handle Parentheses: Just recurse deeper - Expression::Priority(inner) => fold_expression(&inner.node), + Expression::Priority(inner) => fold_expression(&inner.node, scope), // 3. Handle Negation: Recurse, then negate Expression::Negation(inner) => { - let val = fold_expression(&inner.node)?; + let val = fold_expression(&inner.node, scope)?; Some(-val) // Requires impl Neg for Number } // 4. Handle Binary Ops: Recurse BOTH sides, then combine - Expression::Binary(bin) => fold_binary_expression(&bin.node), + Expression::Binary(bin) => fold_binary_expression(&bin.node, scope), - // 5. Anything else (Variables, Function Calls) cannot be compile-time folded + // 5. Handle Variable Reference: Check if it's a const + Expression::Variable(var_id) => { + if let Ok(var_loc) = scope.get_location_of(var_id, None) { + if let VariableLocation::Constant(Literal::Number(num)) = var_loc { + return Some(num); + } + } + None + } + + // 6. Handle hash() syscall - evaluates to a constant at compile time + Expression::Syscall(Spanned { + node: + SysCall::System(System::Hash(Spanned { + node: Literal::String(str_to_hash), + .. + })), + .. + }) => { + return Some(Number::Integer(crc_hash_signed(str_to_hash), Unit::None)); + } + + // 7. Handle hash() macro as invocation - evaluates to a constant at compile time + Expression::Invocation(inv) => { + if inv.node.name.node == "hash" && inv.node.arguments.len() == 1 { + if let Expression::Literal(Spanned { + node: Literal::String(str_to_hash), + .. + }) = &inv.node.arguments[0].node + { + // hash() takes a string literal and returns a signed integer + return Some(Number::Integer(crc_hash_signed(str_to_hash), Unit::None)); + } + } + None + } + + // 8. Anything else cannot be compile-time folded _ => None, } } - if let Some(const_lit) = fold_binary_expression(&expr.node) { + if let Some(const_lit) = fold_binary_expression(&expr.node, scope) { return Ok(CompileLocation { location: VariableLocation::Constant(Literal::Number(const_lit)), temp_name: None, @@ -2189,6 +2540,24 @@ impl<'a> Compiler<'a> { BinaryExpression::Modulo(l, r) => { (|into, lhs, rhs| Instruction::Mod(into, lhs, rhs), l, r) } + BinaryExpression::BitwiseAnd(l, r) => { + (|into, lhs, rhs| Instruction::And(into, lhs, rhs), l, r) + } + BinaryExpression::BitwiseOr(l, r) => { + (|into, lhs, rhs| Instruction::Or(into, lhs, rhs), l, r) + } + BinaryExpression::BitwiseXor(l, r) => { + (|into, lhs, rhs| Instruction::Xor(into, lhs, rhs), l, r) + } + BinaryExpression::LeftShift(l, r) => { + (|into, lhs, rhs| Instruction::Sll(into, lhs, rhs), l, r) + } + BinaryExpression::RightShiftArithmetic(l, r) => { + (|into, lhs, rhs| Instruction::Sra(into, lhs, rhs), l, r) + } + BinaryExpression::RightShiftLogical(l, r) => { + (|into, lhs, rhs| Instruction::Srl(into, lhs, rhs), l, r) + } }; let span = Self::merge_spans(left_expr.span, right_expr.span); @@ -2633,6 +3002,17 @@ impl<'a> Compiler<'a> { span: Span, scope: &mut VariableScope<'a, '_>, ) -> Result>, Error<'a>> { + // Track the syscall in metadata + let syscall_name = expr.name(); + let doc = expr.docs().into(); + self.metadata.add_syscall_with_doc( + Cow::Borrowed(syscall_name), + crate::SyscallType::System, + expr.arg_count(), + Some(span), + Some(doc), + ); + macro_rules! cleanup { ($($to_clean:expr),*) => { $( @@ -2654,6 +3034,13 @@ impl<'a> Compiler<'a> { cleanup!(var_cleanup); Ok(None) } + System::Clr(device) => { + let (op, var_cleanup) = self.compile_operand(*device, scope)?; + self.write_instruction(Instruction::Clr(op), Some(span))?; + + cleanup!(var_cleanup); + Ok(None) + } System::Hash(hash_arg) => { let Spanned { node: Literal::String(str_lit), @@ -2973,6 +3360,42 @@ impl<'a> Compiler<'a> { cleanup!(reagent_cleanup, reagent_hash_cleanup, device_cleanup); + Ok(Some(CompileLocation { + location: VariableLocation::Persistant(VariableScope::RETURN_REGISTER), + temp_name: None, + })) + } + System::Rmap(device, reagent_hash) => { + let Spanned { + node: LiteralOrVariable::Variable(device_spanned), + .. + } = device + else { + return Err(Error::AgrumentMismatch( + "Arg1 expected to be a variable".into(), + span, + )); + }; + + let (device, device_cleanup) = self.compile_literal_or_variable( + LiteralOrVariable::Variable(device_spanned), + scope, + )?; + + let (reagent_hash, reagent_hash_cleanup) = + self.compile_operand(*reagent_hash, scope)?; + + self.write_instruction( + Instruction::Rmap( + Operand::Register(VariableScope::RETURN_REGISTER), + device, + reagent_hash, + ), + Some(span), + )?; + + cleanup!(reagent_hash_cleanup, device_cleanup); + Ok(Some(CompileLocation { location: VariableLocation::Persistant(VariableScope::RETURN_REGISTER), temp_name: None, @@ -2987,6 +3410,17 @@ impl<'a> Compiler<'a> { span: Span, scope: &mut VariableScope<'a, '_>, ) -> Result>, Error<'a>> { + // Track the syscall in metadata + let syscall_name = expr.name(); + let doc = expr.docs().into(); + self.metadata.add_syscall_with_doc( + Cow::Borrowed(syscall_name), + crate::SyscallType::Math, + expr.arg_count(), + Some(span), + Some(doc), + ); + macro_rules! cleanup { ($($to_clean:expr),*) => { $( @@ -3247,6 +3681,19 @@ impl<'a> Compiler<'a> { let span = expr.span; + // Track the function definition in metadata + let param_names: Vec> = arguments.iter().map(|a| a.node.clone()).collect(); + let doc_comment = self + .parser + .get_declaration_doc(name.node.as_ref()) + .map(Cow::Owned); + self.metadata.add_function_with_doc( + name.node.clone(), + param_names, + Some(name.span), + doc_comment, + ); + if self.function_meta.locations.contains_key(&name.node) { self.errors .push(Error::DuplicateIdentifier(name.node.clone(), name.span)); diff --git a/rust_compiler/libs/helpers/src/syscall.rs b/rust_compiler/libs/helpers/src/syscall.rs index e329129..b01f615 100644 --- a/rust_compiler/libs/helpers/src/syscall.rs +++ b/rust_compiler/libs/helpers/src/syscall.rs @@ -5,12 +5,14 @@ macro_rules! with_syscalls { // Big names "yield", "sleep", + "clr", "hash", "load", "loadBatched", "loadBatchedNamed", "loadSlot", "loadReagent", + "rmap", "set", "setBatched", "setBatchedNamed", diff --git a/rust_compiler/libs/il/src/lib.rs b/rust_compiler/libs/il/src/lib.rs index 54aadfb..5520d7e 100644 --- a/rust_compiler/libs/il/src/lib.rs +++ b/rust_compiler/libs/il/src/lib.rs @@ -195,6 +195,9 @@ pub enum Instruction<'a> { /// `lr register device reagentMode int` LoadReagent(Operand<'a>, Operand<'a>, Operand<'a>, Operand<'a>), + /// `rmap register device reagentHash` - Resolve Reagent to Item Hash + Rmap(Operand<'a>, Operand<'a>, Operand<'a>), + /// `j label` - Unconditional Jump Jump(Operand<'a>), /// `jal label` - Jump and Link (Function Call) @@ -232,12 +235,22 @@ pub enum Instruction<'a> { /// `sle dst a b` - Set if Less or Equal SetLe(Operand<'a>, Operand<'a>, Operand<'a>), - /// `and dst a b` - Logical AND + /// `and dst a b` - Bitwise AND And(Operand<'a>, Operand<'a>, Operand<'a>), - /// `or dst a b` - Logical OR + /// `or dst a b` - Bitwise OR Or(Operand<'a>, Operand<'a>, Operand<'a>), - /// `xor dst a b` - Logical XOR + /// `xor dst a b` - Bitwise XOR Xor(Operand<'a>, Operand<'a>, Operand<'a>), + /// `nor dst a b` - Bitwise NOR + Nor(Operand<'a>, Operand<'a>, Operand<'a>), + /// `not dst a` - Bitwise NOT + Not(Operand<'a>, Operand<'a>), + /// `sll dst a b` - Logical Left Shift + Sll(Operand<'a>, Operand<'a>, Operand<'a>), + /// `sra dst a b` - Arithmetic Right Shift + Sra(Operand<'a>, Operand<'a>, Operand<'a>), + /// `srl dst a b` - Logical Right Shift + Srl(Operand<'a>, Operand<'a>, Operand<'a>), /// `push val` - Push to Stack Push(Operand<'a>), @@ -257,6 +270,8 @@ pub enum Instruction<'a> { Yield, /// `sleep val` - Sleep for seconds Sleep(Operand<'a>), + /// `clr val` - Clear stack memory on device + Clr(Operand<'a>), /// `alias name target` - Define Alias (Usually handled by compiler, but good for IR) Alias(Cow<'a, str>, Operand<'a>), @@ -318,6 +333,9 @@ impl<'a> fmt::Display for Instruction<'a> { Instruction::LoadReagent(reg, device, reagent_mode, reagent_hash) => { write!(f, "lr {} {} {} {}", reg, device, reagent_mode, reagent_hash) } + Instruction::Rmap(reg, device, reagent_hash) => { + write!(f, "rmap {} {} {}", reg, device, reagent_hash) + } Instruction::Jump(lbl) => write!(f, "j {}", lbl), Instruction::JumpAndLink(lbl) => write!(f, "jal {}", lbl), Instruction::JumpRelative(off) => write!(f, "jr {}", off), @@ -338,6 +356,11 @@ impl<'a> fmt::Display for Instruction<'a> { Instruction::And(dst, a, b) => write!(f, "and {} {} {}", dst, a, b), Instruction::Or(dst, a, b) => write!(f, "or {} {} {}", dst, a, b), Instruction::Xor(dst, a, b) => write!(f, "xor {} {} {}", dst, a, b), + Instruction::Nor(dst, a, b) => write!(f, "nor {} {} {}", dst, a, b), + Instruction::Not(dst, a) => write!(f, "not {} {}", dst, a), + Instruction::Sll(dst, a, b) => write!(f, "sll {} {} {}", dst, a, b), + Instruction::Sra(dst, a, b) => write!(f, "sra {} {} {}", dst, a, b), + Instruction::Srl(dst, a, b) => write!(f, "srl {} {} {}", dst, a, b), Instruction::Push(val) => write!(f, "push {}", val), Instruction::Pop(dst) => write!(f, "pop {}", dst), Instruction::Peek(dst) => write!(f, "peek {}", dst), @@ -348,6 +371,7 @@ impl<'a> fmt::Display for Instruction<'a> { } Instruction::Yield => write!(f, "yield"), Instruction::Sleep(val) => write!(f, "sleep {}", val), + Instruction::Clr(val) => write!(f, "clr {}", val), Instruction::Alias(name, target) => write!(f, "alias {} {}", name, target), Instruction::Define(name, val) => write!(f, "define {} {}", name, val), Instruction::LabelDef(lbl) => write!(f, "{}:", lbl), diff --git a/rust_compiler/libs/integration_tests/src/bitwise_tests.rs b/rust_compiler/libs/integration_tests/src/bitwise_tests.rs new file mode 100644 index 0000000..cd6df21 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/bitwise_tests.rs @@ -0,0 +1,112 @@ +#[cfg(test)] +mod bitwise_tests { + use crate::common::compile_with_and_without_optimization; + use indoc::indoc; + + #[test] + fn test_bitwise_operations() { + let source = indoc! {" + let a = 5; + let b = 3; + let and_result = a & b; + let or_result = a | b; + let xor_result = a ^ b; + let not_result = ~a; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_bitwise_shifts() { + let source = indoc! {" + let x = 8; + let left_shift = x << 2; + let arithmetic_shift = x >> 1; + let logical_shift = x >>> 1; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_bitwise_constant_folding() { + let source = indoc! {" + let packed = (1 << 16) & (255 << 8) & 2; + let mask = 0xFF & 0x0F; + let combined = (15 | 4096); + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_bitwise_with_variables() { + let source = indoc! {" + fn pack_bits(high, low) { + let packed = (high << 8) | low; + return packed; + } + fn extract_bits(value) { + let high = (value >> 8) & 0xFF; + let low = value & 0xFF; + return (high, low); + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_complex_bit_manipulation() { + let source = indoc! {" + fn encode_flags(enabled, mode, priority) { + let flag_byte = (enabled << 7) | (mode << 4) | priority; + return flag_byte; + } + fn decode_flags(byte) { + let enabled = (byte >> 7) & 1; + let mode = (byte >> 4) & 0x7; + let priority = byte & 0xF; + return (enabled, mode, priority); + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_sorter_bitwise_operations() { + let source = indoc! {r#" + device self = "db"; + device sorter = "d0"; + + loop { + yield(); + // allow Hay with an op_code of `1` + sorter[0] = (hash("ItemCropHay") << 8) | 1; + } + "#}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_bitwise_with_const() { + let source = indoc! {r#" + device sorterOutput = "d0"; + + const ingotSortClass = 19; + const equals = 0; + + loop { + yield(); + clr(sorterOutput); + sorterOutput[0] = (ingotSortClass << 16) | (equals << 8) | 3; + } + "#}; + + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } +} diff --git a/rust_compiler/libs/integration_tests/src/common.rs b/rust_compiler/libs/integration_tests/src/common.rs new file mode 100644 index 0000000..9998892 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/common.rs @@ -0,0 +1,46 @@ +use compiler::Compiler; +use parser::Parser; +use tokenizer::Tokenizer; + +/// Compile Slang source code and return both unoptimized and optimized output +pub fn compile_with_and_without_optimization(source: &str) -> String { + // Compile for unoptimized output + let tokenizer = Tokenizer::from(source); + let parser = Parser::new(tokenizer); + let compiler = Compiler::new(parser, None); + let result = compiler.compile(); + + // Get unoptimized output + let mut unoptimized_writer = std::io::BufWriter::new(Vec::new()); + result + .instructions + .write(&mut unoptimized_writer) + .expect("Failed to write unoptimized output"); + let unoptimized_bytes = unoptimized_writer + .into_inner() + .expect("Failed to get bytes"); + let unoptimized = String::from_utf8(unoptimized_bytes).expect("Invalid UTF-8"); + + // Compile again for optimized output + let tokenizer2 = Tokenizer::from(source); + let parser2 = Parser::new(tokenizer2); + let compiler2 = Compiler::new(parser2, None); + let result2 = compiler2.compile(); + + // Apply optimizations + let optimized_instructions = optimizer::optimize(result2.instructions); + + // Get optimized output + let mut optimized_writer = std::io::BufWriter::new(Vec::new()); + optimized_instructions + .write(&mut optimized_writer) + .expect("Failed to write optimized output"); + let optimized_bytes = optimized_writer.into_inner().expect("Failed to get bytes"); + let optimized = String::from_utf8(optimized_bytes).expect("Invalid UTF-8"); + + // Combine both outputs with clear separators + format!( + "## Unoptimized Output\n\n{}\n## Optimized Output\n\n{}", + unoptimized, optimized + ) +} diff --git a/rust_compiler/libs/integration_tests/src/device_indexing_tests.rs b/rust_compiler/libs/integration_tests/src/device_indexing_tests.rs new file mode 100644 index 0000000..378b63e --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/device_indexing_tests.rs @@ -0,0 +1,107 @@ +#[cfg(test)] +mod device_indexing_tests { + use crate::common::compile_with_and_without_optimization; + use indoc::indoc; + + #[test] + fn test_device_indexing_with_hash_and_binary_literals() { + let source = indoc! {" + device printer = \"d0\"; + + let item_type = hash(\"ItemIronIngot\"); + let quality = 16; + let quantity = 7; + + // Pack into a single value using bitwise operations + // Format: (itemHash << 16) | (quality << 8) | quantity + let packed = (item_type << 16) | (quality << 8) | quantity; + + // Write to device stack at address 255 + let addr = 255; + printer[addr] = packed; + + // Read it back + let result = printer[addr]; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_device_indexing_with_computed_index() { + let source = indoc! {" + device storage = \"d1\"; + + let base_addr = 10; + let offset = 5; + let index = base_addr + offset; + + let value = 42; + storage[index] = value; + + let retrieved = storage[index]; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_device_indexing_with_binary_literals() { + let source = indoc! {" + device mem = \"d0\"; + + // Binary literals for bitwise operations + let flags = 0b1010_0101; + let mask = 0b1111_0000; + let masked = flags & mask; + + // Write to device + mem[0] = masked; + + // Read back + let read_value = mem[0]; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_device_indexing_complex_expression() { + let source = indoc! {" + device db = \"d0\"; + + let item = hash(\"ItemCopper\"); + let quality = 5; + let quantity = 100; + + // Complex bitwise expression + let packed = (item << 16) | ((quality & 0xFF) << 8) | (quantity & 0xFF); + + // Index with computed address + let slot = 1; + let address = slot * 256 + 100; + db[address] = packed; + + // Read back with different computation + let read_addr = (slot + 0) * 256 + 100; + let stored_value = db[read_addr]; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_device_indexing_optimization_folds_constants() { + let source = indoc! {" + device cache = \"d0\"; + + // This should optimize to a single constant + let packed_constant = (hash(\"ItemSilver\") << 16) | (10 << 8) | 50; + + cache[255] = packed_constant; + let result = cache[255]; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } +} diff --git a/rust_compiler/libs/integration_tests/src/function_tests.rs b/rust_compiler/libs/integration_tests/src/function_tests.rs new file mode 100644 index 0000000..d44d61a --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/function_tests.rs @@ -0,0 +1,48 @@ +#[cfg(test)] +mod function_tests { + use crate::common::compile_with_and_without_optimization; + use indoc::indoc; + + #[test] + fn test_simple_leaf_function() { + let source = "fn test() { let x = 10; }"; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_function_with_call() { + let source = indoc! {" + fn add(a, b) { return a + b; } + fn main() { let x = add(5, 10); } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_leaf_function_no_stack_frame() { + let source = indoc! {" + fn increment(x) { + x = x + 1; + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_nested_function_calls() { + let source = indoc! {" + fn add(a, b) { return a + b; } + fn multiply(x, y) { return x * 2; } + fn complex(a, b) { + let sum = add(a, b); + let doubled = multiply(sum, 2); + return doubled; + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } +} diff --git a/rust_compiler/libs/integration_tests/src/lib.rs b/rust_compiler/libs/integration_tests/src/lib.rs index 292eeaf..5b58111 100644 --- a/rust_compiler/libs/integration_tests/src/lib.rs +++ b/rust_compiler/libs/integration_tests/src/lib.rs @@ -4,180 +4,22 @@ //! and optimization passes work correctly together using snapshot testing. #[cfg(test)] -mod tests { - use compiler::Compiler; +mod bitwise_tests; +#[cfg(test)] +mod common; +#[cfg(test)] +mod device_indexing_tests; +#[cfg(test)] +mod function_tests; +#[cfg(test)] +mod number_literal_tests; +#[cfg(test)] +mod optimization_tests; + +#[cfg(test)] +mod integration_tests { + use crate::common::compile_with_and_without_optimization; use indoc::indoc; - use parser::Parser; - use tokenizer::Tokenizer; - - /// Compile Slang source code and return both unoptimized and optimized output - fn compile_with_and_without_optimization(source: &str) -> String { - // Compile for unoptimized output - let tokenizer = Tokenizer::from(source); - let parser = Parser::new(tokenizer); - let compiler = Compiler::new(parser, None); - let result = compiler.compile(); - - assert!( - result.errors.is_empty(), - "Compilation errors: {:?}", - result.errors - ); - - // Get unoptimized output - let mut unoptimized_writer = std::io::BufWriter::new(Vec::new()); - result - .instructions - .write(&mut unoptimized_writer) - .expect("Failed to write unoptimized output"); - let unoptimized_bytes = unoptimized_writer - .into_inner() - .expect("Failed to get bytes"); - let unoptimized = String::from_utf8(unoptimized_bytes).expect("Invalid UTF-8"); - - // Compile again for optimized output - let tokenizer2 = Tokenizer::from(source); - let parser2 = Parser::new(tokenizer2); - let compiler2 = Compiler::new(parser2, None); - let result2 = compiler2.compile(); - - // Apply optimizations - let optimized_instructions = optimizer::optimize(result2.instructions); - - // Get optimized output - let mut optimized_writer = std::io::BufWriter::new(Vec::new()); - optimized_instructions - .write(&mut optimized_writer) - .expect("Failed to write optimized output"); - let optimized_bytes = optimized_writer.into_inner().expect("Failed to get bytes"); - let optimized = String::from_utf8(optimized_bytes).expect("Invalid UTF-8"); - - // Combine both outputs with clear separators - format!( - "## Unoptimized Output\n\n{}\n## Optimized Output\n\n{}", - unoptimized, optimized - ) - } - - #[test] - fn test_simple_leaf_function() { - let source = "fn test() { let x = 10; }"; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_function_with_call() { - let source = indoc! {" - fn add(a, b) { return a + b; } - fn main() { let x = add(5, 10); } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_constant_folding() { - let source = "let x = 5 + 10;"; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_algebraic_simplification() { - let source = "let x = 5; let y = x * 1;"; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_strength_reduction() { - let source = "fn double(x) { return x * 2; }"; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_dead_code_elimination() { - let source = indoc! {" - fn compute(x) { - let unused = 20; - return x + 1; - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_peephole_comparison_fusion() { - let source = indoc! {" - fn compare(x, y) { - if (x > y) { - let z = 1; - } - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_select_optimization() { - let source = indoc! {" - fn ternary(cond) { - let result = 0; - if (cond) { - result = 10; - } else { - result = 20; - } - return result; - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_leaf_function_no_stack_frame() { - let source = indoc! {" - fn increment(x) { - x = x + 1; - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_complex_arithmetic() { - let source = indoc! {" - fn compute(a, b, c) { - let x = a * 2; - let y = b + 0; - let z = c * 1; - return x + y + z; - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } - - #[test] - fn test_nested_function_calls() { - let source = indoc! {" - fn add(a, b) { return a + b; } - fn multiply(x, y) { return x * 2; } - fn complex(a, b) { - let sum = add(a, b); - let doubled = multiply(sum, 2); - return doubled; - } - "}; - let output = compile_with_and_without_optimization(source); - insta::assert_snapshot!(output); - } #[test] fn test_tuples() { diff --git a/rust_compiler/libs/integration_tests/src/number_literal_tests.rs b/rust_compiler/libs/integration_tests/src/number_literal_tests.rs new file mode 100644 index 0000000..fafeeb5 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/number_literal_tests.rs @@ -0,0 +1,29 @@ +#[cfg(test)] +mod number_literal_tests { + use crate::common::compile_with_and_without_optimization; + use indoc::indoc; + + #[test] + fn test_binary_literals() { + let source = indoc! {" + let binary = 0b1010_1100; + let octal = 0o755; + let hex_upper = 0xDEAD_BEEF; + let hex_lower = 0xcafe; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_number_literal_optimization() { + let source = indoc! {" + let decimal = 42_000; + let negative_hex = -0xFF; + let negative_binary = -0b1111_0000; + let temp_c = 100c; + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } +} diff --git a/rust_compiler/libs/integration_tests/src/optimization_tests.rs b/rust_compiler/libs/integration_tests/src/optimization_tests.rs new file mode 100644 index 0000000..07bf416 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/optimization_tests.rs @@ -0,0 +1,82 @@ +#[cfg(test)] +mod optimization_tests { + use crate::common::compile_with_and_without_optimization; + use indoc::indoc; + + #[test] + fn test_constant_folding() { + let source = "let x = 5 + 10;"; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_algebraic_simplification() { + let source = "let x = 5; let y = x * 1;"; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_strength_reduction() { + let source = "fn double(x) { return x * 2; }"; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_dead_code_elimination() { + let source = indoc! {" + fn compute(x) { + let unused = 20; + return x + 1; + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_peephole_comparison_fusion() { + let source = indoc! {" + fn compare(x, y) { + if (x > y) { + let z = 1; + } + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_select_optimization() { + let source = indoc! {" + fn ternary(cond) { + let result = 0; + if (cond) { + result = 10; + } else { + result = 20; + } + return result; + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } + + #[test] + fn test_complex_arithmetic() { + let source = indoc! {" + fn compute(a, b, c) { + let x = a * 2; + let y = b + 0; + let z = c * 1; + return x + y + z; + } + "}; + let output = compile_with_and_without_optimization(source); + insta::assert_snapshot!(output); + } +} diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_constant_folding.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_constant_folding.snap new file mode 100644 index 0000000..8abc3c4 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_constant_folding.snap @@ -0,0 +1,18 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 40 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 0 +move r9 15 +move r10 4111 + +## Optimized Output + +move r8 0 +move r9 15 +move r10 4111 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_operations.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_operations.snap new file mode 100644 index 0000000..597b6bd --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_operations.snap @@ -0,0 +1,32 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 17 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 5 +move r9 3 +and r1 r8 r9 +move r10 r1 +or r2 r8 r9 +move r11 r2 +xor r3 r8 r9 +move r12 r3 +not r4 r8 +move r13 r4 + +## Optimized Output + +move r8 5 +move r9 3 +move r1 1 +move r10 1 +move r2 7 +move r11 7 +move r3 6 +move r12 6 +not r4 r8 +move r13 r4 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_shifts.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_shifts.snap new file mode 100644 index 0000000..bd3d19f --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_shifts.snap @@ -0,0 +1,26 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 29 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 8 +sll r1 r8 2 +move r9 r1 +sra r2 r8 1 +move r10 r2 +srl r3 r8 1 +move r11 r3 + +## Optimized Output + +move r8 8 +move r1 32 +move r9 32 +move r2 4 +move r10 4 +move r3 4 +move r11 4 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_const.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_const.snap new file mode 100644 index 0000000..897a733 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_const.snap @@ -0,0 +1,21 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +expression: output +--- +## Unoptimized Output + +j main +main: +__internal_L1: +yield +clr d0 +put d0 0 1245187 +j __internal_L1 +__internal_L2: + +## Optimized Output + +yield +clr d0 +put d0 0 1245187 +j 0 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_variables.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_variables.snap new file mode 100644 index 0000000..2ffdc35 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__bitwise_with_variables.snap @@ -0,0 +1,67 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 57 +expression: output +--- +## Unoptimized Output + +j main +pack_bits: +pop r8 +pop r9 +push sp +push ra +sll r1 r9 8 +or r2 r1 r8 +move r10 r2 +move r15 r10 +j __internal_L1 +__internal_L1: +pop ra +pop sp +j ra +extract_bits: +pop r8 +push sp +push ra +sra r1 r8 8 +and r2 r1 255 +move r9 r2 +and r3 r8 255 +move r10 r3 +push r9 +push r10 +sub r0 sp 4 +get r0 db r0 +move r15 r0 +j __internal_L2 +__internal_L2: +sub r0 sp 3 +get ra db r0 +j ra + +## Optimized Output + +j main +pop r8 +pop r9 +push sp +push ra +sll r1 r9 8 +or r15 r1 r8 +pop ra +pop sp +j ra +pop r8 +push sp +push ra +sra r1 r8 8 +and r9 r1 255 +and r10 r8 255 +push r9 +push r10 +sub r0 sp 4 +get r15 db r0 +sub r0 sp 3 +get ra db r0 +j ra diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__complex_bit_manipulation.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__complex_bit_manipulation.snap new file mode 100644 index 0000000..6518acd --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__complex_bit_manipulation.snap @@ -0,0 +1,80 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 75 +expression: output +--- +## Unoptimized Output + +j main +encode_flags: +pop r8 +pop r9 +pop r10 +push sp +push ra +sll r1 r10 7 +sll r2 r9 4 +or r3 r1 r2 +or r4 r3 r8 +move r11 r4 +move r15 r11 +j __internal_L1 +__internal_L1: +pop ra +pop sp +j ra +decode_flags: +pop r8 +push sp +push ra +sra r1 r8 7 +and r2 r1 1 +move r9 r2 +sra r3 r8 4 +and r4 r3 7 +move r10 r4 +and r5 r8 15 +move r11 r5 +push r9 +push r10 +push r11 +sub r0 sp 5 +get r0 db r0 +move r15 r0 +j __internal_L2 +__internal_L2: +sub r0 sp 4 +get ra db r0 +j ra + +## Optimized Output + +j main +pop r8 +pop r9 +pop r10 +push sp +push ra +sll r1 r10 7 +sll r2 r9 4 +or r3 r1 r2 +or r15 r3 r8 +pop ra +pop sp +j ra +pop r8 +push sp +push ra +sra r1 r8 7 +and r9 r1 1 +sra r3 r8 4 +and r10 r3 7 +and r11 r8 15 +push r9 +push r10 +push r11 +sub r0 sp 5 +get r15 db r0 +sub r0 sp 4 +get ra db r0 +j ra diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__sorter_bitwise_operations.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__sorter_bitwise_operations.snap new file mode 100644 index 0000000..0b3b60a --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__bitwise_tests__bitwise_tests__sorter_bitwise_operations.snap @@ -0,0 +1,20 @@ +--- +source: libs/integration_tests/src/bitwise_tests.rs +assertion_line: 91 +expression: output +--- +## Unoptimized Output + +j main +main: +__internal_L1: +yield +put d0 0 55164456193 +j __internal_L1 +__internal_L2: + +## Optimized Output + +yield +put d0 0 55164456193 +j 0 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_complex_expression.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_complex_expression.snap new file mode 100644 index 0000000..b9ef518 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_complex_expression.snap @@ -0,0 +1,54 @@ +--- +source: libs/integration_tests/src/device_indexing_tests.rs +assertion_line: 90 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 r15 +move r9 5 +move r10 100 +sll r1 r8 16 +and r2 r9 255 +sll r3 r2 8 +or r4 r1 r3 +and r5 r10 255 +or r6 r4 r5 +move r11 r6 +move r12 1 +mul r7 r12 256 +add r2 r7 100 +move r13 r2 +put d0 r13 r11 +add r1 r12 0 +mul r3 r1 256 +add r4 r3 100 +move r14 r4 +get r5 d0 r14 +push r5 +sub sp sp 1 + +## Optimized Output + +move r8 r15 +move r9 5 +move r10 100 +sll r1 r8 16 +move r3 1280 +or r4 r1 r3 +move r5 100 +or r11 r4 r5 +move r12 1 +move r7 256 +move r2 356 +move r13 356 +put d0 r13 r11 +move r1 1 +move r3 256 +move r4 356 +move r14 356 +get r5 d0 r14 +push r5 +sub sp sp 1 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_optimization_folds_constants.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_optimization_folds_constants.snap new file mode 100644 index 0000000..fc8984f --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_optimization_folds_constants.snap @@ -0,0 +1,19 @@ +--- +source: libs/integration_tests/src/device_indexing_tests.rs +assertion_line: 105 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 -62440604628430 +put d0 255 r8 +get r1 d0 255 +move r9 r1 + +## Optimized Output + +move r8 -62440604628430 +put d0 255 r8 +get r9 d0 255 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_binary_literals.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_binary_literals.snap new file mode 100644 index 0000000..f7ce277 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_binary_literals.snap @@ -0,0 +1,25 @@ +--- +source: libs/integration_tests/src/device_indexing_tests.rs +assertion_line: 65 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 165 +move r9 240 +and r1 r8 r9 +move r10 r1 +put d0 0 r10 +get r2 d0 0 +move r11 r2 + +## Optimized Output + +move r8 165 +move r9 240 +move r1 160 +move r10 160 +put d0 0 r10 +get r11 d0 0 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_computed_index.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_computed_index.snap new file mode 100644 index 0000000..839f776 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_computed_index.snap @@ -0,0 +1,27 @@ +--- +source: libs/integration_tests/src/device_indexing_tests.rs +assertion_line: 45 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 10 +move r9 5 +add r1 r8 r9 +move r10 r1 +move r11 42 +put d1 r10 r11 +get r2 d1 r10 +move r12 r2 + +## Optimized Output + +move r8 10 +move r9 5 +move r1 15 +move r10 15 +move r11 42 +put d1 r10 r11 +get r12 d1 r10 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_hash_and_binary_literals.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_hash_and_binary_literals.snap new file mode 100644 index 0000000..a7829b6 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__device_indexing_tests__device_indexing_tests__device_indexing_with_hash_and_binary_literals.snap @@ -0,0 +1,34 @@ +--- +source: libs/integration_tests/src/device_indexing_tests.rs +assertion_line: 27 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 r15 +move r9 16 +move r10 7 +sll r1 r8 16 +sll r2 r9 8 +or r3 r1 r2 +or r4 r3 r10 +move r11 r4 +move r12 255 +put d0 r12 r11 +get r5 d0 r12 +move r13 r5 + +## Optimized Output + +move r8 r15 +move r9 16 +move r10 7 +sll r1 r8 16 +move r2 4096 +or r3 r1 r2 +or r11 r3 r10 +move r12 255 +put d0 r12 r11 +get r13 d0 r12 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__function_with_call.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__function_with_call.snap similarity index 85% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__function_with_call.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__function_with_call.snap index 264b371..c0c0c7e 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__function_with_call.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__function_with_call.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 70 +source: libs/integration_tests/src/function_tests.rs +assertion_line: 20 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__leaf_function_no_stack_frame.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__leaf_function_no_stack_frame.snap similarity index 77% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__leaf_function_no_stack_frame.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__leaf_function_no_stack_frame.snap index 3362dcd..bfe66cd 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__leaf_function_no_stack_frame.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__leaf_function_no_stack_frame.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 144 +source: libs/integration_tests/src/function_tests.rs +assertion_line: 31 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__nested_function_calls.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__nested_function_calls.snap similarity index 92% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__nested_function_calls.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__nested_function_calls.snap index a9fbdd7..67fa4b6 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__nested_function_calls.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__nested_function_calls.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 173 +source: libs/integration_tests/src/function_tests.rs +assertion_line: 46 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__simple_leaf_function.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__simple_leaf_function.snap similarity index 73% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__simple_leaf_function.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__simple_leaf_function.snap index f093e06..a4f5c5d 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__simple_leaf_function.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__function_tests__function_tests__simple_leaf_function.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 60 +source: libs/integration_tests/src/function_tests.rs +assertion_line: 10 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__larre_script.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__larre_script.snap new file mode 100644 index 0000000..5173db1 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__larre_script.snap @@ -0,0 +1,224 @@ +--- +source: libs/integration_tests/src/lib.rs +assertion_line: 54 +expression: output +--- +## Unoptimized Output + +j main +waitForIdle: +push sp +push ra +yield +__internal_L2: +l r1 d0 Idle +seq r2 r1 0 +beqz r2 __internal_L3 +yield +j __internal_L2 +__internal_L3: +__internal_L1: +pop ra +pop sp +j ra +deposit: +push sp +push ra +s d0 Setting 1 +jal waitForIdle +move r1 r15 +s d0 Activate 1 +jal waitForIdle +move r2 r15 +s d1 Open 0 +__internal_L4: +pop ra +pop sp +j ra +checkAndHarvest: +pop r8 +push sp +push ra +sle r1 r8 1 +ls r15 d0 255 Seeding +slt r2 r15 1 +or r3 r1 r2 +beqz r3 __internal_L6 +j __internal_L5 +__internal_L6: +__internal_L7: +ls r15 d0 255 Mature +beqz r15 __internal_L8 +yield +s d0 Activate 1 +j __internal_L7 +__internal_L8: +ls r15 d0 255 Occupied +move r9 r15 +s d0 Setting 1 +push r8 +push r9 +jal waitForIdle +pop r9 +pop r8 +move r4 r15 +push r8 +push r9 +jal deposit +pop r9 +pop r8 +move r5 r15 +beqz r9 __internal_L9 +push r8 +push r9 +jal deposit +pop r9 +pop r8 +move r6 r15 +__internal_L9: +s d0 Setting r8 +push r8 +push r9 +jal waitForIdle +pop r9 +pop r8 +move r6 r15 +ls r15 d0 0 Occupied +beqz r15 __internal_L10 +s d0 Activate 1 +__internal_L10: +push r8 +push r9 +jal waitForIdle +pop r9 +pop r8 +move r7 r15 +__internal_L5: +pop ra +pop sp +j ra +main: +move r8 0 +__internal_L11: +yield +l r1 d0 Idle +seq r2 r1 0 +beqz r2 __internal_L13 +j __internal_L11 +__internal_L13: +add r3 r8 1 +sgt r4 r3 19 +add r5 r8 1 +select r6 r4 2 r5 +move r9 r6 +push r8 +push r9 +push r8 +jal checkAndHarvest +pop r9 +pop r8 +move r7 r15 +s d0 Setting r9 +move r8 r9 +j __internal_L11 +__internal_L12: + +## Optimized Output + +j 77 +push sp +push ra +yield +l r1 d0 Idle +bne r1 0 8 +yield +j 4 +pop ra +pop sp +j ra +push sp +push ra +s d0 Setting 1 +jal 1 +move r1 r15 +s d0 Activate 1 +jal 1 +move r2 r15 +s d1 Open 0 +pop ra +pop sp +j ra +pop r8 +push sp +push ra +sle r1 r8 1 +ls r15 d0 255 Seeding +slt r2 r15 1 +or r3 r1 r2 +beqz r3 32 +j 74 +ls r15 d0 255 Mature +beqz r15 37 +yield +s d0 Activate 1 +j 32 +ls r9 d0 255 Occupied +s d0 Setting 1 +push r8 +push r9 +jal 1 +pop r9 +pop r8 +move r4 r15 +push r8 +push r9 +jal 11 +pop r9 +pop r8 +move r5 r15 +beqz r9 58 +push r8 +push r9 +jal 11 +pop r9 +pop r8 +move r6 r15 +s d0 Setting r8 +push r8 +push r9 +jal 1 +pop r9 +pop r8 +move r6 r15 +ls r15 d0 0 Occupied +beqz r15 68 +s d0 Activate 1 +push r8 +push r9 +jal 1 +pop r9 +pop r8 +move r7 r15 +pop ra +pop sp +j ra +move r8 0 +yield +l r1 d0 Idle +bne r1 0 82 +j 78 +add r3 r8 1 +sgt r4 r3 19 +add r5 r8 1 +select r6 r4 2 r5 +move r9 r6 +push r8 +push r9 +push r8 +jal 23 +pop r9 +pop r8 +move r7 r15 +s d0 Setting r9 +move r8 r9 +j 78 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__reagent_processing.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__reagent_processing.snap new file mode 100644 index 0000000..ae4b277 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__reagent_processing.snap @@ -0,0 +1,112 @@ +--- +source: libs/integration_tests/src/lib.rs +assertion_line: 61 +expression: output +--- +## Unoptimized Output + +j main +main: +s d2 Mode 1 +s d2 On 0 +move r8 0 +move r9 0 +__internal_L1: +yield +l r1 d0 Reagents +move r10 r1 +sge r2 r10 100 +sge r3 r9 2 +or r4 r2 r3 +beqz r4 __internal_L3 +move r8 1 +__internal_L3: +slt r5 r10 100 +ls r15 d0 0 Occupied +seq r6 r15 0 +and r7 r5 r6 +add r1 r9 1 +select r2 r7 r1 0 +move r9 r2 +l r3 d0 Rpm +slt r4 r3 1 +and r5 r8 r4 +beqz r5 __internal_L4 +s d0 Open 1 +seq r6 r10 0 +ls r15 d0 0 Occupied +and r7 r6 r15 +seq r1 r7 0 +move r8 r1 +__internal_L4: +seq r6 r8 0 +s d0 On r6 +ls r15 d1 0 Quantity +move r11 r15 +l r7 d3 Pressure +sgt r1 r7 200 +beqz r1 __internal_L5 +j __internal_L1 +__internal_L5: +sgt r2 r11 0 +s d1 On r2 +sgt r3 r11 0 +s d1 Activate r3 +l r4 d3 Pressure +sgt r5 r4 0 +l r6 d1 Activate +or r7 r5 r6 +s d2 On r7 +l r1 d1 Activate +s db Setting r1 +j __internal_L1 +__internal_L2: + +## Optimized Output + +s d2 Mode 1 +s d2 On 0 +move r8 0 +move r9 0 +yield +l r10 d0 Reagents +sge r2 r10 100 +sge r3 r9 2 +or r4 r2 r3 +beqz r4 11 +move r8 1 +slt r5 r10 100 +ls r15 d0 0 Occupied +seq r6 r15 0 +and r7 r5 r6 +add r1 r9 1 +select r2 r7 r1 0 +move r9 r2 +l r3 d0 Rpm +slt r4 r3 1 +and r5 r8 r4 +beqz r5 27 +s d0 Open 1 +seq r6 r10 0 +ls r15 d0 0 Occupied +and r7 r6 r15 +seq r8 r7 0 +seq r6 r8 0 +s d0 On r6 +ls r15 d1 0 Quantity +move r11 r15 +l r7 d3 Pressure +ble r7 200 34 +j 4 +sgt r2 r11 0 +s d1 On r2 +sgt r3 r11 0 +s d1 Activate r3 +l r4 d3 Pressure +sgt r5 r4 0 +l r6 d1 Activate +or r7 r5 r6 +s d2 On r7 +l r1 d1 Activate +s db Setting r1 +j 4 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__setbatched_with_member_access.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__setbatched_with_member_access.snap similarity index 95% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__setbatched_with_member_access.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__setbatched_with_member_access.snap index 0bfb100..c2ae181 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__setbatched_with_member_access.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__setbatched_with_member_access.snap @@ -1,6 +1,5 @@ --- source: libs/integration_tests/src/lib.rs -assertion_line: 242 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__larre_script.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_larre_script.snap similarity index 100% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__larre_script.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_larre_script.snap diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__reagent_processing.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_reagent_processing.snap similarity index 100% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__reagent_processing.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_reagent_processing.snap diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__tuples.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_tuples.snap similarity index 100% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__tuples.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__test_tuples.snap diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__tuples.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__tuples.snap new file mode 100644 index 0000000..da3c808 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__integration_tests__tuples.snap @@ -0,0 +1,93 @@ +--- +source: libs/integration_tests/src/lib.rs +assertion_line: 47 +expression: output +--- +## Unoptimized Output + +j main +getSomethingElse: +pop r8 +push sp +push ra +add r1 r8 1 +move r15 r1 +j __internal_L1 +__internal_L1: +pop ra +pop sp +j ra +getSensorData: +push sp +push ra +l r1 d0 Vertical +push r1 +l r2 d0 Horizontal +push r2 +push 3 +jal getSomethingElse +move r3 r15 +push r3 +sub r0 sp 5 +get r0 db r0 +move r15 r0 +j __internal_L2 +__internal_L2: +sub r0 sp 4 +get ra db r0 +j ra +main: +__internal_L3: +yield +jal getSensorData +pop r0 +pop r9 +pop r8 +move sp r15 +jal getSensorData +pop r0 +pop r0 +pop r9 +move sp r15 +s db Setting r9 +j __internal_L3 +__internal_L4: + +## Optimized Output + +j 23 +pop r8 +push sp +push ra +add r15 r8 1 +pop ra +pop sp +j ra +push sp +push ra +l r1 d0 Vertical +push r1 +l r2 d0 Horizontal +push r2 +push 3 +jal 1 +move r3 r15 +push r3 +sub r0 sp 5 +get r15 db r0 +sub r0 sp 4 +get ra db r0 +j ra +yield +jal 8 +pop r0 +pop r9 +pop r8 +move sp r15 +jal 8 +pop r0 +pop r0 +pop r9 +move sp r15 +s db Setting r9 +j 23 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__binary_literals.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__binary_literals.snap new file mode 100644 index 0000000..4fe37e5 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__binary_literals.snap @@ -0,0 +1,20 @@ +--- +source: libs/integration_tests/src/number_literal_tests.rs +assertion_line: 15 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 172 +move r9 493 +move r10 3735928559 +move r11 51966 + +## Optimized Output + +move r8 172 +move r9 493 +move r10 3735928559 +move r11 51966 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__number_literal_optimization.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__number_literal_optimization.snap new file mode 100644 index 0000000..aae49e6 --- /dev/null +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__number_literal_tests__number_literal_tests__number_literal_optimization.snap @@ -0,0 +1,20 @@ +--- +source: libs/integration_tests/src/number_literal_tests.rs +assertion_line: 27 +expression: output +--- +## Unoptimized Output + +j main +main: +move r8 42000 +move r9 -255 +move r10 -240 +move r11 373.15 + +## Optimized Output + +move r8 42000 +move r9 -255 +move r10 -240 +move r11 373.15 diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__algebraic_simplification.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__algebraic_simplification.snap similarity index 66% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__algebraic_simplification.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__algebraic_simplification.snap index ce3aa70..1772850 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__algebraic_simplification.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__algebraic_simplification.snap @@ -1,5 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 17 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__complex_arithmetic.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__complex_arithmetic.snap similarity index 84% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__complex_arithmetic.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__complex_arithmetic.snap index f18794c..981241e 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__complex_arithmetic.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__complex_arithmetic.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 158 +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 80 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__constant_folding.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__constant_folding.snap similarity index 58% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__constant_folding.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__constant_folding.snap index 075159a..0963bc5 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__constant_folding.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__constant_folding.snap @@ -1,5 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 10 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__dead_code_elimination.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__dead_code_elimination.snap similarity index 77% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__dead_code_elimination.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__dead_code_elimination.snap index 9404104..da82825 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__dead_code_elimination.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__dead_code_elimination.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 103 +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 36 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__peephole_comparison_fusion.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__peephole_comparison_fusion.snap similarity index 79% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__peephole_comparison_fusion.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__peephole_comparison_fusion.snap index 880034c..aa3675d 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__peephole_comparison_fusion.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__peephole_comparison_fusion.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 116 +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 49 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__select_optimization.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__select_optimization.snap similarity index 81% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__select_optimization.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__select_optimization.snap index 20172da..069229f 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__select_optimization.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__select_optimization.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 133 +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 66 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__strength_reduction.snap b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__strength_reduction.snap similarity index 76% rename from rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__strength_reduction.snap rename to rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__strength_reduction.snap index a2615e0..85b3443 100644 --- a/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__tests__strength_reduction.snap +++ b/rust_compiler/libs/integration_tests/src/snapshots/integration_tests__optimization_tests__optimization_tests__strength_reduction.snap @@ -1,6 +1,6 @@ --- -source: libs/integration_tests/src/lib.rs -assertion_line: 91 +source: libs/integration_tests/src/optimization_tests.rs +assertion_line: 24 expression: output --- ## Unoptimized Output diff --git a/rust_compiler/libs/optimizer/OPTIMIZATION_IDEAS.md b/rust_compiler/libs/optimizer/OPTIMIZATION_IDEAS.md deleted file mode 100644 index 8d970ab..0000000 --- a/rust_compiler/libs/optimizer/OPTIMIZATION_IDEAS.md +++ /dev/null @@ -1,212 +0,0 @@ -# Additional Optimization Opportunities for Slang IL Optimizer - -## Currently Implemented ✓ - -1. Constant Propagation - Folds math operations with known values -2. Register Forwarding - Eliminates intermediate moves -3. Function Call Optimization - Removes unnecessary push/pop around calls -4. Leaf Function Optimization - Removes RA save/restore for non-calling functions -5. Redundant Move Elimination - Removes `move rx rx` -6. Dead Code Elimination - Removes unreachable code after jumps - -## Proposed Additional Optimizations - -### 1. **Algebraic Simplification** 🔥 HIGH IMPACT - -Simplify mathematical identities: - -- `x + 0` → `x` (move) -- `x - 0` → `x` (move) -- `x * 1` → `x` (move) -- `x * 0` → `0` (move to constant) -- `x / 1` → `x` (move) -- `x - x` → `0` (move to constant) -- `x % 1` → `0` (move to constant) - -**Example:** - -``` -add r1 r2 0 → move r1 r2 -mul r3 r4 1 → move r3 r4 -mul r5 r6 0 → move r5 0 -``` - -### 2. **Strength Reduction** 🔥 HIGH IMPACT - -Replace expensive operations with cheaper ones: - -- `x * 2` → `add x x x` (addition is cheaper than multiplication) -- `x * power_of_2` → bit shifts (if IC10 supports) -- `x / 2` → bit shifts (if IC10 supports) - -**Example:** - -``` -mul r1 r2 2 → add r1 r2 r2 -``` - -### 3. **Peephole Optimization - Instruction Sequences** 🔥 MEDIUM-HIGH IMPACT - -Recognize and optimize common instruction patterns: - -#### Pattern: Conditional Branch Simplification - -``` -seq r1 ra rb → beq ra rb label -beqz r1 label (remove the seq entirely) - -sne r1 ra rb → bne ra rb label -beqz r1 label (remove the sne entirely) -``` - -#### Pattern: Double Move Elimination - -``` -move r1 r2 → move r1 r3 -move r1 r3 (remove first move if r1 not used between) -``` - -#### Pattern: Redundant Load Elimination - -If a register's value is already loaded and hasn't been clobbered: - -``` -l r1 d0 Temperature -... (no writes to r1) -l r1 d0 Temperature → (remove second load) -``` - -### 4. **Copy Propagation Enhancement** 🔥 MEDIUM IMPACT - -Current register forwarding is good, but we can extend it: - -- Track `move` chains: if `r1 = r2` and `r2 = 5`, propagate the `5` directly -- Eliminate the intermediate register if possible - -### 5. **Dead Store Elimination** 🔥 MEDIUM IMPACT - -Remove writes to registers that are never read before being overwritten: - -``` -move r1 5 -move r1 10 → move r1 10 - (first write is dead) -``` - -### 6. **Common Subexpression Elimination (CSE)** 🔥 MEDIUM-HIGH IMPACT - -Recognize when the same computation is done multiple times: - -``` -add r1 r8 r9 -add r2 r8 r9 → add r1 r8 r9 - move r2 r1 -``` - -This is especially valuable for expensive operations like: - -- Device loads (`l`) -- Math functions (sqrt, sin, cos, etc.) - -### 7. **Jump Threading** 🔥 LOW-MEDIUM IMPACT - -Optimize jump-to-jump sequences: - -``` -j label1 -... -label1: -j label2 → j label2 (rewrite first jump) -``` - -### 8. **Branch Folding** 🔥 LOW-MEDIUM IMPACT - -Merge consecutive branches to the same target: - -``` -bgt r1 r2 label -bgt r3 r4 label → Could potentially be optimized based on conditions -``` - -### 9. **Loop Invariant Code Motion** 🔥 MEDIUM-HIGH IMPACT - -Move calculations out of loops if they don't change: - -``` -loop: - mul r2 5 10 → mul r2 5 10 (hoisted before loop) - add r1 r1 r2 loop: - ... add r1 r1 r2 - j loop ... - j loop -``` - -### 10. **Select Instruction Optimization** 🔥 LOW-MEDIUM IMPACT - -The `select` instruction can sometimes replace branch patterns: - -``` -beq r1 r2 else -move r3 r4 -j end -else: -move r3 r5 → seq r6 r1 r2 -end: select r3 r6 r5 r4 -``` - -### 11. **Stack Access Pattern Optimization** 🔥 LOW IMPACT - -If we see repeated `sub r0 sp N` + `get`, we might be able to optimize by: - -- Caching the stack address in a register if used multiple times -- Combining sequential gets from adjacent stack slots - -### 12. **Inline Small Functions** 🔥 HIGH IMPACT (Complex to implement) - -For very small leaf functions (1-2 instructions), inline them at the call site: - -``` -calculateSum: - add r15 r8 r9 - j ra - -main: - push 5 → main: - push 10 add r15 5 10 - jal calculateSum -``` - -### 13. **Branch Prediction Hints** 🔥 LOW IMPACT - -Reorganize code to put likely branches inline (fall-through) and unlikely branches as jumps. - -### 14. **Register Coalescing** 🔥 MEDIUM IMPACT - -Reduce register pressure by reusing registers that have non-overlapping lifetimes. - -## Priority Implementation Order - -### Phase 1 (Quick Wins): - -1. Algebraic Simplification (easy, high impact) -2. Strength Reduction (easy, high impact) -3. Dead Store Elimination (medium complexity, good impact) - -### Phase 2 (Medium Effort): - -4. Peephole Optimizations - seq/beq pattern (medium, high impact) -5. Common Subexpression Elimination (medium, high impact) -6. Copy Propagation Enhancement (medium, medium impact) - -### Phase 3 (Advanced): - -7. Loop Invariant Code Motion (complex, high impact for loop-heavy code) -8. Function Inlining (complex, high impact) -9. Register Coalescing (complex, medium impact) - -## Testing Strategy - -- Add test cases for each optimization -- Ensure optimization preserves semantics (run existing tests after each) -- Measure code size reduction -- Consider adding benchmarks to measure game performance impact diff --git a/rust_compiler/libs/optimizer/src/constant_propagation.rs b/rust_compiler/libs/optimizer/src/constant_propagation.rs index c9fb9cd..20e3079 100644 --- a/rust_compiler/libs/optimizer/src/constant_propagation.rs +++ b/rust_compiler/libs/optimizer/src/constant_propagation.rs @@ -31,6 +31,26 @@ pub fn constant_propagation<'a>( Instruction::Mod(dst, a, b) => try_fold_math(dst, a, b, ®isters, |x, y| { if y.is_zero() { Decimal::ZERO } else { x % y } }), + Instruction::And(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| x & y), + Instruction::Or(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| x | y), + Instruction::Xor(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| x ^ y), + Instruction::Sll(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| { + if y >= 64 { 0 } else { x << y as u32 } + }), + Instruction::Sra(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| { + if y >= 64 { + if x < 0 { -1 } else { 0 } + } else { + x >> y as u32 + } + }), + Instruction::Srl(dst, a, b) => try_fold_bitwise(dst, a, b, ®isters, |x, y| { + if y >= 64 { + 0 + } else { + (x as u64 >> y as u32) as i64 + } + }), Instruction::BranchEq(a, b, l) => { try_resolve_branch(a, b, l, ®isters, |x, y| x == y) } @@ -110,6 +130,43 @@ where )) } +fn decimal_to_i64(d: Decimal) -> i64 { + // Convert decimal to i64, truncating if needed + if let Ok(int_val) = d.try_into() { + int_val + } else { + // For very large or very small values, use a default + if d.is_sign_negative() { + i64::MIN + } else { + i64::MAX + } + } +} + +fn i64_to_decimal(i: i64) -> Decimal { + Decimal::from(i) +} + +fn try_fold_bitwise<'a, F>( + dst: &Operand<'a>, + a: &Operand<'a>, + b: &Operand<'a>, + regs: &[Option; 16], + op: F, +) -> Option> +where + F: Fn(i64, i64) -> i64, +{ + let val_a = resolve_value(a, regs)?; + let val_b = resolve_value(b, regs)?; + let result = op(decimal_to_i64(val_a), decimal_to_i64(val_b)); + Some(Instruction::Move( + dst.clone(), + Operand::Number(i64_to_decimal(result)), + )) +} + fn try_resolve_branch<'a, F>( a: &Operand<'a>, b: &Operand<'a>, diff --git a/rust_compiler/libs/optimizer/src/helpers.rs b/rust_compiler/libs/optimizer/src/helpers.rs index 6bb8d23..efe5350 100644 --- a/rust_compiler/libs/optimizer/src/helpers.rs +++ b/rust_compiler/libs/optimizer/src/helpers.rs @@ -43,6 +43,7 @@ pub fn get_destination_reg(instr: &Instruction) -> Option { | Instruction::Tan(Operand::Register(r), _) | Instruction::Trunc(Operand::Register(r), _) | Instruction::LoadReagent(Operand::Register(r), _, _, _) + | Instruction::Rmap(Operand::Register(r), _, _) | Instruction::Pop(Operand::Register(r)) => Some(*r), _ => None, } @@ -107,6 +108,7 @@ pub fn set_destination_reg<'a>(instr: &Instruction<'a>, new_reg: u8) -> Option Some(Instruction::Sqrt(r, a.clone())), Instruction::Tan(_, a) => Some(Instruction::Tan(r, a.clone())), Instruction::Trunc(_, a) => Some(Instruction::Trunc(r, a.clone())), + Instruction::Rmap(_, a, b) => Some(Instruction::Rmap(r, a.clone(), b.clone())), _ => None, } } @@ -136,6 +138,7 @@ pub fn reg_is_read(instr: &Instruction, reg: u8) -> bool { | Instruction::BranchLe(a, b, _) => check(a) || check(b), Instruction::BranchEqZero(a, _) | Instruction::BranchNeZero(a, _) => check(a), Instruction::LoadReagent(_, device, _, item_hash) => check(device) || check(item_hash), + Instruction::Rmap(_, device, reagent_hash) => check(device) || check(reagent_hash), Instruction::LoadSlot(_, dev, slot, _) => check(dev) || check(slot), Instruction::LoadBatch(_, dev, _, mode) => check(dev) || check(mode), Instruction::LoadBatchNamed(_, d_hash, n_hash, _, mode) => { diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index 2c45cdc..373eca2 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -43,8 +43,11 @@ pub enum Error<'a> { #[error("Unsupported Keyword: {1}")] UnsupportedKeyword(Span, Token<'a>), + #[error("Expected semicolon")] + MissingSemicolon(Span), + #[error("Unexpected End of File")] - UnexpectedEOF, + UnexpectedEOF(Option), } impl<'a> From> for lsp_types::Diagnostic { @@ -56,17 +59,22 @@ impl<'a> From> for lsp_types::Diagnostic { UnexpectedToken(span, _) | DuplicateIdentifier(span, _) | InvalidSyntax(span, _) - | UnsupportedKeyword(span, _) => Diagnostic { + | UnsupportedKeyword(span, _) + | MissingSemicolon(span) => Diagnostic { message: value.to_string(), severity: Some(DiagnosticSeverity::ERROR), range: span.into(), ..Default::default() }, - UnexpectedEOF => Diagnostic { - message: value.to_string(), - severity: Some(DiagnosticSeverity::ERROR), - ..Default::default() - }, + UnexpectedEOF(span) => { + let range = span.map(|s| s.into()).unwrap_or_default(); + Diagnostic { + message: value.to_string(), + severity: Some(DiagnosticSeverity::ERROR), + range, + ..Default::default() + } + } } } } @@ -107,7 +115,12 @@ macro_rules! self_matches_current { pub struct Parser<'a> { tokenizer: TokenizerBuffer<'a>, current_token: Option>, + last_token_span: Option, pub errors: Vec>, + /// Caches the most recent doc comment for attaching to the next declaration + cached_doc_comment: Option, + /// Maps variable/declaration names to their doc comments + pub declaration_docs: std::collections::HashMap, } impl<'a> Parser<'a> { @@ -115,7 +128,10 @@ impl<'a> Parser<'a> { Parser { tokenizer: TokenizerBuffer::new(tokenizer), current_token: None, + last_token_span: None, errors: Vec::new(), + cached_doc_comment: None, + declaration_docs: std::collections::HashMap::new(), } } @@ -141,6 +157,30 @@ impl<'a> Parser<'a> { }) } + /// Pops and returns the cached doc comment, if any + pub fn pop_doc_comment(&mut self) -> Option { + self.cached_doc_comment.take() + } + + /// Caches a doc comment for attachment to the next declaration + pub fn cache_doc_comment(&mut self, comment: String) { + self.cached_doc_comment = Some(comment); + } + + /// Stores a doc comment for a declaration (by name) + pub fn store_declaration_doc(&mut self, name: String, doc: String) { + self.declaration_docs.insert(name, doc); + } + + /// Retrieves and removes a doc comment for a declaration + pub fn get_declaration_doc(&mut self, name: &str) -> Option { + self.declaration_docs.get(name).cloned() + } + + fn unexpected_eof(&self) -> Error<'a> { + Error::UnexpectedEOF(self.last_token_span) + } + /// Helper to run a parsing closure and wrap the result in a Spanned struct fn spanned(&mut self, parser: F) -> Result, Error<'a>> where @@ -271,7 +311,39 @@ impl<'a> Parser<'a> { } fn assign_next(&mut self) -> Result<(), Error<'a>> { - self.current_token = self.tokenizer.next_token()?; + if let Some(token) = &self.current_token { + self.last_token_span = Some(Self::token_to_span(token)); + } + + // Keep reading tokens, caching doc comments and skipping them + loop { + self.current_token = self.tokenizer.next_token_with_comments()?; + + match &self.current_token { + Some(token) => { + if let TokenType::Comment(comment) = &token.token_type { + // Cache doc comments for attachment to the next declaration + if let tokenizer::token::Comment::Doc(doc_text) = comment { + self.cache_doc_comment(doc_text.to_string()); + } + // Skip all comments (both doc and regular) + continue; + } + + // If we have a cached doc comment and encounter an identifier, associate them + if let TokenType::Identifier(ref id) = token.token_type { + if let Some(doc) = self.cached_doc_comment.take() { + self.store_declaration_doc(id.to_string(), doc); + } + } + + // Non-comment token, use it as current + break; + } + None => break, // EOF + } + } + Ok(()) } @@ -294,12 +366,12 @@ impl<'a> Parser<'a> { // Handle Infix operators (Binary, Logical, Assignment) if self_matches_peek!( self, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign | Symbol::Question) + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || s.is_bitwise() || matches!(s, Symbol::Assign | Symbol::Question) ) { return Ok(Some(self.infix(lhs)?)); } else if self_matches_current!( self, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign | Symbol::Question) + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || s.is_bitwise() || matches!(s, Symbol::Assign | Symbol::Question) ) { self.tokenizer.seek(SeekFrom::Current(-1))?; return Ok(Some(self.infix(lhs)?)); @@ -308,7 +380,7 @@ impl<'a> Parser<'a> { Ok(Some(lhs)) } - /// Handles dot notation chains: x.y.z() + /// Handles dot notation chains: x.y.z() and bracket notation: x[i] fn parse_postfix( &mut self, mut lhs: Spanned>, @@ -317,7 +389,7 @@ impl<'a> Parser<'a> { if self_matches_peek!(self, TokenType::Symbol(Symbol::Dot)) { self.assign_next()?; // consume Dot - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -336,10 +408,10 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; // Block expressions not allowed in args if let Expression::Block(_) = expression.node { @@ -353,7 +425,8 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next_token = + self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken( Self::token_to_span(&next_token), next_token, @@ -411,6 +484,40 @@ impl<'a> Parser<'a> { }), }; } + } else if self_matches_peek!(self, TokenType::Symbol(Symbol::LBracket)) { + // Index Access + self.assign_next()?; // consume '[', now current is '[' + self.assign_next()?; // advance to the expression, now current is first token of index expr + let index = self.expression()?.ok_or_else(|| self.unexpected_eof())?; + + // After expression(), current is still on the last token of the index expression + // We need to get the next token and verify it's ']' + let rbracket_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; + if !token_matches!(rbracket_token, TokenType::Symbol(Symbol::RBracket)) { + return Err(Error::UnexpectedToken( + Self::token_to_span(&rbracket_token), + rbracket_token.clone(), + )); + } + + let end_span = Self::token_to_span(&rbracket_token); + let combined_span = Span { + start_line: lhs.span.start_line, + start_col: lhs.span.start_col, + end_line: end_span.end_line, + end_col: end_span.end_col, + }; + + lhs = Spanned { + span: combined_span, + node: Expression::IndexAccess(Spanned { + span: combined_span, + node: IndexAccessExpression { + object: boxed!(lhs), + index: boxed!(index), + }, + }), + }; } else { break; } @@ -459,7 +566,6 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Const) => { let spanned_const = self.spanned(|p| p.const_declaration())?; - Some(Spanned { span: spanned_const.span, node: Expression::ConstDeclaration(spanned_const), @@ -500,9 +606,9 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Break) => { let span = self.current_span(); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(span)); } Some(Spanned { span, @@ -512,9 +618,9 @@ impl<'a> Parser<'a> { TokenType::Keyword(Keyword::Continue) => { let span = self.current_span(); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(span)); } Some(Spanned { span, @@ -572,7 +678,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::Minus) => { let start_span = self.current_span(); self.assign_next()?; - let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; let inner_with_postfix = self.parse_postfix(inner_expr)?; @@ -591,7 +697,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::LogicalNot) => { let start_span = self.current_span(); self.assign_next()?; - let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; let inner_with_postfix = self.parse_postfix(inner_expr)?; let combined_span = Span { start_line: start_span.start_line, @@ -608,6 +714,23 @@ impl<'a> Parser<'a> { }) } + TokenType::Symbol(Symbol::BitwiseNot) => { + let start_span = self.current_span(); + self.assign_next()?; + let inner_expr = self.unary()?.ok_or_else(|| self.unexpected_eof())?; + let inner_with_postfix = self.parse_postfix(inner_expr)?; + let combined_span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner_with_postfix.span.end_line, + end_col: inner_with_postfix.span.end_col, + }; + Some(Spanned { + span: combined_span, + node: Expression::BitwiseNot(boxed!(inner_with_postfix)), + }) + } + _ => { return Err(Error::UnexpectedToken( self.current_span(), @@ -620,7 +743,10 @@ impl<'a> Parser<'a> { } fn get_infix_child_node(&mut self) -> Result>, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let start_span = self.current_span(); @@ -648,7 +774,7 @@ impl<'a> Parser<'a> { TokenType::Symbol(Symbol::LParen) => *self .parenthesized_or_tuple()? .map(Box::new) - .ok_or(Error::UnexpectedEOF)?, + .ok_or_else(|| self.unexpected_eof())?, TokenType::Identifier(ref id) if SysCall::is_syscall(id) => { let spanned_call = self.spanned(|p| p.syscall())?; @@ -699,6 +825,20 @@ impl<'a> Parser<'a> { }), } } + TokenType::Symbol(Symbol::BitwiseNot) => { + self.assign_next()?; + let inner = self.get_infix_child_node()?; + let span = Span { + start_line: start_span.start_line, + start_col: start_span.start_col, + end_line: inner.span.end_line, + end_col: inner.span.end_col, + }; + Spanned { + span, + node: Expression::BitwiseNot(boxed!(inner)), + } + } _ => { return Err(Error::UnexpectedToken( self.current_span(), @@ -713,7 +853,10 @@ impl<'a> Parser<'a> { } fn device(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Device)) { return Err(Error::UnexpectedToken( self.current_span(), @@ -721,7 +864,7 @@ impl<'a> Parser<'a> { )); } - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -733,7 +876,7 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -741,7 +884,7 @@ impl<'a> Parser<'a> { )); } - let device_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let device_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let device = match device_token.token_type { TokenType::String(ref id) => id.clone(), _ => { @@ -765,7 +908,10 @@ impl<'a> Parser<'a> { &mut self, previous: Spanned>, ) -> Result>, Error<'a>> { - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let current_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); match previous.node { Expression::Binary(_) @@ -777,8 +923,10 @@ impl<'a> Parser<'a> { | Expression::Variable(_) | Expression::Ternary(_) | Expression::Negation(_) + | Expression::BitwiseNot(_) | Expression::MemberAccess(_) | Expression::MethodCall(_) + | Expression::IndexAccess(_) | Expression::Tuple(_) => {} _ => { return Err(Error::InvalidSyntax( @@ -796,7 +944,7 @@ impl<'a> Parser<'a> { // Include Assign in the operator loop while token_matches!( temp_token, - TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign | Symbol::Question | Symbol::Colon) + TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || s.is_bitwise() || matches!(s, Symbol::Assign | Symbol::Question | Symbol::Colon) ) { let operator = match temp_token.token_type { TokenType::Symbol(s) => s, @@ -806,7 +954,10 @@ impl<'a> Parser<'a> { self.assign_next()?; expressions.push(self.get_infix_child_node()?); - temp_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + temp_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); } if operators.len() != expressions.len() - 1 { @@ -869,6 +1020,24 @@ impl<'a> Parser<'a> { Symbol::Minus => { BinaryExpression::Subtract(boxed!(left), boxed!(right)) } + Symbol::LeftShift => { + BinaryExpression::LeftShift(boxed!(left), boxed!(right)) + } + Symbol::RightShiftArithmetic => { + BinaryExpression::RightShiftArithmetic(boxed!(left), boxed!(right)) + } + Symbol::RightShiftLogical => { + BinaryExpression::RightShiftLogical(boxed!(left), boxed!(right)) + } + Symbol::BitwiseAnd => { + BinaryExpression::BitwiseAnd(boxed!(left), boxed!(right)) + } + Symbol::BitwiseOr => { + BinaryExpression::BitwiseOr(boxed!(left), boxed!(right)) + } + Symbol::Caret => { + BinaryExpression::BitwiseXor(boxed!(left), boxed!(right)) + } _ => unreachable!(), }; @@ -895,7 +1064,22 @@ impl<'a> Parser<'a> { // --- PRECEDENCE LEVEL 3: Additive (+, -) --- process_binary_ops!(Symbol::Plus | Symbol::Minus, BinaryExpression); - // --- PRECEDENCE LEVEL 4: Comparison (<, >, <=, >=) --- + // --- PRECEDENCE LEVEL 4: Shift Operations (<<, >>, >>>) --- + process_binary_ops!( + Symbol::LeftShift | Symbol::RightShiftArithmetic | Symbol::RightShiftLogical, + BinaryExpression + ); + + // --- PRECEDENCE LEVEL 5: Bitwise AND (&) --- + process_binary_ops!(Symbol::BitwiseAnd, BinaryExpression); + + // --- PRECEDENCE LEVEL 6: Bitwise XOR (^) --- + process_binary_ops!(Symbol::Caret, BinaryExpression); + + // --- PRECEDENCE LEVEL 7: Bitwise OR (|) --- + process_binary_ops!(Symbol::BitwiseOr, BinaryExpression); + + // --- PRECEDENCE LEVEL 8: Comparison (<, >, <=, >=) --- let mut current_iteration = 0; for (i, operator) in operators.iter().enumerate() { if operator.is_comparison() && !matches!(operator, Symbol::Equal | Symbol::NotEqual) { @@ -1139,14 +1323,17 @@ impl<'a> Parser<'a> { self.tokenizer.seek(SeekFrom::Current(-1))?; } - expressions.pop().ok_or(Error::UnexpectedEOF) + expressions.pop().ok_or_else(|| self.unexpected_eof()) } fn parenthesized_or_tuple( &mut self, ) -> Result>>, Error<'a>> { let start_span = self.current_span(); - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( @@ -1173,7 +1360,7 @@ impl<'a> Parser<'a> { })); } - let first_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let first_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; if self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) { // It is a tuple @@ -1182,10 +1369,10 @@ impl<'a> Parser<'a> { // Next toekn is a comma, we need to consume it and advance 1 more time. self.assign_next()?; self.assign_next()?; - items.push(self.expression()?.ok_or(Error::UnexpectedEOF)?); + items.push(self.expression()?.ok_or_else(|| self.unexpected_eof())?); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1204,7 +1391,7 @@ impl<'a> Parser<'a> { })) } else { // It is just priority - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1219,14 +1406,14 @@ impl<'a> Parser<'a> { fn tuple_declaration(&mut self) -> Result, Error<'a>> { // 'let' is consumed before this call // expect '(' - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } let mut names = Vec::new(); while !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let span = Self::token_to_span(&token); if let TokenType::Identifier(id) = token.token_type { names.push(Spanned { span, node: id }); @@ -1240,7 +1427,7 @@ impl<'a> Parser<'a> { } self.assign_next()?; // consume ')' - let assign = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let assign = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(assign, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken(Self::token_to_span(&assign), assign)); @@ -1248,11 +1435,12 @@ impl<'a> Parser<'a> { self.assign_next()?; // Consume the `=` - let value = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let value = self.expression()?.ok_or_else(|| self.unexpected_eof())?; + let value_span = value.span; - let semi = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let semi = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(semi, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&semi), semi)); + return Err(Error::MissingSemicolon(value_span)); } Ok(Expression::TupleDeclaration(Spanned { @@ -1265,19 +1453,24 @@ impl<'a> Parser<'a> { } fn invocation(&mut self) -> Result, Error<'a>> { - let identifier_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let identifier_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1288,10 +1481,10 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; if let Expression::Block(_) = expression.node { return Err(Error::InvalidSyntax( @@ -1305,7 +1498,7 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken( Self::token_to_span(&next_token), next_token, @@ -1328,7 +1521,10 @@ impl<'a> Parser<'a> { fn block(&mut self) -> Result, Error<'a>> { let mut expressions = Vec::>::new(); - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken( @@ -1341,11 +1537,11 @@ impl<'a> Parser<'a> { self, TokenType::Symbol(Symbol::RBrace) | TokenType::Keyword(Keyword::Return) ) { - let expression = self.parse()?.ok_or(Error::UnexpectedEOF)?; + let expression = self.parse()?.ok_or_else(|| self.unexpected_eof())?; expressions.push(expression); } - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if token_matches!(current_token, TokenType::Keyword(Keyword::Return)) { // Need to capture return span @@ -1353,14 +1549,16 @@ impl<'a> Parser<'a> { self.assign_next()?; let expr = if token_matches!( - self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?, + self.current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::Semicolon) ) { // rewind 1 token so we can check for the semicolon at the bottom of this function. self.tokenizer.seek(SeekFrom::Current(-1))?; None } else { - Some(self.expression()?.ok_or(Error::UnexpectedEOF)?) + Some(self.expression()?.ok_or_else(|| self.unexpected_eof())?) }; let ret_span = Span { @@ -1382,12 +1580,12 @@ impl<'a> Parser<'a> { }; expressions.push(return_expr); - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); + return Err(Error::MissingSemicolon(ret_span)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1398,7 +1596,10 @@ impl<'a> Parser<'a> { fn const_declaration(&mut self) -> Result, Error<'a>> { // const - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Const)) { return Err(Error::UnexpectedToken( self.current_span(), @@ -1407,7 +1608,7 @@ impl<'a> Parser<'a> { } // variable_name - let ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let ident_span = Self::token_to_span(&ident_token); let ident = match ident_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1415,7 +1616,10 @@ impl<'a> Parser<'a> { }; // `=` - let assign_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let assign_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); if !token_matches!(assign_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( Self::token_to_span(&assign_token), @@ -1439,7 +1643,7 @@ impl<'a> Parser<'a> { } else { // we need to rewind our tokenizer to our previous location self.tokenizer.seek(SeekFrom::Current( - self.tokenizer.loc() - current_token_index, + current_token_index - self.tokenizer.loc(), ))?; let syscall = self.spanned(|p| p.syscall())?; @@ -1452,7 +1656,9 @@ impl<'a> Parser<'a> { ) { return Err(Error::UnexpectedToken( syscall.span, - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } @@ -1467,14 +1673,17 @@ impl<'a> Parser<'a> { } fn declaration(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) { return Err(Error::UnexpectedToken( self.current_span(), current_token.clone(), )); } - let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let identifier_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let identifier_span = Self::token_to_span(&identifier_token); let identifier = match identifier_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1486,7 +1695,10 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone(); + let current_token = self + .get_next()? + .ok_or_else(|| self.unexpected_eof())? + .clone(); if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) { return Err(Error::UnexpectedToken( @@ -1496,14 +1708,12 @@ impl<'a> Parser<'a> { } self.assign_next()?; - let assignment_expression = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let assignment_expression = self.expression()?.ok_or_else(|| self.unexpected_eof())?; + let expr_span = assignment_expression.span; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::Semicolon)) { - return Err(Error::UnexpectedToken( - Self::token_to_span(¤t_token), - current_token, - )); + return Err(Error::MissingSemicolon(expr_span)); } Ok(Expression::Declaration( @@ -1516,7 +1726,10 @@ impl<'a> Parser<'a> { } fn literal(&mut self) -> Result, Error<'a>> { - let current_token = self.current_token.clone().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?; let literal = match current_token.token_type { TokenType::Number(num) => Literal::Number(num), TokenType::String(ref string) => Literal::String(string.clone()), @@ -1532,7 +1745,7 @@ impl<'a> Parser<'a> { wrong_token, )); } - None => return Err(Error::UnexpectedEOF), + None => return Err(self.unexpected_eof()), }, _ => { return Err(Error::UnexpectedToken( @@ -1547,20 +1760,20 @@ impl<'a> Parser<'a> { fn if_expression(&mut self) -> Result, Error<'a>> { // 'if' is current - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } self.assign_next()?; - let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?; - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1586,7 +1799,7 @@ impl<'a> Parser<'a> { node: Expression::Block(block), })) } else { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } } else { @@ -1601,7 +1814,7 @@ impl<'a> Parser<'a> { } fn loop_expression(&mut self) -> Result, Error<'a>> { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1612,20 +1825,20 @@ impl<'a> Parser<'a> { } fn while_expression(&mut self) -> Result, Error<'a>> { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } self.assign_next()?; - let condition = self.expression()?.ok_or(Error::UnexpectedEOF)?; + let condition = self.expression()?.ok_or_else(|| self.unexpected_eof())?; - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::RParen)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(next, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1640,7 +1853,7 @@ impl<'a> Parser<'a> { fn function(&mut self) -> Result, Error<'a>> { // 'fn' is current - let fn_ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let fn_ident_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; let fn_ident_span = Self::token_to_span(&fn_ident_token); let fn_ident = match fn_ident_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1649,7 +1862,7 @@ impl<'a> Parser<'a> { } }; - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LParen)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1660,10 +1873,13 @@ impl<'a> Parser<'a> { let mut arguments = Vec::>>::new(); while !token_matches!( - self.get_next()?.ok_or(Error::UnexpectedEOF)?, + self.get_next()?.ok_or_else(|| self.unexpected_eof())?, TokenType::Symbol(Symbol::RParen) ) { - let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?; + let current_token = self + .current_token + .as_ref() + .ok_or_else(|| self.unexpected_eof())?; let arg_span = Self::token_to_span(current_token); let argument = match current_token.token_type { TokenType::Identifier(ref id) => id.clone(), @@ -1689,7 +1905,7 @@ impl<'a> Parser<'a> { if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma)) && !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) { - let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let next = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; return Err(Error::UnexpectedToken(Self::token_to_span(&next), next)); } @@ -1698,7 +1914,7 @@ impl<'a> Parser<'a> { } } - let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?; + let current_token = self.get_next()?.ok_or_else(|| self.unexpected_eof())?; if !token_matches!(current_token, TokenType::Symbol(Symbol::LBrace)) { return Err(Error::UnexpectedToken( Self::token_to_span(¤t_token), @@ -1761,7 +1977,9 @@ impl<'a> Parser<'a> { _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )) } } @@ -1793,9 +2011,14 @@ impl<'a> Parser<'a> { } "sleep" => { let mut args = args!(1); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::Sleep(boxed!(expr)))) } + "clr" => { + let mut args = args!(1); + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; + Ok(SysCall::System(System::Clr(boxed!(expr)))) + } "hash" => { let mut args = args!(1); let lit_str = literal_or_variable!(args.next()); @@ -1847,7 +2070,9 @@ impl<'a> Parser<'a> { _ => { return Err(Error::UnexpectedToken( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )); } }; @@ -1903,7 +2128,7 @@ impl<'a> Parser<'a> { let tmp = args.next(); let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); - let variable = args.next().ok_or(Error::UnexpectedEOF)?; + let variable = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(sys_call::System::SetOnDevice( device, Spanned { @@ -1922,7 +2147,7 @@ impl<'a> Parser<'a> { let tmp = args.next(); let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); - let variable = args.next().ok_or(Error::UnexpectedEOF)?; + let variable = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(sys_call::System::SetOnDeviceBatched( device_hash, @@ -1945,7 +2170,7 @@ impl<'a> Parser<'a> { let logic_type = get_arg!(Literal, literal_or_variable!(tmp)); let tmp = args.next(); - let expr = Box::new(tmp.ok_or(Error::UnexpectedEOF)?); + let expr = Box::new(tmp.ok_or_else(|| self.unexpected_eof())?); Ok(SysCall::System(System::SetOnDeviceBatchedNamed( device_hash, @@ -1958,7 +2183,7 @@ impl<'a> Parser<'a> { let mut args = args!(3); let next = args.next(); let dev_name = literal_or_variable!(next); - let slot_index = args.next().ok_or(Error::UnexpectedEOF)?; + let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?; let next = args.next(); let slot_logic = get_arg!(Literal, literal_or_variable!(next)); @@ -1985,7 +2210,7 @@ impl<'a> Parser<'a> { let mut args = args!(4); let next = args.next(); let dev_name = literal_or_variable!(next); - let slot_index = args.next().ok_or(Error::UnexpectedEOF)?; + let slot_index = args.next().ok_or_else(|| self.unexpected_eof())?; let next = args.next(); let slot_logic = get_arg!(Literal, literal_or_variable!(next)); @@ -2002,7 +2227,7 @@ impl<'a> Parser<'a> { )); } let next = args.next(); - let expr = next.ok_or(Error::UnexpectedEOF)?; + let expr = next.ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::SetSlot( dev_name, @@ -2017,7 +2242,7 @@ impl<'a> Parser<'a> { let device = literal_or_variable!(next); let next = args.next(); let reagent_mode = get_arg!(Literal, literal_or_variable!(next)); - let reagent_hash = args.next().ok_or(Error::UnexpectedEOF)?; + let reagent_hash = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::System(System::LoadReagent( device, @@ -2025,85 +2250,96 @@ impl<'a> Parser<'a> { Box::new(reagent_hash), ))) } + "rmap" => { + let mut args = args!(2); + let next = args.next(); + let device = literal_or_variable!(next); + let reagent_hash = args.next().ok_or_else(|| self.unexpected_eof())?; + + Ok(SysCall::System(System::Rmap( + device, + Box::new(reagent_hash), + ))) + } // Math SysCalls "acos" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let tmp = args.next().ok_or(Error::UnexpectedEOF)?; + let tmp = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Acos(boxed!(tmp)))) } "asin" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let tmp = args.next().ok_or(Error::UnexpectedEOF)?; + let tmp = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Asin(boxed!(tmp)))) } "atan" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Atan(boxed!(expr)))) } "atan2" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Atan2(boxed!(arg1), boxed!(arg2)))) } "abs" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let expr = args.next().ok_or(Error::UnexpectedEOF)?; + let expr = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Abs(boxed!(expr)))) } "ceil" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Ceil(boxed!(arg)))) } "cos" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Cos(boxed!(arg)))) } "floor" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Floor(boxed!(arg)))) } "log" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Log(boxed!(arg)))) } "max" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Max(boxed!(arg1), boxed!(arg2)))) } "min" => { check_length(2)?; let mut args = invocation.arguments.into_iter(); - let arg1 = args.next().ok_or(Error::UnexpectedEOF)?; - let arg2 = args.next().ok_or(Error::UnexpectedEOF)?; + let arg1 = args.next().ok_or_else(|| self.unexpected_eof())?; + let arg2 = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Min(boxed!(arg1), boxed!(arg2)))) } @@ -2114,34 +2350,36 @@ impl<'a> Parser<'a> { "sin" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Sin(boxed!(arg)))) } "sqrt" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Sqrt(boxed!(arg)))) } "tan" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Tan(boxed!(arg)))) } "trunc" => { check_length(1)?; let mut args = invocation.arguments.into_iter(); - let arg = args.next().ok_or(Error::UnexpectedEOF)?; + let arg = args.next().ok_or_else(|| self.unexpected_eof())?; Ok(SysCall::Math(Math::Trunc(boxed!(arg)))) } _ => Err(Error::UnsupportedKeyword( self.current_span(), - self.current_token.clone().ok_or(Error::UnexpectedEOF)?, + self.current_token + .clone() + .ok_or_else(|| self.unexpected_eof())?, )), } } diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index 67e03a9..d437e33 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -127,6 +127,52 @@ impl<'a> std::fmt::Display for Math<'a> { } } +impl<'a> Math<'a> { + /// Returns the name of this math function (e.g., "acos", "sin", "sqrt", etc.) + pub fn name(&self) -> &'static str { + match self { + Math::Acos(_) => "acos", + Math::Asin(_) => "asin", + Math::Atan(_) => "atan", + Math::Atan2(_, _) => "atan2", + Math::Abs(_) => "abs", + Math::Ceil(_) => "ceil", + Math::Cos(_) => "cos", + Math::Floor(_) => "floor", + Math::Log(_) => "log", + Math::Max(_, _) => "max", + Math::Min(_, _) => "min", + Math::Rand => "rand", + Math::Sin(_) => "sin", + Math::Sqrt(_) => "sqrt", + Math::Tan(_) => "tan", + Math::Trunc(_) => "trunc", + } + } + + /// Returns the number of arguments this math function expects + pub fn arg_count(&self) -> usize { + match self { + Math::Acos(_) => 1, + Math::Asin(_) => 1, + Math::Atan(_) => 1, + Math::Atan2(_, _) => 2, + Math::Abs(_) => 1, + Math::Ceil(_) => 1, + Math::Cos(_) => 1, + Math::Floor(_) => 1, + Math::Log(_) => 1, + Math::Max(_, _) => 2, + Math::Min(_, _) => 2, + Math::Rand => 0, + Math::Sin(_) => 1, + Math::Sqrt(_) => 1, + Math::Tan(_) => 1, + Math::Trunc(_) => 1, + } + } +} + documented! { #[derive(Debug, PartialEq, Eq)] pub enum System<'a> { @@ -142,6 +188,12 @@ documented! { /// ## Slang /// `sleep(number|var);` Sleep(Box>>), + /// Clears stack memory on the provided device. + /// ## IC10 + /// `clr d?` + /// ## Slang + /// `clr(device);` + Clr(Box>>), /// Gets the in-game hash for a specific prefab name. NOTE! This call is COMPLETELY /// optimized away unless you bind it to a `let` variable. If you use a `const` variable /// however, the hash is correctly computed at compile time and substitued automatically. @@ -249,6 +301,17 @@ documented! { Spanned>, Spanned>, Box>> + ), + /// Maps a reagent hash to the item hash that fulfills it on a device + /// + /// ## IC10 + /// `rmap r? d? reagentHash(r?|num)` + /// ## Slang + /// `let itemHash = rmap(device, reagentHash);` + /// `let itemHash = rmap(device, reagentHashValue);` + Rmap( + Spanned>, + Box>> ) } } @@ -258,6 +321,7 @@ impl<'a> std::fmt::Display for System<'a> { match self { System::Yield => write!(f, "yield()"), System::Sleep(a) => write!(f, "sleep({})", a), + System::Clr(a) => write!(f, "clr({})", a), System::Hash(a) => write!(f, "hash({})", a), System::LoadFromDevice(a, b) => write!(f, "loadFromDevice({}, {})", a, b), System::LoadBatch(a, b, c) => write!(f, "loadBatch({}, {}, {})", a, b, c), @@ -274,6 +338,49 @@ impl<'a> std::fmt::Display for System<'a> { System::LoadSlot(a, b, c) => write!(f, "loadSlot({}, {}, {})", a, b, c), System::SetSlot(a, b, c, d) => write!(f, "setSlot({}, {}, {}, {})", a, b, c, d), System::LoadReagent(a, b, c) => write!(f, "loadReagent({}, {}, {})", a, b, c), + System::Rmap(a, b) => write!(f, "rmap({}, {})", a, b), + } + } +} + +impl<'a> System<'a> { + /// Returns the name of this syscall (e.g., "yield", "sleep", "hash", etc.) + pub fn name(&self) -> &'static str { + match self { + System::Yield => "yield", + System::Sleep(_) => "sleep", + System::Clr(_) => "clr", + System::Hash(_) => "hash", + System::LoadFromDevice(_, _) => "loadFromDevice", + System::LoadBatch(_, _, _) => "loadBatch", + System::LoadBatchNamed(_, _, _, _) => "loadBatchNamed", + System::SetOnDevice(_, _, _) => "setOnDevice", + System::SetOnDeviceBatched(_, _, _) => "setOnDeviceBatched", + System::SetOnDeviceBatchedNamed(_, _, _, _) => "setOnDeviceBatchedNamed", + System::LoadSlot(_, _, _) => "loadSlot", + System::SetSlot(_, _, _, _) => "setSlot", + System::LoadReagent(_, _, _) => "loadReagent", + System::Rmap(_, _) => "rmap", + } + } + + /// Returns the number of arguments this syscall expects + pub fn arg_count(&self) -> usize { + match self { + System::Yield => 0, + System::Sleep(_) => 1, + System::Clr(_) => 1, + System::Hash(_) => 1, + System::LoadFromDevice(_, _) => 2, + System::LoadBatch(_, _, _) => 3, + System::LoadBatchNamed(_, _, _, _) => 4, + System::SetOnDevice(_, _, _) => 3, + System::SetOnDeviceBatched(_, _, _) => 3, + System::SetOnDeviceBatchedNamed(_, _, _, _) => 4, + System::LoadSlot(_, _, _) => 3, + System::SetSlot(_, _, _, _) => 4, + System::LoadReagent(_, _, _) => 3, + System::Rmap(_, _) => 2, } } } diff --git a/rust_compiler/libs/parser/src/test/mod.rs b/rust_compiler/libs/parser/src/test/mod.rs index a08f6ce..1834bae 100644 --- a/rust_compiler/libs/parser/src/test/mod.rs +++ b/rust_compiler/libs/parser/src/test/mod.rs @@ -149,6 +149,23 @@ fn test_const_hash_expression() -> Result<()> { Ok(()) } +#[test] +fn test_const_hash() -> Result<()> { + // This test explicitly validates the tokenizer rewind logic. + // When parsing "const h = hash(...)", the parser: + // 1. Consumes "const", identifier, "=" + // 2. Attempts to parse "hash(...)" as a literal - this fails + // 3. Must rewind the tokenizer to before "hash" + // 4. Then parse it as a syscall + // If the rewind offset is wrong (e.g., positive instead of negative), + // the tokenizer will be at the wrong position and parsing will fail. + let expr = parser!(r#"const h = hash("ComponentComputer")"#) + .parse()? + .unwrap(); + assert_eq!(r#"(const h = hash("ComponentComputer"))"#, expr.to_string()); + Ok(()) +} + #[test] fn test_negative_literal_const() -> Result<()> { let expr = parser!(r#"const i = -123"#).parse()?.unwrap(); @@ -287,3 +304,36 @@ fn test_tuple_declaration_all_complex_expressions() -> Result<()> { Ok(()) } +#[test] +fn test_eof_error_has_span() -> Result<()> { + // Test that UnexpectedEOF errors capture the span of the last token + let mut parser = parser!("let x = 5"); + let result = parser.parse(); + + // Should have an error + assert!(result.is_err()); + + let err = result.unwrap_err(); + + // Check that it's an UnexpectedEOF error + match err { + super::Error::UnexpectedEOF(Some(span)) => { + // Verify the span points to somewhere in the code (not zero defaults) + assert!( + span.start_line > 0 || span.start_col > 0 || span.end_line > 0 || span.end_col > 0, + "Span should not be all zeros: {:?}", + span + ); + } + super::Error::UnexpectedEOF(None) => { + eprintln!("ERROR: UnexpectedEOF captured None span instead of previous token span"); + eprintln!("This means unexpected_eof() is being called when current_token is None"); + panic!("UnexpectedEOF should have captured the previous token's span"); + } + other => { + panic!("Expected UnexpectedEOF error, got: {:?}", other); + } + } + + Ok(()) +} diff --git a/rust_compiler/libs/parser/src/tree_node.rs b/rust_compiler/libs/parser/src/tree_node.rs index 3da1305..961c9df 100644 --- a/rust_compiler/libs/parser/src/tree_node.rs +++ b/rust_compiler/libs/parser/src/tree_node.rs @@ -45,6 +45,12 @@ pub enum BinaryExpression<'a> { Subtract(Box>>, Box>>), Exponent(Box>>, Box>>), Modulo(Box>>, Box>>), + BitwiseAnd(Box>>, Box>>), + BitwiseOr(Box>>, Box>>), + BitwiseXor(Box>>, Box>>), + LeftShift(Box>>, Box>>), + RightShiftArithmetic(Box>>, Box>>), + RightShiftLogical(Box>>, Box>>), } impl<'a> std::fmt::Display for BinaryExpression<'a> { @@ -56,6 +62,12 @@ impl<'a> std::fmt::Display for BinaryExpression<'a> { BinaryExpression::Subtract(l, r) => write!(f, "({} - {})", l, r), BinaryExpression::Exponent(l, r) => write!(f, "({} ** {})", l, r), BinaryExpression::Modulo(l, r) => write!(f, "({} % {})", l, r), + BinaryExpression::BitwiseAnd(l, r) => write!(f, "({} & {})", l, r), + BinaryExpression::BitwiseOr(l, r) => write!(f, "({} | {})", l, r), + BinaryExpression::BitwiseXor(l, r) => write!(f, "({} ^ {})", l, r), + BinaryExpression::LeftShift(l, r) => write!(f, "({} << {})", l, r), + BinaryExpression::RightShiftArithmetic(l, r) => write!(f, "({} >> {})", l, r), + BinaryExpression::RightShiftLogical(l, r) => write!(f, "({} >>> {})", l, r), } } } @@ -197,6 +209,18 @@ impl<'a> std::fmt::Display for MethodCallExpression<'a> { } } +#[derive(Debug, PartialEq, Eq)] +pub struct IndexAccessExpression<'a> { + pub object: Box>>, + pub index: Box>>, +} + +impl<'a> std::fmt::Display for IndexAccessExpression<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}[{}]", self.object, self.index) + } +} + #[derive(Debug, PartialEq, Eq)] pub enum LiteralOrVariable<'a> { Literal(Literal<'a>), @@ -367,6 +391,7 @@ pub enum Expression<'a> { Binary(Spanned>), Block(Spanned>), Break(Span), + BitwiseNot(Box>>), ConstDeclaration(Spanned>), Continue(Span), Declaration(Spanned>, Box>>), @@ -389,6 +414,7 @@ pub enum Expression<'a> { TupleDeclaration(Spanned>), Variable(Spanned>), While(Spanned>), + IndexAccess(Spanned>), } impl<'a> std::fmt::Display for Expression<'a> { @@ -398,6 +424,7 @@ impl<'a> std::fmt::Display for Expression<'a> { Expression::Binary(e) => write!(f, "{}", e), Expression::Block(e) => write!(f, "{}", e), Expression::Break(_) => write!(f, "break"), + Expression::BitwiseNot(e) => write!(f, "(~{})", e), Expression::ConstDeclaration(e) => write!(f, "{}", e), Expression::Continue(_) => write!(f, "continue"), Expression::Declaration(id, e) => write!(f, "(let {} = {})", id, e), @@ -436,7 +463,7 @@ impl<'a> std::fmt::Display for Expression<'a> { Expression::TupleDeclaration(e) => write!(f, "{}", e), Expression::Variable(id) => write!(f, "{}", id), Expression::While(e) => write!(f, "{}", e), + Expression::IndexAccess(e) => write!(f, "{}", e), } } } - diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index c991047..50a05a4 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -68,6 +68,12 @@ impl<'a> Tokenizer<'a> { Ok(current.map(|t| t.map(|t| self.get_token(t)))?) } + + /// Returns the next token, including comments. Used to preserve doc comments. + pub fn next_token_with_comments(&mut self) -> Result>, Error> { + let current = self.lexer.next().transpose(); + Ok(current.map(|t| t.map(|t| self.get_token(t)))?) + } } // ... Iterator and TokenizerBuffer implementations remain unchanged ... @@ -127,12 +133,28 @@ impl<'a> TokenizerBuffer<'a> { self.index += 1; Ok(token) } + + pub fn next_token_with_comments(&mut self) -> Result>, Error> { + if let Some(token) = self.buffer.pop_front() { + self.history.push_back(token.clone()); + self.index += 1; + return Ok(Some(token)); + } + let token = self.tokenizer.next_token_with_comments()?; + + if let Some(ref token) = token { + self.history.push_back(token.clone()); + } + + self.index += 1; + Ok(token) + } pub fn peek(&mut self) -> Result>, Error> { if let Some(token) = self.buffer.front() { return Ok(Some(token.clone())); } - let Some(new_token) = self.tokenizer.next_token()? else { + let Some(new_token) = self.tokenizer.next_token_with_comments()? else { return Ok(None); }; self.buffer.push_front(new_token.clone()); @@ -145,8 +167,20 @@ impl<'a> TokenizerBuffer<'a> { use Ordering::*; match seek_to_int.cmp(&0) { Greater => { - let mut tokens = Vec::with_capacity(seek_to_int as usize); - for _ in 0..seek_to_int { + let mut seek_remaining = seek_to_int as usize; + + // First, consume tokens from the buffer (peeked but not yet consumed) + while seek_remaining > 0 && !self.buffer.is_empty() { + if let Some(token) = self.buffer.pop_front() { + self.history.push_back(token); + seek_remaining -= 1; + self.index += 1; + } + } + + // Then get tokens from tokenizer if needed + let mut tokens = Vec::with_capacity(seek_remaining); + for _ in 0..seek_remaining { if let Some(token) = self.tokenizer.next_token()? { tokens.push(token); } else { @@ -157,6 +191,7 @@ impl<'a> TokenizerBuffer<'a> { } } self.history.extend(tokens); + self.index += seek_remaining as i64; } Less => { let seek_to = seek_to_int.unsigned_abs() as usize; diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index 8ca6e21..e81c5d7 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -135,6 +135,9 @@ pub enum TokenType<'a> { /// Represents a string token String(Cow<'a, str>), + #[regex(r"0[xX][0-9a-fA-F][0-9a-fA-F_]*", parse_number)] + #[regex(r"0[oO][0-7][0-7_]*", parse_number)] + #[regex(r"0[bB][01][01_]*", parse_number)] #[regex(r"[0-9][0-9_]*(\.[0-9][0-9_]*)?([cfk])?", parse_number)] /// Represents a number token Number(Number), @@ -172,6 +175,23 @@ pub enum TokenType<'a> { #[token(";", symbol!(Semicolon))] #[token(":", symbol!(Colon))] #[token(",", symbol!(Comma))] + #[token("?", symbol!(Question))] + #[token(".", symbol!(Dot))] + #[token("%", symbol!(Percent))] + #[token("~", symbol!(BitwiseNot))] + // Multi-character tokens must be defined before their single-character prefixes + // For tokens like >> and >>>, define >>> before >> to ensure correct matching + #[token(">>>", symbol!(RightShiftLogical))] + #[token(">>", symbol!(RightShiftArithmetic))] + #[token("<<", symbol!(LeftShift))] + #[token("==", symbol!(Equal))] + #[token("!=", symbol!(NotEqual))] + #[token("&&", symbol!(LogicalAnd))] + #[token("||", symbol!(LogicalOr))] + #[token("<=", symbol!(LessThanOrEqual))] + #[token(">=", symbol!(GreaterThanOrEqual))] + #[token("**", symbol!(Exp))] + // Single-character tokens #[token("+", symbol!(Plus))] #[token("-", symbol!(Minus))] #[token("*", symbol!(Asterisk))] @@ -180,17 +200,9 @@ pub enum TokenType<'a> { #[token(">", symbol!(GreaterThan))] #[token("=", symbol!(Assign))] #[token("!", symbol!(LogicalNot))] - #[token(".", symbol!(Dot))] #[token("^", symbol!(Caret))] - #[token("%", symbol!(Percent))] - #[token("?", symbol!(Question))] - #[token("==", symbol!(Equal))] - #[token("!=", symbol!(NotEqual))] - #[token("&&", symbol!(LogicalAnd))] - #[token("||", symbol!(LogicalOr))] - #[token("<=", symbol!(LessThanOrEqual))] - #[token(">=", symbol!(GreaterThanOrEqual))] - #[token("**", symbol!(Exp))] + #[token("&", symbol!(BitwiseAnd))] + #[token("|", symbol!(BitwiseOr))] /// Represents a symbol token Symbol(Symbol), @@ -221,44 +233,75 @@ pub enum Comment<'a> { fn parse_number<'a>(lexer: &mut Lexer<'a, TokenType<'a>>) -> Result { let slice = lexer.slice(); - let last_char = slice.chars().last().unwrap_or_default(); - let (num_str, suffix) = match last_char { - 'c' | 'k' | 'f' => (&slice[..slice.len() - 1], Some(last_char)), - _ => (slice, None), - }; - - let clean_str = if num_str.contains('_') { - num_str.replace('_', "") - } else { - num_str.to_string() - }; let line = lexer.extras.line_count; let mut span = lexer.span(); span.end -= lexer.extras.line_start_index; span.start -= lexer.extras.line_start_index; - let unit = match suffix { - Some('c') => Unit::Celsius, - Some('f') => Unit::Fahrenheit, - Some('k') => Unit::Kelvin, - _ => Unit::None, - }; - - if clean_str.contains('.') { - Ok(Number::Decimal( - clean_str - .parse::() + // Determine the base and parse accordingly + if slice.starts_with("0x") || slice.starts_with("0X") { + // Hexadecimal - no temperature suffix allowed + let clean_str = slice[2..].replace('_', ""); + Ok(Number::Integer( + i128::from_str_radix(&clean_str, 16) .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, - unit, + Unit::None, + )) + } else if slice.starts_with("0o") || slice.starts_with("0O") { + // Octal - no temperature suffix allowed + let clean_str = slice[2..].replace('_', ""); + Ok(Number::Integer( + i128::from_str_radix(&clean_str, 8) + .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, + Unit::None, + )) + } else if slice.starts_with("0b") || slice.starts_with("0B") { + // Binary - no temperature suffix allowed + let clean_str = slice[2..].replace('_', ""); + Ok(Number::Integer( + i128::from_str_radix(&clean_str, 2) + .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, + Unit::None, )) } else { - Ok(Number::Integer( - clean_str - .parse::() - .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, - unit, - )) + // Decimal (with optional temperature suffix) + let last_char = slice.chars().last().unwrap_or_default(); + let (num_str, suffix) = match last_char { + 'c' | 'k' | 'f' => (&slice[..slice.len() - 1], Some(last_char)), + _ => (slice, None), + }; + + let clean_str = if num_str.contains('_') { + num_str.replace('_', "") + } else { + num_str.to_string() + }; + + let unit = match suffix { + Some('c') => Unit::Celsius, + Some('f') => Unit::Fahrenheit, + Some('k') => Unit::Kelvin, + _ => Unit::None, + }; + + if clean_str.contains('.') { + // Decimal floating point + Ok(Number::Decimal( + clean_str + .parse::() + .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, + unit, + )) + } else { + // Decimal integer + Ok(Number::Integer( + clean_str + .parse::() + .map_err(|_| LexError::NumberParse(line, span, slice.to_string()))?, + unit, + )) + } } } @@ -615,6 +658,12 @@ pub enum Symbol { Percent, /// Represents the `?` symbol Question, + /// Represents the `&` symbol (bitwise AND) + BitwiseAnd, + /// Represents the `|` symbol (bitwise OR) + BitwiseOr, + /// Represents the `~` symbol (bitwise NOT) + BitwiseNot, // Double Character Symbols /// Represents the `==` symbol @@ -629,6 +678,12 @@ pub enum Symbol { LessThanOrEqual, /// Represents the `>=` symbol GreaterThanOrEqual, + /// Represents the `<<` symbol (left shift) + LeftShift, + /// Represents the `>>` symbol (arithmetic right shift) + RightShiftArithmetic, + /// Represents the `>>>` symbol (logical right shift) + RightShiftLogical, /// Represents the `**` symbol Exp, } @@ -643,6 +698,19 @@ impl Symbol { | Symbol::Slash | Symbol::Exp | Symbol::Percent + | Symbol::Caret + ) + } + + pub fn is_bitwise(&self) -> bool { + matches!( + self, + Symbol::BitwiseAnd + | Symbol::BitwiseOr + | Symbol::BitwiseNot + | Symbol::LeftShift + | Symbol::RightShiftArithmetic + | Symbol::RightShiftLogical ) } @@ -693,6 +761,12 @@ impl std::fmt::Display for Symbol { Self::NotEqual => write!(f, "!="), Self::Dot => write!(f, "."), Self::Caret => write!(f, "^"), + Self::BitwiseAnd => write!(f, "&"), + Self::BitwiseOr => write!(f, "|"), + Self::BitwiseNot => write!(f, "~"), + Self::LeftShift => write!(f, "<<"), + Self::RightShiftArithmetic => write!(f, ">>"), + Self::RightShiftLogical => write!(f, ">>>"), Self::Exp => write!(f, "**"), } } @@ -715,7 +789,7 @@ documented! { /// } /// ``` Continue, - /// Prepresents the `const` keyword. This allows you to define a variable that will never + /// Represents the `const` keyword. This allows you to define a variable that will never /// change throughout the lifetime of the program, similar to `define` in IC10. If you are /// not planning on mutating the variable (changing it), it is recommend you store it as a /// const, as the compiler will not assign it to a register or stack variable. @@ -846,6 +920,7 @@ documented! { #[cfg(test)] mod tests { use super::TokenType; + use super::{Number, Unit}; use logos::Logos; #[test] @@ -856,7 +931,141 @@ mod tests { let tokens = lexer.collect::>(); - assert!(!tokens.iter().any(|res| res.is_err())); + assert!( + !tokens.iter().any(|res| res.is_err()), + "Expected no lexing errors for CRLF endings" + ); + Ok(()) + } + + #[test] + fn test_binary_literals() -> anyhow::Result<()> { + let src = "0b1010 0b0 0b1111_0000"; + let lexer = TokenType::lexer(src); + let tokens: Vec<_> = lexer.collect::, _>>()?; + + assert_eq!(tokens.len(), 3); + assert!( + matches!( + &tokens[0], + TokenType::Number(Number::Integer(10, Unit::None)) + ), + "Expected binary 0b1010 = 10" + ); + assert!( + matches!( + &tokens[1], + TokenType::Number(Number::Integer(0, Unit::None)) + ), + "Expected binary 0b0 = 0" + ); + assert!( + matches!( + &tokens[2], + TokenType::Number(Number::Integer(240, Unit::None)) + ), + "Expected binary 0b1111_0000 = 240" + ); + Ok(()) + } + + #[test] + fn test_octal_literals() -> anyhow::Result<()> { + let src = "0o77 0o0 0o7_777"; + let lexer = TokenType::lexer(src); + let tokens: Vec<_> = lexer.collect::, _>>()?; + + assert_eq!(tokens.len(), 3); + assert!( + matches!( + &tokens[0], + TokenType::Number(Number::Integer(63, Unit::None)) + ), + "Expected octal 0o77 = 63" + ); + assert!( + matches!( + &tokens[1], + TokenType::Number(Number::Integer(0, Unit::None)) + ), + "Expected octal 0o0 = 0" + ); + assert!( + matches!( + &tokens[2], + TokenType::Number(Number::Integer(4095, Unit::None)) + ), + "Expected octal 0o7_777 = 4095" + ); + Ok(()) + } + + #[test] + fn test_hex_literals() -> anyhow::Result<()> { + let src = "0xFF 0x0 0xFF_FF 0xFF_FF_FF"; + let lexer = TokenType::lexer(src); + let tokens: Vec<_> = lexer.collect::, _>>()?; + + assert_eq!(tokens.len(), 4); + assert!( + matches!( + &tokens[0], + TokenType::Number(Number::Integer(255, Unit::None)) + ), + "Expected hex 0xFF = 255" + ); + assert!( + matches!( + &tokens[1], + TokenType::Number(Number::Integer(0, Unit::None)) + ), + "Expected hex 0x0 = 0" + ); + assert!( + matches!( + &tokens[2], + TokenType::Number(Number::Integer(65535, Unit::None)) + ), + "Expected hex 0xFF_FF = 65535" + ); + assert!( + matches!( + &tokens[3], + TokenType::Number(Number::Integer(16777215, Unit::None)) + ), + "Expected hex 0xFF_FF_FF = 16777215" + ); + Ok(()) + } + + #[test] + fn test_hex_literals_lowercase() -> anyhow::Result<()> { + let src = "0xff 0xab 0xcd_ef"; + let lexer = TokenType::lexer(src); + let tokens: Vec<_> = lexer.collect::, _>>()?; + + assert_eq!(tokens.len(), 3); + assert!( + matches!( + &tokens[0], + TokenType::Number(Number::Integer(255, Unit::None)) + ), + "Expected hex 0xff = 255" + ); + assert!( + matches!( + &tokens[1], + TokenType::Number(Number::Integer(171, Unit::None)) + ), + "Expected hex 0xab = 171" + ); + assert!( + matches!( + &tokens[2], + TokenType::Number(Number::Integer(52719, Unit::None)) + ), + "Expected hex 0xcd_ef = 52719" + ); Ok(()) } } diff --git a/rust_compiler/src/ffi/mod.rs b/rust_compiler/src/ffi/mod.rs index 76195a3..996a621 100644 --- a/rust_compiler/src/ffi/mod.rs +++ b/rust_compiler/src/ffi/mod.rs @@ -94,6 +94,30 @@ impl From for FfiDiagnostic { } } +#[derive_ReprC] +#[repr(C)] +pub struct FfiSymbolKindData { + pub kind: u32, // 0=Function, 1=Syscall, 2=Variable + pub arg_count: u32, + pub syscall_type: u32, // 0=System, 1=Math (only for Syscall kind) +} + +#[derive_ReprC] +#[repr(C)] +pub struct FfiSymbolInfo { + pub name: safer_ffi::String, + pub kind_data: FfiSymbolKindData, + pub span: FfiRange, + pub description: safer_ffi::String, +} + +#[derive_ReprC] +#[repr(C)] +pub struct FfiDiagnosticsAndSymbols { + pub diagnostics: safer_ffi::Vec, + pub symbols: safer_ffi::Vec, +} + #[ffi_export] pub fn free_ffi_compilation_result(input: FfiCompilationResult) { drop(input) @@ -109,6 +133,11 @@ pub fn free_ffi_diagnostic_vec(v: safer_ffi::Vec) { drop(v) } +#[ffi_export] +pub fn free_ffi_diagnostics_and_symbols(v: FfiDiagnosticsAndSymbols) { + drop(v) +} + #[ffi_export] pub fn free_string(s: safer_ffi::String) { drop(s) @@ -182,6 +211,10 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec = + SysCall::get_all_documentation().into_iter().collect(); + let mut tokens = Vec::new(); for token in tokenizer { @@ -217,13 +250,26 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec tokens.push(FfiToken { - column: span.start as i32, - error: "".into(), - length: (span.end - span.start) as i32, - tooltip: token_type.docs().into(), - token_kind: token_type.into(), - }), + }) => { + let mut tooltip = token_type.docs(); + + // If no docs from token type, check if it's a syscall + if tooltip.is_empty() { + if let TokenType::Identifier(id) = &token_type { + if let Some(doc) = syscall_docs.get(id.as_ref()) { + tooltip = doc.clone(); + } + } + } + + tokens.push(FfiToken { + column: span.start as i32, + error: "".into(), + length: (span.end - span.start) as i32, + tooltip: tooltip.into(), + token_kind: token_type.into(), + }) + } } } @@ -257,6 +303,88 @@ pub fn diagnose_source(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec< res.unwrap_or(vec![].into()) } +#[ffi_export] +pub fn diagnose_source_with_symbols( + input: safer_ffi::slice::Ref<'_, u16>, +) -> FfiDiagnosticsAndSymbols { + let res = std::panic::catch_unwind(|| { + let input = String::from_utf16_lossy(input.as_slice()); + + let tokenizer = Tokenizer::from(input.as_str()); + let compiler = Compiler::new(Parser::new(tokenizer), None); + + let CompilationResult { + errors: diagnosis, + metadata, + .. + } = compiler.compile(); + + // Convert diagnostics + let mut diagnostics_vec: Vec = Vec::with_capacity(diagnosis.len()); + for err in diagnosis { + diagnostics_vec.push(lsp_types::Diagnostic::from(err).into()); + } + + // Convert symbols + let mut symbols_vec: Vec = Vec::with_capacity(metadata.symbols.len()); + for symbol in &metadata.symbols { + let (kind, arg_count, syscall_type) = match &symbol.kind { + compiler::SymbolKind::Function { parameters, .. } => { + (0, parameters.len() as u32, 0) + } + compiler::SymbolKind::Syscall { + syscall_type, + argument_count, + } => { + let sc_type = match syscall_type { + compiler::SyscallType::System => 0, + compiler::SyscallType::Math => 1, + }; + (1, *argument_count as u32, sc_type) + } + compiler::SymbolKind::Variable { .. } => (2, 0, 0), + }; + + let span = symbol + .span + .as_ref() + .map(|s| (*s).into()) + .unwrap_or(FfiRange { + start_line: 0, + end_line: 0, + start_col: 0, + end_col: 0, + }); + + symbols_vec.push(FfiSymbolInfo { + name: symbol.name.to_string().into(), + kind_data: FfiSymbolKindData { + kind, + arg_count, + syscall_type, + }, + span, + description: symbol + .description + .as_ref() + .map(|d| d.to_string()) + .unwrap_or_default() + .into(), + }); + } + + FfiDiagnosticsAndSymbols { + diagnostics: diagnostics_vec.into(), + symbols: symbols_vec.into(), + } + }); + + res.unwrap_or(FfiDiagnosticsAndSymbols { + diagnostics: vec![].into(), + symbols: vec![].into(), + }) +} + #[ffi_export] pub fn get_docs() -> safer_ffi::Vec { let res = std::panic::catch_unwind(|| { diff --git a/rust_compiler/src/main.rs b/rust_compiler/src/main.rs index 47494bb..734995b 100644 --- a/rust_compiler/src/main.rs +++ b/rust_compiler/src/main.rs @@ -65,8 +65,8 @@ fn run_logic<'a>() -> Result<(), Error<'a>> { let input_string = match input_file { Some(input_path) => { let mut buf = String::new(); - let mut file = std::fs::File::open(input_path).unwrap(); - file.read_to_string(&mut buf).unwrap(); + let mut file = std::fs::File::open(input_path)?; + file.read_to_string(&mut buf)?; buf } None => {