From 6b69cc1459d0f7a7b4a7fb3bf2b7d585ae050964 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 03:17:23 -0700 Subject: [PATCH] wip --- .gitignore | 2 +- build.sh | 3 + rust_compiler/Cargo.lock | 206 ++++++++++++++++++++++ rust_compiler/Cargo.toml | 15 +- rust_compiler/build.rs | 9 + rust_compiler/libs/tokenizer/src/lib.rs | 8 + rust_compiler/libs/tokenizer/src/token.rs | 38 +++- rust_compiler/src/lib.rs | 114 +++++++----- 8 files changed, 344 insertions(+), 51 deletions(-) create mode 100644 rust_compiler/build.rs diff --git a/.gitignore b/.gitignore index 5e30f94..0c2f875 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target *.ic10 release -bin +csharp_mod/bin obj ref diff --git a/build.sh b/build.sh index 436cfbd..ba83ba6 100755 --- a/build.sh +++ b/build.sh @@ -18,6 +18,9 @@ cargo build --release --target=x86_64-unknown-linux-gnu # -- Build for Windows (x86-64) -- cargo build --release --target=x86_64-pc-windows-gnu +# -- Generate C# Headers -- +cargo run --features headers --bin generate-headers + cd .. echo "--------------------" diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 6348b0f..7d0923d 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -265,6 +265,41 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "ext-trait" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5" +dependencies = [ + "ext-trait-proc_macros", +] + +[[package]] +name = "ext-trait-proc_macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "extension-traits" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537" +dependencies = [ + "ext-trait", +] + +[[package]] +name = "extern-c" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320bea982e85d42441eb25c49b41218e7eaa2657e8f90bc4eca7437376751e23" + [[package]] name = "funty" version = "2.0.0" @@ -328,6 +363,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inventory" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" +dependencies = [ + "rustversion", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -356,6 +400,22 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "macro_rules_attribute" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" + [[package]] name = "memchr" version = "2.7.6" @@ -410,6 +470,12 @@ dependencies = [ "tokenizer", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -429,6 +495,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -578,6 +654,15 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -590,12 +675,56 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safer-ffi" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435fdd58b61a6f1d8545274c1dfa458e905ff68c166e65e294a0130ef5e675bd" +dependencies = [ + "extern-c", + "inventory", + "libc", + "macro_rules_attribute", + "paste", + "safer_ffi-proc_macros", + "scopeguard", + "stabby", + "uninit", + "unwind_safe", + "with_builtin_macros", +] + +[[package]] +name = "safer_ffi-proc_macros" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f25be5ba5f319542edb31925517e0380245ae37df50a9752cdbc05ef948156" +dependencies = [ + "macro_rules_attribute", + "prettyplease", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -638,12 +767,53 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "stabby" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b7e94eaf470c2e76b5f15fb2fb49714471a36cc512df5ee231e62e82ec79f8" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc7a63b8276b54e51bfffe3d85da56e7906b2dcfcb29018a8ab666c06734c1a" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eecb7ec5611ec93ec79d120fbe55f31bea234dc1bed1001d4a071bb688651615" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "rand", + "syn 1.0.109", +] + [[package]] name = "stationlang" version = "0.1.0" @@ -654,6 +824,7 @@ dependencies = [ "parser", "quick-error", "rust_decimal", + "safer-ffi", "tokenizer", ] @@ -751,6 +922,21 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "uninit" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e130f2ed46ca5d8ec13c7ff95836827f92f5f5f37fd2b2bf16f33c408d98bb6" +dependencies = [ + "extension-traits", +] + +[[package]] +name = "unwind_safe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0976c77def3f1f75c4ef892a292c31c0bbe9e3d0702c63044d7c76db298171a3" + [[package]] name = "utf8parse" version = "0.2.2" @@ -848,6 +1034,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "with_builtin_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59d55032495429b87f9d69954c6c8602e4d3f3e0a747a12dea6b0b23de685da" +dependencies = [ + "with_builtin_macros-proc_macros", +] + +[[package]] +name = "with_builtin_macros-proc_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15bd7679c15e22924f53aee34d4e448c45b674feb6129689af88593e129f8f42" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/rust_compiler/Cargo.toml b/rust_compiler/Cargo.toml index 5939f95..4f5a6db 100644 --- a/rust_compiler/Cargo.toml +++ b/rust_compiler/Cargo.toml @@ -9,6 +9,10 @@ members = ["libs/*"] [workspace.dependencies] quick-error = "2" rust_decimal = "1" +safer-ffi = { version = "0.1" } + +[features] +headers = ["safer-ffi/headers"] [profile.release] strip = true @@ -17,8 +21,13 @@ strip = true name = "slang" path = "src/main.rs" +[[bin]] +name = "generate-headers" +path = "src/bin/generate_headers.rs" +required-features = ["headers"] + [lib] -name = "slang" +name = "stationlang" path = "src/lib.rs" crate-type = ["cdylib"] @@ -29,7 +38,11 @@ rust_decimal = { workspace = true } tokenizer = { path = "libs/tokenizer" } parser = { path = "libs/parser" } compiler = { path = "libs/compiler" } +safer-ffi = { workspace = true } [dev-dependencies] anyhow = { version = "^1.0", features = ["backtrace"] } + +[build-dependencies] +safer-ffi = { version = "0.1", features = ["headers"] } diff --git a/rust_compiler/build.rs b/rust_compiler/build.rs new file mode 100644 index 0000000..567d489 --- /dev/null +++ b/rust_compiler/build.rs @@ -0,0 +1,9 @@ +fn main() -> ::std::io::Result<()> { + safer_ffi::headers::builder() + .with_language(safer_ffi::headers::Language::CSharp) + .to_file("../csharp_mod/SlangStubs.cs")? + .generate() + .unwrap(); + + Ok(()) +} diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index d360bc0..9494cc3 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -447,6 +447,14 @@ impl Tokenizer { } } +impl Iterator for Tokenizer { + type Item = Result; + + fn next(&mut self) -> Option { + todo!() + } +} + pub struct TokenizerBuffer { tokenizer: Tokenizer, buffer: VecDeque, diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index bea72dc..3d9750a 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -32,7 +32,7 @@ impl std::fmt::Display for Temperature { match self { Temperature::Celsius(n) => write!(f, "{}°C", n), Temperature::Fahrenheit(n) => write!(f, "{}°F", n), - Temperature::Kelvin(n) => write!(f, "{}K", n), + Temperature::Kelvin(n) => write!(f, "{}°K", n), } } } @@ -88,7 +88,7 @@ impl std::fmt::Display for TokenType { TokenType::Boolean(b) => write!(f, "{}", b), TokenType::Keyword(k) => write!(f, "{:?}", k), TokenType::Identifier(i) => write!(f, "{}", i), - TokenType::Symbol(s) => write!(f, "{:?}", s), + TokenType::Symbol(s) => write!(f, "{}", s), TokenType::EOF => write!(f, "EOF"), } } @@ -208,6 +208,40 @@ impl Symbol { } } +impl std::fmt::Display for Symbol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Percent => write!(f, "%"), + Self::LParen => write!(f, "("), + Self::RParen => write!(f, ")"), + Self::LBrace => write!(f, "{{"), + Self::RBrace => write!(f, "}}"), + Self::LBracket => write!(f, "["), + Self::RBracket => write!(f, "]"), + Self::Semicolon => write!(f, ";"), + Self::Colon => write!(f, ":"), + Self::Plus => write!(f, "+"), + Self::Minus => write!(f, "-"), + Self::Comma => write!(f, ","), + Self::Asterisk => write!(f, "*"), + Self::Slash => write!(f, "/"), + Self::LessThan => write!(f, "<"), + Self::LessThanOrEqual => write!(f, "<="), + Self::GreaterThan => write!(f, ">"), + Self::GreaterThanOrEqual => write!(f, ">="), + Self::Assign => write!(f, "="), + Self::Equal => write!(f, "=="), + Self::LogicalAnd => write!(f, "&&"), + Self::LogicalOr => write!(f, "||"), + Self::LogicalNot => write!(f, "!"), + Self::NotEqual => write!(f, "!="), + Self::Dot => write!(f, "."), + Self::Caret => write!(f, "^"), + Self::Exp => write!(f, "**"), + } + } +} + #[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)] pub enum Keyword { /// Represents the `continue` keyword diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index e7c09ad..e5879a5 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -1,58 +1,78 @@ use compiler::Compiler; use parser::Parser; -use std::{ - ffi::{CStr, CString}, - io::BufWriter, -}; -use tokenizer::Tokenizer; +use safer_ffi::prelude::*; +use std::io::BufWriter; +use tokenizer::{Error as TokenizerError, Tokenizer}; -/// Takes a raw pointer to a string and compiles the `slang` code into valid IC10 -/// # Safety -/// This must be called with a valid string pointer from C# (or wherever is calling this function) -#[no_mangle] -pub unsafe extern "C" fn compile_from_string( - input_ptr: *const std::os::raw::c_char, -) -> *mut std::os::raw::c_char { - if input_ptr.is_null() { - return std::ptr::null_mut(); - } - - let c_str = unsafe { CStr::from_ptr(input_ptr) }; - - let Ok(input_str) = c_str.to_str() else { - return std::ptr::null_mut(); - }; +#[derive_ReprC] +#[repr(C)] +pub struct FfiToken { + pub text: safer_ffi::String, + pub tooltip: Option, + pub error: Option, + pub status: Option, + pub column: i32, +} +#[ffi_export] +pub fn compile_from_string(input: safer_ffi::String) -> safer_ffi::String { let mut writer = BufWriter::new(Vec::new()); - let tokenizer = Tokenizer::from(input_str); - let parser = Parser::new(tokenizer); + let tokenizer = Tokenizer::from(String::from(input)); + let parser = Parser::new(tokenizer); let compiler = Compiler::new(parser, &mut writer, None); - let Ok(()) = compiler.compile() else { - return std::ptr::null_mut(); - }; - - let Ok(buffer) = writer.into_inner() else { - return std::ptr::null_mut(); - }; - - let c_string = CString::from_vec_unchecked(buffer); - - c_string.into_raw() -} - -/// Takes ownership of the string pointer and drops it, freeing the memory -/// # Safety -/// Must be called with a valid string pointer -#[no_mangle] -pub unsafe extern "C" fn free_slang_string(input_ptr: *mut std::os::raw::c_char) { - if input_ptr.is_null() { - return; + if compiler.compile().is_err() { + return safer_ffi::String::EMPTY; } - unsafe { - // Takes ownership of the input string, and then drops it immediately - let _ = CString::from_raw(input_ptr); - } + let Ok(compiled_vec) = writer.into_inner() else { + return safer_ffi::String::EMPTY; + }; + + // Safety: I know the compiler only outputs valid utf8 + safer_ffi::String::from(unsafe { String::from_utf8_unchecked(compiled_vec) }) +} + +#[ffi_export] +pub fn tokenize_line(input: safer_ffi::String) -> safer_ffi::Vec { + let tokenizer = Tokenizer::from(String::from(input)); + + let mut tokens = Vec::::new(); + + for token in tokenizer { + match token { + Err(TokenizerError::NumberParseError(_, _, col)) + | Err(TokenizerError::UnknownSymbolError(_, _, col)) + | Err(TokenizerError::DecimalParseError(_, _, col)) + | Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col)) => { + tokens.push(FfiToken { + column: col as i32, + text: "".into(), + tooltip: None, + // Safety: it's okay to unwrap the err here because we are matching on the `Err` variant + error: Some(token.unwrap_err().to_string().into()), + status: None, + }); + } + Err(_) => return safer_ffi::Vec::EMPTY, + Ok(token) => tokens.push(FfiToken { + text: token.token_type.to_string().into(), + tooltip: None, + error: None, + status: None, + column: token.column as i32, + }), + } + } + + tokens.into() +} + +#[cfg(feature = "headers")] +pub fn generate_headers() -> std::io::Result<()> { + ::safer_ffi::headers::builder() + .with_language(safer_ffi::headers::Language::CSharp) + .to_file("SlangGlue.cs")? + .generate() }