From 353dc169446b9ad2c0210bbfc34a6acfadc5ba6f Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Wed, 26 Nov 2025 16:02:00 -0700 Subject: [PATCH 01/11] Unified the C# mod and the Rust compiler into a monorepo --- .gitignore | 7 +- build.sh | 45 +++++-- csharp_mod/Patches.cs | 36 ++++++ csharp_mod/SlangPlugin.cs | 119 ++++++++++++++++++ csharp_mod/stationeersSlang.csproj | 57 +++++++++ Cargo.lock => rust_compiler/Cargo.lock | 0 Cargo.toml => rust_compiler/Cargo.toml | 0 .../libs}/compiler/Cargo.toml | 0 .../libs}/compiler/src/lib.rs | 0 .../compiler/src/test/binary_expression.rs | 0 .../libs}/compiler/src/test/branching.rs | 0 .../test/declaration_function_invocation.rs | 0 .../compiler/src/test/declaration_literal.rs | 0 .../compiler/src/test/function_declaration.rs | 0 .../compiler/src/test/logic_expression.rs | 0 .../libs}/compiler/src/test/loops.rs | 0 .../libs}/compiler/src/test/mod.rs | 0 .../libs}/compiler/src/test/syscall.rs | 0 .../libs}/compiler/src/v1.rs | 0 .../libs}/compiler/src/variable_manager.rs | 0 .../libs}/compiler/test_files/deviceIo.slang | 0 .../libs}/parser/Cargo.toml | 0 .../libs}/parser/src/lib.rs | 3 +- .../libs}/parser/src/sys_call.rs | 0 .../libs}/parser/src/test/blocks.rs | 0 .../libs}/parser/src/test/mod.rs | 0 .../libs}/parser/src/tree_node.rs | 0 .../libs}/tokenizer/Cargo.toml | 0 .../libs}/tokenizer/src/lib.rs | 0 .../libs}/tokenizer/src/token.rs | 0 .../libs}/tokenizer/tests/file.stlg | 0 .../rust-toolchain.toml | 0 {src => rust_compiler/src}/lib.rs | 0 {src => rust_compiler/src}/main.rs | 0 34 files changed, 253 insertions(+), 14 deletions(-) create mode 100644 csharp_mod/Patches.cs create mode 100644 csharp_mod/SlangPlugin.cs create mode 100644 csharp_mod/stationeersSlang.csproj rename Cargo.lock => rust_compiler/Cargo.lock (100%) rename Cargo.toml => rust_compiler/Cargo.toml (100%) rename {libs => rust_compiler/libs}/compiler/Cargo.toml (100%) rename {libs => rust_compiler/libs}/compiler/src/lib.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/binary_expression.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/branching.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/declaration_function_invocation.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/declaration_literal.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/function_declaration.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/logic_expression.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/loops.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/mod.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/test/syscall.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/v1.rs (100%) rename {libs => rust_compiler/libs}/compiler/src/variable_manager.rs (100%) rename {libs => rust_compiler/libs}/compiler/test_files/deviceIo.slang (100%) rename {libs => rust_compiler/libs}/parser/Cargo.toml (100%) rename {libs => rust_compiler/libs}/parser/src/lib.rs (99%) rename {libs => rust_compiler/libs}/parser/src/sys_call.rs (100%) rename {libs => rust_compiler/libs}/parser/src/test/blocks.rs (100%) rename {libs => rust_compiler/libs}/parser/src/test/mod.rs (100%) rename {libs => rust_compiler/libs}/parser/src/tree_node.rs (100%) rename {libs => rust_compiler/libs}/tokenizer/Cargo.toml (100%) rename {libs => rust_compiler/libs}/tokenizer/src/lib.rs (100%) rename {libs => rust_compiler/libs}/tokenizer/src/token.rs (100%) rename {libs => rust_compiler/libs}/tokenizer/tests/file.stlg (100%) rename rust-toolchain.toml => rust_compiler/rust-toolchain.toml (100%) rename {src => rust_compiler/src}/lib.rs (100%) rename {src => rust_compiler/src}/main.rs (100%) diff --git a/.gitignore b/.gitignore index b7c3df2..8764920 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ -/target -*.ic10 \ No newline at end of file +target +*.ic10 +release +bin +obj diff --git a/build.sh b/build.sh index 22273a4..436cfbd 100755 --- a/build.sh +++ b/build.sh @@ -2,21 +2,46 @@ set -e +RUST_DIR="rust_compiler" +CSHARP_DIR="csharp_mod" +RELEASE_DIR="release" + export RUSTFLAGS="--remap-path-prefix=${PWD}=. --remap-path-prefix=${HOME}/.cargo=~/.cargo" -# -- Build for Native (Linux x86-64) -- -echo "Building native (Linux x86-64) executable..." -cargo build --release --target=x86_64-unknown-linux-gnu -echo "Native build complete." echo "--------------------" +cd "$RUST_DIR" +echo "Building native Rust binaries and libs" + +# -- Build for Native (Linux x86-64) -- +cargo build --release --target=x86_64-unknown-linux-gnu # -- Build for Windows (x86-64) -- -echo "Building Windows (x86-64) dll and executable..." cargo build --release --target=x86_64-pc-windows-gnu -echo "Windows build successful." + +cd .. echo "--------------------" -echo "All builds successful" -echo "Linux executable at target/x86_64-unknown-linux-gnu/release/slang" -echo "Windows .exe at target/x86_64-pc-windows-gnu/release/slang.exe" -echo "Windows .dll at target/x86_64-pc-windows-gnu/release/slang.dll" +echo "Building C# mod" +echo "--------------------" + +cd "$CSHARP_DIR" +dotnet build -c Release + +cd .. +echo "--------------------" + +echo "Copying Release files to output directory" +echo "--------------------" + +RUST_WIN_EXE="$RUST_DIR/target/x86_64-pc-windows-gnu/release/slang.exe" +RUST_LINUX_BIN="$RUST_DIR/target/x86_64-unknown-linux-gnu/release/slang" +CHARP_DLL="$CSHARP_DIR/bin/Release/net46/StationeersSlang.dll" + +# Check if the release dir exists, if not: create it. +if [[ ! -d "$RELEASE_DIR" ]]; then + mkdir "$RELEASE_DIR" +fi + +cp "$RUST_WIN_EXE" "$RELEASE_DIR/slang.exe" +cp "$RUST_LINUX_BIN" "$RELEASE_DIR/slang" +cp "$CHARP_DLL" "$RELEASE_DIR/StationeersSlang.dll" diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs new file mode 100644 index 0000000..3ea5f7f --- /dev/null +++ b/csharp_mod/Patches.cs @@ -0,0 +1,36 @@ +using Assets.Scripts.Objects.Motherboards; +using HarmonyLib; + +namespace Slang +{ + [HarmonyPatch] + public static class SlangPatches + { + [HarmonyPatch( + typeof(ProgrammableChipMotherboard), + nameof(ProgrammableChipMotherboard.InputFinished) + )] + [HarmonyPrefix] + public static void PGM_InputFinished(ref string result) + { + if (string.IsNullOrEmpty(result) || !SlangPlugin.IsSlangSource(ref result)) + { + return; + } + + L.Info("Detected Slang source, compiling..."); + + // Compile the Slang source into IC10 + string compiled = SlangPlugin.Compile(result); + + // Ensure that the string is correct + if (string.IsNullOrEmpty(compiled)) + { + return; + } + + // Set the result to be the compiled source so the rest of the function can continue as normal + result = compiled; + } + } +} diff --git a/csharp_mod/SlangPlugin.cs b/csharp_mod/SlangPlugin.cs new file mode 100644 index 0000000..9038960 --- /dev/null +++ b/csharp_mod/SlangPlugin.cs @@ -0,0 +1,119 @@ +using System; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using BepInEx; +using HarmonyLib; + +namespace Slang +{ + class L + { + private static BepInEx.Logging.ManualLogSource _logger; + + public static void SetLogger(BepInEx.Logging.ManualLogSource logger) + { + _logger = logger; + } + + public static void Debug(string message) + { + _logger.LogDebug(message); + } + + public static void Info(string message) + { + _logger.LogInfo(message); + } + + public static void Error(string message) + { + _logger.LogError(message); + } + + public static void Warning(string message) + { + _logger.LogWarning(message); + } + } + + [BepInPlugin(PluginGuid, PluginName, "0.1.0")] + public class SlangPlugin : BaseUnityPlugin + { + public const string PluginGuid = "com.dbidwell94.slang"; + public const string PluginName = "Slang"; + + const string RUST_DLL_NAME = "slang.dll"; + + private readonly string[] SLANG_KEYWORDS = { "let ", "fn " }; + + /// Takes raw `Slang` source code and compiles it into IC10 + [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr compile_from_string(string input); + + /// Frees memory that was allocated by the FFI call to `compile_from_string` + [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + private static extern void free_slang_string(IntPtr ptr); + + public static string Compile(string source) + { + if (string.IsNullOrEmpty(source)) + return ""; + + IntPtr ptr = compile_from_string(source); + try + { + return Marshal.PtrToStringAnsi(ptr); + } + finally + { + free_slang_string(ptr); + } + } + + public static bool IsSlangSource(ref string input) + { + return true; + } + + private void Awake() + { + L.SetLogger(Logger); + ExtractNativeDll(RUST_DLL_NAME); + var harmony = new Harmony("com.dbidwell94.slang"); + harmony.PatchAll(); + } + + private void ExtractNativeDll(string fileName) + { + string destinationPath = Path.Combine(Path.GetDirectoryName(Info.Location), fileName); + + Assembly assembly = Assembly.GetExecutingAssembly(); + + using (Stream stream = assembly.GetManifestResourceStream(fileName)) + { + if (stream == null) + { + Logger.LogError( + $"{RUST_DLL_NAME} compiler not found. This means it was not embedded in the mod. Please contact the mod author!" + ); + return; + } + + try + { + using (FileStream fileStream = new FileStream(destinationPath, FileMode.Create)) + { + stream.CopyTo(fileStream); + } + } + catch (IOException e) + { + Logger.LogWarning( + $"Could not overwrite {fileName} (it might be in use): {e.Message}" + ); + } + } + } + } +} diff --git a/csharp_mod/stationeersSlang.csproj b/csharp_mod/stationeersSlang.csproj new file mode 100644 index 0000000..20ec76f --- /dev/null +++ b/csharp_mod/stationeersSlang.csproj @@ -0,0 +1,57 @@ + + + + net46 + enable + StationeersSlang + Slang Compiler Bridge + 0.1.0 + true + latest + + + + /home/dbidwell/.local/share/Steam/steamapps/common/Stationeers/ + $(GameDir)/rocketstation_Data/Managed + $(GameDir)/BepInEx/core + + + + + $(ManagedDir)/netstandard.dll + False + + + $(BepInExDir)/BepInEx.dll + False + + + $(BepInExDir)/0Harmony.dll + False + + + + $(ManagedDir)/UnityEngine.dll + False + + + $(ManagedDir)/UnityEngine.CoreModule.dll + False + + + $(ManagedDir)/Assembly-CSharp.dll + False + + + $(ManagedDir)/Assembly-CSharp-firstpass.dll + False + + + + + + slang.dll + + + + diff --git a/Cargo.lock b/rust_compiler/Cargo.lock similarity index 100% rename from Cargo.lock rename to rust_compiler/Cargo.lock diff --git a/Cargo.toml b/rust_compiler/Cargo.toml similarity index 100% rename from Cargo.toml rename to rust_compiler/Cargo.toml diff --git a/libs/compiler/Cargo.toml b/rust_compiler/libs/compiler/Cargo.toml similarity index 100% rename from libs/compiler/Cargo.toml rename to rust_compiler/libs/compiler/Cargo.toml diff --git a/libs/compiler/src/lib.rs b/rust_compiler/libs/compiler/src/lib.rs similarity index 100% rename from libs/compiler/src/lib.rs rename to rust_compiler/libs/compiler/src/lib.rs diff --git a/libs/compiler/src/test/binary_expression.rs b/rust_compiler/libs/compiler/src/test/binary_expression.rs similarity index 100% rename from libs/compiler/src/test/binary_expression.rs rename to rust_compiler/libs/compiler/src/test/binary_expression.rs diff --git a/libs/compiler/src/test/branching.rs b/rust_compiler/libs/compiler/src/test/branching.rs similarity index 100% rename from libs/compiler/src/test/branching.rs rename to rust_compiler/libs/compiler/src/test/branching.rs diff --git a/libs/compiler/src/test/declaration_function_invocation.rs b/rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs similarity index 100% rename from libs/compiler/src/test/declaration_function_invocation.rs rename to rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs diff --git a/libs/compiler/src/test/declaration_literal.rs b/rust_compiler/libs/compiler/src/test/declaration_literal.rs similarity index 100% rename from libs/compiler/src/test/declaration_literal.rs rename to rust_compiler/libs/compiler/src/test/declaration_literal.rs diff --git a/libs/compiler/src/test/function_declaration.rs b/rust_compiler/libs/compiler/src/test/function_declaration.rs similarity index 100% rename from libs/compiler/src/test/function_declaration.rs rename to rust_compiler/libs/compiler/src/test/function_declaration.rs diff --git a/libs/compiler/src/test/logic_expression.rs b/rust_compiler/libs/compiler/src/test/logic_expression.rs similarity index 100% rename from libs/compiler/src/test/logic_expression.rs rename to rust_compiler/libs/compiler/src/test/logic_expression.rs diff --git a/libs/compiler/src/test/loops.rs b/rust_compiler/libs/compiler/src/test/loops.rs similarity index 100% rename from libs/compiler/src/test/loops.rs rename to rust_compiler/libs/compiler/src/test/loops.rs diff --git a/libs/compiler/src/test/mod.rs b/rust_compiler/libs/compiler/src/test/mod.rs similarity index 100% rename from libs/compiler/src/test/mod.rs rename to rust_compiler/libs/compiler/src/test/mod.rs diff --git a/libs/compiler/src/test/syscall.rs b/rust_compiler/libs/compiler/src/test/syscall.rs similarity index 100% rename from libs/compiler/src/test/syscall.rs rename to rust_compiler/libs/compiler/src/test/syscall.rs diff --git a/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs similarity index 100% rename from libs/compiler/src/v1.rs rename to rust_compiler/libs/compiler/src/v1.rs diff --git a/libs/compiler/src/variable_manager.rs b/rust_compiler/libs/compiler/src/variable_manager.rs similarity index 100% rename from libs/compiler/src/variable_manager.rs rename to rust_compiler/libs/compiler/src/variable_manager.rs diff --git a/libs/compiler/test_files/deviceIo.slang b/rust_compiler/libs/compiler/test_files/deviceIo.slang similarity index 100% rename from libs/compiler/test_files/deviceIo.slang rename to rust_compiler/libs/compiler/test_files/deviceIo.slang diff --git a/libs/parser/Cargo.toml b/rust_compiler/libs/parser/Cargo.toml similarity index 100% rename from libs/parser/Cargo.toml rename to rust_compiler/libs/parser/Cargo.toml diff --git a/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs similarity index 99% rename from libs/parser/src/lib.rs rename to rust_compiler/libs/parser/src/lib.rs index 8eca23a..481524b 100644 --- a/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -4,6 +4,7 @@ mod test; pub mod sys_call; pub mod tree_node; +use crate::sys_call::System; use quick_error::quick_error; use std::io::SeekFrom; use sys_call::SysCall; @@ -13,8 +14,6 @@ use tokenizer::{ }; use tree_node::*; -use crate::sys_call::System; - #[macro_export] /// A macro to create a boxed value. macro_rules! boxed { diff --git a/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs similarity index 100% rename from libs/parser/src/sys_call.rs rename to rust_compiler/libs/parser/src/sys_call.rs diff --git a/libs/parser/src/test/blocks.rs b/rust_compiler/libs/parser/src/test/blocks.rs similarity index 100% rename from libs/parser/src/test/blocks.rs rename to rust_compiler/libs/parser/src/test/blocks.rs diff --git a/libs/parser/src/test/mod.rs b/rust_compiler/libs/parser/src/test/mod.rs similarity index 100% rename from libs/parser/src/test/mod.rs rename to rust_compiler/libs/parser/src/test/mod.rs diff --git a/libs/parser/src/tree_node.rs b/rust_compiler/libs/parser/src/tree_node.rs similarity index 100% rename from libs/parser/src/tree_node.rs rename to rust_compiler/libs/parser/src/tree_node.rs diff --git a/libs/tokenizer/Cargo.toml b/rust_compiler/libs/tokenizer/Cargo.toml similarity index 100% rename from libs/tokenizer/Cargo.toml rename to rust_compiler/libs/tokenizer/Cargo.toml diff --git a/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs similarity index 100% rename from libs/tokenizer/src/lib.rs rename to rust_compiler/libs/tokenizer/src/lib.rs diff --git a/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs similarity index 100% rename from libs/tokenizer/src/token.rs rename to rust_compiler/libs/tokenizer/src/token.rs diff --git a/libs/tokenizer/tests/file.stlg b/rust_compiler/libs/tokenizer/tests/file.stlg similarity index 100% rename from libs/tokenizer/tests/file.stlg rename to rust_compiler/libs/tokenizer/tests/file.stlg diff --git a/rust-toolchain.toml b/rust_compiler/rust-toolchain.toml similarity index 100% rename from rust-toolchain.toml rename to rust_compiler/rust-toolchain.toml diff --git a/src/lib.rs b/rust_compiler/src/lib.rs similarity index 100% rename from src/lib.rs rename to rust_compiler/src/lib.rs diff --git a/src/main.rs b/rust_compiler/src/main.rs similarity index 100% rename from src/main.rs rename to rust_compiler/src/main.rs From 2e7dc4fd91e6b4958b4626b8cdb6657053bf547e Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Wed, 26 Nov 2025 16:07:34 -0700 Subject: [PATCH 02/11] Added a 'clean' script to cleanup build directories --- clean.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 clean.sh diff --git a/clean.sh b/clean.sh new file mode 100755 index 0000000..c88bbd1 --- /dev/null +++ b/clean.sh @@ -0,0 +1,18 @@ +echo "Cleaning build directories" + +CSHARP_DIR="csharp_mod" +RUST_DIR="rust_compiler" + +cd "$CSHARP_DIR" +dotnet clean + +cd .. + +cd "$RUST_DIR" +cargo clean + +cd .. + +if [[ -d "release" ]]; then + rm -rd release +fi From 611f0f6f714e91ccbf4ae6d0c4b51bba80758870 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 01:14:54 -0700 Subject: [PATCH 03/11] Start bindings and interface implementation for the IC10Editor mod --- .gitignore | 1 + csharp_mod/Patches.cs | 5 +++ csharp_mod/SlangFormatter.cs | 12 +++++++ csharp_mod/SlangPlugin.cs | 48 +++++++++++++++++-------- csharp_mod/StationpediaDocumentation.cs | 22 ++++++++++++ csharp_mod/stationeersSlang.csproj | 6 +++- 6 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 csharp_mod/SlangFormatter.cs create mode 100644 csharp_mod/StationpediaDocumentation.cs diff --git a/.gitignore b/.gitignore index 8764920..5e30f94 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target release bin obj +ref diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs index 3ea5f7f..78d067f 100644 --- a/csharp_mod/Patches.cs +++ b/csharp_mod/Patches.cs @@ -1,3 +1,4 @@ +using System; using Assets.Scripts.Objects.Motherboards; using HarmonyLib; @@ -29,6 +30,10 @@ namespace Slang return; } + var newUuid = Guid.NewGuid().ToString(); + + SlangPlugin.CopySourceToFile(result); + // Set the result to be the compiled source so the rest of the function can continue as normal result = compiled; } diff --git a/csharp_mod/SlangFormatter.cs b/csharp_mod/SlangFormatter.cs new file mode 100644 index 0000000..fd96cba --- /dev/null +++ b/csharp_mod/SlangFormatter.cs @@ -0,0 +1,12 @@ +using StationeersIC10Editor; + +namespace Slang +{ + public class SlangFormatter : ICodeFormatter + { + public override Line ParseLine(string line) + { + throw new System.NotImplementedException(); + } + } +} diff --git a/csharp_mod/SlangPlugin.cs b/csharp_mod/SlangPlugin.cs index 9038960..0225cfa 100644 --- a/csharp_mod/SlangPlugin.cs +++ b/csharp_mod/SlangPlugin.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Reflection; using System.Runtime.InteropServices; +using System.Text.RegularExpressions; using BepInEx; using HarmonyLib; @@ -9,7 +10,7 @@ namespace Slang { class L { - private static BepInEx.Logging.ManualLogSource _logger; + private static BepInEx.Logging.ManualLogSource? _logger; public static void SetLogger(BepInEx.Logging.ManualLogSource logger) { @@ -18,34 +19,32 @@ namespace Slang public static void Debug(string message) { - _logger.LogDebug(message); + _logger?.LogDebug(message); } public static void Info(string message) { - _logger.LogInfo(message); + _logger?.LogInfo(message); } public static void Error(string message) { - _logger.LogError(message); + _logger?.LogError(message); } public static void Warning(string message) { - _logger.LogWarning(message); + _logger?.LogWarning(message); } } [BepInPlugin(PluginGuid, PluginName, "0.1.0")] public class SlangPlugin : BaseUnityPlugin { - public const string PluginGuid = "com.dbidwell94.slang"; + public const string PluginGuid = "com.biddydev.slang"; public const string PluginName = "Slang"; - const string RUST_DLL_NAME = "slang.dll"; - - private readonly string[] SLANG_KEYWORDS = { "let ", "fn " }; + const string RUST_DLL_NAME = "slang.compiler.dll"; /// Takes raw `Slang` source code and compiles it into IC10 [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] @@ -55,6 +54,21 @@ namespace Slang [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] private static extern void free_slang_string(IntPtr ptr); + private static Regex? _slangSourceCheck = null; + + private static Regex SlangSourceCheck + { + get + { + if (_slangSourceCheck is null) + { + _slangSourceCheck = new Regex(@"[;{}()]|\b(let|fn|device)\b|\/\/"); + } + + return _slangSourceCheck; + } + } + public static string Compile(string source) { if (string.IsNullOrEmpty(source)) @@ -71,11 +85,19 @@ namespace Slang } } - public static bool IsSlangSource(ref string input) + /// Take original slang source code and copies it to a file + /// for use in restoring later. + /// + public static bool CopySourceToFile(string source) { return true; } + public static bool IsSlangSource(ref string input) + { + return SlangSourceCheck.IsMatch(input); + } + private void Awake() { L.SetLogger(Logger); @@ -94,7 +116,7 @@ namespace Slang { if (stream == null) { - Logger.LogError( + L.Error( $"{RUST_DLL_NAME} compiler not found. This means it was not embedded in the mod. Please contact the mod author!" ); return; @@ -109,9 +131,7 @@ namespace Slang } catch (IOException e) { - Logger.LogWarning( - $"Could not overwrite {fileName} (it might be in use): {e.Message}" - ); + L.Warning($"Could not overwrite {fileName} (it might be in use): {e.Message}"); } } } diff --git a/csharp_mod/StationpediaDocumentation.cs b/csharp_mod/StationpediaDocumentation.cs new file mode 100644 index 0000000..e03814a --- /dev/null +++ b/csharp_mod/StationpediaDocumentation.cs @@ -0,0 +1,22 @@ +using Assets.Scripts.UI; + +namespace Slang +{ + public static class SlangDocs + { + public static StationpediaPage[] Pages + { + get + { + return + [ + new StationpediaPage( + "slang-init", + "Slang", + "Slang is a new high level language built specifically for Stationeers" + ), + ]; + } + } + } +} diff --git a/csharp_mod/stationeersSlang.csproj b/csharp_mod/stationeersSlang.csproj index 20ec76f..2810128 100644 --- a/csharp_mod/stationeersSlang.csproj +++ b/csharp_mod/stationeersSlang.csproj @@ -46,11 +46,15 @@ $(ManagedDir)/Assembly-CSharp-firstpass.dll False + + ./ref/IC10Editor.dll + False + - slang.dll + slang.compiler.dll From 6b69cc1459d0f7a7b4a7fb3bf2b7d585ae050964 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 03:17:23 -0700 Subject: [PATCH 04/11] wip --- .gitignore | 2 +- build.sh | 3 + rust_compiler/Cargo.lock | 206 ++++++++++++++++++++++ rust_compiler/Cargo.toml | 15 +- rust_compiler/build.rs | 9 + rust_compiler/libs/tokenizer/src/lib.rs | 8 + rust_compiler/libs/tokenizer/src/token.rs | 38 +++- rust_compiler/src/lib.rs | 114 +++++++----- 8 files changed, 344 insertions(+), 51 deletions(-) create mode 100644 rust_compiler/build.rs diff --git a/.gitignore b/.gitignore index 5e30f94..0c2f875 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ target *.ic10 release -bin +csharp_mod/bin obj ref diff --git a/build.sh b/build.sh index 436cfbd..ba83ba6 100755 --- a/build.sh +++ b/build.sh @@ -18,6 +18,9 @@ cargo build --release --target=x86_64-unknown-linux-gnu # -- Build for Windows (x86-64) -- cargo build --release --target=x86_64-pc-windows-gnu +# -- Generate C# Headers -- +cargo run --features headers --bin generate-headers + cd .. echo "--------------------" diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 6348b0f..7d0923d 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -265,6 +265,41 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "ext-trait" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5" +dependencies = [ + "ext-trait-proc_macros", +] + +[[package]] +name = "ext-trait-proc_macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "extension-traits" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537" +dependencies = [ + "ext-trait", +] + +[[package]] +name = "extern-c" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320bea982e85d42441eb25c49b41218e7eaa2657e8f90bc4eca7437376751e23" + [[package]] name = "funty" version = "2.0.0" @@ -328,6 +363,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inventory" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" +dependencies = [ + "rustversion", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -356,6 +400,22 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "macro_rules_attribute" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" + [[package]] name = "memchr" version = "2.7.6" @@ -410,6 +470,12 @@ dependencies = [ "tokenizer", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -429,6 +495,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -578,6 +654,15 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -590,12 +675,56 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safer-ffi" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435fdd58b61a6f1d8545274c1dfa458e905ff68c166e65e294a0130ef5e675bd" +dependencies = [ + "extern-c", + "inventory", + "libc", + "macro_rules_attribute", + "paste", + "safer_ffi-proc_macros", + "scopeguard", + "stabby", + "uninit", + "unwind_safe", + "with_builtin_macros", +] + +[[package]] +name = "safer_ffi-proc_macros" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f25be5ba5f319542edb31925517e0380245ae37df50a9752cdbc05ef948156" +dependencies = [ + "macro_rules_attribute", + "prettyplease", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -638,12 +767,53 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "stabby" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b7e94eaf470c2e76b5f15fb2fb49714471a36cc512df5ee231e62e82ec79f8" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc7a63b8276b54e51bfffe3d85da56e7906b2dcfcb29018a8ab666c06734c1a" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "36.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eecb7ec5611ec93ec79d120fbe55f31bea234dc1bed1001d4a071bb688651615" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "rand", + "syn 1.0.109", +] + [[package]] name = "stationlang" version = "0.1.0" @@ -654,6 +824,7 @@ dependencies = [ "parser", "quick-error", "rust_decimal", + "safer-ffi", "tokenizer", ] @@ -751,6 +922,21 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "uninit" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e130f2ed46ca5d8ec13c7ff95836827f92f5f5f37fd2b2bf16f33c408d98bb6" +dependencies = [ + "extension-traits", +] + +[[package]] +name = "unwind_safe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0976c77def3f1f75c4ef892a292c31c0bbe9e3d0702c63044d7c76db298171a3" + [[package]] name = "utf8parse" version = "0.2.2" @@ -848,6 +1034,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "with_builtin_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59d55032495429b87f9d69954c6c8602e4d3f3e0a747a12dea6b0b23de685da" +dependencies = [ + "with_builtin_macros-proc_macros", +] + +[[package]] +name = "with_builtin_macros-proc_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15bd7679c15e22924f53aee34d4e448c45b674feb6129689af88593e129f8f42" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/rust_compiler/Cargo.toml b/rust_compiler/Cargo.toml index 5939f95..4f5a6db 100644 --- a/rust_compiler/Cargo.toml +++ b/rust_compiler/Cargo.toml @@ -9,6 +9,10 @@ members = ["libs/*"] [workspace.dependencies] quick-error = "2" rust_decimal = "1" +safer-ffi = { version = "0.1" } + +[features] +headers = ["safer-ffi/headers"] [profile.release] strip = true @@ -17,8 +21,13 @@ strip = true name = "slang" path = "src/main.rs" +[[bin]] +name = "generate-headers" +path = "src/bin/generate_headers.rs" +required-features = ["headers"] + [lib] -name = "slang" +name = "stationlang" path = "src/lib.rs" crate-type = ["cdylib"] @@ -29,7 +38,11 @@ rust_decimal = { workspace = true } tokenizer = { path = "libs/tokenizer" } parser = { path = "libs/parser" } compiler = { path = "libs/compiler" } +safer-ffi = { workspace = true } [dev-dependencies] anyhow = { version = "^1.0", features = ["backtrace"] } + +[build-dependencies] +safer-ffi = { version = "0.1", features = ["headers"] } diff --git a/rust_compiler/build.rs b/rust_compiler/build.rs new file mode 100644 index 0000000..567d489 --- /dev/null +++ b/rust_compiler/build.rs @@ -0,0 +1,9 @@ +fn main() -> ::std::io::Result<()> { + safer_ffi::headers::builder() + .with_language(safer_ffi::headers::Language::CSharp) + .to_file("../csharp_mod/SlangStubs.cs")? + .generate() + .unwrap(); + + Ok(()) +} diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index d360bc0..9494cc3 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -447,6 +447,14 @@ impl Tokenizer { } } +impl Iterator for Tokenizer { + type Item = Result; + + fn next(&mut self) -> Option { + todo!() + } +} + pub struct TokenizerBuffer { tokenizer: Tokenizer, buffer: VecDeque, diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index bea72dc..3d9750a 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -32,7 +32,7 @@ impl std::fmt::Display for Temperature { match self { Temperature::Celsius(n) => write!(f, "{}°C", n), Temperature::Fahrenheit(n) => write!(f, "{}°F", n), - Temperature::Kelvin(n) => write!(f, "{}K", n), + Temperature::Kelvin(n) => write!(f, "{}°K", n), } } } @@ -88,7 +88,7 @@ impl std::fmt::Display for TokenType { TokenType::Boolean(b) => write!(f, "{}", b), TokenType::Keyword(k) => write!(f, "{:?}", k), TokenType::Identifier(i) => write!(f, "{}", i), - TokenType::Symbol(s) => write!(f, "{:?}", s), + TokenType::Symbol(s) => write!(f, "{}", s), TokenType::EOF => write!(f, "EOF"), } } @@ -208,6 +208,40 @@ impl Symbol { } } +impl std::fmt::Display for Symbol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Percent => write!(f, "%"), + Self::LParen => write!(f, "("), + Self::RParen => write!(f, ")"), + Self::LBrace => write!(f, "{{"), + Self::RBrace => write!(f, "}}"), + Self::LBracket => write!(f, "["), + Self::RBracket => write!(f, "]"), + Self::Semicolon => write!(f, ";"), + Self::Colon => write!(f, ":"), + Self::Plus => write!(f, "+"), + Self::Minus => write!(f, "-"), + Self::Comma => write!(f, ","), + Self::Asterisk => write!(f, "*"), + Self::Slash => write!(f, "/"), + Self::LessThan => write!(f, "<"), + Self::LessThanOrEqual => write!(f, "<="), + Self::GreaterThan => write!(f, ">"), + Self::GreaterThanOrEqual => write!(f, ">="), + Self::Assign => write!(f, "="), + Self::Equal => write!(f, "=="), + Self::LogicalAnd => write!(f, "&&"), + Self::LogicalOr => write!(f, "||"), + Self::LogicalNot => write!(f, "!"), + Self::NotEqual => write!(f, "!="), + Self::Dot => write!(f, "."), + Self::Caret => write!(f, "^"), + Self::Exp => write!(f, "**"), + } + } +} + #[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)] pub enum Keyword { /// Represents the `continue` keyword diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index e7c09ad..e5879a5 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -1,58 +1,78 @@ use compiler::Compiler; use parser::Parser; -use std::{ - ffi::{CStr, CString}, - io::BufWriter, -}; -use tokenizer::Tokenizer; +use safer_ffi::prelude::*; +use std::io::BufWriter; +use tokenizer::{Error as TokenizerError, Tokenizer}; -/// Takes a raw pointer to a string and compiles the `slang` code into valid IC10 -/// # Safety -/// This must be called with a valid string pointer from C# (or wherever is calling this function) -#[no_mangle] -pub unsafe extern "C" fn compile_from_string( - input_ptr: *const std::os::raw::c_char, -) -> *mut std::os::raw::c_char { - if input_ptr.is_null() { - return std::ptr::null_mut(); - } - - let c_str = unsafe { CStr::from_ptr(input_ptr) }; - - let Ok(input_str) = c_str.to_str() else { - return std::ptr::null_mut(); - }; +#[derive_ReprC] +#[repr(C)] +pub struct FfiToken { + pub text: safer_ffi::String, + pub tooltip: Option, + pub error: Option, + pub status: Option, + pub column: i32, +} +#[ffi_export] +pub fn compile_from_string(input: safer_ffi::String) -> safer_ffi::String { let mut writer = BufWriter::new(Vec::new()); - let tokenizer = Tokenizer::from(input_str); - let parser = Parser::new(tokenizer); + let tokenizer = Tokenizer::from(String::from(input)); + let parser = Parser::new(tokenizer); let compiler = Compiler::new(parser, &mut writer, None); - let Ok(()) = compiler.compile() else { - return std::ptr::null_mut(); - }; - - let Ok(buffer) = writer.into_inner() else { - return std::ptr::null_mut(); - }; - - let c_string = CString::from_vec_unchecked(buffer); - - c_string.into_raw() -} - -/// Takes ownership of the string pointer and drops it, freeing the memory -/// # Safety -/// Must be called with a valid string pointer -#[no_mangle] -pub unsafe extern "C" fn free_slang_string(input_ptr: *mut std::os::raw::c_char) { - if input_ptr.is_null() { - return; + if compiler.compile().is_err() { + return safer_ffi::String::EMPTY; } - unsafe { - // Takes ownership of the input string, and then drops it immediately - let _ = CString::from_raw(input_ptr); - } + let Ok(compiled_vec) = writer.into_inner() else { + return safer_ffi::String::EMPTY; + }; + + // Safety: I know the compiler only outputs valid utf8 + safer_ffi::String::from(unsafe { String::from_utf8_unchecked(compiled_vec) }) +} + +#[ffi_export] +pub fn tokenize_line(input: safer_ffi::String) -> safer_ffi::Vec { + let tokenizer = Tokenizer::from(String::from(input)); + + let mut tokens = Vec::::new(); + + for token in tokenizer { + match token { + Err(TokenizerError::NumberParseError(_, _, col)) + | Err(TokenizerError::UnknownSymbolError(_, _, col)) + | Err(TokenizerError::DecimalParseError(_, _, col)) + | Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col)) => { + tokens.push(FfiToken { + column: col as i32, + text: "".into(), + tooltip: None, + // Safety: it's okay to unwrap the err here because we are matching on the `Err` variant + error: Some(token.unwrap_err().to_string().into()), + status: None, + }); + } + Err(_) => return safer_ffi::Vec::EMPTY, + Ok(token) => tokens.push(FfiToken { + text: token.token_type.to_string().into(), + tooltip: None, + error: None, + status: None, + column: token.column as i32, + }), + } + } + + tokens.into() +} + +#[cfg(feature = "headers")] +pub fn generate_headers() -> std::io::Result<()> { + ::safer_ffi::headers::builder() + .with_language(safer_ffi::headers::Language::CSharp) + .to_file("SlangGlue.cs")? + .generate() } From c97c5763ae7b562e6bb66009c9fb2e767b95282f Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 03:51:05 -0700 Subject: [PATCH 05/11] automated C# glue FFI glue code --- csharp_mod/SlangGlue.cs | 80 +++++++++++++++++++++++ rust_compiler/Cargo.lock | 28 ++++---- rust_compiler/Cargo.toml | 10 +-- rust_compiler/build.rs | 9 --- rust_compiler/libs/compiler/src/v1.rs | 4 +- rust_compiler/libs/parser/src/lib.rs | 8 +-- rust_compiler/libs/tokenizer/src/lib.rs | 32 +++++---- rust_compiler/src/bin/generate_headers.rs | 3 + rust_compiler/src/lib.rs | 10 +-- 9 files changed, 130 insertions(+), 54 deletions(-) create mode 100644 csharp_mod/SlangGlue.cs delete mode 100644 rust_compiler/build.rs create mode 100644 rust_compiler/src/bin/generate_headers.rs diff --git a/csharp_mod/SlangGlue.cs b/csharp_mod/SlangGlue.cs new file mode 100644 index 0000000..8e99ae1 --- /dev/null +++ b/csharp_mod/SlangGlue.cs @@ -0,0 +1,80 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#pragma warning disable IDE0044, IDE0049, IDE0055, IDE1006, +#pragma warning disable SA1004, SA1008, SA1023, SA1028, +#pragma warning disable SA1121, SA1134, +#pragma warning disable SA1201, +#pragma warning disable SA1300, SA1306, SA1307, SA1310, SA1313, +#pragma warning disable SA1500, SA1505, SA1507, +#pragma warning disable SA1600, SA1601, SA1604, SA1605, SA1611, SA1615, SA1649, + +namespace Slang { +using System; +using System.Runtime.InteropServices; + +public unsafe partial class Ffi { +#if IOS + private const string RustLib = "slang.framework/slang"; +#else + private const string RustLib = "slang"; +#endif +} + +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_uint8_t { + public byte * ptr; + + public UIntPtr len; + + public UIntPtr cap; +} + +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_uint8_t compile_from_string ( + byte /*const*/ * input); +} + +[StructLayout(LayoutKind.Sequential, Size = 104)] +public unsafe struct FfiToken_t { + public Vec_uint8_t text; + + public Vec_uint8_t tooltip; + + public Vec_uint8_t error; + + public Vec_uint8_t status; + + public Int32 column; +} + +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_FfiToken_t { + public FfiToken_t * ptr; + + public UIntPtr len; + + public UIntPtr cap; +} + +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_FfiToken_t tokenize_line ( + byte /*const*/ * input); +} + + +} /* Slang */ diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 7d0923d..3b6f10b 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -779,6 +779,20 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "slang" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "compiler", + "parser", + "quick-error", + "rust_decimal", + "safer-ffi", + "tokenizer", +] + [[package]] name = "stabby" version = "36.2.2" @@ -814,20 +828,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "stationlang" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "compiler", - "parser", - "quick-error", - "rust_decimal", - "safer-ffi", - "tokenizer", -] - [[package]] name = "strsim" version = "0.11.1" diff --git a/rust_compiler/Cargo.toml b/rust_compiler/Cargo.toml index 4f5a6db..ae125f6 100644 --- a/rust_compiler/Cargo.toml +++ b/rust_compiler/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "stationlang" +name = "slang" version = "0.1.0" edition = "2021" @@ -27,9 +27,9 @@ path = "src/bin/generate_headers.rs" required-features = ["headers"] [lib] -name = "stationlang" +name = "slang" path = "src/lib.rs" -crate-type = ["cdylib"] +crate-type = ["cdylib", "rlib"] [dependencies] clap = { version = "^4.5", features = ["derive"] } @@ -40,9 +40,5 @@ parser = { path = "libs/parser" } compiler = { path = "libs/compiler" } safer-ffi = { workspace = true } - [dev-dependencies] anyhow = { version = "^1.0", features = ["backtrace"] } - -[build-dependencies] -safer-ffi = { version = "0.1", features = ["headers"] } diff --git a/rust_compiler/build.rs b/rust_compiler/build.rs deleted file mode 100644 index 567d489..0000000 --- a/rust_compiler/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -fn main() -> ::std::io::Result<()> { - safer_ffi::headers::builder() - .with_language(safer_ffi::headers::Language::CSharp) - .to_file("../csharp_mod/SlangStubs.cs")? - .generate() - .unwrap(); - - Ok(()) -} diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index 2b1709d..427ef57 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -68,7 +68,7 @@ struct CompilationResult { } pub struct Compiler<'a, W: std::io::Write> { - parser: ASTParser, + parser: ASTParser<'a>, function_locations: HashMap, function_metadata: HashMap>, devices: HashMap, @@ -83,7 +83,7 @@ pub struct Compiler<'a, W: std::io::Write> { impl<'a, W: std::io::Write> Compiler<'a, W> { pub fn new( - parser: ASTParser, + parser: ASTParser<'a>, writer: &'a mut BufWriter, config: Option, ) -> Self { diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index 481524b..9367959 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -113,13 +113,13 @@ macro_rules! token_matches { }; } -pub struct Parser { - tokenizer: TokenizerBuffer, +pub struct Parser<'a> { + tokenizer: TokenizerBuffer<'a>, current_token: Option, } -impl Parser { - pub fn new(tokenizer: Tokenizer) -> Self { +impl<'a> Parser<'a> { + pub fn new(tokenizer: Tokenizer<'a>) -> Self { Parser { tokenizer: TokenizerBuffer::new(tokenizer), current_token: None, diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index 9494cc3..43670c9 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -39,15 +39,15 @@ pub trait Tokenize: Read + Seek {} impl Tokenize for T where T: Read + Seek {} -pub struct Tokenizer { - reader: BufReader>, +pub struct Tokenizer<'a> { + reader: BufReader>, char_buffer: [u8; 1], line: usize, column: usize, returned_eof: bool, } -impl Tokenizer { +impl<'a> Tokenizer<'a> { pub fn from_path(input_file: impl Into) -> Result { let file = std::fs::File::open(input_file.into())?; let reader = BufReader::new(Box::new(file) as Box); @@ -62,7 +62,7 @@ impl Tokenizer { } } -impl From for Tokenizer { +impl<'a> From for Tokenizer<'a> { fn from(input: String) -> Self { let reader = BufReader::new(Box::new(Cursor::new(input)) as Box); @@ -76,13 +76,19 @@ impl From for Tokenizer { } } -impl From<&str> for Tokenizer { - fn from(value: &str) -> Self { - Self::from(value.to_string()) +impl<'a> From<&'a str> for Tokenizer<'a> { + fn from(value: &'a str) -> Self { + Self { + reader: BufReader::new(Box::new(Cursor::new(value)) as Box), + char_buffer: [0], + column: 1, + line: 1, + returned_eof: false, + } } } -impl Tokenizer { +impl<'a> Tokenizer<'a> { /// Consumes the tokenizer and returns the next token in the stream /// If there are no more tokens in the stream, this function returns None /// If there is an error reading the stream, this function returns an error @@ -447,7 +453,7 @@ impl Tokenizer { } } -impl Iterator for Tokenizer { +impl<'a> Iterator for Tokenizer<'a> { type Item = Result; fn next(&mut self) -> Option { @@ -455,14 +461,14 @@ impl Iterator for Tokenizer { } } -pub struct TokenizerBuffer { - tokenizer: Tokenizer, +pub struct TokenizerBuffer<'a> { + tokenizer: Tokenizer<'a>, buffer: VecDeque, history: VecDeque, } -impl TokenizerBuffer { - pub fn new(tokenizer: Tokenizer) -> Self { +impl<'a> TokenizerBuffer<'a> { + pub fn new(tokenizer: Tokenizer<'a>) -> Self { Self { tokenizer, buffer: VecDeque::new(), diff --git a/rust_compiler/src/bin/generate_headers.rs b/rust_compiler/src/bin/generate_headers.rs new file mode 100644 index 0000000..54cdb6a --- /dev/null +++ b/rust_compiler/src/bin/generate_headers.rs @@ -0,0 +1,3 @@ +fn main() -> std::io::Result<()> { + ::slang::generate_headers() +} diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index e5879a5..f2b5ead 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -15,10 +15,10 @@ pub struct FfiToken { } #[ffi_export] -pub fn compile_from_string(input: safer_ffi::String) -> safer_ffi::String { +pub fn compile_from_string(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::String { let mut writer = BufWriter::new(Vec::new()); - let tokenizer = Tokenizer::from(String::from(input)); + let tokenizer = Tokenizer::from(input.to_str()); let parser = Parser::new(tokenizer); let compiler = Compiler::new(parser, &mut writer, None); @@ -35,8 +35,8 @@ pub fn compile_from_string(input: safer_ffi::String) -> safer_ffi::String { } #[ffi_export] -pub fn tokenize_line(input: safer_ffi::String) -> safer_ffi::Vec { - let tokenizer = Tokenizer::from(String::from(input)); +pub fn tokenize_line(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::Vec { + let tokenizer = Tokenizer::from(input.to_str()); let mut tokens = Vec::::new(); @@ -73,6 +73,6 @@ pub fn tokenize_line(input: safer_ffi::String) -> safer_ffi::Vec { pub fn generate_headers() -> std::io::Result<()> { ::safer_ffi::headers::builder() .with_language(safer_ffi::headers::Language::CSharp) - .to_file("SlangGlue.cs")? + .to_file("../csharp_mod/SlangGlue.cs")? .generate() } From 4ac2e6408f1be042bc9492a93f40aabe1c9e7e3d Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 04:24:34 -0700 Subject: [PATCH 06/11] Extension methods to help with Rust FFI strings and vecs --- csharp_mod/SlangExtensions.cs | 52 +++++++++++++++++++ csharp_mod/SlangGlue.cs | 12 +++++ .../libs/compiler/src/test/branching.rs | 1 - rust_compiler/src/lib.rs | 10 ++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 csharp_mod/SlangExtensions.cs diff --git a/csharp_mod/SlangExtensions.cs b/csharp_mod/SlangExtensions.cs new file mode 100644 index 0000000..be39dcd --- /dev/null +++ b/csharp_mod/SlangExtensions.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Text; +using StationeersIC10Editor; + +namespace Slang +{ + public static unsafe class SlangExtensions + { + // 1. Convert the Rust Byte Vector (Vec_uint8_t) to a C# String + public static string AsString(this Vec_uint8_t vec) + { + if (vec.ptr == null || vec.len == UIntPtr.Zero) + { + return string.Empty; + } + + // Rust strings are UTF-8. Read bytes from raw pointer. + var toReturn = Encoding.UTF8.GetString(vec.ptr, (int)vec.len); + + return toReturn; + } + + public static void Drop(this Vec_uint8_t vec) + { + Ffi.free_string(vec); + } + + // 2. Convert Rust Token Vector to C# List + public static List AsList(this Vec_FfiToken_t vec) + { + var list = new List((int)vec.len); + var currentPtr = vec.ptr; + + // Iterate through the raw memory array + for (int i = 0; i < (int)vec.len; i++) + { + // Dereference pointer to get the struct at index i + FfiToken_t token = currentPtr[i]; + + var newToken = new Token(token.text.AsString(), token.column); + newToken.Error = token.error.AsString(); + + list.Add(newToken); + } + + Ffi.free_ffi_token_vec(vec); + + return list; + } + } +} diff --git a/csharp_mod/SlangGlue.cs b/csharp_mod/SlangGlue.cs index 8e99ae1..bc5b128 100644 --- a/csharp_mod/SlangGlue.cs +++ b/csharp_mod/SlangGlue.cs @@ -70,6 +70,18 @@ public unsafe struct Vec_FfiToken_t { public UIntPtr cap; } +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_ffi_token_vec ( + Vec_FfiToken_t v); +} + +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_string ( + Vec_uint8_t s); +} + public unsafe partial class Ffi { [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern Vec_FfiToken_t tokenize_line ( diff --git a/rust_compiler/libs/compiler/src/test/branching.rs b/rust_compiler/libs/compiler/src/test/branching.rs index fb06024..d23d880 100644 --- a/rust_compiler/libs/compiler/src/test/branching.rs +++ b/rust_compiler/libs/compiler/src/test/branching.rs @@ -155,4 +155,3 @@ fn test_spilled_variable_update_in_branch() -> anyhow::Result<()> { Ok(()) } - diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index f2b5ead..2657211 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -69,6 +69,16 @@ pub fn tokenize_line(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::Vec tokens.into() } +#[ffi_export] +pub fn free_ffi_token_vec(v: safer_ffi::Vec) { + drop(v) +} + +#[ffi_export] +pub fn free_string(s: safer_ffi::String) { + drop(s) +} + #[cfg(feature = "headers")] pub fn generate_headers() -> std::io::Result<()> { ::safer_ffi::headers::builder() From 036be297eacb026ad4e85a8f71f304bf74e5ce0c Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 04:47:35 -0700 Subject: [PATCH 07/11] renamed guid --- csharp_mod/SlangPlugin.cs | 41 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/csharp_mod/SlangPlugin.cs b/csharp_mod/SlangPlugin.cs index 0225cfa..0822c23 100644 --- a/csharp_mod/SlangPlugin.cs +++ b/csharp_mod/SlangPlugin.cs @@ -44,16 +44,6 @@ namespace Slang public const string PluginGuid = "com.biddydev.slang"; public const string PluginName = "Slang"; - const string RUST_DLL_NAME = "slang.compiler.dll"; - - /// Takes raw `Slang` source code and compiles it into IC10 - [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr compile_from_string(string input); - - /// Frees memory that was allocated by the FFI call to `compile_from_string` - [DllImport(RUST_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] - private static extern void free_slang_string(IntPtr ptr); - private static Regex? _slangSourceCheck = null; private static Regex SlangSourceCheck @@ -69,19 +59,28 @@ namespace Slang } } - public static string Compile(string source) + public static unsafe string Compile(string source) { if (string.IsNullOrEmpty(source)) return ""; - IntPtr ptr = compile_from_string(source); - try + // Add a null terminator char at the end of the source string (turns into a CStr) + source += "\0"; + + var bytes = System.Text.Encoding.UTF8.GetBytes(source); + + // don't move my memory around, C#! + fixed (byte* pBytes = bytes) { - return Marshal.PtrToStringAnsi(ptr); - } - finally - { - free_slang_string(ptr); + var compiled = Ffi.compile_from_string(pBytes); + try + { + return compiled.AsString(); + } + finally + { + Ffi.free_string(compiled); + } } } @@ -101,8 +100,8 @@ namespace Slang private void Awake() { L.SetLogger(Logger); - ExtractNativeDll(RUST_DLL_NAME); - var harmony = new Harmony("com.dbidwell94.slang"); + ExtractNativeDll("slang.dll"); + var harmony = new Harmony(PluginGuid); harmony.PatchAll(); } @@ -117,7 +116,7 @@ namespace Slang if (stream == null) { L.Error( - $"{RUST_DLL_NAME} compiler not found. This means it was not embedded in the mod. Please contact the mod author!" + "slang.dll compiler not found. This means it was not embedded in the mod. Please contact the mod author!" ); return; } From e274b3355355d15008c122ff1d7d8d01b750f62f Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 14:44:26 -0700 Subject: [PATCH 08/11] wip -- marshal UTF16 string from C# to Rust to avoid GC in C# --- .../{SlangExtensions.cs => Extensions.cs} | 7 +++-- csharp_mod/{SlangGlue.cs => FfiGlue.cs} | 0 .../{SlangFormatter.cs => Formatter.cs} | 2 +- csharp_mod/Marshal.cs | 30 +++++++++++++++++++ csharp_mod/{SlangPlugin.cs => Plugin.cs} | 4 +-- rust_compiler/libs/parser/src/sys_call.rs | 4 +-- rust_compiler/src/lib.rs | 19 ++++++++---- 7 files changed, 52 insertions(+), 14 deletions(-) rename csharp_mod/{SlangExtensions.cs => Extensions.cs} (89%) rename csharp_mod/{SlangGlue.cs => FfiGlue.cs} (100%) rename csharp_mod/{SlangFormatter.cs => Formatter.cs} (76%) create mode 100644 csharp_mod/Marshal.cs rename csharp_mod/{SlangPlugin.cs => Plugin.cs} (96%) diff --git a/csharp_mod/SlangExtensions.cs b/csharp_mod/Extensions.cs similarity index 89% rename from csharp_mod/SlangExtensions.cs rename to csharp_mod/Extensions.cs index be39dcd..03b8426 100644 --- a/csharp_mod/SlangExtensions.cs +++ b/csharp_mod/Extensions.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Text; using StationeersIC10Editor; @@ -27,9 +26,11 @@ namespace Slang } // 2. Convert Rust Token Vector to C# List - public static List AsList(this Vec_FfiToken_t vec) + public static Line AsList(this Vec_FfiToken_t vec) { - var list = new List((int)vec.len); + var list = new Line(); + list.Capacity = (int)vec.len; + var currentPtr = vec.ptr; // Iterate through the raw memory array diff --git a/csharp_mod/SlangGlue.cs b/csharp_mod/FfiGlue.cs similarity index 100% rename from csharp_mod/SlangGlue.cs rename to csharp_mod/FfiGlue.cs diff --git a/csharp_mod/SlangFormatter.cs b/csharp_mod/Formatter.cs similarity index 76% rename from csharp_mod/SlangFormatter.cs rename to csharp_mod/Formatter.cs index fd96cba..6bd38fd 100644 --- a/csharp_mod/SlangFormatter.cs +++ b/csharp_mod/Formatter.cs @@ -6,7 +6,7 @@ namespace Slang { public override Line ParseLine(string line) { - throw new System.NotImplementedException(); + return Marshal.TokenizeLine(line); } } } diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs new file mode 100644 index 0000000..83f283a --- /dev/null +++ b/csharp_mod/Marshal.cs @@ -0,0 +1,30 @@ +using System; +using System.Text; +using StationeersIC10Editor; + +namespace Slang +{ + public static class Marshal + { + public static unsafe Line TokenizeLine(string input) + { + if (String.IsNullOrEmpty(input)) + { + return new Line(); + } + + // Make sure the string is a null terminated string + if (input[input.Length - 1] != '\0') + { + input += '\0'; + } + + var strBytes = Encoding.UTF8.GetBytes(input); + + fixed (byte* ptrString = strBytes) + { + return Ffi.tokenize_line(ptrString).AsList(); + } + } + } +} diff --git a/csharp_mod/SlangPlugin.cs b/csharp_mod/Plugin.cs similarity index 96% rename from csharp_mod/SlangPlugin.cs rename to csharp_mod/Plugin.cs index 0822c23..78e5c67 100644 --- a/csharp_mod/SlangPlugin.cs +++ b/csharp_mod/Plugin.cs @@ -1,10 +1,9 @@ -using System; using System.IO; using System.Reflection; -using System.Runtime.InteropServices; using System.Text.RegularExpressions; using BepInEx; using HarmonyLib; +using StationeersIC10Editor; namespace Slang { @@ -103,6 +102,7 @@ namespace Slang ExtractNativeDll("slang.dll"); var harmony = new Harmony(PluginGuid); harmony.PatchAll(); + CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true); } private void ExtractNativeDll(string fileName) diff --git a/rust_compiler/libs/parser/src/sys_call.rs b/rust_compiler/libs/parser/src/sys_call.rs index bb2c882..576094f 100644 --- a/rust_compiler/libs/parser/src/sys_call.rs +++ b/rust_compiler/libs/parser/src/sys_call.rs @@ -124,7 +124,7 @@ pub enum System { /// Loads a LogicType from all connected network devices, aggregating them via a /// batchMode /// ## In Game - /// lb r? deviceHash loggicType batchMode + /// lb r? deviceHash logicType batchMode /// ## Examples /// lb r0 HASH("StructureWallLight") On Minimum LoadBatch(LiteralOrVariable, Literal, Literal), @@ -137,7 +137,7 @@ pub enum System { /// Represents a function which stores a setting to all devices that match /// the given deviceHash /// ## In Game - /// `sb deviceHash logictype r?` + /// `sb deviceHash logicType r?` /// ## Example /// `sb HASH("Doors") Lock 1` SetOnDeviceBatched(LiteralOrVariable, Literal, Box), diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index 2657211..991b421 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -14,11 +14,15 @@ pub struct FfiToken { pub column: i32, } +/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because +/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. +/// This should result in the ability to compile many times without triggering frame drops +/// from the GC from a `GetBytes()` call on a string in C#. #[ffi_export] -pub fn compile_from_string(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::String { +pub fn compile_from_string(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::String { let mut writer = BufWriter::new(Vec::new()); - let tokenizer = Tokenizer::from(input.to_str()); + let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice())); let parser = Parser::new(tokenizer); let compiler = Compiler::new(parser, &mut writer, None); @@ -33,10 +37,13 @@ pub fn compile_from_string(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ff // Safety: I know the compiler only outputs valid utf8 safer_ffi::String::from(unsafe { String::from_utf8_unchecked(compiled_vec) }) } - +/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because +/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. +/// This should result in the ability to tokenize many times without triggering frame drops +/// from the GC from a `GetBytes()` call on a string in C#. #[ffi_export] -pub fn tokenize_line(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::Vec { - let tokenizer = Tokenizer::from(input.to_str()); +pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec { + let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice())); let mut tokens = Vec::::new(); @@ -83,6 +90,6 @@ pub fn free_string(s: safer_ffi::String) { pub fn generate_headers() -> std::io::Result<()> { ::safer_ffi::headers::builder() .with_language(safer_ffi::headers::Language::CSharp) - .to_file("../csharp_mod/SlangGlue.cs")? + .to_file("../csharp_mod/FfiGlue.cs")? .generate() } From 9a9fa9517f89f53d461af07398029dbcdbaaf5e5 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 17:11:08 -0700 Subject: [PATCH 09/11] initial integration with ic10editor mod --- csharp_mod/Extensions.cs | 22 ++++++++-- csharp_mod/FfiGlue.cs | 50 +++++++++++++++++++++-- csharp_mod/Marshal.cs | 54 ++++++++++++++++++++----- csharp_mod/Plugin.cs | 47 ++++++++++----------- csharp_mod/stationeersSlang.csproj | 2 +- rust_compiler/Cargo.lock | 24 +++++------ rust_compiler/libs/compiler/src/v1.rs | 4 +- rust_compiler/libs/parser/src/lib.rs | 4 +- rust_compiler/libs/tokenizer/src/lib.rs | 6 ++- rust_compiler/src/lib.rs | 33 ++++++++++----- 10 files changed, 175 insertions(+), 71 deletions(-) diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index 03b8426..c309f00 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -6,7 +6,12 @@ namespace Slang { public static unsafe class SlangExtensions { - // 1. Convert the Rust Byte Vector (Vec_uint8_t) to a C# String + /** + * + * This is a helper method to convert a Rust struct for a string pointer + * into a C# style string. + * + */ public static string AsString(this Vec_uint8_t vec) { if (vec.ptr == null || vec.len == UIntPtr.Zero) @@ -20,16 +25,28 @@ namespace Slang return toReturn; } + /** + * This will free a Rust string struct. Because this is a pointer to a struct, this memory + * is managed by Rust, therefor it must be freed by Rust + * + */ public static void Drop(this Vec_uint8_t vec) { Ffi.free_string(vec); } - // 2. Convert Rust Token Vector to C# List + /** + * This helper converts a Rust vec to a C# List. This handles freeing the + * Rust allocation after the List is created, there is no need to Drop this memory. + * + */ public static Line AsList(this Vec_FfiToken_t vec) { + L.Info("Converting output into a C# List."); var list = new Line(); + L.Info("Created new `Line`."); list.Capacity = (int)vec.len; + L.Info("Changed `Capacity` to be returned Vec's len"); var currentPtr = vec.ptr; @@ -40,7 +57,6 @@ namespace Slang FfiToken_t token = currentPtr[i]; var newToken = new Token(token.text.AsString(), token.column); - newToken.Error = token.error.AsString(); list.Add(newToken); } diff --git a/csharp_mod/FfiGlue.cs b/csharp_mod/FfiGlue.cs index bc5b128..da8fd3b 100644 --- a/csharp_mod/FfiGlue.cs +++ b/csharp_mod/FfiGlue.cs @@ -23,10 +23,42 @@ public unsafe partial class Ffi { #if IOS private const string RustLib = "slang.framework/slang"; #else - private const string RustLib = "slang"; + public const string RustLib = "slang_compiler.dll"; #endif } +/// +/// &'lt [T] but with a guaranteed #[repr(C)] layout. +/// +/// # C layout (for some given type T) +/// +/// ```c +/// typedef struct { +/// // Cannot be NULL +/// T * ptr; +/// size_t len; +/// } slice_T; +/// ``` +/// +/// # Nullable pointer? +/// +/// If you want to support the above typedef, but where the ptr field is +/// allowed to be NULL (with the contents of len then being undefined) +/// use the Option< slice_ptr<_> > type. +/// +[StructLayout(LayoutKind.Sequential, Size = 16)] +public unsafe struct slice_ref_uint16_t { + /// + /// Pointer to the first element (if any). + /// + public UInt16 /*const*/ * ptr; + + /// + /// Element count + /// + public UIntPtr len; +} + /// /// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout /// @@ -40,9 +72,15 @@ public unsafe struct Vec_uint8_t { } public unsafe partial class Ffi { + /// + /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because + /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. + /// This should result in the ability to compile many times without triggering frame drops + /// from the GC from a GetBytes() call on a string in C#. + /// [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern Vec_uint8_t compile_from_string ( - byte /*const*/ * input); + slice_ref_uint16_t input); } [StructLayout(LayoutKind.Sequential, Size = 104)] @@ -83,9 +121,15 @@ public unsafe partial class Ffi { } public unsafe partial class Ffi { + /// + /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because + /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. + /// This should result in the ability to tokenize many times without triggering frame drops + /// from the GC from a GetBytes() call on a string in C#. + /// [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern Vec_FfiToken_t tokenize_line ( - byte /*const*/ * input); + slice_ref_uint16_t input); } diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs index 83f283a..753c41d 100644 --- a/csharp_mod/Marshal.cs +++ b/csharp_mod/Marshal.cs @@ -1,29 +1,63 @@ using System; -using System.Text; using StationeersIC10Editor; namespace Slang { public static class Marshal { - public static unsafe Line TokenizeLine(string input) + public static unsafe Line TokenizeLine(string source) { - if (String.IsNullOrEmpty(input)) + if (String.IsNullOrEmpty(source)) { return new Line(); } - // Make sure the string is a null terminated string - if (input[input.Length - 1] != '\0') + L.Info("Input string not empty"); + + fixed (char* ptrString = source) { - input += '\0'; + L.Info("In `fixed` block."); + var input = new slice_ref_uint16_t + { + ptr = (ushort*)ptrString, + len = (UIntPtr)source.Length, + }; + L.Info("Calling tokenize_line"); + return Ffi.tokenize_line(input).AsList(); + } + } + + public static unsafe bool CompileFromString(string inputString, out string compiledString) + { + if (String.IsNullOrEmpty(inputString)) + { + compiledString = String.Empty; + return false; } - var strBytes = Encoding.UTF8.GetBytes(input); - - fixed (byte* ptrString = strBytes) + fixed (char* ptrString = inputString) { - return Ffi.tokenize_line(ptrString).AsList(); + var input = new slice_ref_uint16_t + { + ptr = (ushort*)ptrString, + len = (UIntPtr)inputString.Length, + }; + + var result = Ffi.compile_from_string(input); + try + { + if ((ulong)result.len < 1) + { + compiledString = String.Empty; + return false; + } + compiledString = result.AsString(); + return true; + } + finally + { + result.Drop(); + } } } } diff --git a/csharp_mod/Plugin.cs b/csharp_mod/Plugin.cs index 78e5c67..b559d1f 100644 --- a/csharp_mod/Plugin.cs +++ b/csharp_mod/Plugin.cs @@ -38,6 +38,7 @@ namespace Slang } [BepInPlugin(PluginGuid, PluginName, "0.1.0")] + [BepInDependency(StationeersIC10Editor.IC10EditorPlugin.PluginGuid)] public class SlangPlugin : BaseUnityPlugin { public const string PluginGuid = "com.biddydev.slang"; @@ -60,26 +61,15 @@ namespace Slang public static unsafe string Compile(string source) { - if (string.IsNullOrEmpty(source)) - return ""; - - // Add a null terminator char at the end of the source string (turns into a CStr) - source += "\0"; - - var bytes = System.Text.Encoding.UTF8.GetBytes(source); - - // don't move my memory around, C#! - fixed (byte* pBytes = bytes) + string compiled; + if (Marshal.CompileFromString(source, out compiled)) { - var compiled = Ffi.compile_from_string(pBytes); - try - { - return compiled.AsString(); - } - finally - { - Ffi.free_string(compiled); - } + // TODO: handle saving the original source code + return compiled; + } + else + { + return compiled; } } @@ -99,13 +89,16 @@ namespace Slang private void Awake() { L.SetLogger(Logger); - ExtractNativeDll("slang.dll"); - var harmony = new Harmony(PluginGuid); - harmony.PatchAll(); - CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true); + + if (ExtractNativeDll(Ffi.RustLib)) + { + var harmony = new Harmony(PluginGuid); + harmony.PatchAll(); + CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true); + } } - private void ExtractNativeDll(string fileName) + private bool ExtractNativeDll(string fileName) { string destinationPath = Path.Combine(Path.GetDirectoryName(Info.Location), fileName); @@ -116,9 +109,9 @@ namespace Slang if (stream == null) { L.Error( - "slang.dll compiler not found. This means it was not embedded in the mod. Please contact the mod author!" + $"{Ffi.RustLib} not found. This means it was not embedded in the mod. Please contact the mod author!" ); - return; + return false; } try @@ -127,10 +120,12 @@ namespace Slang { stream.CopyTo(fileStream); } + return true; } catch (IOException e) { L.Warning($"Could not overwrite {fileName} (it might be in use): {e.Message}"); + return false; } } } diff --git a/csharp_mod/stationeersSlang.csproj b/csharp_mod/stationeersSlang.csproj index 2810128..c281b7f 100644 --- a/csharp_mod/stationeersSlang.csproj +++ b/csharp_mod/stationeersSlang.csproj @@ -54,7 +54,7 @@ - slang.compiler.dll + slang_compiler.dll diff --git a/rust_compiler/Cargo.lock b/rust_compiler/Cargo.lock index 3b6f10b..7a9df42 100644 --- a/rust_compiler/Cargo.lock +++ b/rust_compiler/Cargo.lock @@ -386,9 +386,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -967,9 +967,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", @@ -980,9 +980,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -990,9 +990,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", "proc-macro2", @@ -1003,9 +1003,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] @@ -1027,9 +1027,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ "memchr", ] diff --git a/rust_compiler/libs/compiler/src/v1.rs b/rust_compiler/libs/compiler/src/v1.rs index 427ef57..56b32ca 100644 --- a/rust_compiler/libs/compiler/src/v1.rs +++ b/rust_compiler/libs/compiler/src/v1.rs @@ -1125,9 +1125,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> { })) } - _ => { - todo!() - } + t => Err(Error::Unknown(format!("{t:?}\n\nNot yet implemented"))), } } diff --git a/rust_compiler/libs/parser/src/lib.rs b/rust_compiler/libs/parser/src/lib.rs index 9367959..3789894 100644 --- a/rust_compiler/libs/parser/src/lib.rs +++ b/rust_compiler/libs/parser/src/lib.rs @@ -1260,7 +1260,9 @@ impl<'a> Parser<'a> { let arg = literal_or_variable!(invocation.arguments.first()); Ok(SysCall::Math(sys_call::Math::Trunc(arg))) } - _ => todo!(), + _ => Err(Error::UnsupportedKeyword(token_from_option!( + self.current_token + ))), } } } diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index 43670c9..3df8fc8 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -457,7 +457,11 @@ impl<'a> Iterator for Tokenizer<'a> { type Item = Result; fn next(&mut self) -> Option { - todo!() + match self.next_token() { + Ok(Some(tok)) => Some(Ok(tok)), + Ok(None) => None, + Err(e) => Some(Err(e)), + } } } diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index 991b421..ed994a3 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -8,9 +8,9 @@ use tokenizer::{Error as TokenizerError, Tokenizer}; #[repr(C)] pub struct FfiToken { pub text: safer_ffi::String, - pub tooltip: Option, - pub error: Option, - pub status: Option, + pub tooltip: safer_ffi::String, + pub error: safer_ffi::String, + pub status: safer_ffi::String, pub column: i32, } @@ -56,18 +56,18 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec return safer_ffi::Vec::EMPTY, Ok(token) => tokens.push(FfiToken { text: token.token_type.to_string().into(), - tooltip: None, - error: None, - status: None, + tooltip: "".into(), + error: "".into(), + status: "".into(), column: token.column as i32, }), } @@ -88,8 +88,19 @@ pub fn free_string(s: safer_ffi::String) { #[cfg(feature = "headers")] pub fn generate_headers() -> std::io::Result<()> { + let file_name = "../csharp_mod/FfiGlue.cs"; ::safer_ffi::headers::builder() .with_language(safer_ffi::headers::Language::CSharp) - .to_file("../csharp_mod/FfiGlue.cs")? - .generate() + .to_file(file_name)? + .generate()?; + + let content = std::fs::read_to_string(file_name)?; + + let content = content.replace( + "private const string RustLib = \"slang\";", + "public const string RustLib = \"slang_compiler.dll\";", + ); + + std::fs::write(file_name, content)?; + Ok(()) } From 804bf11d0209ccc0ea4453ae776ffd8a5b673038 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 17:34:38 -0700 Subject: [PATCH 10/11] Don't send EOF token to the C# mod --- csharp_mod/Extensions.cs | 3 --- csharp_mod/Marshal.cs | 4 ---- csharp_mod/Patches.cs | 2 +- rust_compiler/src/lib.rs | 5 +++-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index c309f00..e957a88 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -42,11 +42,8 @@ namespace Slang */ public static Line AsList(this Vec_FfiToken_t vec) { - L.Info("Converting output into a C# List."); var list = new Line(); - L.Info("Created new `Line`."); list.Capacity = (int)vec.len; - L.Info("Changed `Capacity` to be returned Vec's len"); var currentPtr = vec.ptr; diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs index 753c41d..bc52a99 100644 --- a/csharp_mod/Marshal.cs +++ b/csharp_mod/Marshal.cs @@ -12,17 +12,13 @@ namespace Slang return new Line(); } - L.Info("Input string not empty"); - fixed (char* ptrString = source) { - L.Info("In `fixed` block."); var input = new slice_ref_uint16_t { ptr = (ushort*)ptrString, len = (UIntPtr)source.Length, }; - L.Info("Calling tokenize_line"); return Ffi.tokenize_line(input).AsList(); } } diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs index 78d067f..d4c4d82 100644 --- a/csharp_mod/Patches.cs +++ b/csharp_mod/Patches.cs @@ -19,7 +19,7 @@ namespace Slang return; } - L.Info("Detected Slang source, compiling..."); + L.Debug("Detected Slang source, compiling..."); // Compile the Slang source into IC10 string compiled = SlangPlugin.Compile(result); diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index ed994a3..9913506 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -2,7 +2,7 @@ use compiler::Compiler; use parser::Parser; use safer_ffi::prelude::*; use std::io::BufWriter; -use tokenizer::{Error as TokenizerError, Tokenizer}; +use tokenizer::{token::TokenType, Error as TokenizerError, Tokenizer}; #[derive_ReprC] #[repr(C)] @@ -63,13 +63,14 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec return safer_ffi::Vec::EMPTY, - Ok(token) => tokens.push(FfiToken { + Ok(token) if !matches!(token.token_type, TokenType::EOF) => tokens.push(FfiToken { text: token.token_type.to_string().into(), tooltip: "".into(), error: "".into(), status: "".into(), column: token.column as i32, }), + _ => {} } } From f172ac58999fbef1898a9e10c5a90cb3e7c94281 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Fri, 28 Nov 2025 18:01:57 -0700 Subject: [PATCH 11/11] buffer original source data into the Token struct for use in-game --- rust_compiler/libs/tokenizer/src/lib.rs | 103 +++++++++++++++++----- rust_compiler/libs/tokenizer/src/token.rs | 9 +- rust_compiler/src/lib.rs | 15 ++-- 3 files changed, 100 insertions(+), 27 deletions(-) diff --git a/rust_compiler/libs/tokenizer/src/lib.rs b/rust_compiler/libs/tokenizer/src/lib.rs index 3df8fc8..8909d5b 100644 --- a/rust_compiler/libs/tokenizer/src/lib.rs +++ b/rust_compiler/libs/tokenizer/src/lib.rs @@ -18,18 +18,18 @@ quick_error! { display("IO Error: {}", err) source(err) } - NumberParseError(err: std::num::ParseIntError, line: usize, column: usize) { + NumberParseError(err: std::num::ParseIntError, line: usize, column: usize, original: String) { display("Number Parse Error: {}\nLine: {}, Column: {}", err, line, column) source(err) } - DecimalParseError(err: rust_decimal::Error, line: usize, column: usize) { + DecimalParseError(err: rust_decimal::Error, line: usize, column: usize, original: String) { display("Decimal Parse Error: {}\nLine: {}, Column: {}", err, line, column) source(err) } - UnknownSymbolError(char: char, line: usize, column: usize) { + UnknownSymbolError(char: char, line: usize, column: usize, original: String) { display("Unknown Symbol: {}\nLine: {}, Column: {}", char, line, column) } - UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize) { + UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize, original: String) { display("Unknown Keyword or Identifier: {}\nLine: {}, Column: {}", val, line, column) } } @@ -45,6 +45,7 @@ pub struct Tokenizer<'a> { line: usize, column: usize, returned_eof: bool, + string_buffer: String, } impl<'a> Tokenizer<'a> { @@ -58,6 +59,7 @@ impl<'a> Tokenizer<'a> { column: 1, char_buffer: [0], returned_eof: false, + string_buffer: String::new(), }) } } @@ -72,6 +74,7 @@ impl<'a> From for Tokenizer<'a> { column: 1, char_buffer: [0], returned_eof: false, + string_buffer: String::new(), } } } @@ -84,6 +87,7 @@ impl<'a> From<&'a str> for Tokenizer<'a> { column: 1, line: 1, returned_eof: false, + string_buffer: String::new(), } } } @@ -111,6 +115,7 @@ impl<'a> Tokenizer<'a> { self.column += 1; } + self.string_buffer.push(c); Ok(Some(c)) } @@ -177,7 +182,12 @@ impl<'a> Tokenizer<'a> { return self.tokenize_keyword_or_identifier(next_char).map(Some); } _ => { - return Err(Error::UnknownSymbolError(next_char, self.line, self.column)); + return Err(Error::UnknownSymbolError( + next_char, + self.line, + self.column, + std::mem::take(&mut self.string_buffer), + )); } } } @@ -185,7 +195,12 @@ impl<'a> Tokenizer<'a> { Ok(None) } else { self.returned_eof = true; - Ok(Some(Token::new(TokenType::EOF, self.line, self.column))) + Ok(Some(Token::new( + TokenType::EOF, + self.line, + self.column, + Some(std::mem::take(&mut self.string_buffer)), + ))) } } @@ -212,6 +227,7 @@ impl<'a> Tokenizer<'a> { TokenType::Symbol(Symbol::$symbol), self.line, self.column, + Some(std::mem::take(&mut self.string_buffer)), )) }; } @@ -279,6 +295,7 @@ impl<'a> Tokenizer<'a> { first_symbol, self.line, self.column, + std::mem::take(&mut self.string_buffer), )), } } @@ -328,17 +345,28 @@ impl<'a> Tokenizer<'a> { let decimal_scale = decimal.len() as u32; let number = format!("{}{}", primary, decimal) .parse::() - .map_err(|e| Error::NumberParseError(e, self.line, self.column))?; + .map_err(|e| { + Error::NumberParseError( + e, + self.line, + self.column, + std::mem::take(&mut self.string_buffer), + ) + })?; Number::Decimal( - Decimal::try_from_i128_with_scale(number, decimal_scale) - .map_err(|e| Error::DecimalParseError(e, line, column))?, + Decimal::try_from_i128_with_scale(number, decimal_scale).map_err(|e| { + Error::DecimalParseError( + e, + line, + column, + std::mem::take(&mut self.string_buffer), + ) + })?, ) } else { - Number::Integer( - primary - .parse() - .map_err(|e| Error::NumberParseError(e, line, column))?, - ) + Number::Integer(primary.parse().map_err(|e| { + Error::NumberParseError(e, line, column, std::mem::take(&mut self.string_buffer)) + })?) }; // check if the next char is a temperature suffix @@ -347,14 +375,31 @@ impl<'a> Tokenizer<'a> { 'c' => Temperature::Celsius(number), 'f' => Temperature::Fahrenheit(number), 'k' => Temperature::Kelvin(number), - _ => return Ok(Token::new(TokenType::Number(number), line, column)), + _ => { + return Ok(Token::new( + TokenType::Number(number), + line, + column, + Some(std::mem::take(&mut self.string_buffer)), + )); + } } .to_kelvin(); self.next_char()?; - Ok(Token::new(TokenType::Number(temperature), line, column)) + Ok(Token::new( + TokenType::Number(temperature), + line, + column, + Some(std::mem::take(&mut self.string_buffer)), + )) } else { - Ok(Token::new(TokenType::Number(number), line, column)) + Ok(Token::new( + TokenType::Number(number), + line, + column, + Some(std::mem::take(&mut self.string_buffer)), + )) } } @@ -373,7 +418,12 @@ impl<'a> Tokenizer<'a> { buffer.push(next_char); } - Ok(Token::new(TokenType::String(buffer), line, column)) + Ok(Token::new( + TokenType::String(buffer), + line, + column, + Some(std::mem::take(&mut self.string_buffer)), + )) } /// Tokenizes a keyword or an identifier. Also handles boolean literals @@ -384,6 +434,7 @@ impl<'a> Tokenizer<'a> { TokenType::Keyword(Keyword::$keyword), self.line, self.column, + Some(std::mem::take(&mut self.string_buffer)), )); }}; } @@ -426,13 +477,19 @@ impl<'a> Tokenizer<'a> { // boolean literals "true" if next_ws!() => { - return Ok(Token::new(TokenType::Boolean(true), self.line, self.column)); + return Ok(Token::new( + TokenType::Boolean(true), + self.line, + self.column, + Some(std::mem::take(&mut self.string_buffer)), + )); } "false" if next_ws!() => { return Ok(Token::new( TokenType::Boolean(false), self.line, self.column, + Some(std::mem::take(&mut self.string_buffer)), )); } // if the next character is whitespace or not alphanumeric, then we have an identifier @@ -442,6 +499,7 @@ impl<'a> Tokenizer<'a> { TokenType::Identifier(val.to_string()), line, column, + Some(std::mem::take(&mut self.string_buffer)), )); } _ => {} @@ -449,7 +507,12 @@ impl<'a> Tokenizer<'a> { looped_char = self.next_char()?; } - Err(Error::UnknownKeywordOrIdentifierError(buffer, line, column)) + Err(Error::UnknownKeywordOrIdentifierError( + buffer, + line, + column, + std::mem::take(&mut self.string_buffer), + )) } } diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index 3d9750a..c5bed81 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -8,14 +8,21 @@ pub struct Token { pub line: usize, /// The column where the token was found pub column: usize, + pub original_string: Option, } impl Token { - pub fn new(token_type: TokenType, line: usize, column: usize) -> Self { + pub fn new( + token_type: TokenType, + line: usize, + column: usize, + original: Option, + ) -> Self { Self { token_type, line, column, + original_string: original, } } } diff --git a/rust_compiler/src/lib.rs b/rust_compiler/src/lib.rs index 9913506..056f3fc 100644 --- a/rust_compiler/src/lib.rs +++ b/rust_compiler/src/lib.rs @@ -49,13 +49,13 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec { + Err(TokenizerError::NumberParseError(_, _, col, ref original)) + | Err(TokenizerError::UnknownSymbolError(_, _, col, ref original)) + | Err(TokenizerError::DecimalParseError(_, _, col, ref original)) + | Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col, ref original)) => { tokens.push(FfiToken { column: col as i32, - text: "".into(), + text: original.to_string().into(), tooltip: "".into(), // Safety: it's okay to unwrap the err here because we are matching on the `Err` variant error: token.unwrap_err().to_string().into(), @@ -64,7 +64,10 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec return safer_ffi::Vec::EMPTY, Ok(token) if !matches!(token.token_type, TokenType::EOF) => tokens.push(FfiToken { - text: token.token_type.to_string().into(), + text: token + .original_string + .unwrap_or(token.token_type.to_string()) + .into(), tooltip: "".into(), error: "".into(), status: "".into(),