wip -- marshal UTF16 string from C# to Rust to avoid GC in C#

csharp_mod/Extensions.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Text;
 using StationeersIC10Editor;
 
@@ -27,9 +26,11 @@ namespace Slang
         }
 
         // 2. Convert Rust Token Vector to C# List
-        public static List<Token> AsList(this Vec_FfiToken_t vec)
+        public static Line AsList(this Vec_FfiToken_t vec)
         {
-            var list = new List<Token>((int)vec.len);
+            var list = new Line();
+            list.Capacity = (int)vec.len;
+
             var currentPtr = vec.ptr;
 
             // Iterate through the raw memory array

csharp_mod/Formatter.cs

@@ -6,7 +6,7 @@ namespace Slang
     {
         public override Line ParseLine(string line)
         {
-            throw new System.NotImplementedException();
+            return Marshal.TokenizeLine(line);
         }
     }
 }

csharp_mod/Marshal.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Text;
+using StationeersIC10Editor;
+
+namespace Slang
+{
+    public static class Marshal
+    {
+        public static unsafe Line TokenizeLine(string input)
+        {
+            if (String.IsNullOrEmpty(input))
+            {
+                return new Line();
+            }
+
+            // Make sure the string is a null terminated string
+            if (input[input.Length - 1] != '\0')
+            {
+                input += '\0';
+            }
+
+            var strBytes = Encoding.UTF8.GetBytes(input);
+
+            fixed (byte* ptrString = strBytes)
+            {
+                return Ffi.tokenize_line(ptrString).AsList();
+            }
+        }
+    }
+}

csharp_mod/Plugin.cs

@@ -1,10 +1,9 @@
-using System;
 using System.IO;
 using System.Reflection;
-using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using BepInEx;
 using HarmonyLib;
+using StationeersIC10Editor;
 
 namespace Slang
 {
@@ -103,6 +102,7 @@ namespace Slang
             ExtractNativeDll("slang.dll");
             var harmony = new Harmony(PluginGuid);
             harmony.PatchAll();
+            CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true);
         }
 
         private void ExtractNativeDll(string fileName)

rust_compiler/libs/parser/src/sys_call.rs

@@ -124,7 +124,7 @@ pub enum System {
     /// Loads a LogicType from all connected network devices, aggregating them via a
     /// batchMode
     /// ## In Game
-    /// lb r? deviceHash loggicType batchMode
+    /// lb r? deviceHash logicType batchMode
     /// ## Examples
     /// lb r0 HASH("StructureWallLight") On Minimum
     LoadBatch(LiteralOrVariable, Literal, Literal),
@@ -137,7 +137,7 @@ pub enum System {
     /// Represents a function which stores a setting to all devices that match
     /// the given deviceHash
     /// ## In Game
-    /// `sb deviceHash logictype r?`
+    /// `sb deviceHash logicType r?`
     /// ## Example
     /// `sb HASH("Doors") Lock 1`
     SetOnDeviceBatched(LiteralOrVariable, Literal, Box<Expression>),

rust_compiler/src/lib.rs

@@ -14,11 +14,15 @@ pub struct FfiToken {
     pub column: i32,
 }
 
+/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+/// This should result in the ability to compile many times without triggering frame drops
+/// from the GC from a `GetBytes()` call on a string in C#.
 #[ffi_export]
-pub fn compile_from_string(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::String {
+pub fn compile_from_string(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::String {
     let mut writer = BufWriter::new(Vec::new());
 
-    let tokenizer = Tokenizer::from(input.to_str());
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
     let parser = Parser::new(tokenizer);
     let compiler = Compiler::new(parser, &mut writer, None);
 
@@ -33,10 +37,13 @@ pub fn compile_from_string(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ff
     // Safety: I know the compiler only outputs valid utf8
     safer_ffi::String::from(unsafe { String::from_utf8_unchecked(compiled_vec) })
 }
-
+/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+/// This should result in the ability to tokenize many times without triggering frame drops
+/// from the GC from a `GetBytes()` call on a string in C#.
 #[ffi_export]
-pub fn tokenize_line(input: safer_ffi::char_p::char_p_ref<'_>) -> safer_ffi::Vec<FfiToken> {
-    let tokenizer = Tokenizer::from(input.to_str());
+pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<FfiToken> {
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
 
     let mut tokens = Vec::<FfiToken>::new();
 
@@ -83,6 +90,6 @@ pub fn free_string(s: safer_ffi::String) {
 pub fn generate_headers() -> std::io::Result<()> {
     ::safer_ffi::headers::builder()
         .with_language(safer_ffi::headers::Language::CSharp)
-        .to_file("../csharp_mod/SlangGlue.cs")?
+        .to_file("../csharp_mod/FfiGlue.cs")?
         .generate()
 }