initial integration with ic10editor mod

csharp_mod/Extensions.cs

@@ -6,7 +6,12 @@ namespace Slang
 {
     public static unsafe class SlangExtensions
     {
-        // 1. Convert the Rust Byte Vector (Vec_uint8_t) to a C# String
+        /**
+         * <summary>
+         * This is a helper method to convert a Rust struct for a string pointer
+         * into a C# style string.
+         * </summary>
+         */
         public static string AsString(this Vec_uint8_t vec)
         {
             if (vec.ptr == null || vec.len == UIntPtr.Zero)
@@ -20,16 +25,28 @@ namespace Slang
             return toReturn;
         }
 
+        /**
+         * <summary>This will free a Rust string struct. Because this is a pointer to a struct, this memory
+         * is managed by Rust, therefor it must be freed by Rust
+         * </summary>
+         */
         public static void Drop(this Vec_uint8_t vec)
         {
             Ffi.free_string(vec);
         }
 
-        // 2. Convert Rust Token Vector to C# List
+        /**
+         * <summary>This helper converts a Rust vec to a C# List. This handles freeing the
+         * Rust allocation after the List is created, there is no need to Drop this memory.
+         * </summary>
+         */
         public static Line AsList(this Vec_FfiToken_t vec)
         {
+            L.Info("Converting output into a C# List.");
             var list = new Line();
+            L.Info("Created new `Line`.");
             list.Capacity = (int)vec.len;
+            L.Info("Changed `Capacity` to be returned Vec's len");
 
             var currentPtr = vec.ptr;
 
@@ -40,7 +57,6 @@ namespace Slang
                 FfiToken_t token = currentPtr[i];
 
                 var newToken = new Token(token.text.AsString(), token.column);
-                newToken.Error = token.error.AsString();
 
                 list.Add(newToken);
             }

csharp_mod/FfiGlue.cs

@@ -23,10 +23,42 @@ public unsafe partial class Ffi {
 #if IOS
     private const string RustLib = "slang.framework/slang";
 #else
-    private const string RustLib = "slang";
+    public const string RustLib = "slang_compiler.dll";
 #endif
 }
 
+/// <summary>
+/// <c>&'lt [T]</c> but with a guaranteed <c>#[repr(C)]</c> layout.
+///
+/// # C layout (for some given type T)
+///
+/// ```c
+/// typedef struct {
+/// // Cannot be NULL
+/// T * ptr;
+/// size_t len;
+/// } slice_T;
+/// ```
+///
+/// # Nullable pointer?
+///
+/// If you want to support the above typedef, but where the <c>ptr</c> field is
+/// allowed to be <c>NULL</c> (with the contents of <c>len</c> then being undefined)
+/// use the <c>Option< slice_ptr<_> ></c> type.
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 16)]
+public unsafe struct slice_ref_uint16_t {
+    /// <summary>
+    /// Pointer to the first element (if any).
+    /// </summary>
+    public UInt16 /*const*/ * ptr;
+
+    /// <summary>
+    /// Element count
+    /// </summary>
+    public UIntPtr len;
+}
+
 /// <summary>
 /// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
 /// </summary>
@@ -40,9 +72,15 @@ public unsafe struct Vec_uint8_t {
 }
 
 public unsafe partial class Ffi {
+    /// <summary>
+    /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+    /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+    /// This should result in the ability to compile many times without triggering frame drops
+    /// from the GC from a <c>GetBytes()</c> call on a string in C#.
+    /// </summary>
     [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
     Vec_uint8_t compile_from_string (
-        byte /*const*/ * input);
+        slice_ref_uint16_t input);
 }
 
 [StructLayout(LayoutKind.Sequential, Size = 104)]
@@ -83,9 +121,15 @@ public unsafe partial class Ffi {
 }
 
 public unsafe partial class Ffi {
+    /// <summary>
+    /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+    /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+    /// This should result in the ability to tokenize many times without triggering frame drops
+    /// from the GC from a <c>GetBytes()</c> call on a string in C#.
+    /// </summary>
     [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
     Vec_FfiToken_t tokenize_line (
-        byte /*const*/ * input);
+        slice_ref_uint16_t input);
 }
 
 

csharp_mod/Marshal.cs

@@ -1,29 +1,63 @@
 using System;
-using System.Text;
 using StationeersIC10Editor;
 
 namespace Slang
 {
     public static class Marshal
     {
-        public static unsafe Line TokenizeLine(string input)
+        public static unsafe Line TokenizeLine(string source)
         {
-            if (String.IsNullOrEmpty(input))
+            if (String.IsNullOrEmpty(source))
             {
                 return new Line();
             }
 
-            // Make sure the string is a null terminated string
+            L.Info("Input string not empty");
-            if (input[input.Length - 1] != '\0')
+
+            fixed (char* ptrString = source)
             {
-                input += '\0';
+                L.Info("In `fixed` block.");
+                var input = new slice_ref_uint16_t
+                {
+                    ptr = (ushort*)ptrString,
+                    len = (UIntPtr)source.Length,
+                };
+                L.Info("Calling tokenize_line");
+                return Ffi.tokenize_line(input).AsList();
+            }
         }
 
-            var strBytes = Encoding.UTF8.GetBytes(input);
+        public static unsafe bool CompileFromString(string inputString, out string compiledString)
-
-            fixed (byte* ptrString = strBytes)
         {
-                return Ffi.tokenize_line(ptrString).AsList();
+            if (String.IsNullOrEmpty(inputString))
+            {
+                compiledString = String.Empty;
+                return false;
+            }
+
+            fixed (char* ptrString = inputString)
+            {
+                var input = new slice_ref_uint16_t
+                {
+                    ptr = (ushort*)ptrString,
+                    len = (UIntPtr)inputString.Length,
+                };
+
+                var result = Ffi.compile_from_string(input);
+                try
+                {
+                    if ((ulong)result.len < 1)
+                    {
+                        compiledString = String.Empty;
+                        return false;
+                    }
+                    compiledString = result.AsString();
+                    return true;
+                }
+                finally
+                {
+                    result.Drop();
+                }
             }
         }
     }

csharp_mod/Plugin.cs

@@ -38,6 +38,7 @@ namespace Slang
     }
 
     [BepInPlugin(PluginGuid, PluginName, "0.1.0")]
+    [BepInDependency(StationeersIC10Editor.IC10EditorPlugin.PluginGuid)]
     public class SlangPlugin : BaseUnityPlugin
     {
         public const string PluginGuid = "com.biddydev.slang";
@@ -60,26 +61,15 @@ namespace Slang
 
         public static unsafe string Compile(string source)
         {
-            if (string.IsNullOrEmpty(source))
+            string compiled;
-                return "";
+            if (Marshal.CompileFromString(source, out compiled))
-
-            // Add a null terminator char at the end of the source string (turns into a CStr)
-            source += "\0";
-
-            var bytes = System.Text.Encoding.UTF8.GetBytes(source);
-
-            // don't move my memory around, C#!
-            fixed (byte* pBytes = bytes)
             {
-                var compiled = Ffi.compile_from_string(pBytes);
+                // TODO: handle saving the original source code
-                try
+                return compiled;
-                {
-                    return compiled.AsString();
             }
-                finally
+            else
             {
-                    Ffi.free_string(compiled);
+                return compiled;
-                }
             }
         }
 
@@ -99,13 +89,16 @@ namespace Slang
         private void Awake()
         {
             L.SetLogger(Logger);
-            ExtractNativeDll("slang.dll");
+
+            if (ExtractNativeDll(Ffi.RustLib))
+            {
                 var harmony = new Harmony(PluginGuid);
                 harmony.PatchAll();
                 CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true);
             }
+        }
 
-        private void ExtractNativeDll(string fileName)
+        private bool ExtractNativeDll(string fileName)
         {
             string destinationPath = Path.Combine(Path.GetDirectoryName(Info.Location), fileName);
 
@@ -116,9 +109,9 @@ namespace Slang
                 if (stream == null)
                 {
                     L.Error(
-                        "slang.dll compiler not found. This means it was not embedded in the mod. Please contact the mod author!"
+                        $"{Ffi.RustLib} not found. This means it was not embedded in the mod. Please contact the mod author!"
                     );
-                    return;
+                    return false;
                 }
 
                 try
@@ -127,10 +120,12 @@ namespace Slang
                     {
                         stream.CopyTo(fileStream);
                     }
+                    return true;
                 }
                 catch (IOException e)
                 {
                     L.Warning($"Could not overwrite {fileName} (it might be in use): {e.Message}");
+                    return false;
                 }
             }
         }

csharp_mod/stationeersSlang.csproj

@@ -54,7 +54,7 @@
 
   <ItemGroup>
     <EmbeddedResource Include="../rust_compiler/target/x86_64-pc-windows-gnu/release/slang.dll">
-      <LogicalName>slang.compiler.dll</LogicalName>
+      <LogicalName>slang_compiler.dll</LogicalName>
     </EmbeddedResource>
   </ItemGroup>
 

rust_compiler/Cargo.lock

@@ -386,9 +386,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "js-sys"
-version = "0.3.82"
+version = "0.3.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65"
+checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
 dependencies = [
  "once_cell",
  "wasm-bindgen",
@@ -967,9 +967,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60"
+checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -980,9 +980,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2"
+checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -990,9 +990,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc"
+checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -1003,9 +1003,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76"
+checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
 dependencies = [
  "unicode-ident",
 ]
@@ -1027,9 +1027,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
+checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
 dependencies = [
  "memchr",
 ]

rust_compiler/libs/compiler/src/v1.rs

@@ -1125,9 +1125,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
                 }))
             }
 
-            _ => {
+            t => Err(Error::Unknown(format!("{t:?}\n\nNot yet implemented"))),
-                todo!()
-            }
         }
     }
 

rust_compiler/libs/parser/src/lib.rs

@@ -1260,7 +1260,9 @@ impl<'a> Parser<'a> {
                 let arg = literal_or_variable!(invocation.arguments.first());
                 Ok(SysCall::Math(sys_call::Math::Trunc(arg)))
             }
-            _ => todo!(),
+            _ => Err(Error::UnsupportedKeyword(token_from_option!(
+                self.current_token
+            ))),
         }
     }
 }

rust_compiler/libs/tokenizer/src/lib.rs

@@ -457,7 +457,11 @@ impl<'a> Iterator for Tokenizer<'a> {
     type Item = Result<Token, Error>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        todo!()
+        match self.next_token() {
+            Ok(Some(tok)) => Some(Ok(tok)),
+            Ok(None) => None,
+            Err(e) => Some(Err(e)),
+        }
     }
 }
 

rust_compiler/src/lib.rs

@@ -8,9 +8,9 @@ use tokenizer::{Error as TokenizerError, Tokenizer};
 #[repr(C)]
 pub struct FfiToken {
     pub text: safer_ffi::String,
-    pub tooltip: Option<safer_ffi::String>,
+    pub tooltip: safer_ffi::String,
-    pub error: Option<safer_ffi::String>,
+    pub error: safer_ffi::String,
-    pub status: Option<safer_ffi::String>,
+    pub status: safer_ffi::String,
     pub column: i32,
 }
 
@@ -56,18 +56,18 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<Ff
                 tokens.push(FfiToken {
                     column: col as i32,
                     text: "".into(),
-                    tooltip: None,
+                    tooltip: "".into(),
                     // Safety: it's okay to unwrap the err here because we are matching on the `Err` variant
-                    error: Some(token.unwrap_err().to_string().into()),
+                    error: token.unwrap_err().to_string().into(),
-                    status: None,
+                    status: "".into(),
                 });
             }
             Err(_) => return safer_ffi::Vec::EMPTY,
             Ok(token) => tokens.push(FfiToken {
                 text: token.token_type.to_string().into(),
-                tooltip: None,
+                tooltip: "".into(),
-                error: None,
+                error: "".into(),
-                status: None,
+                status: "".into(),
                 column: token.column as i32,
             }),
         }
@@ -88,8 +88,19 @@ pub fn free_string(s: safer_ffi::String) {
 
 #[cfg(feature = "headers")]
 pub fn generate_headers() -> std::io::Result<()> {
+    let file_name = "../csharp_mod/FfiGlue.cs";
     ::safer_ffi::headers::builder()
         .with_language(safer_ffi::headers::Language::CSharp)
-        .to_file("../csharp_mod/FfiGlue.cs")?
+        .to_file(file_name)?
-        .generate()
+        .generate()?;
+
+    let content = std::fs::read_to_string(file_name)?;
+
+    let content = content.replace(
+        "private const string RustLib = \"slang\";",
+        "public const string RustLib = \"slang_compiler.dll\";",
+    );
+
+    std::fs::write(file_name, content)?;
+    Ok(())
 }