Merge pull request #6 from dbidwell94/ic10editor-mod

Ic10editor mod

.gitignore

@@ -1,2 +1,6 @@
-/target
-*.ic10
+target
+*.ic10
+release
+csharp_mod/bin
+obj
+ref

build.sh

@@ -2,21 +2,49 @@
 
 set -e
 
+RUST_DIR="rust_compiler"
+CSHARP_DIR="csharp_mod"
+RELEASE_DIR="release"
+
 export RUSTFLAGS="--remap-path-prefix=${PWD}=. --remap-path-prefix=${HOME}/.cargo=~/.cargo"
 
-# -- Build for Native (Linux x86-64) --
-echo "Building native (Linux x86-64) executable..."
-cargo build --release --target=x86_64-unknown-linux-gnu
-echo "Native build complete."
 echo "--------------------"
+cd "$RUST_DIR"
+echo "Building native Rust binaries and libs"
+
+# -- Build for Native (Linux x86-64) --
+cargo build --release --target=x86_64-unknown-linux-gnu
 
 # -- Build for Windows (x86-64) --
-echo "Building Windows (x86-64) dll and executable..."
 cargo build --release --target=x86_64-pc-windows-gnu
-echo "Windows build successful."
+
+# -- Generate C# Headers --
+cargo run --features headers --bin generate-headers
+
+cd ..
 echo "--------------------"
 
-echo "All builds successful"
-echo "Linux executable at target/x86_64-unknown-linux-gnu/release/slang"
-echo "Windows .exe at target/x86_64-pc-windows-gnu/release/slang.exe"
-echo "Windows .dll at target/x86_64-pc-windows-gnu/release/slang.dll"
+echo "Building C# mod"
+echo "--------------------"
+
+cd "$CSHARP_DIR"
+dotnet build -c Release
+
+cd ..
+echo "--------------------"
+
+echo "Copying Release files to output directory"
+echo "--------------------"
+
+RUST_WIN_EXE="$RUST_DIR/target/x86_64-pc-windows-gnu/release/slang.exe"
+RUST_LINUX_BIN="$RUST_DIR/target/x86_64-unknown-linux-gnu/release/slang"
+CHARP_DLL="$CSHARP_DIR/bin/Release/net46/StationeersSlang.dll"
+
+# Check if the release dir exists, if not: create it.
+if [[ ! -d "$RELEASE_DIR" ]]; then
+  mkdir "$RELEASE_DIR"
+fi
+
+cp "$RUST_WIN_EXE" "$RELEASE_DIR/slang.exe"
+cp "$RUST_LINUX_BIN" "$RELEASE_DIR/slang"
+cp "$CHARP_DLL" "$RELEASE_DIR/StationeersSlang.dll"

clean.sh

@@ -0,0 +1,18 @@
+echo "Cleaning build directories"
+
+CSHARP_DIR="csharp_mod"
+RUST_DIR="rust_compiler"
+
+cd "$CSHARP_DIR"
+dotnet clean
+
+cd ..
+
+cd "$RUST_DIR"
+cargo clean
+
+cd ..
+
+if [[ -d "release" ]]; then
+  rm -rd release
+fi

csharp_mod/Extensions.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Text;
+using StationeersIC10Editor;
+
+namespace Slang
+{
+    public static unsafe class SlangExtensions
+    {
+        /**
+         * <summary>
+         * This is a helper method to convert a Rust struct for a string pointer
+         * into a C# style string.
+         * </summary>
+         */
+        public static string AsString(this Vec_uint8_t vec)
+        {
+            if (vec.ptr == null || vec.len == UIntPtr.Zero)
+            {
+                return string.Empty;
+            }
+
+            // Rust strings are UTF-8. Read bytes from raw pointer.
+            var toReturn = Encoding.UTF8.GetString(vec.ptr, (int)vec.len);
+
+            return toReturn;
+        }
+
+        /**
+         * <summary>This will free a Rust string struct. Because this is a pointer to a struct, this memory
+         * is managed by Rust, therefor it must be freed by Rust
+         * </summary>
+         */
+        public static void Drop(this Vec_uint8_t vec)
+        {
+            Ffi.free_string(vec);
+        }
+
+        /**
+         * <summary>This helper converts a Rust vec to a C# List. This handles freeing the
+         * Rust allocation after the List is created, there is no need to Drop this memory.
+         * </summary>
+         */
+        public static Line AsList(this Vec_FfiToken_t vec)
+        {
+            var list = new Line();
+            list.Capacity = (int)vec.len;
+
+            var currentPtr = vec.ptr;
+
+            // Iterate through the raw memory array
+            for (int i = 0; i < (int)vec.len; i++)
+            {
+                // Dereference pointer to get the struct at index i
+                FfiToken_t token = currentPtr[i];
+
+                var newToken = new Token(token.text.AsString(), token.column);
+
+                list.Add(newToken);
+            }
+
+            Ffi.free_ffi_token_vec(vec);
+
+            return list;
+        }
+    }
+}

csharp_mod/FfiGlue.cs

@@ -0,0 +1,136 @@
+/*! \file */
+/*******************************************
+ *                                         *
+ *  File auto-generated by `::safer_ffi`.  *
+ *                                         *
+ *  Do not manually edit this file.        *
+ *                                         *
+ *******************************************/
+
+#pragma warning disable IDE0044, IDE0049, IDE0055, IDE1006,
+#pragma warning disable SA1004, SA1008, SA1023, SA1028,
+#pragma warning disable SA1121, SA1134,
+#pragma warning disable SA1201,
+#pragma warning disable SA1300, SA1306, SA1307, SA1310, SA1313,
+#pragma warning disable SA1500, SA1505, SA1507,
+#pragma warning disable SA1600, SA1601, SA1604, SA1605, SA1611, SA1615, SA1649,
+
+namespace Slang {
+using System;
+using System.Runtime.InteropServices;
+
+public unsafe partial class Ffi {
+#if IOS
+    private const string RustLib = "slang.framework/slang";
+#else
+    public const string RustLib = "slang_compiler.dll";
+#endif
+}
+
+/// <summary>
+/// <c>&'lt [T]</c> but with a guaranteed <c>#[repr(C)]</c> layout.
+///
+/// # C layout (for some given type T)
+///
+/// ```c
+/// typedef struct {
+/// // Cannot be NULL
+/// T * ptr;
+/// size_t len;
+/// } slice_T;
+/// ```
+///
+/// # Nullable pointer?
+///
+/// If you want to support the above typedef, but where the <c>ptr</c> field is
+/// allowed to be <c>NULL</c> (with the contents of <c>len</c> then being undefined)
+/// use the <c>Option< slice_ptr<_> ></c> type.
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 16)]
+public unsafe struct slice_ref_uint16_t {
+    /// <summary>
+    /// Pointer to the first element (if any).
+    /// </summary>
+    public UInt16 /*const*/ * ptr;
+
+    /// <summary>
+    /// Element count
+    /// </summary>
+    public UIntPtr len;
+}
+
+/// <summary>
+/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 24)]
+public unsafe struct Vec_uint8_t {
+    public byte * ptr;
+
+    public UIntPtr len;
+
+    public UIntPtr cap;
+}
+
+public unsafe partial class Ffi {
+    /// <summary>
+    /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+    /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+    /// This should result in the ability to compile many times without triggering frame drops
+    /// from the GC from a <c>GetBytes()</c> call on a string in C#.
+    /// </summary>
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    Vec_uint8_t compile_from_string (
+        slice_ref_uint16_t input);
+}
+
+[StructLayout(LayoutKind.Sequential, Size = 104)]
+public unsafe struct FfiToken_t {
+    public Vec_uint8_t text;
+
+    public Vec_uint8_t tooltip;
+
+    public Vec_uint8_t error;
+
+    public Vec_uint8_t status;
+
+    public Int32 column;
+}
+
+/// <summary>
+/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 24)]
+public unsafe struct Vec_FfiToken_t {
+    public FfiToken_t * ptr;
+
+    public UIntPtr len;
+
+    public UIntPtr cap;
+}
+
+public unsafe partial class Ffi {
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    void free_ffi_token_vec (
+        Vec_FfiToken_t v);
+}
+
+public unsafe partial class Ffi {
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    void free_string (
+        Vec_uint8_t s);
+}
+
+public unsafe partial class Ffi {
+    /// <summary>
+    /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+    /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+    /// This should result in the ability to tokenize many times without triggering frame drops
+    /// from the GC from a <c>GetBytes()</c> call on a string in C#.
+    /// </summary>
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    Vec_FfiToken_t tokenize_line (
+        slice_ref_uint16_t input);
+}
+
+
+} /* Slang */

csharp_mod/Formatter.cs

@@ -0,0 +1,12 @@
+using StationeersIC10Editor;
+
+namespace Slang
+{
+    public class SlangFormatter : ICodeFormatter
+    {
+        public override Line ParseLine(string line)
+        {
+            return Marshal.TokenizeLine(line);
+        }
+    }
+}

csharp_mod/Marshal.cs

@@ -0,0 +1,60 @@
+using System;
+using StationeersIC10Editor;
+
+namespace Slang
+{
+    public static class Marshal
+    {
+        public static unsafe Line TokenizeLine(string source)
+        {
+            if (String.IsNullOrEmpty(source))
+            {
+                return new Line();
+            }
+
+            fixed (char* ptrString = source)
+            {
+                var input = new slice_ref_uint16_t
+                {
+                    ptr = (ushort*)ptrString,
+                    len = (UIntPtr)source.Length,
+                };
+                return Ffi.tokenize_line(input).AsList();
+            }
+        }
+
+        public static unsafe bool CompileFromString(string inputString, out string compiledString)
+        {
+            if (String.IsNullOrEmpty(inputString))
+            {
+                compiledString = String.Empty;
+                return false;
+            }
+
+            fixed (char* ptrString = inputString)
+            {
+                var input = new slice_ref_uint16_t
+                {
+                    ptr = (ushort*)ptrString,
+                    len = (UIntPtr)inputString.Length,
+                };
+
+                var result = Ffi.compile_from_string(input);
+                try
+                {
+                    if ((ulong)result.len < 1)
+                    {
+                        compiledString = String.Empty;
+                        return false;
+                    }
+                    compiledString = result.AsString();
+                    return true;
+                }
+                finally
+                {
+                    result.Drop();
+                }
+            }
+        }
+    }
+}

csharp_mod/Patches.cs

@@ -0,0 +1,41 @@
+using System;
+using Assets.Scripts.Objects.Motherboards;
+using HarmonyLib;
+
+namespace Slang
+{
+    [HarmonyPatch]
+    public static class SlangPatches
+    {
+        [HarmonyPatch(
+            typeof(ProgrammableChipMotherboard),
+            nameof(ProgrammableChipMotherboard.InputFinished)
+        )]
+        [HarmonyPrefix]
+        public static void PGM_InputFinished(ref string result)
+        {
+            if (string.IsNullOrEmpty(result) || !SlangPlugin.IsSlangSource(ref result))
+            {
+                return;
+            }
+
+            L.Debug("Detected Slang source, compiling...");
+
+            // Compile the Slang source into IC10
+            string compiled = SlangPlugin.Compile(result);
+
+            // Ensure that the string is correct
+            if (string.IsNullOrEmpty(compiled))
+            {
+                return;
+            }
+
+            var newUuid = Guid.NewGuid().ToString();
+
+            SlangPlugin.CopySourceToFile(result);
+
+            // Set the result to be the compiled source so the rest of the function can continue as normal
+            result = compiled;
+        }
+    }
+}

csharp_mod/Plugin.cs

@@ -0,0 +1,133 @@
+using System.IO;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using BepInEx;
+using HarmonyLib;
+using StationeersIC10Editor;
+
+namespace Slang
+{
+    class L
+    {
+        private static BepInEx.Logging.ManualLogSource? _logger;
+
+        public static void SetLogger(BepInEx.Logging.ManualLogSource logger)
+        {
+            _logger = logger;
+        }
+
+        public static void Debug(string message)
+        {
+            _logger?.LogDebug(message);
+        }
+
+        public static void Info(string message)
+        {
+            _logger?.LogInfo(message);
+        }
+
+        public static void Error(string message)
+        {
+            _logger?.LogError(message);
+        }
+
+        public static void Warning(string message)
+        {
+            _logger?.LogWarning(message);
+        }
+    }
+
+    [BepInPlugin(PluginGuid, PluginName, "0.1.0")]
+    [BepInDependency(StationeersIC10Editor.IC10EditorPlugin.PluginGuid)]
+    public class SlangPlugin : BaseUnityPlugin
+    {
+        public const string PluginGuid = "com.biddydev.slang";
+        public const string PluginName = "Slang";
+
+        private static Regex? _slangSourceCheck = null;
+
+        private static Regex SlangSourceCheck
+        {
+            get
+            {
+                if (_slangSourceCheck is null)
+                {
+                    _slangSourceCheck = new Regex(@"[;{}()]|\b(let|fn|device)\b|\/\/");
+                }
+
+                return _slangSourceCheck;
+            }
+        }
+
+        public static unsafe string Compile(string source)
+        {
+            string compiled;
+            if (Marshal.CompileFromString(source, out compiled))
+            {
+                // TODO: handle saving the original source code
+                return compiled;
+            }
+            else
+            {
+                return compiled;
+            }
+        }
+
+        /// <summary>Take original slang source code and copies it to a file
+        /// for use in restoring later.
+        /// </summary>
+        public static bool CopySourceToFile(string source)
+        {
+            return true;
+        }
+
+        public static bool IsSlangSource(ref string input)
+        {
+            return SlangSourceCheck.IsMatch(input);
+        }
+
+        private void Awake()
+        {
+            L.SetLogger(Logger);
+
+            if (ExtractNativeDll(Ffi.RustLib))
+            {
+                var harmony = new Harmony(PluginGuid);
+                harmony.PatchAll();
+                CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true);
+            }
+        }
+
+        private bool ExtractNativeDll(string fileName)
+        {
+            string destinationPath = Path.Combine(Path.GetDirectoryName(Info.Location), fileName);
+
+            Assembly assembly = Assembly.GetExecutingAssembly();
+
+            using (Stream stream = assembly.GetManifestResourceStream(fileName))
+            {
+                if (stream == null)
+                {
+                    L.Error(
+                        $"{Ffi.RustLib} not found. This means it was not embedded in the mod. Please contact the mod author!"
+                    );
+                    return false;
+                }
+
+                try
+                {
+                    using (FileStream fileStream = new FileStream(destinationPath, FileMode.Create))
+                    {
+                        stream.CopyTo(fileStream);
+                    }
+                    return true;
+                }
+                catch (IOException e)
+                {
+                    L.Warning($"Could not overwrite {fileName} (it might be in use): {e.Message}");
+                    return false;
+                }
+            }
+        }
+    }
+}

csharp_mod/StationpediaDocumentation.cs

@@ -0,0 +1,22 @@
+using Assets.Scripts.UI;
+
+namespace Slang
+{
+    public static class SlangDocs
+    {
+        public static StationpediaPage[] Pages
+        {
+            get
+            {
+                return
+                [
+                    new StationpediaPage(
+                        "slang-init",
+                        "Slang",
+                        "Slang is a new high level language built specifically for Stationeers"
+                    ),
+                ];
+            }
+        }
+    }
+}

csharp_mod/stationeersSlang.csproj

@@ -0,0 +1,61 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net46</TargetFramework>
+    <Nullable>enable</Nullable>
+    <AssemblyName>StationeersSlang</AssemblyName>
+    <Description>Slang Compiler Bridge</Description>
+    <Version>0.1.0</Version>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <GameDir>/home/dbidwell/.local/share/Steam/steamapps/common/Stationeers/</GameDir>
+    <ManagedDir>$(GameDir)/rocketstation_Data/Managed</ManagedDir>
+    <BepInExDir>$(GameDir)/BepInEx/core</BepInExDir>
+  </PropertyGroup>
+
+  <ItemGroup>
+      <Reference Include="netstandard">
+        <HintPath>$(ManagedDir)/netstandard.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="BepInEx">
+        <HintPath>$(BepInExDir)/BepInEx.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="0Harmony">
+        <HintPath>$(BepInExDir)/0Harmony.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+
+      <Reference Include="UnityEngine">
+        <HintPath>$(ManagedDir)/UnityEngine.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="UnityEngine.CoreModule">
+        <HintPath>$(ManagedDir)/UnityEngine.CoreModule.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="Assembly-CSharp">
+        <HintPath>$(ManagedDir)/Assembly-CSharp.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="Assembly-CSharp-firstpass">
+        <HintPath>$(ManagedDir)/Assembly-CSharp-firstpass.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+      <Reference Include="IC10Editor.dll">
+        <HintPath>./ref/IC10Editor.dll</HintPath>
+        <Private>False</Private>
+      </Reference>
+    </ItemGroup>
+
+  <ItemGroup>
+    <EmbeddedResource Include="../rust_compiler/target/x86_64-pc-windows-gnu/release/slang.dll">
+      <LogicalName>slang_compiler.dll</LogicalName>
+    </EmbeddedResource>
+  </ItemGroup>
+
+</Project>

rust_compiler/Cargo.lock

@@ -265,6 +265,41 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
+[[package]]
+name = "ext-trait"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5"
+dependencies = [
+ "ext-trait-proc_macros",
+]
+
+[[package]]
+name = "ext-trait-proc_macros"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "extension-traits"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537"
+dependencies = [
+ "ext-trait",
+]
+
+[[package]]
+name = "extern-c"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320bea982e85d42441eb25c49b41218e7eaa2657e8f90bc4eca7437376751e23"
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -328,6 +363,15 @@ dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "inventory"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.2"
@@ -342,9 +386,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
 [[package]]
 name = "js-sys"
-version = "0.3.82"
+version = "0.3.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65"
+checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
 dependencies = [
  "once_cell",
  "wasm-bindgen",
@@ -356,6 +400,22 @@ version = "0.2.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
 
+[[package]]
+name = "macro_rules_attribute"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "paste",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
+
 [[package]]
 name = "memchr"
 version = "2.7.6"
@@ -410,6 +470,12 @@ dependencies = [
  "tokenizer",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.21"
@@ -429,6 +495,16 @@ dependencies = [
  "yansi",
 ]
 
+[[package]]
+name = "prettyplease"
+version = "0.1.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86"
+dependencies = [
+ "proc-macro2",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "3.4.0"
@@ -578,6 +654,15 @@ version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -590,12 +675,56 @@ version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
+[[package]]
+name = "safer-ffi"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "435fdd58b61a6f1d8545274c1dfa458e905ff68c166e65e294a0130ef5e675bd"
+dependencies = [
+ "extern-c",
+ "inventory",
+ "libc",
+ "macro_rules_attribute",
+ "paste",
+ "safer_ffi-proc_macros",
+ "scopeguard",
+ "stabby",
+ "uninit",
+ "unwind_safe",
+ "with_builtin_macros",
+]
+
+[[package]]
+name = "safer_ffi-proc_macros"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0f25be5ba5f319542edb31925517e0380245ae37df50a9752cdbc05ef948156"
+dependencies = [
+ "macro_rules_attribute",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "seahash"
 version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
 
+[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
 [[package]]
 name = "serde"
 version = "1.0.228"
@@ -638,6 +767,12 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "sha2-const-stable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9"
+
 [[package]]
 name = "simdutf8"
 version = "0.1.5"
@@ -645,7 +780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
 [[package]]
-name = "stationlang"
+name = "slang"
 version = "0.1.0"
 dependencies = [
  "anyhow",
@@ -654,9 +789,45 @@ dependencies = [
  "parser",
  "quick-error",
  "rust_decimal",
+ "safer-ffi",
  "tokenizer",
 ]
 
+[[package]]
+name = "stabby"
+version = "36.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b7e94eaf470c2e76b5f15fb2fb49714471a36cc512df5ee231e62e82ec79f8"
+dependencies = [
+ "rustversion",
+ "stabby-abi",
+]
+
+[[package]]
+name = "stabby-abi"
+version = "36.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dc7a63b8276b54e51bfffe3d85da56e7906b2dcfcb29018a8ab666c06734c1a"
+dependencies = [
+ "rustc_version",
+ "rustversion",
+ "sha2-const-stable",
+ "stabby-macros",
+]
+
+[[package]]
+name = "stabby-macros"
+version = "36.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eecb7ec5611ec93ec79d120fbe55f31bea234dc1bed1001d4a071bb688651615"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "rand",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -751,6 +922,21 @@ version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
 
+[[package]]
+name = "uninit"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e130f2ed46ca5d8ec13c7ff95836827f92f5f5f37fd2b2bf16f33c408d98bb6"
+dependencies = [
+ "extension-traits",
+]
+
+[[package]]
+name = "unwind_safe"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0976c77def3f1f75c4ef892a292c31c0bbe9e3d0702c63044d7c76db298171a3"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.2"
@@ -781,9 +967,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60"
+checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -794,9 +980,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2"
+checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -804,9 +990,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc"
+checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -817,9 +1003,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.105"
+version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76"
+checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
 dependencies = [
  "unicode-ident",
 ]
@@ -841,13 +1027,33 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
+checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
 dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "with_builtin_macros"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a59d55032495429b87f9d69954c6c8602e4d3f3e0a747a12dea6b0b23de685da"
+dependencies = [
+ "with_builtin_macros-proc_macros",
+]
+
+[[package]]
+name = "with_builtin_macros-proc_macros"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15bd7679c15e22924f53aee34d4e448c45b674feb6129689af88593e129f8f42"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "wyz"
 version = "0.5.1"

rust_compiler/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "stationlang"
+name = "slang"
 version = "0.1.0"
 edition = "2021"
 
@@ -9,6 +9,10 @@ members = ["libs/*"]
 [workspace.dependencies]
 quick-error = "2"
 rust_decimal = "1"
+safer-ffi = { version = "0.1" }
+
+[features]
+headers = ["safer-ffi/headers"]
 
 [profile.release]
 strip = true
@@ -17,10 +21,15 @@ strip = true
 name = "slang"
 path = "src/main.rs"
 
+[[bin]]
+name = "generate-headers"
+path = "src/bin/generate_headers.rs"
+required-features = ["headers"]
+
 [lib]
 name = "slang"
 path = "src/lib.rs"
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 clap = { version = "^4.5", features = ["derive"] }
@@ -29,7 +38,7 @@ rust_decimal = { workspace = true }
 tokenizer = { path = "libs/tokenizer" }
 parser = { path = "libs/parser" }
 compiler = { path = "libs/compiler" }
-
+safer-ffi = { workspace = true }
 
 [dev-dependencies]
 anyhow = { version = "^1.0", features = ["backtrace"] }

rust_compiler/libs/compiler/src/test/branching.rs

@@ -155,4 +155,3 @@ fn test_spilled_variable_update_in_branch() -> anyhow::Result<()> {
 
     Ok(())
 }
-

rust_compiler/libs/compiler/src/v1.rs

@@ -68,7 +68,7 @@ struct CompilationResult {
 }
 
 pub struct Compiler<'a, W: std::io::Write> {
-    parser: ASTParser,
+    parser: ASTParser<'a>,
     function_locations: HashMap<String, usize>,
     function_metadata: HashMap<String, Vec<String>>,
     devices: HashMap<String, String>,
@@ -83,7 +83,7 @@ pub struct Compiler<'a, W: std::io::Write> {
 
 impl<'a, W: std::io::Write> Compiler<'a, W> {
     pub fn new(
-        parser: ASTParser,
+        parser: ASTParser<'a>,
         writer: &'a mut BufWriter<W>,
         config: Option<CompilerConfig>,
     ) -> Self {
@@ -1125,9 +1125,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
                 }))
             }
 
-            _ => {
-                todo!()
-            }
+            t => Err(Error::Unknown(format!("{t:?}\n\nNot yet implemented"))),
         }
     }
 

rust_compiler/libs/parser/src/lib.rs

@@ -4,6 +4,7 @@ mod test;
 pub mod sys_call;
 pub mod tree_node;
 
+use crate::sys_call::System;
 use quick_error::quick_error;
 use std::io::SeekFrom;
 use sys_call::SysCall;
@@ -13,8 +14,6 @@ use tokenizer::{
 };
 use tree_node::*;
 
-use crate::sys_call::System;
-
 #[macro_export]
 /// A macro to create a boxed value.
 macro_rules! boxed {
@@ -114,13 +113,13 @@ macro_rules! token_matches {
     };
 }
 
-pub struct Parser {
-    tokenizer: TokenizerBuffer,
+pub struct Parser<'a> {
+    tokenizer: TokenizerBuffer<'a>,
     current_token: Option<Token>,
 }
 
-impl Parser {
-    pub fn new(tokenizer: Tokenizer) -> Self {
+impl<'a> Parser<'a> {
+    pub fn new(tokenizer: Tokenizer<'a>) -> Self {
         Parser {
             tokenizer: TokenizerBuffer::new(tokenizer),
             current_token: None,
@@ -1261,7 +1260,9 @@ impl Parser {
                 let arg = literal_or_variable!(invocation.arguments.first());
                 Ok(SysCall::Math(sys_call::Math::Trunc(arg)))
             }
-            _ => todo!(),
+            _ => Err(Error::UnsupportedKeyword(token_from_option!(
+                self.current_token
+            ))),
         }
     }
 }

rust_compiler/libs/parser/src/sys_call.rs

@@ -124,7 +124,7 @@ pub enum System {
     /// Loads a LogicType from all connected network devices, aggregating them via a
     /// batchMode
     /// ## In Game
-    /// lb r? deviceHash loggicType batchMode
+    /// lb r? deviceHash logicType batchMode
     /// ## Examples
     /// lb r0 HASH("StructureWallLight") On Minimum
     LoadBatch(LiteralOrVariable, Literal, Literal),
@@ -137,7 +137,7 @@ pub enum System {
     /// Represents a function which stores a setting to all devices that match
     /// the given deviceHash
     /// ## In Game
-    /// `sb deviceHash logictype r?`
+    /// `sb deviceHash logicType r?`
     /// ## Example
     /// `sb HASH("Doors") Lock 1`
     SetOnDeviceBatched(LiteralOrVariable, Literal, Box<Expression>),

rust_compiler/libs/tokenizer/src/lib.rs

@@ -18,18 +18,18 @@ quick_error! {
             display("IO Error: {}", err)
             source(err)
         }
-        NumberParseError(err: std::num::ParseIntError, line: usize, column: usize) {
+        NumberParseError(err: std::num::ParseIntError, line: usize, column: usize, original: String) {
             display("Number Parse Error: {}\nLine: {}, Column: {}", err, line, column)
             source(err)
         }
-        DecimalParseError(err: rust_decimal::Error, line: usize, column: usize) {
+        DecimalParseError(err: rust_decimal::Error, line: usize, column: usize, original: String) {
             display("Decimal Parse Error: {}\nLine: {}, Column: {}", err, line, column)
             source(err)
         }
-        UnknownSymbolError(char: char, line: usize, column: usize) {
+        UnknownSymbolError(char: char, line: usize, column: usize, original: String) {
             display("Unknown Symbol: {}\nLine: {}, Column: {}", char, line, column)
         }
-        UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize) {
+        UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize, original: String) {
             display("Unknown Keyword or Identifier: {}\nLine: {}, Column: {}", val, line, column)
         }
     }
@@ -39,15 +39,16 @@ pub trait Tokenize: Read + Seek {}
 
 impl<T> Tokenize for T where T: Read + Seek {}
 
-pub struct Tokenizer {
-    reader: BufReader<Box<dyn Tokenize>>,
+pub struct Tokenizer<'a> {
+    reader: BufReader<Box<dyn Tokenize + 'a>>,
     char_buffer: [u8; 1],
     line: usize,
     column: usize,
     returned_eof: bool,
+    string_buffer: String,
 }
 
-impl Tokenizer {
+impl<'a> Tokenizer<'a> {
     pub fn from_path(input_file: impl Into<PathBuf>) -> Result<Self, Error> {
         let file = std::fs::File::open(input_file.into())?;
         let reader = BufReader::new(Box::new(file) as Box<dyn Tokenize>);
@@ -58,11 +59,12 @@ impl Tokenizer {
             column: 1,
             char_buffer: [0],
             returned_eof: false,
+            string_buffer: String::new(),
         })
     }
 }
 
-impl From<String> for Tokenizer {
+impl<'a> From<String> for Tokenizer<'a> {
     fn from(input: String) -> Self {
         let reader = BufReader::new(Box::new(Cursor::new(input)) as Box<dyn Tokenize>);
 
@@ -72,17 +74,25 @@ impl From<String> for Tokenizer {
             column: 1,
             char_buffer: [0],
             returned_eof: false,
+            string_buffer: String::new(),
         }
     }
 }
 
-impl From<&str> for Tokenizer {
-    fn from(value: &str) -> Self {
-        Self::from(value.to_string())
+impl<'a> From<&'a str> for Tokenizer<'a> {
+    fn from(value: &'a str) -> Self {
+        Self {
+            reader: BufReader::new(Box::new(Cursor::new(value)) as Box<dyn Tokenize>),
+            char_buffer: [0],
+            column: 1,
+            line: 1,
+            returned_eof: false,
+            string_buffer: String::new(),
+        }
     }
 }
 
-impl Tokenizer {
+impl<'a> Tokenizer<'a> {
     /// Consumes the tokenizer and returns the next token in the stream
     /// If there are no more tokens in the stream, this function returns None
     /// If there is an error reading the stream, this function returns an error
@@ -105,6 +115,7 @@ impl Tokenizer {
             self.column += 1;
         }
 
+        self.string_buffer.push(c);
         Ok(Some(c))
     }
 
@@ -171,7 +182,12 @@ impl Tokenizer {
                     return self.tokenize_keyword_or_identifier(next_char).map(Some);
                 }
                 _ => {
-                    return Err(Error::UnknownSymbolError(next_char, self.line, self.column));
+                    return Err(Error::UnknownSymbolError(
+                        next_char,
+                        self.line,
+                        self.column,
+                        std::mem::take(&mut self.string_buffer),
+                    ));
                 }
             }
         }
@@ -179,7 +195,12 @@ impl Tokenizer {
             Ok(None)
         } else {
             self.returned_eof = true;
-            Ok(Some(Token::new(TokenType::EOF, self.line, self.column)))
+            Ok(Some(Token::new(
+                TokenType::EOF,
+                self.line,
+                self.column,
+                Some(std::mem::take(&mut self.string_buffer)),
+            )))
         }
     }
 
@@ -206,6 +227,7 @@ impl Tokenizer {
                     TokenType::Symbol(Symbol::$symbol),
                     self.line,
                     self.column,
+                    Some(std::mem::take(&mut self.string_buffer)),
                 ))
             };
         }
@@ -273,6 +295,7 @@ impl Tokenizer {
                 first_symbol,
                 self.line,
                 self.column,
+                std::mem::take(&mut self.string_buffer),
             )),
         }
     }
@@ -322,17 +345,28 @@ impl Tokenizer {
             let decimal_scale = decimal.len() as u32;
             let number = format!("{}{}", primary, decimal)
                 .parse::<i128>()
-                .map_err(|e| Error::NumberParseError(e, self.line, self.column))?;
+                .map_err(|e| {
+                    Error::NumberParseError(
+                        e,
+                        self.line,
+                        self.column,
+                        std::mem::take(&mut self.string_buffer),
+                    )
+                })?;
             Number::Decimal(
-                Decimal::try_from_i128_with_scale(number, decimal_scale)
-                    .map_err(|e| Error::DecimalParseError(e, line, column))?,
+                Decimal::try_from_i128_with_scale(number, decimal_scale).map_err(|e| {
+                    Error::DecimalParseError(
+                        e,
+                        line,
+                        column,
+                        std::mem::take(&mut self.string_buffer),
+                    )
+                })?,
             )
         } else {
-            Number::Integer(
-                primary
-                    .parse()
-                    .map_err(|e| Error::NumberParseError(e, line, column))?,
-            )
+            Number::Integer(primary.parse().map_err(|e| {
+                Error::NumberParseError(e, line, column, std::mem::take(&mut self.string_buffer))
+            })?)
         };
 
         // check if the next char is a temperature suffix
@@ -341,14 +375,31 @@ impl Tokenizer {
                 'c' => Temperature::Celsius(number),
                 'f' => Temperature::Fahrenheit(number),
                 'k' => Temperature::Kelvin(number),
-                _ => return Ok(Token::new(TokenType::Number(number), line, column)),
+                _ => {
+                    return Ok(Token::new(
+                        TokenType::Number(number),
+                        line,
+                        column,
+                        Some(std::mem::take(&mut self.string_buffer)),
+                    ));
+                }
             }
             .to_kelvin();
 
             self.next_char()?;
-            Ok(Token::new(TokenType::Number(temperature), line, column))
+            Ok(Token::new(
+                TokenType::Number(temperature),
+                line,
+                column,
+                Some(std::mem::take(&mut self.string_buffer)),
+            ))
         } else {
-            Ok(Token::new(TokenType::Number(number), line, column))
+            Ok(Token::new(
+                TokenType::Number(number),
+                line,
+                column,
+                Some(std::mem::take(&mut self.string_buffer)),
+            ))
         }
     }
 
@@ -367,7 +418,12 @@ impl Tokenizer {
             buffer.push(next_char);
         }
 
-        Ok(Token::new(TokenType::String(buffer), line, column))
+        Ok(Token::new(
+            TokenType::String(buffer),
+            line,
+            column,
+            Some(std::mem::take(&mut self.string_buffer)),
+        ))
     }
 
     /// Tokenizes a keyword or an identifier. Also handles boolean literals
@@ -378,6 +434,7 @@ impl Tokenizer {
                     TokenType::Keyword(Keyword::$keyword),
                     self.line,
                     self.column,
+                    Some(std::mem::take(&mut self.string_buffer)),
                 ));
             }};
         }
@@ -420,13 +477,19 @@ impl Tokenizer {
 
                 // boolean literals
                 "true" if next_ws!() => {
-                    return Ok(Token::new(TokenType::Boolean(true), self.line, self.column));
+                    return Ok(Token::new(
+                        TokenType::Boolean(true),
+                        self.line,
+                        self.column,
+                        Some(std::mem::take(&mut self.string_buffer)),
+                    ));
                 }
                 "false" if next_ws!() => {
                     return Ok(Token::new(
                         TokenType::Boolean(false),
                         self.line,
                         self.column,
+                        Some(std::mem::take(&mut self.string_buffer)),
                     ));
                 }
                 // if the next character is whitespace or not alphanumeric, then we have an identifier
@@ -436,6 +499,7 @@ impl Tokenizer {
                         TokenType::Identifier(val.to_string()),
                         line,
                         column,
+                        Some(std::mem::take(&mut self.string_buffer)),
                     ));
                 }
                 _ => {}
@@ -443,18 +507,35 @@ impl Tokenizer {
 
             looped_char = self.next_char()?;
         }
-        Err(Error::UnknownKeywordOrIdentifierError(buffer, line, column))
+        Err(Error::UnknownKeywordOrIdentifierError(
+            buffer,
+            line,
+            column,
+            std::mem::take(&mut self.string_buffer),
+        ))
     }
 }
 
-pub struct TokenizerBuffer {
-    tokenizer: Tokenizer,
+impl<'a> Iterator for Tokenizer<'a> {
+    type Item = Result<Token, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.next_token() {
+            Ok(Some(tok)) => Some(Ok(tok)),
+            Ok(None) => None,
+            Err(e) => Some(Err(e)),
+        }
+    }
+}
+
+pub struct TokenizerBuffer<'a> {
+    tokenizer: Tokenizer<'a>,
     buffer: VecDeque<Token>,
     history: VecDeque<Token>,
 }
 
-impl TokenizerBuffer {
-    pub fn new(tokenizer: Tokenizer) -> Self {
+impl<'a> TokenizerBuffer<'a> {
+    pub fn new(tokenizer: Tokenizer<'a>) -> Self {
         Self {
             tokenizer,
             buffer: VecDeque::new(),

rust_compiler/libs/tokenizer/src/token.rs

@@ -8,14 +8,21 @@ pub struct Token {
     pub line: usize,
     /// The column where the token was found
     pub column: usize,
+    pub original_string: Option<String>,
 }
 
 impl Token {
-    pub fn new(token_type: TokenType, line: usize, column: usize) -> Self {
+    pub fn new(
+        token_type: TokenType,
+        line: usize,
+        column: usize,
+        original: Option<String>,
+    ) -> Self {
         Self {
             token_type,
             line,
             column,
+            original_string: original,
         }
     }
 }
@@ -32,7 +39,7 @@ impl std::fmt::Display for Temperature {
         match self {
             Temperature::Celsius(n) => write!(f, "{}°C", n),
             Temperature::Fahrenheit(n) => write!(f, "{}°F", n),
-            Temperature::Kelvin(n) => write!(f, "{}K", n),
+            Temperature::Kelvin(n) => write!(f, "{}°K", n),
         }
     }
 }
@@ -88,7 +95,7 @@ impl std::fmt::Display for TokenType {
             TokenType::Boolean(b) => write!(f, "{}", b),
             TokenType::Keyword(k) => write!(f, "{:?}", k),
             TokenType::Identifier(i) => write!(f, "{}", i),
-            TokenType::Symbol(s) => write!(f, "{:?}", s),
+            TokenType::Symbol(s) => write!(f, "{}", s),
             TokenType::EOF => write!(f, "EOF"),
         }
     }
@@ -208,6 +215,40 @@ impl Symbol {
     }
 }
 
+impl std::fmt::Display for Symbol {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Percent => write!(f, "%"),
+            Self::LParen => write!(f, "("),
+            Self::RParen => write!(f, ")"),
+            Self::LBrace => write!(f, "{{"),
+            Self::RBrace => write!(f, "}}"),
+            Self::LBracket => write!(f, "["),
+            Self::RBracket => write!(f, "]"),
+            Self::Semicolon => write!(f, ";"),
+            Self::Colon => write!(f, ":"),
+            Self::Plus => write!(f, "+"),
+            Self::Minus => write!(f, "-"),
+            Self::Comma => write!(f, ","),
+            Self::Asterisk => write!(f, "*"),
+            Self::Slash => write!(f, "/"),
+            Self::LessThan => write!(f, "<"),
+            Self::LessThanOrEqual => write!(f, "<="),
+            Self::GreaterThan => write!(f, ">"),
+            Self::GreaterThanOrEqual => write!(f, ">="),
+            Self::Assign => write!(f, "="),
+            Self::Equal => write!(f, "=="),
+            Self::LogicalAnd => write!(f, "&&"),
+            Self::LogicalOr => write!(f, "||"),
+            Self::LogicalNot => write!(f, "!"),
+            Self::NotEqual => write!(f, "!="),
+            Self::Dot => write!(f, "."),
+            Self::Caret => write!(f, "^"),
+            Self::Exp => write!(f, "**"),
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)]
 pub enum Keyword {
     /// Represents the `continue` keyword

rust_compiler/src/bin/generate_headers.rs

@@ -0,0 +1,3 @@
+fn main() -> std::io::Result<()> {
+    ::slang::generate_headers()
+}

rust_compiler/src/lib.rs

@@ -0,0 +1,110 @@
+use compiler::Compiler;
+use parser::Parser;
+use safer_ffi::prelude::*;
+use std::io::BufWriter;
+use tokenizer::{token::TokenType, Error as TokenizerError, Tokenizer};
+
+#[derive_ReprC]
+#[repr(C)]
+pub struct FfiToken {
+    pub text: safer_ffi::String,
+    pub tooltip: safer_ffi::String,
+    pub error: safer_ffi::String,
+    pub status: safer_ffi::String,
+    pub column: i32,
+}
+
+/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+/// This should result in the ability to compile many times without triggering frame drops
+/// from the GC from a `GetBytes()` call on a string in C#.
+#[ffi_export]
+pub fn compile_from_string(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::String {
+    let mut writer = BufWriter::new(Vec::new());
+
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
+    let parser = Parser::new(tokenizer);
+    let compiler = Compiler::new(parser, &mut writer, None);
+
+    if compiler.compile().is_err() {
+        return safer_ffi::String::EMPTY;
+    }
+
+    let Ok(compiled_vec) = writer.into_inner() else {
+        return safer_ffi::String::EMPTY;
+    };
+
+    // Safety: I know the compiler only outputs valid utf8
+    safer_ffi::String::from(unsafe { String::from_utf8_unchecked(compiled_vec) })
+}
+/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+/// This should result in the ability to tokenize many times without triggering frame drops
+/// from the GC from a `GetBytes()` call on a string in C#.
+#[ffi_export]
+pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<FfiToken> {
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
+
+    let mut tokens = Vec::<FfiToken>::new();
+
+    for token in tokenizer {
+        match token {
+            Err(TokenizerError::NumberParseError(_, _, col, ref original))
+            | Err(TokenizerError::UnknownSymbolError(_, _, col, ref original))
+            | Err(TokenizerError::DecimalParseError(_, _, col, ref original))
+            | Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col, ref original)) => {
+                tokens.push(FfiToken {
+                    column: col as i32,
+                    text: original.to_string().into(),
+                    tooltip: "".into(),
+                    // Safety: it's okay to unwrap the err here because we are matching on the `Err` variant
+                    error: token.unwrap_err().to_string().into(),
+                    status: "".into(),
+                });
+            }
+            Err(_) => return safer_ffi::Vec::EMPTY,
+            Ok(token) if !matches!(token.token_type, TokenType::EOF) => tokens.push(FfiToken {
+                text: token
+                    .original_string
+                    .unwrap_or(token.token_type.to_string())
+                    .into(),
+                tooltip: "".into(),
+                error: "".into(),
+                status: "".into(),
+                column: token.column as i32,
+            }),
+            _ => {}
+        }
+    }
+
+    tokens.into()
+}
+
+#[ffi_export]
+pub fn free_ffi_token_vec(v: safer_ffi::Vec<FfiToken>) {
+    drop(v)
+}
+
+#[ffi_export]
+pub fn free_string(s: safer_ffi::String) {
+    drop(s)
+}
+
+#[cfg(feature = "headers")]
+pub fn generate_headers() -> std::io::Result<()> {
+    let file_name = "../csharp_mod/FfiGlue.cs";
+    ::safer_ffi::headers::builder()
+        .with_language(safer_ffi::headers::Language::CSharp)
+        .to_file(file_name)?
+        .generate()?;
+
+    let content = std::fs::read_to_string(file_name)?;
+
+    let content = content.replace(
+        "private const string RustLib = \"slang\";",
+        "public const string RustLib = \"slang_compiler.dll\";",
+    );
+
+    std::fs::write(file_name, content)?;
+    Ok(())
+}

src/lib.rs

@@ -1,58 +0,0 @@
-use compiler::Compiler;
-use parser::Parser;
-use std::{
-    ffi::{CStr, CString},
-    io::BufWriter,
-};
-use tokenizer::Tokenizer;
-
-/// Takes a raw pointer to a string and compiles the `slang` code into valid IC10
-/// # Safety
-/// This must be called with a valid string pointer from C# (or wherever is calling this function)
-#[no_mangle]
-pub unsafe extern "C" fn compile_from_string(
-    input_ptr: *const std::os::raw::c_char,
-) -> *mut std::os::raw::c_char {
-    if input_ptr.is_null() {
-        return std::ptr::null_mut();
-    }
-
-    let c_str = unsafe { CStr::from_ptr(input_ptr) };
-
-    let Ok(input_str) = c_str.to_str() else {
-        return std::ptr::null_mut();
-    };
-
-    let mut writer = BufWriter::new(Vec::new());
-    let tokenizer = Tokenizer::from(input_str);
-    let parser = Parser::new(tokenizer);
-
-    let compiler = Compiler::new(parser, &mut writer, None);
-
-    let Ok(()) = compiler.compile() else {
-        return std::ptr::null_mut();
-    };
-
-    let Ok(buffer) = writer.into_inner() else {
-        return std::ptr::null_mut();
-    };
-
-    let c_string = CString::from_vec_unchecked(buffer);
-
-    c_string.into_raw()
-}
-
-/// Takes ownership of the string pointer and drops it, freeing the memory
-/// # Safety
-/// Must be called with a valid string pointer
-#[no_mangle]
-pub unsafe extern "C" fn free_slang_string(input_ptr: *mut std::os::raw::c_char) {
-    if input_ptr.is_null() {
-        return;
-    }
-
-    unsafe {
-        // Takes ownership of the input string, and then drops it immediately
-        let _ = CString::from_raw(input_ptr);
-    }
-}