WIP -- emit compilation errors

csharp_mod/Extensions.cs

@@ -1,6 +1,7 @@
 namespace Slang;
 
 using System;
+using System.Collections.Generic;
 using System.Text;
 using StationeersIC10Editor;
 
@@ -53,7 +54,7 @@ public static unsafe class SlangExtensions
 
             var color = GetColorForKind(token.token_kind);
 
-            int colIndex = token.column;
+            int colIndex = token.column - 1;
             if (colIndex < 0)
                 colIndex = 0;
 
@@ -80,20 +81,50 @@ public static unsafe class SlangExtensions
         return list;
     }
 
+    public static unsafe List<Diagnostic> ToList(this Vec_FfiDiagnostic_t vec)
+    {
+        var toReturn = new List<Diagnostic>((int)vec.len);
+
+        var currentPtr = vec.ptr;
+
+        for (int i = 0; i < (int)vec.len; i++)
+        {
+            var item = currentPtr[i];
+
+            toReturn.Add(
+                new Slang.Diagnostic
+                {
+                    Message = item.message.AsString(),
+                    Severity = item.severity,
+                    Range = new Slang.Range
+                    {
+                        EndCol = item.range.end_col - 1,
+                        EndLine = item.range.end_line - 1,
+                        StartCol = item.range.start_col - 1,
+                        StartLine = item.range.end_line - 1,
+                    },
+                }
+            );
+        }
+
+        Ffi.free_ffi_diagnostic_vec(vec);
+        return toReturn;
+    }
+
     private static uint GetColorForKind(uint kind)
     {
         switch (kind)
         {
             case 1:
-                return SlangFormatter.ColorInstruction; // Keyword
-            case 2:
-                return SlangFormatter.ColorDefault; // Identifier
-            case 3:
-                return SlangFormatter.ColorNumber; // Number
-            case 4:
                 return SlangFormatter.ColorString; // String
-            case 5:
+            case 2:
+                return SlangFormatter.ColorString; // Number
+            case 3:
                 return SlangFormatter.ColorInstruction; // Boolean
+            case 4:
+                return SlangFormatter.ColorInstruction; // Keyword
+            case 5:
+                return SlangFormatter.ColorInstruction; // Identifier
             case 6:
                 return SlangFormatter.ColorDefault; // Symbol
             default:

csharp_mod/FfiGlue.cs

@@ -15,162 +15,160 @@
 #pragma warning disable SA1500, SA1505, SA1507,
 #pragma warning disable SA1600, SA1601, SA1604, SA1605, SA1611, SA1615, SA1649,
 
-namespace Slang
+namespace Slang {
-{
+using System;
-    using System;
+using System.Runtime.InteropServices;
-    using System.Runtime.InteropServices;
 
-    public unsafe partial class Ffi
+public unsafe partial class Ffi {
-    {
 #if IOS
-        private const string RustLib = "slang.framework/slang";
+    private const string RustLib = "slang.framework/slang";
 #else
-        public const string RustLib = "slang_compiler.dll";
+    public const string RustLib = "slang_compiler.dll";
 #endif
-    }
+}
+
+/// <summary>
+/// <c>&'lt [T]</c> but with a guaranteed <c>#[repr(C)]</c> layout.
+///
+/// # C layout (for some given type T)
+///
+/// ```c
+/// typedef struct {
+/// // Cannot be NULL
+/// T * ptr;
+/// size_t len;
+/// } slice_T;
+/// ```
+///
+/// # Nullable pointer?
+///
+/// If you want to support the above typedef, but where the <c>ptr</c> field is
+/// allowed to be <c>NULL</c> (with the contents of <c>len</c> then being undefined)
+/// use the <c>Option< slice_ptr<_> ></c> type.
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 16)]
+public unsafe struct slice_ref_uint16_t {
+    /// <summary>
+    /// Pointer to the first element (if any).
+    /// </summary>
+    public UInt16 /*const*/ * ptr;
 
     /// <summary>
-    /// <c>&'lt [T]</c> but with a guaranteed <c>#[repr(C)]</c> layout.
+    /// Element count
-    ///
-    /// # C layout (for some given type T)
-    ///
-    /// ```c
-    /// typedef struct {
-    /// // Cannot be NULL
-    /// T * ptr;
-    /// size_t len;
-    /// } slice_T;
-    /// ```
-    ///
-    /// # Nullable pointer?
-    ///
-    /// If you want to support the above typedef, but where the <c>ptr</c> field is
-    /// allowed to be <c>NULL</c> (with the contents of <c>len</c> then being undefined)
-    /// use the <c>Option< slice_ptr<_> ></c> type.
     /// </summary>
-    [StructLayout(LayoutKind.Sequential, Size = 16)]
+    public UIntPtr len;
-    public unsafe struct slice_ref_uint16_t
+}
-    {
-        /// <summary>
-        /// Pointer to the first element (if any).
-        /// </summary>
-        public UInt16 /*const*/
-        * ptr;
 
-        /// <summary>
+/// <summary>
-        /// Element count
+/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
-        /// </summary>
+/// </summary>
-        public UIntPtr len;
+[StructLayout(LayoutKind.Sequential, Size = 24)]
-    }
+public unsafe struct Vec_uint8_t {
+    public byte * ptr;
 
+    public UIntPtr len;
+
+    public UIntPtr cap;
+}
+
+public unsafe partial class Ffi {
     /// <summary>
-    /// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+    /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
+    /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
+    /// This should result in the ability to compile many times without triggering frame drops
+    /// from the GC from a <c>GetBytes()</c> call on a string in C#.
     /// </summary>
-    [StructLayout(LayoutKind.Sequential, Size = 24)]
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
-    public unsafe struct Vec_uint8_t
+    Vec_uint8_t compile_from_string (
-    {
+        slice_ref_uint16_t input);
-        public byte* ptr;
+}
 
-        public UIntPtr len;
+[StructLayout(LayoutKind.Sequential, Size = 16)]
+public unsafe struct FfiRange_t {
+    public UInt32 start_col;
 
-        public UIntPtr cap;
+    public UInt32 end_col;
-    }
 
-    public unsafe partial class Ffi
+    public UInt32 start_line;
-    {
-        /// <summary>
-        /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
-        /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
-        /// This should result in the ability to compile many times without triggering frame drops
-        /// from the GC from a <c>GetBytes()</c> call on a string in C#.
-        /// </summary>
-        [DllImport(RustLib, ExactSpelling = true)]
-        public static extern unsafe Vec_uint8_t compile_from_string(slice_ref_uint16_t input);
-    }
 
-    [StructLayout(LayoutKind.Sequential, Size = 16)]
+    public UInt32 end_line;
-    public unsafe struct FfiRange_t
+}
-    {
-        public UInt32 start_col;
 
-        public UInt32 end_col;
+[StructLayout(LayoutKind.Sequential, Size = 48)]
+public unsafe struct FfiDiagnostic_t {
+    public Vec_uint8_t message;
 
-        public UInt32 start_line;
+    public Int32 severity;
 
-        public UInt32 end_line;
+    public FfiRange_t range;
-    }
+}
 
-    [StructLayout(LayoutKind.Sequential, Size = 48)]
+/// <summary>
-    public unsafe struct FfiDiagnostic_t
+/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
-    {
+/// </summary>
-        public Vec_uint8_t message;
+[StructLayout(LayoutKind.Sequential, Size = 24)]
+public unsafe struct Vec_FfiDiagnostic_t {
+    public FfiDiagnostic_t * ptr;
 
-        public Int32 severity;
+    public UIntPtr len;
 
-        public FfiRange_t range;
+    public UIntPtr cap;
-    }
+}
 
-    /// <summary>
+public unsafe partial class Ffi {
-    /// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
-    /// </summary>
+    Vec_FfiDiagnostic_t diagnose_source (
-    [StructLayout(LayoutKind.Sequential, Size = 24)]
+        slice_ref_uint16_t input);
-    public unsafe struct Vec_FfiDiagnostic_t
+}
-    {
-        public FfiDiagnostic_t* ptr;
 
-        public UIntPtr len;
+public unsafe partial class Ffi {
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    void free_ffi_diagnostic_vec (
+        Vec_FfiDiagnostic_t v);
+}
 
-        public UIntPtr cap;
+[StructLayout(LayoutKind.Sequential, Size = 64)]
-    }
+public unsafe struct FfiToken_t {
+    public Vec_uint8_t tooltip;
 
-    public unsafe partial class Ffi
+    public Vec_uint8_t error;
-    {
-        [DllImport(RustLib, ExactSpelling = true)]
-        public static extern unsafe Vec_FfiDiagnostic_t diagnose_source();
-    }
 
-    public unsafe partial class Ffi
+    public Int32 column;
-    {
-        [DllImport(RustLib, ExactSpelling = true)]
-        public static extern unsafe void free_ffi_diagnostic_vec(Vec_FfiDiagnostic_t v);
-    }
 
-    [StructLayout(LayoutKind.Sequential, Size = 64)]
+    public Int32 length;
-    public unsafe struct FfiToken_t
-    {
-        public Vec_uint8_t tooltip;
 
-        public Vec_uint8_t error;
+    public UInt32 token_kind;
+}
 
-        public Int32 column;
+/// <summary>
+/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+/// </summary>
+[StructLayout(LayoutKind.Sequential, Size = 24)]
+public unsafe struct Vec_FfiToken_t {
+    public FfiToken_t * ptr;
 
-        public Int32 length;
+    public UIntPtr len;
 
-        public UInt32 token_kind;
+    public UIntPtr cap;
-    }
+}
 
-    /// <summary>
+public unsafe partial class Ffi {
-    /// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
-    /// </summary>
+    void free_ffi_token_vec (
-    [StructLayout(LayoutKind.Sequential, Size = 24)]
+        Vec_FfiToken_t v);
-    public unsafe struct Vec_FfiToken_t
+}
-    {
-        public FfiToken_t* ptr;
 
-        public UIntPtr len;
+public unsafe partial class Ffi {
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    void free_string (
+        Vec_uint8_t s);
+}
 
-        public UIntPtr cap;
+public unsafe partial class Ffi {
-    }
+    [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
+    Vec_FfiToken_t tokenize_line (
+        slice_ref_uint16_t input);
+}
 
-    public unsafe partial class Ffi
-    {
-        [DllImport(RustLib, ExactSpelling = true)]
-        public static extern unsafe void free_ffi_token_vec(Vec_FfiToken_t v);
-    }
 
-    public unsafe partial class Ffi
-    {
-        [DllImport(RustLib, ExactSpelling = true)]
-        public static extern unsafe void free_string(Vec_uint8_t s);
-    }
 } /* Slang */

csharp_mod/Formatter.cs

@@ -1,40 +1,138 @@
 namespace Slang;
 
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 using System.Timers;
 using StationeersIC10Editor;
 
 public class SlangFormatter : ICodeFormatter
 {
-    private Timer _timer;
+    private System.Timers.Timer _timer;
+    private CancellationTokenSource? _lspCancellationToken;
+    private readonly SynchronizationContext? _mainThreadContext;
+    private volatile bool IsDiagnosing = false;
 
     public static readonly uint ColorInstruction = ColorFromHTML("#ffff00");
     public static readonly uint ColorString = ColorFromHTML("#ce9178");
 
+    private object _textLock = new();
+
     public SlangFormatter()
     {
-        _timer = new Timer(250);
+        // 1. Capture the Main Thread context.
+        // This works because the Editor instantiates this class on the main thread.
+        _mainThreadContext = SynchronizationContext.Current;
 
-        this.OnCodeChanged += HandleCodeChanged;
+        _timer = new System.Timers.Timer(250);
+        _timer.AutoReset = false;
     }
 
     public override string Compile()
     {
-        L.Info("ICodeFormatter attempted to compile source code.");
         return this.Lines.RawText;
     }
 
     public override Line ParseLine(string line)
     {
-        return new Line(line);
+        HandleCodeChanged();
+        return Marshal.TokenizeLine(line);
     }
 
     private void HandleCodeChanged()
     {
-        _timer.Stop();
+        if (IsDiagnosing)
-        _timer.Dispose();
+            return;
-        _timer = new Timer(250);
+
-        _timer.Elapsed += (_, _) => HandleLsp();
+        _lspCancellationToken?.Cancel();
+        _lspCancellationToken?.Dispose();
+
+        _lspCancellationToken = new CancellationTokenSource();
+
+        _ = HandleLsp(_lspCancellationToken.Token, this.RawText);
     }
 
-    private void HandleLsp() { }
+    private void OnTimerElapsed(object sender, ElapsedEventArgs e) { }
+
+    private async Task HandleLsp(CancellationToken cancellationToken, string text)
+    {
+        try
+        {
+            await Task.Delay(500, cancellationToken);
+
+            if (cancellationToken.IsCancellationRequested)
+                return;
+
+            List<Diagnostic> diagnosis = Marshal.DiagnoseSource(text);
+
+            var dict = diagnosis
+                .GroupBy(d => d.Range.StartLine)
+                .ToDictionary(g => g.Key, g => g.ToList());
+
+            // 3. Dispatch the UI update to the Main Thread
+            if (_mainThreadContext != null)
+            {
+                // Post ensures ApplyDiagnostics runs on the captured thread (Main Thread)
+                _mainThreadContext.Post(_ => ApplyDiagnostics(dict), null);
+            }
+            else
+            {
+                // Fallback: If context is null (rare in Unity), try running directly
+                // but warn, as this might crash if not thread-safe.
+                L.Warning("SynchronizationContext was null. Attempting direct update (risky).");
+                ApplyDiagnostics(dict);
+            }
+        }
+        finally { }
+    }
+
+    // This runs on the Main Thread
+    private void ApplyDiagnostics(Dictionary<uint, List<Diagnostic>> dict)
+    {
+        IsDiagnosing = true;
+        // Standard LSP uses 0-based indexing.
+        for (int i = 0; i < this.Lines.Count; i++)
+        {
+            uint lineIndex = (uint)i;
+
+            if (dict.TryGetValue(lineIndex, out var lineDiagnostics))
+            {
+                var line = this.Lines[i];
+                if (line is null)
+                {
+                    continue;
+                }
+
+                var tokenMap = line.Tokens.ToDictionary((t) => t.Column);
+
+                foreach (var diag in lineDiagnostics)
+                {
+                    var newToken = new SemanticToken
+                    {
+                        Column = (int)diag.Range.StartCol,
+                        Length = (int)(diag.Range.EndCol - diag.Range.StartCol),
+                        Line = i,
+                        IsError = true,
+                        Data = diag.Message,
+                        Color = ICodeFormatter.ColorError,
+                    };
+
+                    L.Info(
+                        $"Col: {newToken.Column} -- Length: {newToken.Length} -- Msg: {newToken.Data}"
+                    );
+
+                    tokenMap[newToken.Column] = newToken;
+                }
+
+                line.ClearTokens();
+
+                foreach (var token in tokenMap.Values)
+                {
+                    line.AddToken(token);
+                }
+            }
+        }
+        IsDiagnosing = false;
+    }
 }

csharp_mod/Marshal.cs

@@ -1,11 +1,27 @@
 namespace Slang;
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
 using System.Runtime.InteropServices;
 using StationeersIC10Editor;
 
+public struct Range
+{
+    public uint StartCol;
+    public uint EndCol;
+    public uint StartLine;
+    public uint EndLine;
+}
+
+public struct Diagnostic
+{
+    public string Message;
+    public int Severity;
+    public Range Range;
+}
+
 public static class Marshal
 {
     private static IntPtr _libraryHandle = IntPtr.Zero;
@@ -63,13 +79,7 @@ public static class Marshal
 
     public static unsafe bool CompileFromString(string inputString, out string compiledString)
     {
-        if (String.IsNullOrEmpty(inputString))
+        if (String.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
-        {
-            compiledString = String.Empty;
-            return false;
-        }
-
-        if (!EnsureLibLoaded())
         {
             compiledString = String.Empty;
             return false;
@@ -101,6 +111,46 @@ public static class Marshal
         }
     }
 
+    public static unsafe List<Diagnostic> DiagnoseSource(string inputString)
+    {
+        if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
+        {
+            return new();
+        }
+
+        fixed (char* ptrInput = inputString)
+        {
+            var input = new slice_ref_uint16_t
+            {
+                ptr = (ushort*)ptrInput,
+                len = (UIntPtr)inputString.Length,
+            };
+
+            return Ffi.diagnose_source(input).ToList();
+        }
+    }
+
+    public static unsafe Line TokenizeLine(string inputString)
+    {
+        if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
+        {
+            return new Line(inputString);
+        }
+
+        fixed (char* ptrInputStr = inputString)
+        {
+            var strRef = new slice_ref_uint16_t
+            {
+                len = (UIntPtr)inputString.Length,
+                ptr = (ushort*)ptrInputStr,
+            };
+
+            var tokens = Ffi.tokenize_line(strRef);
+
+            return tokens.ToLine(inputString);
+        }
+    }
+
     private static string ExtractNativeLibrary(string libName)
     {
         string destinationPath = Path.Combine(Path.GetTempPath(), libName);

csharp_mod/Patches.cs

@@ -1,11 +1,9 @@
 namespace Slang;
 
 using System;
-using Assets.Scripts;
 using Assets.Scripts.Objects;
 using Assets.Scripts.Objects.Electrical;
 using Assets.Scripts.Objects.Motherboards;
-using Assets.Scripts.UI;
 using HarmonyLib;
 
 [HarmonyPatch]

csharp_mod/Plugin.cs

@@ -1,7 +1,3 @@
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Text;
 using System.Text.RegularExpressions;
 using BepInEx;
 using HarmonyLib;
@@ -65,28 +61,6 @@ namespace Slang
             }
         }
 
-        /// <summary>
-        /// Encodes the original slang source code as base64 and uses gzip to compress it, returning the resulting string.
-        /// </summary>
-        public static string EncodeSource(string source)
-        {
-            if (string.IsNullOrEmpty(source))
-            {
-                return "";
-            }
-
-            byte[] bytes = Encoding.UTF8.GetBytes(source);
-
-            using (var memoryStream = new MemoryStream())
-            {
-                using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress))
-                {
-                    gzipStream.Write(bytes, 0, bytes.Length);
-                }
-                return Convert.ToBase64String(memoryStream.ToArray());
-            }
-        }
-
         public static bool IsSlangSource(ref string input)
         {
             return SlangSourceCheck.IsMatch(input);

rust_compiler/libs/tokenizer/src/token.rs

@@ -87,6 +87,21 @@ pub enum TokenType {
     EOF,
 }
 
+impl From<TokenType> for u32 {
+    fn from(value: TokenType) -> Self {
+        use TokenType::*;
+        match value {
+            String(_) => 1,
+            Number(_) => 2,
+            Boolean(_) => 3,
+            Keyword(_) => 4,
+            Identifier(_) => 5,
+            Symbol(_) => 6,
+            EOF => 0,
+        }
+    }
+}
+
 impl std::fmt::Display for TokenType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {

rust_compiler/src/ffi/mod.rs

@@ -2,7 +2,10 @@ use compiler::Compiler;
 use parser::Parser;
 use safer_ffi::prelude::*;
 use std::io::BufWriter;
-use tokenizer::Tokenizer;
+use tokenizer::{
+    token::{Token, TokenType},
+    Tokenizer,
+};
 
 #[derive_ReprC]
 #[repr(C)]
@@ -98,6 +101,56 @@ pub fn compile_from_string(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::
 }
 
 #[ffi_export]
-pub fn diagnose_source() -> safer_ffi::Vec<FfiDiagnostic> {
+pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<FfiToken> {
-    vec![].into()
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
+
+    let mut tokens = Vec::new();
+
+    // Error reporting is handled in `diagnose_source`. We only care about successful tokens here
+    // for syntax highlighting
+    for token in tokenizer {
+        if matches!(
+            token,
+            Ok(Token {
+                token_type: TokenType::EOF,
+                ..
+            })
+        ) {
+            continue;
+        }
+        match token {
+            Err(_) => {}
+            Ok(Token {
+                column,
+                original_string,
+                token_type,
+                ..
+            }) => tokens.push(FfiToken {
+                column: column as i32,
+                error: "".into(),
+                length: (original_string.unwrap_or_default().len()) as i32,
+                token_kind: token_type.into(),
+                tooltip: "".into(),
+            }),
+        }
+    }
+
+    tokens.into()
+}
+
+#[ffi_export]
+pub fn diagnose_source(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<FfiDiagnostic> {
+    let mut writer = BufWriter::new(Vec::new());
+    let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice()));
+    let compiler = Compiler::new(Parser::new(tokenizer), &mut writer, None);
+
+    let diagnosis = compiler.compile();
+
+    let mut result_vec: Vec<FfiDiagnostic> = Vec::with_capacity(diagnosis.len());
+
+    for err in diagnosis {
+        result_vec.push(lsp_types::Diagnostic::from(err).into());
+    }
+
+    result_vec.into()
 }