From 25d9222bd43c910a9aa2496b95bc68a311191075 Mon Sep 17 00:00:00 2001 From: Devin Bidwell Date: Mon, 1 Dec 2025 02:54:53 -0700 Subject: [PATCH] WIP -- emit compilation errors --- csharp_mod/Extensions.cs | 47 ++++- csharp_mod/FfiGlue.cs | 242 +++++++++++----------- csharp_mod/Formatter.cs | 118 ++++++++++- csharp_mod/Marshal.cs | 64 +++++- csharp_mod/Patches.cs | 2 - csharp_mod/Plugin.cs | 26 --- rust_compiler/libs/tokenizer/src/token.rs | 15 ++ rust_compiler/src/ffi/mod.rs | 59 +++++- 8 files changed, 395 insertions(+), 178 deletions(-) diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs index 764b6c9..02ebae3 100644 --- a/csharp_mod/Extensions.cs +++ b/csharp_mod/Extensions.cs @@ -1,6 +1,7 @@ namespace Slang; using System; +using System.Collections.Generic; using System.Text; using StationeersIC10Editor; @@ -53,7 +54,7 @@ public static unsafe class SlangExtensions var color = GetColorForKind(token.token_kind); - int colIndex = token.column; + int colIndex = token.column - 1; if (colIndex < 0) colIndex = 0; @@ -80,20 +81,50 @@ public static unsafe class SlangExtensions return list; } + public static unsafe List ToList(this Vec_FfiDiagnostic_t vec) + { + var toReturn = new List((int)vec.len); + + var currentPtr = vec.ptr; + + for (int i = 0; i < (int)vec.len; i++) + { + var item = currentPtr[i]; + + toReturn.Add( + new Slang.Diagnostic + { + Message = item.message.AsString(), + Severity = item.severity, + Range = new Slang.Range + { + EndCol = item.range.end_col - 1, + EndLine = item.range.end_line - 1, + StartCol = item.range.start_col - 1, + StartLine = item.range.end_line - 1, + }, + } + ); + } + + Ffi.free_ffi_diagnostic_vec(vec); + return toReturn; + } + private static uint GetColorForKind(uint kind) { switch (kind) { case 1: - return SlangFormatter.ColorInstruction; // Keyword - case 2: - return SlangFormatter.ColorDefault; // Identifier - case 3: - return SlangFormatter.ColorNumber; // Number - case 4: return SlangFormatter.ColorString; // String - case 5: + case 2: + return SlangFormatter.ColorString; // Number + case 3: return SlangFormatter.ColorInstruction; // Boolean + case 4: + return SlangFormatter.ColorInstruction; // Keyword + case 5: + return SlangFormatter.ColorInstruction; // Identifier case 6: return SlangFormatter.ColorDefault; // Symbol default: diff --git a/csharp_mod/FfiGlue.cs b/csharp_mod/FfiGlue.cs index 668bb96..5c49ef5 100644 --- a/csharp_mod/FfiGlue.cs +++ b/csharp_mod/FfiGlue.cs @@ -15,162 +15,160 @@ #pragma warning disable SA1500, SA1505, SA1507, #pragma warning disable SA1600, SA1601, SA1604, SA1605, SA1611, SA1615, SA1649, -namespace Slang -{ - using System; - using System.Runtime.InteropServices; +namespace Slang { +using System; +using System.Runtime.InteropServices; - public unsafe partial class Ffi - { +public unsafe partial class Ffi { #if IOS - private const string RustLib = "slang.framework/slang"; + private const string RustLib = "slang.framework/slang"; #else - public const string RustLib = "slang_compiler.dll"; + public const string RustLib = "slang_compiler.dll"; #endif - } +} + +/// +/// &'lt [T] but with a guaranteed #[repr(C)] layout. +/// +/// # C layout (for some given type T) +/// +/// ```c +/// typedef struct { +/// // Cannot be NULL +/// T * ptr; +/// size_t len; +/// } slice_T; +/// ``` +/// +/// # Nullable pointer? +/// +/// If you want to support the above typedef, but where the ptr field is +/// allowed to be NULL (with the contents of len then being undefined) +/// use the Option< slice_ptr<_> > type. +/// +[StructLayout(LayoutKind.Sequential, Size = 16)] +public unsafe struct slice_ref_uint16_t { + /// + /// Pointer to the first element (if any). + /// + public UInt16 /*const*/ * ptr; /// - /// &'lt [T] but with a guaranteed #[repr(C)] layout. - /// - /// # C layout (for some given type T) - /// - /// ```c - /// typedef struct { - /// // Cannot be NULL - /// T * ptr; - /// size_t len; - /// } slice_T; - /// ``` - /// - /// # Nullable pointer? - /// - /// If you want to support the above typedef, but where the ptr field is - /// allowed to be NULL (with the contents of len then being undefined) - /// use the Option< slice_ptr<_> > type. + /// Element count /// - [StructLayout(LayoutKind.Sequential, Size = 16)] - public unsafe struct slice_ref_uint16_t - { - /// - /// Pointer to the first element (if any). - /// - public UInt16 /*const*/ - * ptr; + public UIntPtr len; +} - /// - /// Element count - /// - public UIntPtr len; - } +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_uint8_t { + public byte * ptr; + public UIntPtr len; + + public UIntPtr cap; +} + +public unsafe partial class Ffi { /// - /// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout + /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because + /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. + /// This should result in the ability to compile many times without triggering frame drops + /// from the GC from a GetBytes() call on a string in C#. /// - [StructLayout(LayoutKind.Sequential, Size = 24)] - public unsafe struct Vec_uint8_t - { - public byte* ptr; + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_uint8_t compile_from_string ( + slice_ref_uint16_t input); +} - public UIntPtr len; +[StructLayout(LayoutKind.Sequential, Size = 16)] +public unsafe struct FfiRange_t { + public UInt32 start_col; - public UIntPtr cap; - } + public UInt32 end_col; - public unsafe partial class Ffi - { - /// - /// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because - /// we want to avoid GC. So we pass it to Rust to handle all the memory allocations. - /// This should result in the ability to compile many times without triggering frame drops - /// from the GC from a GetBytes() call on a string in C#. - /// - [DllImport(RustLib, ExactSpelling = true)] - public static extern unsafe Vec_uint8_t compile_from_string(slice_ref_uint16_t input); - } + public UInt32 start_line; - [StructLayout(LayoutKind.Sequential, Size = 16)] - public unsafe struct FfiRange_t - { - public UInt32 start_col; + public UInt32 end_line; +} - public UInt32 end_col; +[StructLayout(LayoutKind.Sequential, Size = 48)] +public unsafe struct FfiDiagnostic_t { + public Vec_uint8_t message; - public UInt32 start_line; + public Int32 severity; - public UInt32 end_line; - } + public FfiRange_t range; +} - [StructLayout(LayoutKind.Sequential, Size = 48)] - public unsafe struct FfiDiagnostic_t - { - public Vec_uint8_t message; +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_FfiDiagnostic_t { + public FfiDiagnostic_t * ptr; - public Int32 severity; + public UIntPtr len; - public FfiRange_t range; - } + public UIntPtr cap; +} - /// - /// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout - /// - [StructLayout(LayoutKind.Sequential, Size = 24)] - public unsafe struct Vec_FfiDiagnostic_t - { - public FfiDiagnostic_t* ptr; +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_FfiDiagnostic_t diagnose_source ( + slice_ref_uint16_t input); +} - public UIntPtr len; +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_ffi_diagnostic_vec ( + Vec_FfiDiagnostic_t v); +} - public UIntPtr cap; - } +[StructLayout(LayoutKind.Sequential, Size = 64)] +public unsafe struct FfiToken_t { + public Vec_uint8_t tooltip; - public unsafe partial class Ffi - { - [DllImport(RustLib, ExactSpelling = true)] - public static extern unsafe Vec_FfiDiagnostic_t diagnose_source(); - } + public Vec_uint8_t error; - public unsafe partial class Ffi - { - [DllImport(RustLib, ExactSpelling = true)] - public static extern unsafe void free_ffi_diagnostic_vec(Vec_FfiDiagnostic_t v); - } + public Int32 column; - [StructLayout(LayoutKind.Sequential, Size = 64)] - public unsafe struct FfiToken_t - { - public Vec_uint8_t tooltip; + public Int32 length; - public Vec_uint8_t error; + public UInt32 token_kind; +} - public Int32 column; +/// +/// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout +/// +[StructLayout(LayoutKind.Sequential, Size = 24)] +public unsafe struct Vec_FfiToken_t { + public FfiToken_t * ptr; - public Int32 length; + public UIntPtr len; - public UInt32 token_kind; - } + public UIntPtr cap; +} - /// - /// Same as [Vec][rust::Vec], but with guaranteed #[repr(C)] layout - /// - [StructLayout(LayoutKind.Sequential, Size = 24)] - public unsafe struct Vec_FfiToken_t - { - public FfiToken_t* ptr; +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_ffi_token_vec ( + Vec_FfiToken_t v); +} - public UIntPtr len; +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + void free_string ( + Vec_uint8_t s); +} - public UIntPtr cap; - } +public unsafe partial class Ffi { + [DllImport(RustLib, ExactSpelling = true)] public static unsafe extern + Vec_FfiToken_t tokenize_line ( + slice_ref_uint16_t input); +} - public unsafe partial class Ffi - { - [DllImport(RustLib, ExactSpelling = true)] - public static extern unsafe void free_ffi_token_vec(Vec_FfiToken_t v); - } - public unsafe partial class Ffi - { - [DllImport(RustLib, ExactSpelling = true)] - public static extern unsafe void free_string(Vec_uint8_t s); - } } /* Slang */ diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs index b26d2cf..0db13ac 100644 --- a/csharp_mod/Formatter.cs +++ b/csharp_mod/Formatter.cs @@ -1,40 +1,138 @@ namespace Slang; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using System.Timers; using StationeersIC10Editor; public class SlangFormatter : ICodeFormatter { - private Timer _timer; + private System.Timers.Timer _timer; + private CancellationTokenSource? _lspCancellationToken; + private readonly SynchronizationContext? _mainThreadContext; + private volatile bool IsDiagnosing = false; public static readonly uint ColorInstruction = ColorFromHTML("#ffff00"); public static readonly uint ColorString = ColorFromHTML("#ce9178"); + private object _textLock = new(); + public SlangFormatter() { - _timer = new Timer(250); + // 1. Capture the Main Thread context. + // This works because the Editor instantiates this class on the main thread. + _mainThreadContext = SynchronizationContext.Current; - this.OnCodeChanged += HandleCodeChanged; + _timer = new System.Timers.Timer(250); + _timer.AutoReset = false; } public override string Compile() { - L.Info("ICodeFormatter attempted to compile source code."); return this.Lines.RawText; } public override Line ParseLine(string line) { - return new Line(line); + HandleCodeChanged(); + return Marshal.TokenizeLine(line); } private void HandleCodeChanged() { - _timer.Stop(); - _timer.Dispose(); - _timer = new Timer(250); - _timer.Elapsed += (_, _) => HandleLsp(); + if (IsDiagnosing) + return; + + _lspCancellationToken?.Cancel(); + _lspCancellationToken?.Dispose(); + + _lspCancellationToken = new CancellationTokenSource(); + + _ = HandleLsp(_lspCancellationToken.Token, this.RawText); } - private void HandleLsp() { } + private void OnTimerElapsed(object sender, ElapsedEventArgs e) { } + + private async Task HandleLsp(CancellationToken cancellationToken, string text) + { + try + { + await Task.Delay(500, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + return; + + List diagnosis = Marshal.DiagnoseSource(text); + + var dict = diagnosis + .GroupBy(d => d.Range.StartLine) + .ToDictionary(g => g.Key, g => g.ToList()); + + // 3. Dispatch the UI update to the Main Thread + if (_mainThreadContext != null) + { + // Post ensures ApplyDiagnostics runs on the captured thread (Main Thread) + _mainThreadContext.Post(_ => ApplyDiagnostics(dict), null); + } + else + { + // Fallback: If context is null (rare in Unity), try running directly + // but warn, as this might crash if not thread-safe. + L.Warning("SynchronizationContext was null. Attempting direct update (risky)."); + ApplyDiagnostics(dict); + } + } + finally { } + } + + // This runs on the Main Thread + private void ApplyDiagnostics(Dictionary> dict) + { + IsDiagnosing = true; + // Standard LSP uses 0-based indexing. + for (int i = 0; i < this.Lines.Count; i++) + { + uint lineIndex = (uint)i; + + if (dict.TryGetValue(lineIndex, out var lineDiagnostics)) + { + var line = this.Lines[i]; + if (line is null) + { + continue; + } + + var tokenMap = line.Tokens.ToDictionary((t) => t.Column); + + foreach (var diag in lineDiagnostics) + { + var newToken = new SemanticToken + { + Column = (int)diag.Range.StartCol, + Length = (int)(diag.Range.EndCol - diag.Range.StartCol), + Line = i, + IsError = true, + Data = diag.Message, + Color = ICodeFormatter.ColorError, + }; + + L.Info( + $"Col: {newToken.Column} -- Length: {newToken.Length} -- Msg: {newToken.Data}" + ); + + tokenMap[newToken.Column] = newToken; + } + + line.ClearTokens(); + + foreach (var token in tokenMap.Values) + { + line.AddToken(token); + } + } + } + IsDiagnosing = false; + } } diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs index ffb6a58..c058e6b 100644 --- a/csharp_mod/Marshal.cs +++ b/csharp_mod/Marshal.cs @@ -1,11 +1,27 @@ namespace Slang; using System; +using System.Collections.Generic; using System.IO; using System.Reflection; using System.Runtime.InteropServices; using StationeersIC10Editor; +public struct Range +{ + public uint StartCol; + public uint EndCol; + public uint StartLine; + public uint EndLine; +} + +public struct Diagnostic +{ + public string Message; + public int Severity; + public Range Range; +} + public static class Marshal { private static IntPtr _libraryHandle = IntPtr.Zero; @@ -63,13 +79,7 @@ public static class Marshal public static unsafe bool CompileFromString(string inputString, out string compiledString) { - if (String.IsNullOrEmpty(inputString)) - { - compiledString = String.Empty; - return false; - } - - if (!EnsureLibLoaded()) + if (String.IsNullOrEmpty(inputString) || !EnsureLibLoaded()) { compiledString = String.Empty; return false; @@ -101,6 +111,46 @@ public static class Marshal } } + public static unsafe List DiagnoseSource(string inputString) + { + if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded()) + { + return new(); + } + + fixed (char* ptrInput = inputString) + { + var input = new slice_ref_uint16_t + { + ptr = (ushort*)ptrInput, + len = (UIntPtr)inputString.Length, + }; + + return Ffi.diagnose_source(input).ToList(); + } + } + + public static unsafe Line TokenizeLine(string inputString) + { + if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded()) + { + return new Line(inputString); + } + + fixed (char* ptrInputStr = inputString) + { + var strRef = new slice_ref_uint16_t + { + len = (UIntPtr)inputString.Length, + ptr = (ushort*)ptrInputStr, + }; + + var tokens = Ffi.tokenize_line(strRef); + + return tokens.ToLine(inputString); + } + } + private static string ExtractNativeLibrary(string libName) { string destinationPath = Path.Combine(Path.GetTempPath(), libName); diff --git a/csharp_mod/Patches.cs b/csharp_mod/Patches.cs index d5fc8bc..276f1cc 100644 --- a/csharp_mod/Patches.cs +++ b/csharp_mod/Patches.cs @@ -1,11 +1,9 @@ namespace Slang; using System; -using Assets.Scripts; using Assets.Scripts.Objects; using Assets.Scripts.Objects.Electrical; using Assets.Scripts.Objects.Motherboards; -using Assets.Scripts.UI; using HarmonyLib; [HarmonyPatch] diff --git a/csharp_mod/Plugin.cs b/csharp_mod/Plugin.cs index 23d24e1..d4a8740 100644 --- a/csharp_mod/Plugin.cs +++ b/csharp_mod/Plugin.cs @@ -1,7 +1,3 @@ -using System; -using System.IO; -using System.IO.Compression; -using System.Text; using System.Text.RegularExpressions; using BepInEx; using HarmonyLib; @@ -65,28 +61,6 @@ namespace Slang } } - /// - /// Encodes the original slang source code as base64 and uses gzip to compress it, returning the resulting string. - /// - public static string EncodeSource(string source) - { - if (string.IsNullOrEmpty(source)) - { - return ""; - } - - byte[] bytes = Encoding.UTF8.GetBytes(source); - - using (var memoryStream = new MemoryStream()) - { - using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Compress)) - { - gzipStream.Write(bytes, 0, bytes.Length); - } - return Convert.ToBase64String(memoryStream.ToArray()); - } - } - public static bool IsSlangSource(ref string input) { return SlangSourceCheck.IsMatch(input); diff --git a/rust_compiler/libs/tokenizer/src/token.rs b/rust_compiler/libs/tokenizer/src/token.rs index c5bed81..b471d6d 100644 --- a/rust_compiler/libs/tokenizer/src/token.rs +++ b/rust_compiler/libs/tokenizer/src/token.rs @@ -87,6 +87,21 @@ pub enum TokenType { EOF, } +impl From for u32 { + fn from(value: TokenType) -> Self { + use TokenType::*; + match value { + String(_) => 1, + Number(_) => 2, + Boolean(_) => 3, + Keyword(_) => 4, + Identifier(_) => 5, + Symbol(_) => 6, + EOF => 0, + } + } +} + impl std::fmt::Display for TokenType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/rust_compiler/src/ffi/mod.rs b/rust_compiler/src/ffi/mod.rs index 4754a20..84159cf 100644 --- a/rust_compiler/src/ffi/mod.rs +++ b/rust_compiler/src/ffi/mod.rs @@ -2,7 +2,10 @@ use compiler::Compiler; use parser::Parser; use safer_ffi::prelude::*; use std::io::BufWriter; -use tokenizer::Tokenizer; +use tokenizer::{ + token::{Token, TokenType}, + Tokenizer, +}; #[derive_ReprC] #[repr(C)] @@ -98,6 +101,56 @@ pub fn compile_from_string(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi:: } #[ffi_export] -pub fn diagnose_source() -> safer_ffi::Vec { - vec![].into() +pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec { + let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice())); + + let mut tokens = Vec::new(); + + // Error reporting is handled in `diagnose_source`. We only care about successful tokens here + // for syntax highlighting + for token in tokenizer { + if matches!( + token, + Ok(Token { + token_type: TokenType::EOF, + .. + }) + ) { + continue; + } + match token { + Err(_) => {} + Ok(Token { + column, + original_string, + token_type, + .. + }) => tokens.push(FfiToken { + column: column as i32, + error: "".into(), + length: (original_string.unwrap_or_default().len()) as i32, + token_kind: token_type.into(), + tooltip: "".into(), + }), + } + } + + tokens.into() +} + +#[ffi_export] +pub fn diagnose_source(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec { + let mut writer = BufWriter::new(Vec::new()); + let tokenizer = Tokenizer::from(String::from_utf16_lossy(input.as_slice())); + let compiler = Compiler::new(Parser::new(tokenizer), &mut writer, None); + + let diagnosis = compiler.compile(); + + let mut result_vec: Vec = Vec::with_capacity(diagnosis.len()); + + for err in diagnosis { + result_vec.push(lsp_types::Diagnostic::from(err).into()); + } + + result_vec.into() }