From 979d32b413677647c6f03f6d1dc6e9cbaa25978c Mon Sep 17 00:00:00 2001
From: Devin Bidwell <dbidwell94@biddydev.com>
Date: Fri, 5 Dec 2025 14:53:56 -0700
Subject: [PATCH] Working syntax highlighting again

---
 csharp_mod/Extensions.cs |  4 ++--
 csharp_mod/Formatter.cs  | 12 ++++++++----
 csharp_mod/Marshal.cs    |  6 +++---
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/csharp_mod/Extensions.cs b/csharp_mod/Extensions.cs
index 0e1f8db..6a7e8f2 100644
--- a/csharp_mod/Extensions.cs
+++ b/csharp_mod/Extensions.cs
@@ -42,7 +42,7 @@ public static unsafe class SlangExtensions
      * Rust allocation after the List is created, there is no need to Drop this memory.
      * </summary>
      */
-    public static StyledLine ToLine(this Vec_FfiToken_t vec, string sourceText)
+    public static List<SemanticToken> ToTokenList(this Vec_FfiToken_t vec)
     {
         var tokens = new List<SemanticToken>();
 
@@ -80,7 +80,7 @@ public static unsafe class SlangExtensions
 
         Ffi.free_ffi_token_vec(vec);
 
-        return new StyledLine(sourceText, tokens);
+        return tokens;
     }
 
     public static unsafe List<Diagnostic> ToList(this Vec_FfiDiagnostic_t vec)
diff --git a/csharp_mod/Formatter.cs b/csharp_mod/Formatter.cs
index 791826d..e4fd74a 100644
--- a/csharp_mod/Formatter.cs
+++ b/csharp_mod/Formatter.cs
@@ -72,7 +72,7 @@ public class SlangFormatter : ICodeFormatter
     public override StyledLine ParseLine(string line)
     {
         L.Debug($"Parsing line for syntax highlighting: {line}");
-        return Marshal.TokenizeLine(line);
+        return new StyledLine(line, Marshal.TokenizeLine(line));
     }
 
     private void HandleCodeChanged()
@@ -137,12 +137,15 @@ public class SlangFormatter : ICodeFormatter
             if (line is null)
                 continue;
 
+            // 1. Get base syntax tokens
+            var allTokens = Marshal.TokenizeLine(line.Text);
+
+            // 2. Overlay error tokens if diagnostics exist for this line
             if (dict.ContainsKey(lineIndex))
             {
-                var tokens = new List<SemanticToken>();
                 foreach (var lineDiagnostic in dict[lineIndex])
                 {
-                    tokens.Add(
+                    allTokens.Add(
                         new SemanticToken(
                             line: (int)lineIndex,
                             column: Math.Abs((int)lineDiagnostic.Range.StartCol),
@@ -156,8 +159,9 @@ public class SlangFormatter : ICodeFormatter
                         )
                     );
                 }
-                line.Update(tokens);
             }
+
+            line.Update(allTokens);
         }
 
         _linesWithErrors = new HashSet<uint>(dict.Keys);
diff --git a/csharp_mod/Marshal.cs b/csharp_mod/Marshal.cs
index e64e964..b03696f 100644
--- a/csharp_mod/Marshal.cs
+++ b/csharp_mod/Marshal.cs
@@ -131,11 +131,11 @@ public static class Marshal
         }
     }
 
-    public static unsafe StyledLine TokenizeLine(string inputString)
+    public static unsafe List<SemanticToken> TokenizeLine(string inputString)
     {
         if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
         {
-            return new StyledLine(inputString ?? "");
+            return new List<SemanticToken>();
         }
 
         fixed (char* ptrInputStr = inputString)
@@ -148,7 +148,7 @@ public static class Marshal
 
             var tokens = Ffi.tokenize_line(strRef);
             L.Debug($"Tokenized line '{inputString}' into {tokens.len} tokens.");
-            return tokens.ToLine(inputString);
+            return tokens.ToTokenList();
         }
     }