Working syntax highlighting again

csharp_mod/Extensions.cs

@@ -42,7 +42,7 @@ public static unsafe class SlangExtensions
      * Rust allocation after the List is created, there is no need to Drop this memory.
      * </summary>
      */
-    public static StyledLine ToLine(this Vec_FfiToken_t vec, string sourceText)
+    public static List<SemanticToken> ToTokenList(this Vec_FfiToken_t vec)
     {
         var tokens = new List<SemanticToken>();
 
@@ -80,7 +80,7 @@ public static unsafe class SlangExtensions
 
         Ffi.free_ffi_token_vec(vec);
 
-        return new StyledLine(sourceText, tokens);
+        return tokens;
     }
 
     public static unsafe List<Diagnostic> ToList(this Vec_FfiDiagnostic_t vec)

csharp_mod/Formatter.cs

@@ -72,7 +72,7 @@ public class SlangFormatter : ICodeFormatter
     public override StyledLine ParseLine(string line)
     {
         L.Debug($"Parsing line for syntax highlighting: {line}");
-        return Marshal.TokenizeLine(line);
+        return new StyledLine(line, Marshal.TokenizeLine(line));
     }
 
     private void HandleCodeChanged()
@@ -137,12 +137,15 @@ public class SlangFormatter : ICodeFormatter
             if (line is null)
                 continue;
 
+            // 1. Get base syntax tokens
+            var allTokens = Marshal.TokenizeLine(line.Text);
+
+            // 2. Overlay error tokens if diagnostics exist for this line
             if (dict.ContainsKey(lineIndex))
             {
-                var tokens = new List<SemanticToken>();
                 foreach (var lineDiagnostic in dict[lineIndex])
                 {
-                    tokens.Add(
+                    allTokens.Add(
                         new SemanticToken(
                             line: (int)lineIndex,
                             column: Math.Abs((int)lineDiagnostic.Range.StartCol),
@@ -156,8 +159,9 @@ public class SlangFormatter : ICodeFormatter
                         )
                     );
                 }
-                line.Update(tokens);
             }
+
+            line.Update(allTokens);
         }
 
         _linesWithErrors = new HashSet<uint>(dict.Keys);

csharp_mod/Marshal.cs

@@ -131,11 +131,11 @@ public static class Marshal
         }
     }
 
-    public static unsafe StyledLine TokenizeLine(string inputString)
+    public static unsafe List<SemanticToken> TokenizeLine(string inputString)
     {
         if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
         {
-            return new StyledLine(inputString ?? "");
+            return new List<SemanticToken>();
         }
 
         fixed (char* ptrInputStr = inputString)
@@ -148,7 +148,7 @@ public static class Marshal
 
             var tokens = Ffi.tokenize_line(strRef);
             L.Debug($"Tokenized line '{inputString}' into {tokens.len} tokens.");
-            return tokens.ToLine(inputString);
+            return tokens.ToTokenList();
         }
     }