wip fixes for the new IC10Editor changes

csharp_mod/Extensions.cs

@@ -42,9 +42,9 @@ public static unsafe class SlangExtensions
      * Rust allocation after the List is created, there is no need to Drop this memory.
      * </summary>
      */
-    public static Line ToLine(this Vec_FfiToken_t vec, string sourceText)
+    public static StyledLine ToLine(this Vec_FfiToken_t vec, string sourceText)
     {
-        var list = new Line(sourceText);
+        var tokens = new List<SemanticToken>();
 
         var currentPtr = vec.ptr;
 
@@ -63,9 +63,8 @@ public static unsafe class SlangExtensions
                 0,
                 colIndex,
                 token.length,
-                color,
                 token.token_kind,
-                0,
+                color,
                 token.tooltip.AsString()
             );
 
@@ -76,12 +75,12 @@ public static unsafe class SlangExtensions
                 semanticToken.Data = errMsg;
                 semanticToken.Color = ICodeFormatter.ColorError;
             }
-            list.AddToken(semanticToken);
+            tokens.Add(semanticToken);
         }
 
         Ffi.free_ffi_token_vec(vec);
 
-        return list;
+        return new StyledLine(sourceText, tokens);
     }
 
     public static unsafe List<Diagnostic> ToList(this Vec_FfiDiagnostic_t vec)

csharp_mod/Formatter.cs

@@ -4,7 +4,6 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
-using System.Timers;
 using Cysharp.Threading.Tasks;
 using StationeersIC10Editor;
 
@@ -70,8 +69,9 @@ public class SlangFormatter : ICodeFormatter
         return this.Lines.RawText;
     }
 
-    public override Line ParseLine(string line)
+    public override StyledLine ParseLine(string line)
     {
+        L.Debug($"Parsing line for syntax highlighting: {line}");
         return Marshal.TokenizeLine(line);
     }
 
@@ -90,8 +90,6 @@ public class SlangFormatter : ICodeFormatter
         HandleLsp(inputSrc, token).Forget();
     }
 
-    private void OnTimerElapsed(object sender, ElapsedEventArgs e) { }
-
     private async UniTaskVoid HandleLsp(string inputSrc, CancellationToken cancellationToken)
     {
         try
@@ -139,33 +137,26 @@ public class SlangFormatter : ICodeFormatter
             if (line is null)
                 continue;
 
-            line.ClearTokens();
-
-            Dictionary<int, SemanticToken> lineDict = Marshal
-                .TokenizeLine(line.Text)
-                .Tokens.ToDictionary((t) => t.Column);
-
             if (dict.ContainsKey(lineIndex))
             {
+                var tokens = new List<SemanticToken>();
                 foreach (var lineDiagnostic in dict[lineIndex])
                 {
-                    lineDict[(int)lineDiagnostic.Range.StartCol] = new SemanticToken
+                    tokens.Add(
-                    {
+                        new SemanticToken(
-                        Column = Math.Abs((int)lineDiagnostic.Range.StartCol),
+                            line: (int)lineIndex,
-                        Length = Math.Abs(
+                            column: Math.Abs((int)lineDiagnostic.Range.StartCol),
+                            length: Math.Abs(
                                 (int)(lineDiagnostic.Range.EndCol - lineDiagnostic.Range.StartCol)
                             ),
-                        Line = (int)lineIndex,
+                            type: 0,
-                        IsError = true,
+                            style: ICodeFormatter.ColorError,
-                        Data = lineDiagnostic.Message,
+                            data: lineDiagnostic.Message,
-                        Color = SlangFormatter.ColorError,
+                            isError: true
-                    };
+                        )
+                    );
                 }
-            }
+                line.Update(tokens);
-
-            foreach (var token in lineDict.Values)
-            {
-                line.AddToken(token);
             }
         }
 

csharp_mod/Marshal.cs

@@ -131,11 +131,11 @@ public static class Marshal
         }
     }
 
-    public static unsafe Line TokenizeLine(string inputString)
+    public static unsafe StyledLine TokenizeLine(string inputString)
     {
         if (string.IsNullOrEmpty(inputString) || !EnsureLibLoaded())
         {
-            return new Line(inputString);
+            return new StyledLine(inputString ?? "");
         }
 
         fixed (char* ptrInputStr = inputString)
@@ -147,7 +147,7 @@ public static class Marshal
             };
 
             var tokens = Ffi.tokenize_line(strRef);
-
+            L.Debug($"Tokenized line '{inputString}' into {tokens.len} tokens.");
             return tokens.ToLine(inputString);
         }
     }