Merge pull request 'Added support for CRLF windows line endings' (#11) from 41-support-windows-crlf-line-endings into master

Reviewed-on: #11

Changelog.md

@@ -1,5 +1,25 @@
 # Changelog
 
+[0.4.7]
+
+- Added support for Windows CRLF endings
+
+[0.4.6]
+
+- Fixed bug in compiler where you were unable to assign a `const` value to
+  a `let` variable
+
+[0.4.5]
+
+- Fixed issue where after clicking "Cancel" on the IC10 Editor, the side-by-side
+  IC10 output would no longer update with highlighting or code updates.
+- Added ability to live-reload the mod while developing using the `ScriptEngine`
+  mod from BepInEx
+  - This required adding in cleanup code to cleanup references to the Rust DLL
+    before destroying the mod instance.
+- Added BepInEx debug logging. This will ONLY show if you have debug logs
+  enabled in the BepInEx configuration file.
+
 [0.4.4]
 
 - Added Stationpedia docs back after removing all harmony patches from the mod

ModData/About/About.xml

@@ -2,11 +2,11 @@
 <ModMetadata xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <Name>Slang</Name>
   <Author>JoeDiertay</Author>
-  <Version>0.4.4</Version>
+  <Version>0.4.7</Version>
   <Description>
 [h1]Slang: High-Level Programming for Stationeers[/h1]
 
-Stop writing assembly. Start writing code.
+Iterate faster. Stop writing assembly. Start writing code.
 
 Slang (Stationeers Language) brings modern programming to Stationeers. It allows you to write scripts using a familiar C-style syntax (variables, functions, if/else, loops) directly in the in-game editor. When you hit confirm, Slang compiles your code into IC10 instantly.
 
@@ -15,7 +15,7 @@ Slang (Stationeers Language) brings modern programming to Stationeers. It allows
 [h2]Features[/h2]
 [list]
 [*] [b]In-Game Compilation:[/b] No external tools needed. Write Slang directly in the IC editor.
-[*] [b]No More Register Juggling:[/b] Define variables with let (e.g., let temp = 300). The compiler manages r0-r15 for you.
+[*] [b]No More Register Juggling:[/b] Define variables with let (e.g., let temp = 200c). The compiler manages r0-r15 for you.
 [*] [b]Control Flow:[/b] Write readable logic with if, else, while, loop, break, and continue.
 [*] [b]Functions:[/b] Create reusable code blocks with arguments.
 [*] [b]Smart Editor:[/b] Get real-time syntax highlighting and error checking (red text) as you type.
@@ -53,14 +53,13 @@ loop {
 [h2]Known Issues (Beta)[/h2]
 [list]
 [*] [b]Stack Access:[/b] Direct stack memory access is disabled to prevent conflicts with the compiler's internal memory management. A workaround is being planned.
-[*] [b]Documentation:[/b] In-game tooltips for syscalls (like load, set) are WIP. Check the "Slang" entry in the Stationpedia (F1) for help.
+[*] [b]Documentation:[/b] In-game tooltips for syscalls (like load, set) are WIP. Check the "Slang" entry in the Stationpedia (F1) for help, or checkout [url=https://github.com/dbidwell94/stationeers_lang/blob/master/docs/getting-started.md]The Docs[/url] for guides on how to get started.
 [/list]
 
 [h2]Planned Features[/h2]
 [list]
 [*] Enhanced LSP features (Autocomplete, Go to Definition).
 [*] Full feature parity with all IC10 instructions.
-[*] Tutorials and beginner script examples.
 [/list]
 
 [h2]FAQ[/h2]
@@ -70,10 +69,18 @@ A: The Slang compiler is built in Rust for performance and reliability. It is co
 [b]Q: Is this compatible with my current save?[/b]
 A: Yes! Slang does not modify any existing IC10 code, it is only a compiler. As a matter of fact: if you wish to stop using Slang at any time, your compiled IC10 will still exist on the chip. However: Slang adds a comment at the bottom of your compiled IC10 which is a GZIP and base64 encoded version of your Slang source code. This might break line limits with Slang not installed. If you wish to no longer use Slang, I recommend you remove this comment from the source code after uninstalling Slang.
 
+[b]Q: Does this modify the in-game scripting language[/b]
+A: No! Slang compiles directly to IC10. Any valid Slang file will produce valid IC10. The goal of this mod is twofold:
+[list]
+[*] Allow experienced users to quickly iterate on their scripts to get back into base upgrades faster
+[*] Allow newcomers who already know C-style languages (JS/C#/Java/Rust/C/etc) to see how it maps to IC10 so they can get to understand IC10 better
+[/list]
+
 [h2]Useful Links[/h2]
 [url=https://github.com/dbidwell94/stationeers_lang]Source Code on GitHub[/url]
 [url=https://discord.gg/stationeers]Stationeers Official Discord[/url]
 [url=https://discord.gg/M4sCfYMacs]Stationeers Modding Discord[/url]
+[url=https://github.com/dbidwell94/stationeers_lang/blob/master/docs/getting-started.md]Getting Started Guide[/url]
   </Description>
   <ChangeLog xsi:nil="true" />
   <WorkshopHandle>3619985558</WorkshopHandle>
@@ -116,5 +123,7 @@ A: Yes! Slang does not modify any existing IC10 code, it is only a compiler. As
         See: https://github.com/StationeersLaunchPad/StationeersLaunchPad
 
         Source Code: https://github.com/dbidwell94/stationeers_lang
+
+        Documentation: https://github.com/dbidwell94/stationeers_lang/blob/master/docs/getting-started.md
         ]]></InGameDescription>
 </ModMetadata>

csharp_mod/Formatter.cs

@@ -6,7 +6,6 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using StationeersIC10Editor;
-using StationeersIC10Editor.IC10;
 
 public class SlangFormatter : ICodeFormatter
 {
@@ -14,8 +13,25 @@ public class SlangFormatter : ICodeFormatter
     private CancellationTokenSource? _lspCancellationToken;
     private object _tokenLock = new();
 
-    protected Editor? Ic10Editor = null;
-    private IC10CodeFormatter iC10CodeFormatter = new IC10CodeFormatter();
+    protected Editor? __Ic10Editor = null;
+
+    protected Editor Ic10Editor
+    {
+        get
+        {
+            if (__Ic10Editor == null)
+            {
+                var tab = Editor.ParentTab;
+                tab.ClearExtraEditors();
+                __Ic10Editor = new Editor(Editor.KeyHandler);
+                Ic10Editor.IsReadOnly = true;
+                tab.AddEditor(__Ic10Editor);
+            }
+
+            return __Ic10Editor;
+        }
+    }
+
     private string ic10CompilationResult = "";
     private List<SourceMapEntry> ic10SourceMap = new();
 
@@ -80,7 +96,7 @@ public class SlangFormatter : ICodeFormatter
     {
         if (!Marshal.CompileFromString(RawText, out var compilationResult, out var sourceMap))
         {
-            return "Compilation Error";
+            return "# Compilation Error";
         }
 
         return compilationResult + $"\n{EncodeSource(RawText, SLANG_SRC)}";
@@ -118,6 +134,10 @@ public class SlangFormatter : ICodeFormatter
         return styledLine;
     }
 
+    /// <summary>
+    /// This handles calling the `HandleLsp` function by creating a new `CancellationToken` and
+    /// cancelling the current call if applicable.
+    /// </summary>
     private void HandleCodeChanged()
     {
         CancellationToken token;
@@ -133,6 +153,11 @@ public class SlangFormatter : ICodeFormatter
         _ = HandleLsp(inputSrc, token);
     }
 
+    /// <summary>
+    /// Takes a copy of the current source code and sends it to the Rust compiler in a background thread
+    /// to get diagnostic data. This also handles getting a compilation response of optimized IC10 for the
+    /// side-by-side IC10Editor to show with sourcemap highlighting.
+    /// </summary>
     private async Task HandleLsp(string inputSrc, CancellationToken cancellationToken)
     {
         try
@@ -165,24 +190,21 @@ public class SlangFormatter : ICodeFormatter
                 return;
             }
 
-            var (compilationSuccess, compiled, sourceMap) = await Task.Run(
-                () =>
-                {
-                    var successful = Marshal.CompileFromString(
-                        inputSrc,
-                        out var compiled,
-                        out var sourceMap
-                    );
-                    return (successful, compiled, sourceMap);
-                },
-                cancellationToken
-            );
+            var (compilationSuccess, compiled, sourceMap) = await Task.Run(() =>
+            {
+                var successful = Marshal.CompileFromString(
+                    inputSrc,
+                    out var compiled,
+                    out var sourceMap
+                );
+                return (successful, compiled, sourceMap);
+            });
 
             if (compilationSuccess)
             {
                 ic10CompilationResult = compiled;
                 ic10SourceMap = sourceMap;
-                UpdateIc10Formatter();
+                UpdateIc10Content(Ic10Editor);
             }
         }
         catch (OperationCanceledException) { }
@@ -192,22 +214,24 @@ public class SlangFormatter : ICodeFormatter
         }
     }
 
+    /// <summary>
+    /// Updates the underlying code in the IC10 Editor, after which will call `UpdateIc10Formatter` to
+    /// update highlighting of relavent fields.
+    /// </summary>
+    private void UpdateIc10Content(Editor editor)
+    {
+        editor.ResetCode(ic10CompilationResult);
+        UpdateIc10Formatter();
+    }
+
+    // This runs on the main thread. This function ONLY updates the highlighting of the IC10 code.
+    // If you need to update the code in the editor itself, you should use `UpdateIc10Content`.
     private void UpdateIc10Formatter()
     {
-        var tab = Editor.ParentTab;
-        if (Ic10Editor == null)
-        {
-            iC10CodeFormatter = new IC10CodeFormatter();
-            Ic10Editor = new Editor(Editor.KeyHandler);
-            Ic10Editor.IsReadOnly = true;
-            iC10CodeFormatter.Editor = Ic10Editor;
-        }
-
-        if (tab.Editors.Count < 2)
-        {
-            tab.AddEditor(Ic10Editor);
-        }
-
+        // Bail if our backing field is null. We don't want to set the field in this function. It
+        // runs way too much and we might not even have source code to use.
+        if (__Ic10Editor == null)
+            return;
         var caretPos = Editor.CaretPos.Line;
 
         // get the slang sourceMap at the current editor line
@@ -215,10 +239,6 @@ public class SlangFormatter : ICodeFormatter
             entry.SlangSource.StartLine == caretPos || entry.SlangSource.EndLine == caretPos
         );
 
-        // extract the current "context" of the ic10 compilation. The current Slang source line
-        // should be directly next to the compiled IC10 source line, and we should highlight the
-        // IC10 code that directly represents the Slang source
-
         Ic10Editor.ResetCode(ic10CompilationResult);
 
         if (lines.Count() < 1)
@@ -245,7 +265,11 @@ public class SlangFormatter : ICodeFormatter
         };
     }
 
-    // This runs on the Main Thread
+    /// <summary>
+    /// Takes diagnostics from the Rust FFI compiler and applies it as semantic tokens to the
+    /// source in this editor.
+    /// This runs on the Main Thread
+    /// </summary>
     private void ApplyDiagnostics(Dictionary<uint, IGrouping<uint, Diagnostic>> dict)
     {
         HashSet<uint> linesToRefresh;

csharp_mod/Marshal.cs

@@ -67,7 +67,9 @@ public static class Marshal
         try
         {
             _libraryHandle = LoadLibrary(ExtractNativeLibrary(Ffi.RustLib));
+            L.Debug("Rust DLL loaded successfully. Enjoy native speed compilations!");
             CodeFormatters.RegisterFormatter("Slang", typeof(SlangFormatter), true);
+
             return true;
         }
         catch (Exception ex)
@@ -91,8 +93,13 @@ public static class Marshal
 
         try
         {
-            FreeLibrary(_libraryHandle);
+            CodeFormatters.RegisterFormatter("Slang", typeof(PlainTextFormatter), true);
+            if (!FreeLibrary(_libraryHandle))
+            {
+                L.Warning("Unable to free Rust library");
+            }
             _libraryHandle = IntPtr.Zero;
+            L.Debug("Rust DLL library freed");
             return true;
         }
         catch (Exception ex)
@@ -191,9 +198,9 @@ public static class Marshal
 
         Assembly assembly = Assembly.GetExecutingAssembly();
 
-        using (Stream stream = assembly.GetManifestResourceStream(libName))
+        using (Stream resourceStream = assembly.GetManifestResourceStream(libName))
         {
-            if (stream == null)
+            if (resourceStream == null)
             {
                 L.Error(
                     $"{libName} not found. This means it was not embedded in the mod. Please contact the mod author!"
@@ -201,18 +208,85 @@ public static class Marshal
                 return "";
             }
 
+            // Check if file exists and contents are identical to avoid overwriting locked files
+            if (File.Exists(destinationPath))
+            {
+                try
+                {
+                    using (
+                        FileStream fileStream = new FileStream(
+                            destinationPath,
+                            FileMode.Open,
+                            FileAccess.Read,
+                            FileShare.ReadWrite
+                        )
+                    )
+                    {
+                        if (resourceStream.Length == fileStream.Length)
+                        {
+                            if (StreamsContentsAreEqual(resourceStream, fileStream))
+                            {
+                                L.Debug(
+                                    $"DLL {libName} already exists and matches. Skipping extraction."
+                                );
+                                return destinationPath;
+                            }
+                        }
+                    }
+                }
+                catch (IOException ex)
+                {
+                    L.Warning(
+                        $"Could not verify existing {libName}, attempting overwrite. {ex.Message}"
+                    );
+                }
+            }
+
+            resourceStream.Position = 0;
+
+            // Attempt to overwrite if missing or different
             try
             {
                 using (FileStream fileStream = new FileStream(destinationPath, FileMode.Create))
                 {
-                    stream.CopyTo(fileStream);
+                    resourceStream.CopyTo(fileStream);
                 }
                 return destinationPath;
             }
             catch (IOException e)
             {
-                L.Warning($"Could not overwrite {libName} (it might be in use): {e.Message}");
-                return "";
+                // If we fail here, the file is likely locked.
+                // However, if we are here, it means the file is DIFFERENT or we couldn't read it.
+                // As a fallback for live-reload, we can try returning the path anyway
+                // assuming the existing locked file might still work.
+                L.Warning(
+                    $"Could not overwrite {libName} (it might be in use): {e.Message}. Attempting to use existing file."
+                );
+                return destinationPath;
+            }
+        }
+    }
+
+    private static bool StreamsContentsAreEqual(Stream stream1, Stream stream2)
+    {
+        const int bufferSize = 4096;
+        byte[] buffer1 = new byte[bufferSize];
+        byte[] buffer2 = new byte[bufferSize];
+
+        while (true)
+        {
+            int count1 = stream1.Read(buffer1, 0, bufferSize);
+            int count2 = stream2.Read(buffer2, 0, bufferSize);
+
+            if (count1 != count2)
+                return false;
+            if (count1 == 0)
+                return true;
+
+            for (int i = 0; i < count1; i++)
+            {
+                if (buffer1[i] != buffer2[i])
+                    return false;
             }
         }
     }

csharp_mod/Plugin.cs

@@ -1,4 +1,3 @@
-using System.Text.RegularExpressions;
 using BepInEx;
 using HarmonyLib;
 
@@ -40,43 +39,32 @@ namespace Slang
     {
         public const string PluginGuid = "com.biddydev.slang";
         public const string PluginName = "Slang";
-        public const string PluginVersion = "0.4.4";
+        public const string PluginVersion = "0.4.7";
 
-        private Harmony? _harmony;
+        private static Harmony? _harmony;
 
-        private static Regex? _slangSourceCheck = null;
-
-        private static Regex SlangSourceCheck
-        {
-            get
-            {
-                if (_slangSourceCheck is null)
-                {
-                    _slangSourceCheck = new Regex(@"[;{}()]|\b(let|fn|device)\b|\/\/");
-                }
-
-                return _slangSourceCheck;
-            }
-        }
-
-        public static bool IsSlangSource(ref string input)
-        {
-            return SlangSourceCheck.IsMatch(input);
-        }
-
-        private void Awake()
+        public void Awake()
         {
             L.SetLogger(Logger);
-            this._harmony = new Harmony(PluginGuid);
+            _harmony = new Harmony(PluginGuid);
 
             // If we failed to load the compiler, bail from the rest of the patches. It won't matter,
             // as the compiler itself has failed to load.
             if (!Marshal.Init())
             {
+                L.Error("Marshal failed to init");
                 return;
             }
 
-            this._harmony.PatchAll();
+            _harmony.PatchAll();
+            L.Debug("Ran Harmony patches");
+        }
+
+        public void OnDestroy()
+        {
+            Marshal.Destroy();
+            _harmony?.UnpatchSelf();
+            L.Debug("Cleaned up Harmony patches");
         }
     }
 }

rust_compiler/Cargo.lock

@@ -930,7 +930,7 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
 [[package]]
 name = "slang"
-version = "0.4.3"
+version = "0.4.7"
 dependencies = [
  "anyhow",
  "clap",

rust_compiler/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "slang"
-version = "0.4.3"
+version = "0.4.7"
 edition = "2021"
 
 [workspace]

rust_compiler/libs/compiler/src/test/declaration_literal.rs

@@ -168,3 +168,28 @@ fn test_const_hash_expr() -> anyhow::Result<()> {
     );
     Ok(())
 }
+
+#[test]
+fn test_declaration_is_const() -> anyhow::Result<()> {
+    let compiled = compile! {
+        debug
+        r#"
+            const MAX = 100;
+
+            let max = MAX;
+        "#
+    };
+
+    assert_eq!(
+        compiled,
+        indoc! {
+            "
+            j main
+            main:
+            move r8 100
+            "
+        }
+    );
+
+    Ok(())
+}

rust_compiler/libs/compiler/src/v1.rs

@@ -714,7 +714,12 @@ impl<'a> Compiler<'a> {
 
                         Operand::Register(VariableScope::TEMP_STACK_REGISTER)
                     }
-                    VariableLocation::Constant(_) | VariableLocation::Device(_) => unreachable!(),
+                    VariableLocation::Constant(Literal::Number(num)) => Operand::Number(num.into()),
+                    VariableLocation::Constant(Literal::Boolean(b)) => {
+                        Operand::Number(Number::from(b).into())
+                    }
+                    VariableLocation::Device(_)
+                    | VariableLocation::Constant(Literal::String(_)) => unreachable!(),
                 };
                 self.emit_variable_assignment(&var_loc, src)?;
                 (var_loc, None)

rust_compiler/libs/tokenizer/src/token.rs

@@ -115,7 +115,7 @@ macro_rules! keyword {
 }
 
 #[derive(Debug, PartialEq, Hash, Eq, Clone, Logos)]
-#[logos(skip r"[ \t\f]+")]
+#[logos(skip r"[ \r\t\f]+")]
 #[logos(extras = Extras)]
 #[logos(error(LexError, LexError::from_lexer))]
 pub enum TokenType<'a> {
@@ -843,3 +843,20 @@ documented! {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::TokenType;
+    use logos::Logos;
+
+    #[test]
+    fn test_windows_crlf_endings() -> anyhow::Result<()> {
+        let src = "let i = 0;\r\n";
+
+        let lexer = TokenType::lexer(src);
+
+        let tokens = lexer.collect::<Vec<_>>();
+
+        assert!(!tokens.iter().any(|res| res.is_err()));
+        Ok(())
+    }
+}