dbidwell/stationeers_lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #11 from dbidwell94/dot-notation

Browse Source 
Dot notation and bug improvements
This commit is contained in:
Devin Bidwell
2025-12-05 02:49:35 -07:00
committed by GitHub
parent c911b0af18 4357bf5263
commit df4c2274d5
32 changed files with 1658 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
.github/workflows/build.yml vendored Normal file
View File

@@ -0,0 +1,69 @@
name: CI/CD Pipeline
# Trigger conditions
on:
push:
branches: ["master"]
pull_request:
branches: ["master"]
jobs:
# JOB 1: RUN TESTS
# This runs on every PR and every push to master.
# It validates that the Rust code is correct.
test:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
# 1. Build the Docker Image (Cached)
- name: Build Docker Image
run: docker build -t slang-builder -f Dockerfile.build .
# 2. Run Rust Tests
# --manifest-path: Point to the nested Cargo.toml
# --workspace: Test all crates (compiler, parser, tokenizer, helpers)
# --all-targets: Test lib, bin, and tests folder (skips doc-tests because they are not valid Rust)
- name: Run Rust Tests
run: |
docker run --rm \
-u $(id -u):$(id -g) \
-v "$PWD":/app \
slang-builder \
cargo test --manifest-path rust_compiler/Cargo.toml --workspace --all-targets
build:
needs: test
runs-on: self-hosted
if: github.event_name == 'push'
steps:
- uses: actions/checkout@v4
# 1. Build Image (Fast, uses cache from previous job if available/local)
- name: Build Docker Image
run: docker build -t slang-builder -f Dockerfile.build .
# 2. Run the Build Script
# Mounts the references from the server's secure storage
- name: Build Release Artifacts
run: |
docker run --rm \
-u $(id -u):$(id -g) \
-v "$PWD":/app \
-v "/home/github-runner/permanent-refs":/app/csharp_mod/refs \
slang-builder \
./build.sh
# 3. Fix Permissions
# Docker writes files as root. We need to own them to upload them.
- name: Fix Permissions
if: always()
run: sudo chown -R $USER:$USER release/
# 4. Upload to GitHub
- name: Upload Release Artifacts
uses: actions/upload-artifact@v4
with:
name: StationeersSlang-Release
path: release/

19
Dockerfile.build Normal file
View File

@@ -0,0 +1,19 @@
FROM rust:latest
RUN apt-get update && apt-get install -y \
mingw-w64 \
wget \
apt-transport-https \
&& rm -rf /var/lib/apt/lists/*
RUN wget --progress=dot:giga https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \
&& chmod +x ./dotnet-install.sh \
&& ./dotnet-install.sh --channel 8.0 --install-dir /usr/share/dotnet \
&& ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
RUN rustup target add x86_64-pc-windows-gnu && rustup target add x86_64-unknown-linux-gnu
WORKDIR /app
# The command will be provided at runtime
CMD ["./build.sh"]

2
build.sh
View File

@@ -39,7 +39,6 @@ echo "--------------------"
RUST_WIN_EXE="$RUST_DIR/target/x86_64-pc-windows-gnu/release/slang.exe"
RUST_LINUX_BIN="$RUST_DIR/target/x86_64-unknown-linux-gnu/release/slang"
CHARP_DLL="$CSHARP_DIR/bin/Release/net48/StationeersSlang.dll"
CHARP_PDB="$CSHARP_DIR/bin/Release/net48/StationeersSlang.pdb"
# Check if the release dir exists, if not: create it.
if [[ ! -d "$RELEASE_DIR" ]]; then
@@ -49,4 +48,3 @@ fi
cp "$RUST_WIN_EXE" "$RELEASE_DIR/slang.exe"
cp "$RUST_LINUX_BIN" "$RELEASE_DIR/slang"
cp "$CHARP_DLL" "$RELEASE_DIR/StationeersSlang.dll"
cp "$CHARP_PDB" "$RELEASE_DIR/StationeersSlang.pdb"

72
csharp_mod/Extensions.cs
View File

@@ -3,6 +3,7 @@ namespace Slang;
using System;
using System.Collections.Generic;
using System.Text;
using Assets.Scripts.UI;
using StationeersIC10Editor;
public static unsafe class SlangExtensions
@@ -63,7 +64,9 @@ public static unsafe class SlangExtensions
colIndex,
token.length,
color,
token.token_kind
token.token_kind,
0,
token.tooltip.AsString()
);
string errMsg = token.error.AsString();
@@ -115,20 +118,63 @@ public static unsafe class SlangExtensions
{
switch (kind)
{
case 1:
return SlangFormatter.ColorString; // String
case 2:
return SlangFormatter.ColorString; // Number
case 3:
return SlangFormatter.ColorInstruction; // Boolean
case 4:
return SlangFormatter.ColorSelection; // Keyword
case 5:
return SlangFormatter.ColorLineNumber; // Identifier
case 6:
return SlangFormatter.ColorDefault; // Symbol
case 1: // Strings
return SlangFormatter.ColorString;
case 2: // Numbers
return SlangFormatter.ColorNumber;
case 3: // Booleans
return SlangFormatter.ColorBoolean;
case 4: // (if, else, loop)
return SlangFormatter.ColorControl;
case 5: // (let, const, device)
return SlangFormatter.ColorDeclaration;
case 6: // (variables)
return SlangFormatter.ColorIdentifier;
case 7: // (punctuation)
return SlangFormatter.ColorDefault;
case 10: // (syscalls)
return SlangFormatter.ColorFunction;
case 11: // Comparisons
case 12: // Math
case 13: // Logic
return SlangFormatter.ColorOperator;
default:
return SlangFormatter.ColorDefault;
}
}
public static unsafe List<StationpediaPage> ToList(this Vec_FfiDocumentedItem_t vec)
{
var toReturn = new List<StationpediaPage>((int)vec.len);
var currentPtr = vec.ptr;
for (int i = 0; i < (int)vec.len; i++)
{
var doc = currentPtr[i];
var docItemName = doc.item_name.AsString();
var formattedText = TextMeshProFormatter.FromMarkdown(doc.docs.AsString());
var pediaPage = new StationpediaPage(
$"slang-item-{docItemName}",
docItemName,
formattedText
);
pediaPage.Text = formattedText;
pediaPage.Description = formattedText;
pediaPage.ParsePage();
toReturn.Add(pediaPage);
}
Ffi.free_docs_vec(vec);
return toReturn;
}
}

30
csharp_mod/FfiGlue.cs
View File

@@ -121,6 +121,31 @@ public unsafe partial class Ffi {
slice_ref_uint16_t input);
}
[StructLayout(LayoutKind.Sequential, Size = 48)]
public unsafe struct FfiDocumentedItem_t {
public Vec_uint8_t item_name;
public Vec_uint8_t docs;
}
/// <summary>
/// Same as [<c>Vec<T></c>][<c>rust::Vec</c>], but with guaranteed <c>#[repr(C)]</c> layout
/// </summary>
[StructLayout(LayoutKind.Sequential, Size = 24)]
public unsafe struct Vec_FfiDocumentedItem_t {
public FfiDocumentedItem_t * ptr;
public UIntPtr len;
public UIntPtr cap;
}
public unsafe partial class Ffi {
[DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
void free_docs_vec (
Vec_FfiDocumentedItem_t v);
}
public unsafe partial class Ffi {
[DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
void free_ffi_diagnostic_vec (
@@ -164,6 +189,11 @@ public unsafe partial class Ffi {
Vec_uint8_t s);
}
public unsafe partial class Ffi {
[DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
Vec_FfiDocumentedItem_t get_docs ();
}
public unsafe partial class Ffi {
[DllImport(RustLib, ExactSpelling = true)] public static unsafe extern
Vec_FfiToken_t tokenize_line (

116
csharp_mod/Formatter.cs
View File

@@ -4,26 +4,65 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Timers;
using Cysharp.Threading.Tasks;
using StationeersIC10Editor;
public class SlangFormatter : ICodeFormatter
{
private CancellationTokenSource? _lspCancellationToken;
private readonly SynchronizationContext? _mainThreadContext;
private volatile bool IsDiagnosing = false;
private object _tokenLock = new();
public static readonly uint ColorInstruction = ColorFromHTML("#ffff00");
public static readonly uint ColorString = ColorFromHTML("#ce9178");
// VS Code Dark Theme Palette
public static readonly uint ColorControl = ColorFromHTML("#C586C0"); // Pink (if, return, loop)
public static readonly uint ColorDeclaration = ColorFromHTML("#569CD6"); // Blue (let, device, fn)
public static readonly uint ColorFunction = ColorFromHTML("#DCDCAA"); // Yellow (syscalls)
public static readonly uint ColorString = ColorFromHTML("#CE9178"); // Orange
public static new readonly uint ColorNumber = ColorFromHTML("#B5CEA8"); // Light Green
public static readonly uint ColorBoolean = ColorFromHTML("#569CD6"); // Blue (true/false)
public static readonly uint ColorIdentifier = ColorFromHTML("#9CDCFE"); // Light Blue (variables)
public static new readonly uint ColorDefault = ColorFromHTML("#D4D4D4"); // White (punctuation ; { } )
// Operators are often the same color as default text in VS Code Dark,
// but having a separate definition lets you tweak it (e.g. make them slightly darker or distinct)
public static readonly uint ColorOperator = ColorFromHTML("#D4D4D4");
private HashSet<uint> _linesWithErrors = new();
public SlangFormatter()
: base()
{
// 1. Capture the Main Thread context.
// This works because the Editor instantiates this class on the main thread.
_mainThreadContext = SynchronizationContext.Current;
OnCodeChanged += HandleCodeChanged;
}
public static double MatchingScore(string input)
{
// Empty input is not valid Slang
if (string.IsNullOrWhiteSpace(input))
return 0d;
// Run the compiler to get diagnostics
var diagnostics = Marshal.DiagnoseSource(input);
// Count the number of actual Errors (Severity 1).
// We ignore Warnings (2), Info (3), etc.
double errorCount = diagnostics.Count(d => d.Severity == 1);
// Get the total line count to calculate error density
double lineCount = input.Split('\n').Length;
// Prevent division by zero
if (lineCount == 0)
return 0d;
// Calculate score: Start at 1.0 (100%) and subtract the ratio of errors per line.
// Example: 10 lines with 0 errors = 1.0
// Example: 10 lines with 2 errors = 0.8
// Example: 10 lines with 10+ errors = 0.0
double score = 1.0d - (errorCount / lineCount);
// Clamp the result between 0 and 1
return Math.Max(0d, Math.Min(1d, score));
}
public override string Compile()
@@ -33,65 +72,62 @@ public class SlangFormatter : ICodeFormatter
public override Line ParseLine(string line)
{
HandleCodeChanged();
return Marshal.TokenizeLine(line);
}
private void HandleCodeChanged()
{
if (IsDiagnosing)
return;
CancellationToken token;
string inputSrc;
lock (_tokenLock)
{
_lspCancellationToken?.Cancel();
_lspCancellationToken?.Dispose();
_lspCancellationToken = new CancellationTokenSource();
token = _lspCancellationToken.Token;
inputSrc = this.RawText;
}
_ = Task.Run(() => HandleLsp(_lspCancellationToken.Token), _lspCancellationToken.Token);
HandleLsp(inputSrc, token).Forget();
}
private void OnTimerElapsed(object sender, ElapsedEventArgs e) { }
private async Task HandleLsp(CancellationToken cancellationToken)
private async UniTaskVoid HandleLsp(string inputSrc, CancellationToken cancellationToken)
{
try
{
await Task.Delay(200, cancellationToken);
await UniTask.SwitchToThreadPool();
if (cancellationToken.IsCancellationRequested)
{
return;
}
// 3. Dispatch the UI update to the Main Thread
if (_mainThreadContext != null)
await System.Threading.Tasks.Task.Delay(200, cancellationToken: cancellationToken);
if (cancellationToken.IsCancellationRequested)
return;
var dict = Marshal
.DiagnoseSource(inputSrc)
.GroupBy(d => d.Range.StartLine)
.ToDictionary(g => g.Key);
await UniTask.Yield(PlayerLoopTiming.Update, cancellationToken);
ApplyDiagnostics(dict);
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
// Post ensures ApplyDiagnostics runs on the captured thread (Main Thread)
_mainThreadContext.Post(_ => ApplyDiagnostics(), null);
L.Error(ex.Message);
}
else
{
// Fallback: If context is null (rare in Unity), try running directly
// but warn, as this might crash if not thread-safe.
L.Warning("SynchronizationContext was null. Attempting direct update (risky).");
ApplyDiagnostics();
}
}
finally { }
}
// This runs on the Main Thread
private void ApplyDiagnostics()
private void ApplyDiagnostics(Dictionary<uint, IGrouping<uint, Diagnostic>> dict)
{
List<Diagnostic> diagnosis = Marshal.DiagnoseSource(this.RawText);
var dict = diagnosis.GroupBy(d => d.Range.StartLine).ToDictionary(g => g.Key);
var linesToRefresh = new HashSet<uint>(dict.Keys);
linesToRefresh.UnionWith(_linesWithErrors);
IsDiagnosing = true;
foreach (var lineIndex in linesToRefresh)
{
// safety check for out of bounds (in case lines were deleted)
@@ -134,7 +170,5 @@ public class SlangFormatter : ICodeFormatter
}
_linesWithErrors = new HashSet<uint>(dict.Keys);
IsDiagnosing = false;
}
}

9
csharp_mod/Marshal.cs
View File

@@ -5,6 +5,7 @@ using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
using Assets.Scripts.UI;
using StationeersIC10Editor;
public struct Range
@@ -151,6 +152,14 @@ public static class Marshal
}
}
/// <summary>
/// Gets the currently documented items from the Slang compiler and returns new StationpediaPages with correct formatting.
/// </summary>
public static unsafe List<StationpediaPage> GetSlangDocs()
{
return Ffi.get_docs().ToList();
}
private static string ExtractNativeLibrary(string libName)
{
string destinationPath = Path.Combine(Path.GetTempPath(), libName);

11
csharp_mod/Patches.cs
View File

@@ -214,7 +214,6 @@ public static class SlangPatches
[HarmonyPrefix]
public static void isc_ButtonInputCancel()
{
L.Info("ButtonInputCancel called on the InputSourceCode static instance.");
if (_currentlyEditingMotherboard is null || _motherboardCachedCode is null)
{
return;
@@ -225,4 +224,14 @@ public static class SlangPatches
_currentlyEditingMotherboard = null;
_motherboardCachedCode = null;
}
[HarmonyPatch(typeof(Stationpedia), nameof(Stationpedia.Regenerate))]
[HarmonyPostfix]
public static void Stationpedia_Regenerate()
{
foreach (var page in Marshal.GetSlangDocs())
{
Stationpedia.Register(page);
}
}
}

5
csharp_mod/Plugin.cs
View File

@@ -35,14 +35,15 @@ namespace Slang
}
}
[BepInPlugin(PluginGuid, PluginName, "0.1.0")]
[BepInPlugin(PluginGuid, PluginName, PluginVersion)]
[BepInDependency(StationeersIC10Editor.IC10EditorPlugin.PluginGuid)]
public class SlangPlugin : BaseUnityPlugin
{
public const string PluginGuid = "com.biddydev.slang";
public const string PluginName = "Slang";
public const string PluginVersion = "0.1.0";
public static Mod MOD = new Mod(PluginName, "0.1.0");
public static Mod MOD = new Mod(PluginName, PluginVersion);
private Harmony? _harmony;

22
csharp_mod/StationpediaDocumentation.cs
View File

@@ -1,22 +0,0 @@
using Assets.Scripts.UI;
namespace Slang
{
public static class SlangDocs
{
public static StationpediaPage[] Pages
{
get
{
return
[
new StationpediaPage(
"slang-init",
"Slang",
"Slang is a new high level language built specifically for Stationeers"
),
];
}
}
}
}

54
csharp_mod/TmpFormatter.cs Normal file
View File

@@ -0,0 +1,54 @@
using System.Text.RegularExpressions;
namespace Slang;
public static class TextMeshProFormatter
{
private const string CODE_COLOR = "#FFD700";
public static string FromMarkdown(string markdown)
{
if (string.IsNullOrEmpty(markdown))
return "";
// 1. Normalize Line Endings
string text = markdown.Replace("\r\n", "\n");
// 2. Handle Code Blocks (```)
text = Regex.Replace(
text,
@"```\s*(.*?)\s*```",
match =>
{
var codeContent = match.Groups[1].Value;
return $"<color={CODE_COLOR}>{codeContent}</color>"; // Gold color for code
},
RegexOptions.Singleline
);
// 3. Handle Headers (## Header)
// Convert ## Header to large bold text
text = Regex.Replace(
text,
@"^##(\s+)?(.+)$",
"<size=120%><b>$1</b></size>",
RegexOptions.Multiline
);
// 4. Handle Inline Code (`code`)
text = Regex.Replace(text, @"`([^`]+)`", $"<color={CODE_COLOR}>$1</color>");
// 5. Handle Bold (**text**)
text = Regex.Replace(text, @"\*\*(.+?)\*\*", "<b>$1</b>");
// 6. Handle Italics (*text*)
text = Regex.Replace(text, @"\*(.+?)\*", "<i>$1</i>");
// 7. Convert Newlines to TMP Line Breaks
// Stationpedia needs <br> or explicit newlines.
// Often just ensuring \n is preserved is enough, but <br> is safer for HTML-like parsers.
text = text.Replace("\n", "<br>");
return text;
}
}

22
csharp_mod/stationeersSlang.csproj
View File

@@ -11,34 +11,36 @@
</PropertyGroup>
<PropertyGroup>
<ManagedDir>$(STATIONEERS_DIR)/rocketstation_Data/Managed</ManagedDir>
<BepInExDir>$(STATIONEERS_DIR)/BepInEx/core</BepInExDir>
</PropertyGroup>
<ItemGroup>
<Reference Include="netstandard">
<HintPath>$(ManagedDir)/netstandard.dll</HintPath>
<HintPath>./ref/netstandard.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="BepInEx">
<HintPath>$(BepInExDir)/BepInEx.dll</HintPath>
<HintPath>./ref/BepInEx.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="0Harmony">
<HintPath>$(BepInExDir)/0Harmony.dll</HintPath>
<HintPath>./ref/0Harmony.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="UnityEngine">
<HintPath>$(ManagedDir)/UnityEngine.dll</HintPath>
<HintPath>./ref/UnityEngine.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="UnityEngine.CoreModule">
<HintPath>$(ManagedDir)/UnityEngine.CoreModule.dll</HintPath>
<HintPath>./ref/UnityEngine.CoreModule.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="Assembly-CSharp">
<HintPath>$(ManagedDir)/Assembly-CSharp.dll</HintPath>
<HintPath>./ref/Assembly-CSharp.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="UniTask">
<HintPath>./ref/UniTask.dll</HintPath>
<Private>False</Private>
</Reference>
@@ -47,11 +49,11 @@
<Private>False</Private>
</Reference>
<Reference Include="LaunchPadBooster.dll">
<HintPath>$(STATIONEERS_DIR)/BepInEx/plugins/StationeersLaunchPad/LaunchPadBooster.dll</HintPath>
<HintPath>./ref/LaunchPadBooster.dll</HintPath>
<Private>False</Private>
</Reference>
<Reference Include="StationeersMods.Interface.dll">
<HintPath>$(STATIONEERS_DIR)/BepInEx/plugins/StationeersLaunchPad/StationeersMods.Interface.dll</HintPath>
<HintPath>./ref/StationeersMods.Interface.dll</HintPath>
<Private>False</Private>
</Reference>
</ItemGroup>

16
rust_compiler/Cargo.lock generated
View File

@@ -360,6 +360,10 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "helpers"
version = "0.1.0"
[[package]]
name = "indexmap"
version = "2.12.1"
@@ -412,9 +416,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.177"
version = "0.2.178"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
[[package]]
name = "lsp-types"
@@ -495,7 +499,9 @@ name = "parser"
version = "0.1.0"
dependencies = [
"anyhow",
"helpers",
"lsp-types",
"pretty_assertions",
"quick-error",
"tokenizer",
]
@@ -828,6 +834,7 @@ dependencies = [
"anyhow",
"clap",
"compiler",
"helpers",
"lsp-types",
"parser",
"quick-error",
@@ -925,6 +932,7 @@ name = "tokenizer"
version = "0.1.0"
dependencies = [
"anyhow",
"helpers",
"lsp-types",
"quick-error",
"rust_decimal",
@@ -989,9 +997,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.18.1"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
dependencies = [
"js-sys",
"wasm-bindgen",

1
rust_compiler/Cargo.toml
View File

@@ -40,6 +40,7 @@ rust_decimal = { workspace = true }
tokenizer = { path = "libs/tokenizer" }
parser = { path = "libs/parser" }
compiler = { path = "libs/compiler" }
helpers = { path = "libs/helpers" }
safer-ffi = { workspace = true }
[dev-dependencies]

1
rust_compiler/libs/compiler/src/test/branching.rs
View File

@@ -149,6 +149,7 @@ fn test_spilled_variable_update_in_branch() -> anyhow::Result<()> {
sub r0 sp 1
put db r0 99 #h
L1:
sub sp sp 1
"
}
);

5
rust_compiler/libs/compiler/src/test/declaration_function_invocation.rs
View File

@@ -34,6 +34,8 @@ fn no_arguments() -> anyhow::Result<()> {
#[test]
fn let_var_args() -> anyhow::Result<()> {
// !IMPORTANT this needs to be stabilized as it currently incorrectly calculates sp offset at
// both ends of the cleanup lifecycle
let compiled = compile! {
debug
"
@@ -64,6 +66,7 @@ fn let_var_args() -> anyhow::Result<()> {
get r8 db r0
sub sp sp 1
move r9 r15 #i
sub sp sp 1
"
}
);
@@ -123,6 +126,7 @@ fn inline_literal_args() -> anyhow::Result<()> {
get r8 db r0
sub sp sp 1
move r9 r15 #returnedValue
sub sp sp 1
"
}
);
@@ -164,6 +168,7 @@ fn mixed_args() -> anyhow::Result<()> {
get r8 db r0
sub sp sp 1
move r9 r15 #returnValue
sub sp sp 1
"
}
);

1
rust_compiler/libs/compiler/src/test/declaration_literal.rs
View File

@@ -56,6 +56,7 @@ fn variable_declaration_numeric_literal_stack_spillover() -> anyhow::Result<()>
push 7 #h
push 8 #i
push 9 #j
sub sp sp 3
"
}
);

260
rust_compiler/libs/compiler/src/v1.rs
View File

@@ -4,9 +4,10 @@ use parser::{
Parser as ASTParser,
sys_call::{SysCall, System},
tree_node::{
AssignmentExpression, BinaryExpression, BlockExpression, DeviceDeclarationExpression,
Expression, FunctionExpression, IfExpression, InvocationExpression, Literal,
LiteralOrVariable, LogicalExpression, LoopExpression, Span, Spanned, WhileExpression,
AssignmentExpression, BinaryExpression, BlockExpression, ConstDeclarationExpression,
DeviceDeclarationExpression, Expression, FunctionExpression, IfExpression,
InvocationExpression, Literal, LiteralOrVariable, LogicalExpression, LoopExpression,
MemberAccessExpression, Span, Spanned, WhileExpression,
},
};
use quick_error::quick_error;
@@ -33,6 +34,20 @@ macro_rules! debug {
};
}
fn extract_literal(literal: Literal, allow_strings: bool) -> Result<String, Error> {
if !allow_strings && matches!(literal, Literal::String(_)) {
return Err(Error::Unknown(
"Literal strings are not allowed in this context".to_string(),
None,
));
}
Ok(match literal {
Literal::String(s) => s,
Literal::Number(n) => n.to_string(),
Literal::Boolean(b) => if b { "1" } else { "0" }.into(),
})
}
quick_error! {
#[derive(Debug)]
pub enum Error {
@@ -57,6 +72,9 @@ quick_error! {
AgrumentMismatch(func_name: String, span: Span) {
display("Incorrect number of arguments passed into `{func_name}`")
}
ConstAssignment(ident: String, span: Span) {
display("Attempted to re-assign a value to const variable `{ident}`")
}
Unknown(reason: String, span: Option<Span>) {
display("{reason}")
}
@@ -83,6 +101,7 @@ impl From<Error> for lsp_types::Diagnostic {
DuplicateIdentifier(_, span)
| UnknownIdentifier(_, span)
| InvalidDevice(_, span)
| ConstAssignment(_, span)
| AgrumentMismatch(_, span) => Diagnostic {
range: span.into(),
message: value.to_string(),
@@ -266,6 +285,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
// decl_expr is Box<Spanned<Expression>>
self.expression_declaration(var_name, *decl_expr, scope)
}
Expression::ConstDeclaration(const_decl_expr) => {
self.expression_const_declaration(const_decl_expr.node, scope)?;
Ok(None)
}
Expression::Assignment(assign_expr) => {
self.expression_assignment(assign_expr.node, scope)?;
Ok(None)
@@ -332,6 +355,42 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
}
}
}
Expression::MemberAccess(access) => {
// "load" behavior (e.g. `let x = d0.On`)
let MemberAccessExpression { object, member } = access.node;
// 1. Resolve the object to a device string (e.g., "d0" or "rX")
let (device_str, cleanup) = self.resolve_device(*object, scope)?;
// 2. Allocate a temp register for the result
let result_name = self.next_temp_name();
let loc = scope.add_variable(&result_name, LocationRequest::Temp)?;
let reg = self.resolve_register(&loc)?;
// 3. Emit load instruction: l rX device member
self.write_output(format!("l {} {} {}", reg, device_str, member.node))?;
// 4. Cleanup
if let Some(c) = cleanup {
scope.free_temp(c)?;
}
Ok(Some(CompilationResult {
location: loc,
temp_name: Some(result_name),
}))
}
Expression::MethodCall(call) => {
// Methods are not yet fully supported (e.g. `d0.SomeFunc()`).
// This would likely map to specialized syscalls or batch instructions.
Err(Error::Unknown(
format!(
"Method calls are not yet supported: {}",
call.node.method.node
),
Some(call.span),
))
}
Expression::Priority(inner_expr) => self.expression(*inner_expr, scope),
Expression::Negation(inner_expr) => {
// Compile negation as 0 - inner
@@ -361,6 +420,24 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
}
}
/// Resolves an expression to a device identifier string for use in instructions like `s` or `l`.
/// Returns (device_string, optional_cleanup_temp_name).
fn resolve_device<'v>(
&mut self,
expr: Spanned<Expression>,
scope: &mut VariableScope<'v>,
) -> Result<(String, Option<String>), Error> {
// If it's a direct variable reference, check if it's a known device alias first
if let Expression::Variable(ref name) = expr.node
&& let Some(device_id) = self.devices.get(&name.node)
{
return Ok((device_id.clone(), None));
}
// Otherwise, compile it as an operand (e.g. it might be a register holding a device hash/id)
self.compile_operand(expr, scope)
}
fn emit_variable_assignment(
&mut self,
var_name: &str,
@@ -380,6 +457,14 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
VariableLocation::Stack(_) => {
self.write_output(format!("push {}{debug_tag}", source_value.into()))?;
}
VariableLocation::Constant(_) => {
return Err(Error::Unknown(
r#"Attempted to emit a variable assignent for a constant value.
This is a Compiler bug and should be reported to the developer."#
.into(),
None,
));
}
}
Ok(())
@@ -526,6 +611,7 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
))?;
format!("r{}", VariableScope::TEMP_STACK_REGISTER)
}
VariableLocation::Constant(_) => unreachable!(),
};
self.emit_variable_assignment(&name_str, &var_loc, src_str)?;
(var_loc, None)
@@ -540,6 +626,35 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
scope,
);
}
Expression::MemberAccess(access) => {
// Compile the member access (load instruction)
let result = self.expression(
Spanned {
node: Expression::MemberAccess(access),
span: name_span, // Use declaration span roughly
},
scope,
)?;
// Result is in a temp register
let Some(comp_res) = result else {
return Err(Error::Unknown(
"Member access did not return a value".into(),
Some(name_span),
));
};
let var_loc = scope.add_variable(&name_str, LocationRequest::Persist)?;
let result_reg = self.resolve_register(&comp_res.location)?;
self.emit_variable_assignment(&name_str, &var_loc, result_reg)?;
if let Some(temp) = comp_res.temp_name {
scope.free_temp(temp)?;
}
(var_loc, None)
}
_ => {
return Err(Error::Unknown(
format!("`{name_str}` declaration of this type is not supported/implemented."),
@@ -554,16 +669,34 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
}))
}
fn expression_const_declaration<'v>(
&mut self,
expr: ConstDeclarationExpression,
scope: &mut VariableScope<'v>,
) -> Result<CompilationResult, Error> {
let ConstDeclarationExpression {
name: const_name,
value: const_value,
} = expr;
Ok(CompilationResult {
location: scope.define_const(const_name.node, const_value.node)?,
temp_name: None,
})
}
fn expression_assignment<'v>(
&mut self,
expr: AssignmentExpression,
scope: &mut VariableScope<'v>,
) -> Result<(), Error> {
let AssignmentExpression {
identifier,
assignee,
expression,
} = expr;
match assignee.node {
Expression::Variable(identifier) => {
let location = match scope.get_location_of(&identifier.node) {
Ok(l) => l,
Err(_) => {
@@ -599,11 +732,39 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
VariableScope::TEMP_STACK_REGISTER
))?;
}
VariableLocation::Constant(_) => {
return Err(Error::ConstAssignment(identifier.node, identifier.span));
}
}
if let Some(name) = cleanup {
scope.free_temp(name)?;
}
}
Expression::MemberAccess(access) => {
// Set instruction: s device member value
let MemberAccessExpression { object, member } = access.node;
let (device_str, dev_cleanup) = self.resolve_device(*object, scope)?;
let (val_str, val_cleanup) = self.compile_operand(*expression, scope)?;
self.write_output(format!("s {} {} {}", device_str, member.node, val_str))?;
if let Some(c) = dev_cleanup {
scope.free_temp(c)?;
}
if let Some(c) = val_cleanup {
scope.free_temp(c)?;
}
}
_ => {
return Err(Error::Unknown(
"Invalid assignment target. Only variables and member access are supported."
.into(),
Some(assignee.span),
));
}
}
Ok(())
}
@@ -671,6 +832,9 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
VariableLocation::Persistant(reg) | VariableLocation::Temporary(reg) => {
self.write_output(format!("push r{reg}"))?;
}
VariableLocation::Constant(lit) => {
self.write_output(format!("push {}", extract_literal(lit, false)?))?;
}
VariableLocation::Stack(stack_offset) => {
self.write_output(format!(
"sub r{0} sp {stack_offset}",
@@ -705,6 +869,31 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
stack.free_temp(name)?;
}
}
Expression::MemberAccess(access) => {
// Compile member access to temp and push
let result_opt = self.expression(
Spanned {
node: Expression::MemberAccess(access),
span: Span {
start_col: 0,
end_col: 0,
start_line: 0,
end_line: 0,
}, // Dummy span
},
stack,
)?;
if let Some(result) = result_opt {
let reg_str = self.resolve_register(&result.location)?;
self.write_output(format!("push {reg_str}"))?;
if let Some(name) = result.temp_name {
stack.free_temp(name)?;
}
} else {
self.write_output("push 0")?; // Should fail ideally
}
}
_ => {
return Err(Error::Unknown(
format!(
@@ -905,6 +1094,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
fn resolve_register(&self, loc: &VariableLocation) -> Result<String, Error> {
match loc {
VariableLocation::Temporary(r) | VariableLocation::Persistant(r) => Ok(format!("r{r}")),
VariableLocation::Constant(_) => Err(Error::Unknown(
"Cannot resolve a constant value to register".into(),
None,
)),
VariableLocation::Stack(_) => Err(Error::Unknown(
"Cannot resolve Stack location directly to register string without context".into(),
None,
@@ -954,6 +1147,11 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
VariableLocation::Temporary(r) | VariableLocation::Persistant(r) => {
Ok((format!("r{r}"), result.temp_name))
}
VariableLocation::Constant(lit) => match lit {
Literal::Number(n) => Ok((n.to_string(), None)),
Literal::Boolean(b) => Ok((if b { "1" } else { "0" }.to_string(), None)),
Literal::String(s) => Ok((s, None)),
},
VariableLocation::Stack(offset) => {
// If it's on the stack, we must load it into a temp to use it as an operand
let temp_name = self.next_temp_name();
@@ -1116,25 +1314,34 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
fn expression_block<'v>(
&mut self,
mut expr: BlockExpression,
scope: &mut VariableScope<'v>,
parent_scope: &mut VariableScope<'v>,
) -> Result<(), Error> {
// First, sort the expressions to ensure functions are hoisted
expr.0.sort_by(|a, b| {
if matches!(b.node, Expression::Function(_))
&& matches!(a.node, Expression::Function(_))
{
if matches!(
b.node,
Expression::Function(_) | Expression::ConstDeclaration(_)
) && matches!(
a.node,
Expression::Function(_) | Expression::ConstDeclaration(_)
) {
std::cmp::Ordering::Equal
} else if matches!(a.node, Expression::Function(_)) {
} else if matches!(
a.node,
Expression::Function(_) | Expression::ConstDeclaration(_)
) {
std::cmp::Ordering::Less
} else {
std::cmp::Ordering::Greater
}
});
let mut scope = VariableScope::scoped(parent_scope);
for expr in expr.0 {
if !self.declared_main
&& !matches!(expr.node, Expression::Function(_))
&& !scope.has_parent()
&& !parent_scope.has_parent()
{
self.write_output("main:")?;
self.declared_main = true;
@@ -1142,11 +1349,11 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
match expr.node {
Expression::Return(ret_expr) => {
self.expression_return(*ret_expr, scope)?;
self.expression_return(*ret_expr, &mut scope)?;
}
_ => {
// Swallow errors within expressions so block can continue
if let Err(e) = self.expression(expr, scope).and_then(|result| {
if let Err(e) = self.expression(expr, &mut scope).and_then(|result| {
// If the expression was a statement that returned a temp result (e.g. `1 + 2;` line),
// we must free it to avoid leaking registers.
if let Some(comp_res) = result
@@ -1162,6 +1369,10 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
}
}
if scope.stack_offset() > 0 {
self.write_output(format!("sub sp sp {}", scope.stack_offset()))?;
}
Ok(())
}
@@ -1190,6 +1401,14 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
debug!(self, "#returnValue")
))?;
}
VariableLocation::Constant(lit) => {
let str = extract_literal(lit, false)?;
self.write_output(format!(
"move r{} {str} {}",
VariableScope::RETURN_REGISTER,
debug!(self, "#returnValue")
))?
}
VariableLocation::Stack(offset) => {
self.write_output(format!(
"sub r{} sp {offset}",
@@ -1252,6 +1471,23 @@ impl<'a, W: std::io::Write> Compiler<'a, W> {
scope.free_temp(name)?;
}
}
Expression::MemberAccess(access) => {
// Return result of member access
let res_opt = self.expression(
Spanned {
node: Expression::MemberAccess(access),
span: expr.span,
},
scope,
)?;
if let Some(res) = res_opt {
let reg = self.resolve_register(&res.location)?;
self.write_output(format!("move r{} {}", VariableScope::RETURN_REGISTER, reg))?;
if let Some(temp) = res.temp_name {
scope.free_temp(temp)?;
}
}
}
_ => {
return Err(Error::Unknown(
format!("Unsupported `return` statement: {:?}", expr),

57
rust_compiler/libs/compiler/src/variable_manager.rs
View File

@@ -3,6 +3,7 @@
// r1 - r7 : Temporary Variables
// r8 - r14 : Persistant Variables
use parser::tree_node::Literal;
use quick_error::quick_error;
use std::collections::{HashMap, VecDeque};
@@ -43,6 +44,8 @@ pub enum VariableLocation {
Persistant(u8),
/// Represents a a stack offset (current stack - offset = variable loc)
Stack(u16),
/// Represents a constant value and should be directly substituted as such.
Constant(Literal),
}
pub struct VariableScope<'a> {
@@ -91,6 +94,8 @@ impl<'a> VariableScope<'a> {
pub fn scoped(parent: &'a VariableScope<'a>) -> Self {
Self {
parent: Option::Some(parent),
temporary_vars: parent.temporary_vars.clone(),
persistant_vars: parent.persistant_vars.clone(),
..Default::default()
}
}
@@ -140,24 +145,48 @@ impl<'a> VariableScope<'a> {
Ok(var_location)
}
pub fn get_location_of(
pub fn define_const(
&mut self,
var_name: impl Into<String>,
value: Literal,
) -> Result<VariableLocation, Error> {
let var_name = var_name.into();
let var = self
.var_lookup_table
.get(var_name.as_str())
.cloned()
.ok_or(Error::UnknownVariable(var_name))?;
if let VariableLocation::Stack(inserted_at_offset) = var {
Ok(VariableLocation::Stack(
self.stack_offset - inserted_at_offset,
))
} else {
Ok(var)
if self.var_lookup_table.contains_key(&var_name) {
return Err(Error::DuplicateVariable(var_name));
}
let new_value = VariableLocation::Constant(value);
self.var_lookup_table.insert(var_name, new_value.clone());
Ok(new_value)
}
pub fn get_location_of(&self, var_name: impl Into<String>) -> Result<VariableLocation, Error> {
let var_name = var_name.into();
// 1. Check this scope
if let Some(var) = self.var_lookup_table.get(var_name.as_str()) {
if let VariableLocation::Stack(inserted_at_offset) = var {
// Return offset relative to CURRENT sp
return Ok(VariableLocation::Stack(
self.stack_offset - inserted_at_offset,
));
} else {
return Ok(var.clone());
}
}
// 2. Recursively check parent
if let Some(parent) = self.parent {
let loc = parent.get_location_of(var_name)?;
if let VariableLocation::Stack(parent_offset) = loc {
return Ok(VariableLocation::Stack(parent_offset + self.stack_offset));
}
return Ok(loc);
}
Err(Error::UnknownVariable(var_name))
}
pub fn has_parent(&self) -> bool {
@@ -180,7 +209,7 @@ impl<'a> VariableScope<'a> {
"Attempted to free a `let` variable.",
)));
}
VariableLocation::Stack(_) => {}
_ => {}
};
Ok(())

6
rust_compiler/libs/helpers/Cargo.toml Normal file
View File

@@ -0,0 +1,6 @@
[package]
name = "helpers"
version = "0.1.0"
edition = "2024"
[dependencies]

15
rust_compiler/libs/helpers/src/lib.rs Normal file
View File

@@ -0,0 +1,15 @@
mod macros;
mod syscall;
/// This trait will allow the LSP to emit documentation for various tokens and expressions.
/// You can easily create documentation for large enums with the `documented!` macro.
pub trait Documentation {
/// Retreive documentation for this specific item.
fn docs(&self) -> String;
fn get_all_documentation() -> Vec<(&'static str, String)>;
}
pub mod prelude {
pub use super::{Documentation, documented, with_syscalls};
}

111
rust_compiler/libs/helpers/src/macros.rs Normal file
View File

@@ -0,0 +1,111 @@
#[macro_export]
macro_rules! documented {
// -------------------------------------------------------------------------
// Internal Helper: Filter doc comments
// -------------------------------------------------------------------------
// Case 1: Doc comment. Return Some("string").
// We match the specific structure of a doc attribute.
(@doc_filter #[doc = $doc:expr]) => {
Some($doc)
};
// Case 2: Other attributes (derives, etc.). Return None.
// We catch any other token sequence inside the brackets.
(@doc_filter #[$($attr:tt)*]) => {
None
};
// -------------------------------------------------------------------------
// Internal Helper: Match patterns for `match self`
// -------------------------------------------------------------------------
(@arm $name:ident $variant:ident) => {
$name::$variant
};
(@arm $name:ident $variant:ident ( $($tuple:tt)* )) => {
$name::$variant(..)
};
(@arm $name:ident $variant:ident { $($structure:tt)* }) => {
$name::$variant{..}
};
// -------------------------------------------------------------------------
// Main Macro Entry Point
// -------------------------------------------------------------------------
(
$(#[$enum_attr:meta])* $vis:vis enum $name:ident {
$(
// Capture attributes as a sequence of token trees inside brackets
// to avoid "local ambiguity" and handle multi-token attributes (like doc="...").
$(#[ $($variant_attr:tt)* ])*
$variant:ident
$( ($($tuple:tt)*) )?
$( {$($structure:tt)*} )?
),* $(,)?
}
) => {
// 1. Generate the actual Enum definition
$(#[$enum_attr])*
$vis enum $name {
$(
$(#[ $($variant_attr)* ])*
$variant
$( ($($tuple)*) )?
$( {$($structure)*} )?,
)*
}
// 2. Implement the Documentation Trait
impl Documentation for $name {
fn docs(&self) -> String {
match self {
$(
documented!(@arm $name $variant $( ($($tuple)*) )? $( {$($structure)*} )? ) => {
// Create a temporary array of Option<&str> for all attributes
let doc_lines: &[Option<&str>] = &[
$(
documented!(@doc_filter #[ $($variant_attr)* ])
),*
];
// Filter out the Nones (non-doc attributes), join, and return
doc_lines.iter()
.filter_map(|&d| d)
.collect::<Vec<_>>()
.join("\n")
.trim()
.to_string()
}
)*
}
}
// 3. Implement Static Documentation Provider
#[allow(dead_code)]
fn get_all_documentation() -> Vec<(&'static str, String)> {
vec![
$(
(
stringify!($variant),
{
// Re-use the same extraction logic
let doc_lines: &[Option<&str>] = &[
$(
documented!(@doc_filter #[ $($variant_attr)* ])
),*
];
doc_lines.iter()
.filter_map(|&d| d)
.collect::<Vec<_>>()
.join("\n")
.trim()
.to_string()
}
)
),*
]
}
}
};
}

32
rust_compiler/libs/helpers/src/syscall.rs Normal file
View File

@@ -0,0 +1,32 @@
#[macro_export]
macro_rules! with_syscalls {
($matcher:ident) => {
$matcher!(
"yield",
"sleep",
"hash",
"loadFromDevice",
"loadBatchNamed",
"loadBatch",
"setOnDevice",
"setOnDeviceBatched",
"setOnDeviceBatchedNamed",
"acos",
"asin",
"atan",
"atan2",
"abs",
"ceil",
"cos",
"floor",
"log",
"max",
"min",
"rand",
"sin",
"sqrt",
"tan",
"trunc"
);
};
}

2
rust_compiler/libs/parser/Cargo.toml
View File

@@ -6,8 +6,10 @@ edition = "2024"
[dependencies]
quick-error = { workspace = true }
tokenizer = { path = "../tokenizer" }
helpers = { path = "../helpers" }
lsp-types = { workspace = true }
[dev-dependencies]
anyhow = { version = "1" }
pretty_assertions = "1.4"

370
rust_compiler/libs/parser/src/lib.rs
View File

@@ -14,6 +14,10 @@ use tokenizer::{
};
use tree_node::*;
pub trait Documentation {
fn docs(&self) -> String;
}
#[macro_export]
/// A macro to create a boxed value.
macro_rules! boxed {
@@ -121,7 +125,6 @@ impl<'a> Parser<'a> {
}
/// Calculates a Span from a given Token reference.
/// This is a static helper to avoid borrowing `self` when we already have a token ref.
fn token_to_span(t: &Token) -> Span {
let len = t.original_string.as_ref().map(|s| s.len()).unwrap_or(0);
Span {
@@ -149,7 +152,6 @@ impl<'a> Parser<'a> {
where
F: FnOnce(&mut Self) -> Result<T, Error>,
{
// Peek at the start token. If no current token (parsing hasn't started), peek the buffer.
let start_token = if self.current_token.is_some() {
self.current_token.clone()
} else {
@@ -163,7 +165,6 @@ impl<'a> Parser<'a> {
let node = parser(self)?;
// The end token is the current_token after parsing.
let end_token = self.current_token.as_ref();
let (end_line, end_col) = end_token
@@ -184,26 +185,15 @@ impl<'a> Parser<'a> {
})
}
/// Skips tokens until a statement boundary is found to recover from errors.
fn synchronize(&mut self) -> Result<(), Error> {
// We advance once to consume the error-causing token if we haven't already
// But often the error happens after we consumed something.
// Safe bet: consume current, then look.
// If we assign next, we might be skipping the very token we want to sync on if the error didn't consume it?
// Usually, in recursive descent, the error is raised when `current` is unexpected.
// We want to discard `current` and move on.
self.assign_next()?;
while let Some(token) = &self.current_token {
if token.token_type == TokenType::Symbol(Symbol::Semicolon) {
// Consuming the semicolon is a good place to stop and resume parsing next statement
self.assign_next()?;
return Ok(());
}
// Check if the token looks like the start of a statement.
// If so, we don't consume it; we return so the loop in parse_all can try to parse it.
match token.token_type {
TokenType::Keyword(Keyword::Fn)
| TokenType::Keyword(Keyword::Let)
@@ -231,7 +221,6 @@ impl<'a> Parser<'a> {
let mut expressions = Vec::<Spanned<Expression>>::new();
loop {
// Check EOF without unwrapping error
match self.tokenizer.peek() {
Ok(None) => break,
Err(e) => {
@@ -248,19 +237,13 @@ impl<'a> Parser<'a> {
Ok(None) => break,
Err(e) => {
self.errors.push(e);
// Recover
if self.synchronize().is_err() {
// If sync failed (e.g. EOF during sync), break
break;
}
}
}
}
// Even if we had errors, we return whatever partial AST we managed to build.
// If expressions is empty and we had errors, it's a failed parse, but we return a block.
// Use the last token position for end span, or start if nothing parsed
let end_token_opt = self.tokenizer.peek().unwrap_or(None);
let (end_line, end_col) = end_token_opt
.map(|tok| {
@@ -285,7 +268,6 @@ impl<'a> Parser<'a> {
pub fn parse(&mut self) -> Result<Option<Spanned<tree_node::Expression>>, Error> {
self.assign_next()?;
// If assign_next hit EOF or error?
if self.current_token.is_none() {
return Ok(None);
}
@@ -317,15 +299,18 @@ impl<'a> Parser<'a> {
return Ok(None);
};
// check if the next or current token is an operator, comparison, or logical symbol
// Handle Postfix operators (Member Access, Method Call) immediately after unary
let lhs = self.parse_postfix(lhs)?;
// Handle Infix operators (Binary, Logical, Assignment)
if self_matches_peek!(
self,
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical()
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign)
) {
return Ok(Some(self.infix(lhs)?));
} else if self_matches_current!(
self,
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical()
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign)
) {
self.tokenizer.seek(SeekFrom::Current(-1))?;
return Ok(Some(self.infix(lhs)?));
@@ -334,6 +319,116 @@ impl<'a> Parser<'a> {
Ok(Some(lhs))
}
/// Handles dot notation chains: x.y.z()
fn parse_postfix(
&mut self,
mut lhs: Spanned<Expression>,
) -> Result<Spanned<Expression>, Error> {
loop {
if self_matches_peek!(self, TokenType::Symbol(Symbol::Dot)) {
self.assign_next()?; // consume Dot
let identifier_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let identifier_span = Self::token_to_span(identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
_ => {
return Err(Error::UnexpectedToken(
identifier_span,
identifier_token.clone(),
));
}
};
// Check for Method Call '()'
if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) {
// Method Call
self.assign_next()?; // consume '('
let mut arguments = Vec::<Spanned<Expression>>::new();
while !token_matches!(
self.get_next()?.ok_or(Error::UnexpectedEOF)?,
TokenType::Symbol(Symbol::RParen)
) {
let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
// Block expressions not allowed in args
if let Expression::Block(_) = expression.node {
return Err(Error::InvalidSyntax(
self.current_span(),
String::from("Block expressions are not allowed in method calls"),
));
}
arguments.push(expression);
if !self_matches_peek!(self, TokenType::Symbol(Symbol::Comma))
&& !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen))
{
let next_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
return Err(Error::UnexpectedToken(
Self::token_to_span(next_token),
next_token.clone(),
));
}
if !self_matches_peek!(self, TokenType::Symbol(Symbol::RParen)) {
self.assign_next()?;
}
}
// End span is the ')'
let end_span = self.current_span();
let combined_span = Span {
start_line: lhs.span.start_line,
start_col: lhs.span.start_col,
end_line: end_span.end_line,
end_col: end_span.end_col,
};
lhs = Spanned {
span: combined_span,
node: Expression::MethodCall(Spanned {
span: combined_span,
node: MethodCallExpression {
object: boxed!(lhs),
method: Spanned {
span: identifier_span,
node: identifier,
},
arguments,
},
}),
};
} else {
// Member Access
let combined_span = Span {
start_line: lhs.span.start_line,
start_col: lhs.span.start_col,
end_line: identifier_span.end_line,
end_col: identifier_span.end_col,
};
lhs = Spanned {
span: combined_span,
node: Expression::MemberAccess(Spanned {
span: combined_span,
node: MemberAccessExpression {
object: boxed!(lhs),
member: Spanned {
span: identifier_span,
node: identifier,
},
},
}),
};
}
} else {
break;
}
}
Ok(lhs)
}
fn unary(&mut self) -> Result<Option<Spanned<tree_node::Expression>>, Error> {
macro_rules! matches_keyword {
($keyword:expr, $($pattern:pat),+) => {
@@ -357,10 +452,7 @@ impl<'a> Parser<'a> {
));
}
TokenType::Keyword(Keyword::Let) => {
// declaration is wrapped in spanned inside the function, but expects 'let' to be current
Some(self.spanned(|p| p.declaration())?)
}
TokenType::Keyword(Keyword::Let) => Some(self.spanned(|p| p.declaration())?),
TokenType::Keyword(Keyword::Device) => {
let spanned_dev = self.spanned(|p| p.device())?;
@@ -370,6 +462,15 @@ impl<'a> Parser<'a> {
})
}
TokenType::Keyword(Keyword::Const) => {
let spanned_const = self.spanned(|p| p.const_declaration())?;
Some(Spanned {
span: spanned_const.span,
node: Expression::ConstDeclaration(spanned_const),
})
}
TokenType::Keyword(Keyword::Fn) => {
let spanned_fn = self.spanned(|p| p.function())?;
Some(Spanned {
@@ -404,7 +505,6 @@ impl<'a> Parser<'a> {
TokenType::Keyword(Keyword::Break) => {
let span = self.current_span();
// make sure the next token is a semi-colon
let next = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
if !token_matches!(next, TokenType::Symbol(Symbol::Semicolon)) {
return Err(Error::UnexpectedToken(
@@ -451,16 +551,6 @@ impl<'a> Parser<'a> {
})
}
TokenType::Identifier(_)
if self_matches_peek!(self, TokenType::Symbol(Symbol::Assign)) =>
{
let spanned_assign = self.spanned(|p| p.assignment())?;
Some(Spanned {
span: spanned_assign.span,
node: Expression::Assignment(spanned_assign),
})
}
TokenType::Identifier(ref id) => {
let span = self.current_span();
Some(Spanned {
@@ -489,24 +579,36 @@ impl<'a> Parser<'a> {
}
TokenType::Symbol(Symbol::LParen) => {
// Priority handles its own spanning
self.spanned(|p| p.priority())?.node.map(|node| *node)
}
TokenType::Symbol(Symbol::Minus) => {
// Need to handle span manually because unary call is next
let start_span = self.current_span();
self.assign_next()?;
let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?;
// NOTE: Unary negation can also have postfix applied to the inner expression
// But generally -a.b parses as -(a.b), which is what parse_postfix ensures if called here.
// However, we call parse_postfix on the RESULT of unary in expression(), so
// `expression` sees `Negation`. `parse_postfix` doesn't apply to Negation node unless we allow it?
// Actually, `x.y` binds tighter than `-`. `postfix` logic belongs inside `unary` logic or
// `expression` logic.
// If I have `-x.y`, standard precedence says `-(x.y)`.
// `unary` returns `Negation(x)`. Then `expression` calls `postfix` on `Negation(x)`.
// `postfix` loop runs on `Negation`. This implies `(-x).y`. This is usually WRONG.
// `.` binds tighter than `-`.
// So `unary` must call `postfix` on the *operand* of the negation.
let inner_with_postfix = self.parse_postfix(inner_expr)?;
let combined_span = Span {
start_line: start_span.start_line,
start_col: start_span.start_col,
end_line: inner_expr.span.end_line,
end_col: inner_expr.span.end_col,
end_line: inner_with_postfix.span.end_line,
end_col: inner_with_postfix.span.end_col,
};
Some(Spanned {
span: combined_span,
node: Expression::Negation(boxed!(inner_expr)),
node: Expression::Negation(boxed!(inner_with_postfix)),
})
}
@@ -514,17 +616,18 @@ impl<'a> Parser<'a> {
let start_span = self.current_span();
self.assign_next()?;
let inner_expr = self.unary()?.ok_or(Error::UnexpectedEOF)?;
let inner_with_postfix = self.parse_postfix(inner_expr)?;
let combined_span = Span {
start_line: start_span.start_line,
start_col: start_span.start_col,
end_line: inner_expr.span.end_line,
end_col: inner_expr.span.end_col,
end_line: inner_with_postfix.span.end_line,
end_col: inner_with_postfix.span.end_col,
};
Some(Spanned {
span: combined_span,
node: Expression::Logical(Spanned {
span: combined_span,
node: LogicalExpression::Not(boxed!(inner_expr)),
node: LogicalExpression::Not(boxed!(inner_with_postfix)),
}),
})
}
@@ -543,41 +646,44 @@ impl<'a> Parser<'a> {
fn get_infix_child_node(&mut self) -> Result<Spanned<tree_node::Expression>, Error> {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
match current_token.token_type {
let start_span = self.current_span();
let expr = match current_token.token_type {
TokenType::Number(_) | TokenType::Boolean(_) => {
let lit = self.spanned(|p| p.literal())?;
Ok(Spanned {
Spanned {
span: lit.span,
node: Expression::Literal(lit),
})
}
}
TokenType::Identifier(ref ident)
if !self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) =>
{
// This is a Variable. We need to check for Postfix operations on it.
let span = self.current_span();
Ok(Spanned {
Spanned {
span,
node: Expression::Variable(Spanned {
span,
node: ident.clone(),
}),
})
}
TokenType::Symbol(Symbol::LParen) => Ok(*self
}
TokenType::Symbol(Symbol::LParen) => *self
.spanned(|p| p.priority())?
.node
.ok_or(Error::UnexpectedEOF)?),
.ok_or(Error::UnexpectedEOF)?,
TokenType::Identifier(_)
if self_matches_peek!(self, TokenType::Symbol(Symbol::LParen)) =>
{
let inv = self.spanned(|p| p.invocation())?;
Ok(Spanned {
Spanned {
span: inv.span,
node: Expression::Invocation(inv),
})
}
}
TokenType::Symbol(Symbol::Minus) => {
let start_span = self.current_span();
self.assign_next()?;
let inner = self.get_infix_child_node()?;
let span = Span {
@@ -586,13 +692,12 @@ impl<'a> Parser<'a> {
end_line: inner.span.end_line,
end_col: inner.span.end_col,
};
Ok(Spanned {
Spanned {
span,
node: Expression::Negation(boxed!(inner)),
})
}
}
TokenType::Symbol(Symbol::LogicalNot) => {
let start_span = self.current_span();
self.assign_next()?;
let inner = self.get_infix_child_node()?;
let span = Span {
@@ -601,19 +706,25 @@ impl<'a> Parser<'a> {
end_line: inner.span.end_line,
end_col: inner.span.end_col,
};
Ok(Spanned {
Spanned {
span,
node: Expression::Logical(Spanned {
span,
node: LogicalExpression::Not(boxed!(inner)),
}),
})
}
_ => Err(Error::UnexpectedToken(
}
_ => {
return Err(Error::UnexpectedToken(
self.current_span(),
current_token.clone(),
)),
));
}
};
// Important: We must check for postfix operations here too
// e.g. a + b.c
self.parse_postfix(expr)
}
fn device(&mut self) -> Result<DeviceDeclarationExpression, Error> {
@@ -665,39 +776,6 @@ impl<'a> Parser<'a> {
})
}
fn assignment(&mut self) -> Result<AssignmentExpression, Error> {
let identifier_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
let identifier_span = Self::token_to_span(identifier_token);
let identifier = match identifier_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
_ => {
return Err(Error::UnexpectedToken(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
));
}
};
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
if !token_matches!(current_token, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&current_token),
current_token.clone(),
));
}
self.assign_next()?;
let expression = self.expression()?.ok_or(Error::UnexpectedEOF)?;
Ok(AssignmentExpression {
identifier: Spanned {
span: identifier_span,
node: identifier,
},
expression: boxed!(expression),
})
}
fn infix(&mut self, previous: Spanned<Expression>) -> Result<Spanned<Expression>, Error> {
let current_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
@@ -708,7 +786,9 @@ impl<'a> Parser<'a> {
| Expression::Priority(_)
| Expression::Literal(_)
| Expression::Variable(_)
| Expression::Negation(_) => {}
| Expression::Negation(_)
| Expression::MemberAccess(_)
| Expression::MethodCall(_) => {}
_ => {
return Err(Error::InvalidSyntax(
self.current_span(),
@@ -722,9 +802,10 @@ impl<'a> Parser<'a> {
let mut temp_token = current_token.clone();
// Include Assign in the operator loop
while token_matches!(
temp_token,
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical()
TokenType::Symbol(s) if s.is_operator() || s.is_comparison() || s.is_logical() || matches!(s, Symbol::Assign)
) {
let operator = match temp_token.token_type {
TokenType::Symbol(s) => s,
@@ -955,6 +1036,37 @@ impl<'a> Parser<'a> {
}
operators.retain(|symbol| !matches!(symbol, Symbol::LogicalOr));
// --- PRECEDENCE LEVEL 8: Assignment (=) ---
// Assignment is Right Associative: a = b = c => a = (b = c)
// We iterate Right to Left
for (i, operator) in operators.iter().enumerate().rev() {
if matches!(operator, Symbol::Assign) {
let right = expressions.remove(i + 1);
let left = expressions.remove(i);
let span = Span {
start_line: left.span.start_line,
start_col: left.span.start_col,
end_line: right.span.end_line,
end_col: right.span.end_col,
};
expressions.insert(
i,
Spanned {
span,
node: Expression::Assignment(Spanned {
span,
node: AssignmentExpression {
assignee: boxed!(left),
expression: boxed!(right),
},
}),
},
);
}
}
operators.retain(|symbol| !matches!(symbol, Symbol::Assign));
if expressions.len() != 1 || !operators.is_empty() {
return Err(Error::InvalidSyntax(
self.current_span(),
@@ -1117,6 +1229,46 @@ impl<'a> Parser<'a> {
Ok(BlockExpression(expressions))
}
fn const_declaration(&mut self) -> Result<ConstDeclarationExpression, Error> {
// const
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
if !self_matches_current!(self, TokenType::Keyword(Keyword::Const)) {
return Err(Error::UnexpectedToken(
self.current_span(),
current_token.clone(),
));
}
// variable_name
let ident_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?;
let ident_span = Self::token_to_span(ident_token);
let ident = match ident_token.token_type {
TokenType::Identifier(ref id) => id.clone(),
_ => return Err(Error::UnexpectedToken(ident_span, ident_token.clone())),
};
// `=`
let assign_token = self.get_next()?.ok_or(Error::UnexpectedEOF)?.clone();
if !token_matches!(assign_token, TokenType::Symbol(Symbol::Assign)) {
return Err(Error::UnexpectedToken(
Self::token_to_span(&assign_token),
assign_token,
));
}
// literal value
self.assign_next()?;
let lit = self.spanned(|p| p.literal())?;
Ok(ConstDeclarationExpression {
name: Spanned {
span: ident_span,
node: ident,
},
value: lit,
})
}
fn declaration(&mut self) -> Result<Expression, Error> {
let current_token = self.current_token.as_ref().ok_or(Error::UnexpectedEOF)?;
if !self_matches_current!(self, TokenType::Keyword(Keyword::Let)) {
@@ -1506,7 +1658,6 @@ impl<'a> Parser<'a> {
Literal::String(variable),
)))
}
// ... (implementing other syscalls similarly using patterns above)
"setOnDevice" => {
check_length(self, &invocation.arguments, 3)?;
let mut args = invocation.arguments.into_iter();
@@ -1531,23 +1682,10 @@ impl<'a> Parser<'a> {
boxed!(variable),
)))
}
_ => {
// For Math functions or unknown functions
if SysCall::is_syscall(&invocation.name.node) {
// Attempt to parse as math if applicable, or error if strict
// Here we are falling back to simple handling or error.
// Since Math isn't fully expanded in this snippet, we return Unsupported.
Err(Error::UnsupportedKeyword(
_ => Err(Error::UnsupportedKeyword(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
))
} else {
Err(Error::UnsupportedKeyword(
self.current_span(),
self.current_token.clone().ok_or(Error::UnexpectedEOF)?,
))
}
}
)),
}
}
}

171
rust_compiler/libs/parser/src/sys_call.rs
View File

@@ -1,74 +1,108 @@
use crate::tree_node::{Expression, Literal, Spanned};
use super::LiteralOrVariable;
use crate::tree_node::{Expression, Literal, Spanned};
use helpers::prelude::*;
documented! {
#[derive(Debug, PartialEq, Eq)]
pub enum Math {
/// Returns the angle in radians whose cosine is the specified number.
/// ## In Game
/// ## IC10
/// `acos r? a(r?|num)`
/// ## Slang
/// `(number|var).acos();`
Acos(LiteralOrVariable),
/// Returns the angle in radians whose sine is the specified number.
/// ## In Game
/// ## IC10
/// `asin r? a(r?|num)`
/// ## Slang
/// `(number|var).asin();`
Asin(LiteralOrVariable),
/// Returns the angle in radians whose tangent is the specified number.
/// ## In Game
/// ## IC10
/// `atan r? a(r?|num)`
/// ## Slang
/// `(number|var).atan();`
Atan(LiteralOrVariable),
/// Returns the angle in radians whose tangent is the quotient of the specified numbers.
/// ## In Game
/// ## IC10
/// `atan2 r? a(r?|num) b(r?|num)`
/// ## Slang
/// `(number|var).atan2((number|var));`
Atan2(LiteralOrVariable, LiteralOrVariable),
/// Gets the absolute value of a number.
/// ## In Game
/// ## IC10
/// `abs r? a(r?|num)`
/// ## Slang
/// `(number|var).abs();`
Abs(LiteralOrVariable),
/// Rounds a number up to the nearest whole number.
/// ## In Game
/// ## IC10
/// `ceil r? a(r?|num)`
/// ## Slang
/// `(number|var).ceil();`
Ceil(LiteralOrVariable),
/// Returns the cosine of the specified angle in radians.
/// ## In Game
/// cos r? a(r?|num)
/// ## IC10
/// `cos r? a(r?|num)`
/// ## Slang
/// `(number|var).cos();`
Cos(LiteralOrVariable),
/// Rounds a number down to the nearest whole number.
/// ## In Game
/// ## IC10
/// `floor r? a(r?|num)`
/// ## Slang
/// `(number|var).floor();`
Floor(LiteralOrVariable),
/// Computes the natural logarithm of a number.
/// ## In Game
/// ## IC10
/// `log r? a(r?|num)`
/// ## Slang
/// `(number|var).log();`
Log(LiteralOrVariable),
/// Computes the maximum of two numbers.
/// ## In Game
/// ## IC10
/// `max r? a(r?|num) b(r?|num)`
/// ## Slang
/// `(number|var).max((number|var));`
Max(LiteralOrVariable, LiteralOrVariable),
/// Computes the minimum of two numbers.
/// ## In Game
/// ## IC10
/// `min r? a(r?|num) b(r?|num)`
/// ## Slang
/// `(number|var).min((number|var));`
Min(LiteralOrVariable, LiteralOrVariable),
/// Gets a random number between 0 and 1.
/// ## In Game
/// ## IC10
/// `rand r?`
/// ## Slang
/// `rand();`
Rand,
/// Returns the sine of the specified angle in radians.
/// ## In Game
/// ## IC10
/// `sin r? a(r?|num)`
/// ## Slang
/// `(number|var).sin();`
Sin(LiteralOrVariable),
/// Computes the square root of a number.
/// ## In Game
/// ## IC10
/// `sqrt r? a(r?|num)`
/// ## Slang
/// `(number|var).sqrt();`
Sqrt(LiteralOrVariable),
/// Returns the tangent of the specified angle in radians.
/// ## In Game
/// ## IC10
/// `tan r? a(r?|num)`
/// ## Slang
/// `(number|var).tan();`
Tan(LiteralOrVariable),
/// Truncates a number by removing the decimal portion.
/// ## In Game
/// ## IC10
/// `trunc r? a(r?|num)`
/// ## Slang
/// `(number|var).trunc();`
Trunc(LiteralOrVariable),
}
}
impl std::fmt::Display for Math {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -93,33 +127,39 @@ impl std::fmt::Display for Math {
}
}
documented! {
#[derive(Debug, PartialEq, Eq)]
pub enum System {
/// Pauses execution for exactly 1 tick and then resumes.
/// ## In Game
/// yield
/// ## IC10
/// `yield`
/// ## Slang
/// `yield();`
Yield,
/// Represents a function that can be called to sleep for a certain amount of time.
/// ## In Game
/// ## IC10
/// `sleep a(r?|num)`
/// ## Slang
/// `sleep(number|var);`
Sleep(Box<Spanned<Expression>>),
/// Gets the in-game hash for a specific prefab name.
/// ## In Game
/// ## IC10
/// `HASH("prefabName")`
/// ## Slang
/// `HASH("prefabName");`
Hash(Literal),
/// Represents a function which loads a device variable into a register.
/// ## In Game
/// ## IC10
/// `l r? d? var`
/// ## Examples
/// `l r0 d0 Setting`
/// `l r1 d5 Pressure`
/// ## Slang
/// `loadFromDevice(deviceType, "LogicType");`
LoadFromDevice(LiteralOrVariable, Literal),
/// Function which gets a LogicType from all connected network devices that match
/// the provided device hash and name, aggregating them via a batchMode
/// ## In Game
/// lbn r? deviceHash nameHash logicType batchMode
/// ## Examples
/// lbn r0 HASH("StructureWallLight") HASH("wallLight") On Minimum
/// ## IC10
/// `lbn r? deviceHash nameHash logicType batchMode`
/// ## Slang
/// `loadFromDeviceBatchedNamed(deviceHash, deviceName, "LogicType", "BatchMode");`
LoadBatchNamed(
LiteralOrVariable,
Box<Spanned<Expression>>,
@@ -128,30 +168,28 @@ pub enum System {
),
/// Loads a LogicType from all connected network devices, aggregating them via a
/// batchMode
/// ## In Game
/// lb r? deviceHash logicType batchMode
/// ## Examples
/// lb r0 HASH("StructureWallLight") On Minimum
/// ## IC10
/// `lb r? deviceHash logicType batchMode`
/// ## Slang
/// `loadFromDeviceBatched(deviceHash, "Variable", "LogicType");`
LoadBatch(LiteralOrVariable, Literal, Literal),
/// Represents a function which stores a setting into a specific device.
/// ## In Game
/// ## IC10
/// `s d? logicType r?`
/// ## Example
/// `s d0 Setting r0`
/// ## Slang
/// `setOnDevice(deviceType, "Variable", (number|var));`
SetOnDevice(LiteralOrVariable, Literal, Box<Spanned<Expression>>),
/// Represents a function which stores a setting to all devices that match
/// the given deviceHash
/// ## In Game
/// ## IC10
/// `sb deviceHash logicType r?`
/// ## Example
/// `sb HASH("Doors") Lock 1`
SetOnDeviceBatched(LiteralOrVariable, Literal, Box<Spanned<Expression>>),
/// Represents a function which stores a setting to all devices that match
/// both the given deviceHash AND the given nameHash
/// ## In Game
/// ## IC10
/// `sbn deviceHash nameHash logicType r?`
/// ## Example
/// `sbn HASH("Doors") HASH("Exterior") Lock 1`
/// ## Slang
/// `setOnDeviceBatchedNamed(deviceType, nameHash, "LogicType", (number|var))`
SetOnDeviceBatchedNamed(
LiteralOrVariable,
LiteralOrVariable,
@@ -159,6 +197,7 @@ pub enum System {
Box<Spanned<Expression>>,
),
}
}
impl std::fmt::Display for System {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -190,6 +229,22 @@ pub enum SysCall {
Math(Math),
}
impl Documentation for SysCall {
fn docs(&self) -> String {
match self {
Self::System(s) => s.docs(),
Self::Math(m) => m.docs(),
}
}
fn get_all_documentation() -> Vec<(&'static str, String)> {
let mut all_docs = System::get_all_documentation();
all_docs.extend(Math::get_all_documentation());
all_docs
}
}
impl std::fmt::Display for SysCall {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
@@ -201,32 +256,6 @@ impl std::fmt::Display for SysCall {
impl SysCall {
pub fn is_syscall(identifier: &str) -> bool {
matches!(
identifier,
"yield"
| "sleep"
| "hash"
| "loadFromDevice"
| "setOnDevice"
| "setOnDeviceBatched"
| "setOnDeviceBatchedNamed"
| "acos"
| "asin"
| "atan"
| "atan2"
| "abs"
| "ceil"
| "cos"
| "floor"
| "log"
| "max"
| "min"
| "rand"
| "sin"
| "sqrt"
| "tan"
| "trunc"
)
tokenizer::token::is_syscall(identifier)
}
}

49
rust_compiler/libs/parser/src/test/mod.rs
View File

@@ -1,7 +1,7 @@
#[macro_export]
macro_rules! parser {
($input:expr) => {
Parser::new(Tokenizer::from($input.to_owned()))
Parser::new(Tokenizer::from($input))
};
}
@@ -9,6 +9,7 @@ mod blocks;
use super::Parser;
use super::Tokenizer;
use anyhow::Result;
use pretty_assertions::assert_eq;
#[test]
fn test_unsupported_keywords() -> Result<()> {
@@ -31,7 +32,7 @@ fn test_declarations() -> Result<()> {
// The below line should fail
let y = 234
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let tokenizer = Tokenizer::from(input);
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
@@ -43,6 +44,36 @@ fn test_declarations() -> Result<()> {
Ok(())
}
#[test]
fn test_const_declaration() -> Result<()> {
let input = r#"
const item = 20c;
const decimal = 200.15;
const nameConst = "str_lit";
"#;
let tokenizer = Tokenizer::from(input);
let mut parser = Parser::new(tokenizer);
assert_eq!(
"(const item = 293.15)",
parser.parse()?.unwrap().to_string()
);
assert_eq!(
"(const decimal = 200.15)",
parser.parse()?.unwrap().to_string()
);
assert_eq!(
r#"(const nameConst = "str_lit")"#,
parser.parse()?.unwrap().to_string()
);
assert_eq!(None, parser.parse()?);
Ok(())
}
#[test]
fn test_function_expression() -> Result<()> {
let input = r#"
@@ -52,7 +83,7 @@ fn test_function_expression() -> Result<()> {
}
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let tokenizer = Tokenizer::from(input);
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
@@ -71,7 +102,7 @@ fn test_function_invocation() -> Result<()> {
add();
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let tokenizer = Tokenizer::from(input);
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
@@ -87,7 +118,7 @@ fn test_priority_expression() -> Result<()> {
let x = (4);
"#;
let tokenizer = Tokenizer::from(input.to_owned());
let tokenizer = Tokenizer::from(input);
let mut parser = Parser::new(tokenizer);
let expression = parser.parse()?.unwrap();
@@ -99,16 +130,16 @@ fn test_priority_expression() -> Result<()> {
#[test]
fn test_binary_expression() -> Result<()> {
let expr = parser!("4 ** 2 + 5 ** 2").parse()?.unwrap();
let expr = parser!("4 ** 2 + 5 ** 2;").parse()?.unwrap();
assert_eq!("((4 ** 2) + (5 ** 2))", expr.to_string());
let expr = parser!("2 ** 3 ** 4").parse()?.unwrap();
let expr = parser!("2 ** 3 ** 4;").parse()?.unwrap();
assert_eq!("(2 ** (3 ** 4))", expr.to_string());
let expr = parser!("45 * 2 - 15 / 5 + 5 ** 2").parse()?.unwrap();
let expr = parser!("45 * 2 - 15 / 5 + 5 ** 2;").parse()?.unwrap();
assert_eq!("(((45 * 2) - (15 / 5)) + (5 ** 2))", expr.to_string());
let expr = parser!("(5 - 2) * 10").parse()?.unwrap();
let expr = parser!("(5 - 2) * 10;").parse()?.unwrap();
assert_eq!("((5 - 2) * 10)", expr.to_string());
Ok(())

57
rust_compiler/libs/parser/src/tree_node.rs
View File

@@ -74,13 +74,13 @@ impl std::fmt::Display for LogicalExpression {
#[derive(Debug, PartialEq, Eq)]
pub struct AssignmentExpression {
pub identifier: Spanned<String>,
pub assignee: Box<Spanned<Expression>>,
pub expression: Box<Spanned<Expression>>,
}
impl std::fmt::Display for AssignmentExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "({} = {})", self.identifier, self.expression)
write!(f, "({} = {})", self.assignee, self.expression)
}
}
@@ -145,6 +145,41 @@ impl std::fmt::Display for InvocationExpression {
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct MemberAccessExpression {
pub object: Box<Spanned<Expression>>,
pub member: Spanned<String>,
}
impl std::fmt::Display for MemberAccessExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.object, self.member)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct MethodCallExpression {
pub object: Box<Spanned<Expression>>,
pub method: Spanned<String>,
pub arguments: Vec<Spanned<Expression>>,
}
impl std::fmt::Display for MethodCallExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}.{}({})",
self.object,
self.method,
self.arguments
.iter()
.map(|e| e.to_string())
.collect::<Vec<String>>()
.join(", ")
)
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum LiteralOrVariable {
Literal(Literal),
@@ -160,6 +195,18 @@ impl std::fmt::Display for LiteralOrVariable {
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct ConstDeclarationExpression {
pub name: Spanned<String>,
pub value: Spanned<Literal>,
}
impl std::fmt::Display for ConstDeclarationExpression {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "(const {} = {})", self.name, self.value)
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct DeviceDeclarationExpression {
/// any variable-like name
@@ -281,6 +328,7 @@ pub enum Expression {
Binary(Spanned<BinaryExpression>),
Block(Spanned<BlockExpression>),
Break(Span),
ConstDeclaration(Spanned<ConstDeclarationExpression>),
Continue(Span),
Declaration(Spanned<String>, Box<Spanned<Expression>>),
DeviceDeclaration(Spanned<DeviceDeclarationExpression>),
@@ -290,6 +338,8 @@ pub enum Expression {
Literal(Spanned<Literal>),
Logical(Spanned<LogicalExpression>),
Loop(Spanned<LoopExpression>),
MemberAccess(Spanned<MemberAccessExpression>),
MethodCall(Spanned<MethodCallExpression>),
Negation(Box<Spanned<Expression>>),
Priority(Box<Spanned<Expression>>),
Return(Box<Spanned<Expression>>),
@@ -305,6 +355,7 @@ impl std::fmt::Display for Expression {
Expression::Binary(e) => write!(f, "{}", e),
Expression::Block(e) => write!(f, "{}", e),
Expression::Break(_) => write!(f, "break"),
Expression::ConstDeclaration(e) => write!(f, "{}", e),
Expression::Continue(_) => write!(f, "continue"),
Expression::Declaration(id, e) => write!(f, "(let {} = {})", id, e),
Expression::DeviceDeclaration(e) => write!(f, "{}", e),
@@ -314,6 +365,8 @@ impl std::fmt::Display for Expression {
Expression::Literal(l) => write!(f, "{}", l),
Expression::Logical(e) => write!(f, "{}", e),
Expression::Loop(e) => write!(f, "{}", e),
Expression::MemberAccess(e) => write!(f, "{}", e),
Expression::MethodCall(e) => write!(f, "{}", e),
Expression::Negation(e) => write!(f, "(-{})", e),
Expression::Priority(e) => write!(f, "({})", e),
Expression::Return(e) => write!(f, "(return {})", e),

1
rust_compiler/libs/tokenizer/Cargo.toml
View File

@@ -7,6 +7,7 @@ edition = "2024"
rust_decimal = { workspace = true }
quick-error = { workspace = true }
lsp-types = { workspace = true }
helpers = { path = "../helpers" }
[dev-dependencies]
anyhow = { version = "^1" }

24
rust_compiler/libs/tokenizer/src/lib.rs
View File

@@ -199,7 +199,7 @@ impl<'a> Tokenizer<'a> {
.tokenize_symbol(next_char, start_line, start_col)
.map(Some);
}
char if char.is_alphabetic() => {
char if char.is_alphabetic() || char == '_' => {
return self
.tokenize_keyword_or_identifier(next_char, start_line, start_col)
.map(Some);
@@ -439,14 +439,15 @@ impl<'a> Tokenizer<'a> {
}};
}
macro_rules! next_ws {
() => { matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || !x.is_alphanumeric()) || self.peek_next_char()?.is_none() };
() => { matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || (!x.is_alphanumeric()) && x != '_') || self.peek_next_char()?.is_none() };
}
let mut buffer = String::with_capacity(16);
let mut looped_char = Some(first_char);
while let Some(next_char) = looped_char {
if next_char.is_whitespace() || !next_char.is_alphanumeric() {
// allow UNDERSCORE_IDENTS
if next_char.is_whitespace() || (!next_char.is_alphanumeric() && next_char != '_') {
break;
}
buffer.push(next_char);
@@ -463,6 +464,7 @@ impl<'a> Tokenizer<'a> {
"break" if next_ws!() => keyword!(Break),
"while" if next_ws!() => keyword!(While),
"continue" if next_ws!() => keyword!(Continue),
"const" if next_ws!() => keyword!(Const),
"true" if next_ws!() => {
return Ok(Token::new(
TokenType::Boolean(true),
@@ -837,7 +839,9 @@ mod tests {
#[test]
fn test_keyword_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("let fn if else return enum"));
let mut tokenizer = Tokenizer::from(String::from(
"let fn if else return enum continue break const",
));
let expected_tokens = vec![
TokenType::Keyword(Keyword::Let),
@@ -846,6 +850,9 @@ mod tests {
TokenType::Keyword(Keyword::Else),
TokenType::Keyword(Keyword::Return),
TokenType::Keyword(Keyword::Enum),
TokenType::Keyword(Keyword::Continue),
TokenType::Keyword(Keyword::Break),
TokenType::Keyword(Keyword::Const),
];
for expected_token in expected_tokens {
@@ -859,7 +866,7 @@ mod tests {
#[test]
fn test_identifier_parse() -> Result<()> {
let mut tokenizer = Tokenizer::from(String::from("fn test"));
let mut tokenizer = Tokenizer::from(String::from("fn test fn test_underscores"));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn));
@@ -868,6 +875,13 @@ mod tests {
token.token_type,
TokenType::Identifier(String::from("test"))
);
let token = tokenizer.next_token()?.unwrap();
assert_eq!(token.token_type, TokenType::Keyword(Keyword::Fn));
let token = tokenizer.next_token()?.unwrap();
assert_eq!(
token.token_type,
TokenType::Identifier(String::from("test_underscores"))
);
Ok(())
}

206
rust_compiler/libs/tokenizer/src/token.rs
View File

@@ -1,5 +1,15 @@
use helpers::prelude::*;
use rust_decimal::Decimal;
// Define a local macro to consume the list
macro_rules! generate_check {
($($name:literal),*) => {
pub fn is_syscall(s: &str) -> bool {
matches!(s, $($name)|*)
}
}
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token {
/// The type of the token
@@ -87,17 +97,56 @@ pub enum TokenType {
EOF,
}
impl Documentation for TokenType {
fn docs(&self) -> String {
match self {
Self::Keyword(k) => k.docs(),
_ => "".into(),
}
}
fn get_all_documentation() -> Vec<(&'static str, String)> {
Keyword::get_all_documentation()
}
}
helpers::with_syscalls!(generate_check);
impl From<TokenType> for u32 {
fn from(value: TokenType) -> Self {
use TokenType::*;
match value {
String(_) => 1,
Number(_) => 2,
Boolean(_) => 3,
Keyword(_) => 4,
Identifier(_) => 5,
Symbol(_) => 6,
EOF => 0,
TokenType::String(_) => 1,
TokenType::Number(_) => 2,
TokenType::Boolean(_) => 3,
TokenType::Keyword(k) => match k {
Keyword::If
| Keyword::Else
| Keyword::Loop
| Keyword::While
| Keyword::Break
| Keyword::Continue
| Keyword::Return => 4,
_ => 5,
},
TokenType::Identifier(s) => {
if is_syscall(&s) {
10
} else {
6
}
}
TokenType::Symbol(s) => {
if s.is_comparison() {
11
} else if s.is_operator() {
12
} else if s.is_logical() {
13
} else {
7
}
}
TokenType::EOF => 0,
}
}
}
@@ -264,28 +313,147 @@ impl std::fmt::Display for Symbol {
}
}
documented! {
#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy)]
pub enum Keyword {
/// Represents the `continue` keyword
/// Represents the `continue` keyword. This will allow you to bypass the current iteration in a loop and start the next one.
/// ## Example
/// ```
/// let item = 0;
/// loop {
/// if (item % 2 == 0) {
/// // This will NOT increment `item` and will continue with the next iteration of the
/// // loop
/// continue;
/// }
/// item = item + 1;
/// }
/// ```
Continue,
/// Represents the `let` keyword
/// Prepresents the `const` keyword. This allows you to define a variable that will never
/// change throughout the lifetime of the program, similar to `define` in IC10. If you are
/// not planning on mutating the variable (changing it), it is recommend you store it as a
/// const, as the compiler will not assign it to a register or stack variable.
///
/// ## Example
/// ```
/// const targetTemp = 20c;
/// device gasSensor = "d0";
/// device airCon = "d1";
///
/// airCon.On = gasSensor.Temperature > targetTemp;
/// ```
Const,
/// Represents the `let` keyword, used to declare variables within Slang.
/// ## Example
/// ```
/// // This variable now exists either in a register or the stack depending on how many
/// // free registers were available when declaring it.
/// let item = 0;
/// ```
Let,
/// Represents the `fn` keyword
/// Represents the `fn` keyword, used to declare functions within Slang.
/// # WARNING
/// Functions are currently unstable and are subject to change until stabilized. Use at
/// your own risk! (They are also heavily not optimized and produce a LOT of code bloat)
/// ## Example
/// ```
/// // This allows you to now call `doSomething` with specific arguments.
/// fn doSomething(arg1, arg2) {
///
/// }
/// ```
Fn,
/// Represents the `if` keyword
/// Represents the `if` keyword, allowing you to create branched logic.
/// ## Example
/// ```
/// let i = 0;
/// if (i == 0) {
/// i = 1;
/// }
/// // At this line, `i` is now `1`
/// ```
If,
/// Represents the `device` keyword. Useful for defining a device at a specific address (ex. d0, d1, d2, etc.)
/// Represents the `device` keyword. Useful for defining a device at a specific address
/// (ex. d0, d1, d2, etc.). This also allows you to perform direct operations ON a device.
/// ## Example
/// ```
/// device self = "db";
///
/// // This is the same as `s db Setting 123`
/// self.Setting = 123;
/// ```
Device,
/// Represents the `else` keyword
/// Represents the `else` keyword. Useful if you want to check a condition but run run
/// seperate logic in case that condition fails.
/// ## Example
/// ```
/// device self = "db";
/// let i = 0;
/// if (i < 0) {
/// self.Setting = 0;
/// } else {
/// self.Setting = 1;
/// }
/// // Here, the `Setting` on the current housing is `1` because i was NOT less than 0
/// ```
Else,
/// Represents the `return` keyword
/// Represents the `return` keyword. Allows you to pass values from a function back to
/// the caller.
/// ## Example
/// ```
/// fn doSomething() {
/// return 1 + 2;
/// }
///
/// // `returnedValue` now holds the value `3`
/// let returnedValue = doSomething();
/// ```
Return,
/// Represents the `enum` keyword
/// Represents the `enum` keyword. This is currently not supported, but is kept as a
/// reserved keyword in the future case that this is implemented.
Enum,
/// Represents the `loop` keyword
/// Represents the `loop` keyword. This allows you to create an infinate loop, but can be
/// broken with the `break` keyword.
/// ## Example
/// ```
/// device self = "db";
/// let i = 0;
/// loop {
/// i = i + 1;
/// // The current housing will infinately increment it's `Setting` value.
/// self.Setting = i;
/// }
/// ```
Loop,
/// Represents the `break` keyword
/// Represents the `break` keyword. This allows you to "break out of" a loop prematurely,
/// such as when an if() conditon is true, etc.
/// ## Example
/// ```
/// let i = 0;
/// // This loop will run until the value of `i` is greater than 10,000,
/// // which will then trigger the `break` keyword and it will stop looping
/// loop {
/// if (i > 10_000) {
/// break;
/// }
/// i = i + 1;
/// }
/// ```
Break,
/// Represents the `while` keyword
/// Represents the `while` keyword. This is similar to the `loop` keyword but different in
/// that you don't need an `if` statement to break out of a loop, that is handled
/// automatically when invoking `while`
/// ## Example
/// ```
/// let i = 0;
/// // This loop will run until the value of `i` is greater than 10,000, in which case the
/// // while loop will automatically stop running and code will continue AFTER the last
/// // bracket.
/// while (i < 10_000) {
/// i = i + 1;
/// }
/// ```
While,
}
}

40
rust_compiler/src/ffi/mod.rs
View File

@@ -1,5 +1,6 @@
use compiler::Compiler;
use parser::Parser;
use helpers::Documentation;
use parser::{sys_call::SysCall, Parser};
use safer_ffi::prelude::*;
use std::io::BufWriter;
use tokenizer::{
@@ -26,6 +27,13 @@ pub struct FfiRange {
end_line: u32,
}
#[derive_ReprC]
#[repr(C)]
pub struct FfiDocumentedItem {
item_name: safer_ffi::String,
docs: safer_ffi::String,
}
impl From<lsp_types::Range> for FfiRange {
fn from(value: lsp_types::Range) -> Self {
Self {
@@ -76,6 +84,11 @@ pub fn free_string(s: safer_ffi::String) {
drop(s)
}
#[ffi_export]
pub fn free_docs_vec(v: safer_ffi::Vec<FfiDocumentedItem>) {
drop(v)
}
/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
/// we want to avoid GC. So we pass it to Rust to handle all the memory allocations.
/// This should result in the ability to compile many times without triggering frame drops
@@ -151,8 +164,8 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<Ff
column: column as i32,
error: "".into(),
length: (original_string.unwrap_or_default().len()) as i32,
tooltip: token_type.docs().into(),
token_kind: token_type.into(),
tooltip: "".into(),
}),
}
}
@@ -183,3 +196,26 @@ pub fn diagnose_source(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<
res.unwrap_or(vec![].into())
}
#[ffi_export]
pub fn get_docs() -> safer_ffi::Vec<FfiDocumentedItem> {
let res = std::panic::catch_unwind(|| {
let mut docs = SysCall::get_all_documentation();
docs.extend(TokenType::get_all_documentation());
docs
});
let Ok(result) = res else {
return vec![].into();
};
result
.into_iter()
.map(|(key, doc)| FfiDocumentedItem {
item_name: key.into(),
docs: doc.into(),
})
.collect::<Vec<_>>()
.into()
}