dbidwell/stationeers_lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #7 from dbidwell94/tokenization-refactor

Browse Source 
Tokenization refactor
This commit is contained in:
Devin Bidwell
2025-11-29 15:36:21 -07:00
committed by GitHub
parent 502c60d45e 1a9a15c474
commit 668c3a5d41
8 changed files with 409 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.sh
View File

@@ -39,6 +39,7 @@ echo "--------------------"
RUST_WIN_EXE="$RUST_DIR/target/x86_64-pc-windows-gnu/release/slang.exe"
RUST_LINUX_BIN="$RUST_DIR/target/x86_64-unknown-linux-gnu/release/slang"
CHARP_DLL="$CSHARP_DIR/bin/Release/net46/StationeersSlang.dll"
CHARP_PDB="$CSHARP_DIR/bin/Release/net46/StationeersSlang.pdb"
# Check if the release dir exists, if not: create it.
if [[ ! -d "$RELEASE_DIR" ]]; then
@@ -48,3 +49,4 @@ fi
cp "$RUST_WIN_EXE" "$RELEASE_DIR/slang.exe"
cp "$RUST_LINUX_BIN" "$RELEASE_DIR/slang"
cp "$CHARP_DLL" "$RELEASE_DIR/StationeersSlang.dll"
cp "$CHARP_PDB" "$RELEASE_DIR/StationeersSlang.pdb"

137
csharp_mod/Extensions.cs
View File

@@ -1,66 +1,103 @@
namespace Slang;
using System;
using System.Text;
using StationeersIC10Editor;
namespace Slang
public static unsafe class SlangExtensions
{
public static unsafe class SlangExtensions
/**
* <summary>
* This is a helper method to convert a Rust struct for a string pointer
* into a C# style string.
* </summary>
*/
public static string AsString(this Vec_uint8_t vec)
{
/**
* <summary>
* This is a helper method to convert a Rust struct for a string pointer
* into a C# style string.
* </summary>
*/
public static string AsString(this Vec_uint8_t vec)
if (vec.ptr == null || vec.len == UIntPtr.Zero)
{
if (vec.ptr == null || vec.len == UIntPtr.Zero)
{
return string.Empty;
}
// Rust strings are UTF-8. Read bytes from raw pointer.
var toReturn = Encoding.UTF8.GetString(vec.ptr, (int)vec.len);
return toReturn;
return string.Empty;
}
/**
* <summary>This will free a Rust string struct. Because this is a pointer to a struct, this memory
* is managed by Rust, therefor it must be freed by Rust
* </summary>
*/
public static void Drop(this Vec_uint8_t vec)
// Rust strings are UTF-8. Read bytes from raw pointer.
var toReturn = Encoding.UTF8.GetString(vec.ptr, (int)vec.len);
return toReturn;
}
/**
* <summary>This will free a Rust string struct. Because this is a pointer to a struct, this memory
* is managed by Rust, therefor it must be freed by Rust
* </summary>
*/
public static void Drop(this Vec_uint8_t vec)
{
Ffi.free_string(vec);
}
/**
* <summary>This helper converts a Rust vec to a C# List. This handles freeing the
* Rust allocation after the List is created, there is no need to Drop this memory.
* </summary>
*/
public static Line ToLine(this Vec_FfiToken_t vec, string sourceText)
{
var list = new Line(sourceText);
var currentPtr = vec.ptr;
// Iterate through the raw memory array
for (int i = 0; i < (int)vec.len; i++)
{
Ffi.free_string(vec);
var token = currentPtr[i];
var color = GetColorForKind(token.token_kind);
int colIndex = token.column;
if (colIndex < 0)
colIndex = 0;
var semanticToken = new SemanticToken(
0,
colIndex,
token.length,
color,
token.token_kind
);
string errMsg = token.error.AsString();
if (!string.IsNullOrEmpty(errMsg))
{
semanticToken.IsError = true;
semanticToken.Data = errMsg;
semanticToken.Color = ICodeFormatter.ColorError;
}
list.AddToken(semanticToken);
}
/**
* <summary>This helper converts a Rust vec to a C# List. This handles freeing the
* Rust allocation after the List is created, there is no need to Drop this memory.
* </summary>
*/
public static Line AsList(this Vec_FfiToken_t vec)
Ffi.free_ffi_token_vec(vec);
return list;
}
private static uint GetColorForKind(uint kind)
{
switch (kind)
{
var list = new Line();
list.Capacity = (int)vec.len;
var currentPtr = vec.ptr;
// Iterate through the raw memory array
for (int i = 0; i < (int)vec.len; i++)
{
// Dereference pointer to get the struct at index i
FfiToken_t token = currentPtr[i];
var newToken = new Token(token.text.AsString(), token.column);
list.Add(newToken);
}
Ffi.free_ffi_token_vec(vec);
return list;
case 1:
return SlangFormatter.ColorInstruction; // Keyword
case 2:
return SlangFormatter.ColorDefault; // Identifier
case 3:
return SlangFormatter.ColorNumber; // Number
case 4:
return SlangFormatter.ColorString; // String
case 5:
return SlangFormatter.ColorInstruction; // Boolean
case 6:
return SlangFormatter.ColorDefault; // Symbol
default:
return SlangFormatter.ColorDefault;
}
}
}

10
csharp_mod/FfiGlue.cs
View File

@@ -83,17 +83,17 @@ public unsafe partial class Ffi {
slice_ref_uint16_t input);
}
[StructLayout(LayoutKind.Sequential, Size = 104)]
[StructLayout(LayoutKind.Sequential, Size = 64)]
public unsafe struct FfiToken_t {
public Vec_uint8_t text;
public Vec_uint8_t tooltip;
public Vec_uint8_t error;
public Vec_uint8_t status;
public Int32 column;
public Int32 length;
public UInt32 token_kind;
}
/// <summary>

20
csharp_mod/Formatter.cs
View File

@@ -1,12 +1,24 @@
namespace Slang;
using StationeersIC10Editor;
namespace Slang
public class SlangFormatter : ICodeFormatter
{
public class SlangFormatter : ICodeFormatter
public static readonly uint ColorInstruction = ColorFromHTML("#ffff00");
public static readonly uint ColorString = ColorFromHTML("#ce9178");
public override Line ParseLine(string line)
{
public override Line ParseLine(string line)
return Marshal.TokenizeLine(line);
}
public override string Compile()
{
if (Marshal.CompileFromString(this.Lines.RawText, out string compiled))
{
return Marshal.TokenizeLine(line);
return compiled;
}
return string.Empty;
}
}

182
csharp_mod/Marshal.cs
View File

@@ -1,59 +1,157 @@
namespace Slang;
using System;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
using StationeersIC10Editor;
namespace Slang
public static class Marshal
{
public static class Marshal
{
public static unsafe Line TokenizeLine(string source)
{
if (String.IsNullOrEmpty(source))
{
return new Line();
}
private static IntPtr _libraryHandle = IntPtr.Zero;
fixed (char* ptrString = source)
{
var input = new slice_ref_uint16_t
{
ptr = (ushort*)ptrString,
len = (UIntPtr)source.Length,
};
return Ffi.tokenize_line(input).AsList();
}
[DllImport("kernel32", SetLastError = true, CharSet = CharSet.Ansi)]
private static extern IntPtr LoadLibrary([MarshalAs(UnmanagedType.LPStr)] string lpFileName);
[DllImport("kernel32", SetLastError = true)]
private static extern bool FreeLibrary(IntPtr hModule);
private static bool EnsureLibLoaded()
{
if (_libraryHandle != IntPtr.Zero)
{
return true;
}
public static unsafe bool CompileFromString(string inputString, out string compiledString)
try
{
if (String.IsNullOrEmpty(inputString))
_libraryHandle = LoadLibrary(ExtractNativeLibrary(Ffi.RustLib));
CodeFormatters.RegisterFormatter("Slang", typeof(SlangFormatter), true);
return true;
}
catch (Exception ex)
{
L.Error($"Failed to init slang compiler: {ex.Message}");
return false;
}
}
public static bool Init()
{
return EnsureLibLoaded();
}
public static bool Destroy()
{
if (_libraryHandle == IntPtr.Zero)
{
return true;
}
try
{
FreeLibrary(_libraryHandle);
_libraryHandle = IntPtr.Zero;
return true;
}
catch (Exception ex)
{
L.Warning($"Unable to free handle to slang compiler's dll. {ex.Message}");
return false;
}
}
public static unsafe Line TokenizeLine(string source)
{
if (String.IsNullOrEmpty(source))
{
return new Line(source);
}
if (!EnsureLibLoaded())
{
return new Line(source);
}
fixed (char* ptrString = source)
{
var input = new slice_ref_uint16_t
{
compiledString = String.Empty;
return false;
ptr = (ushort*)ptrString,
len = (UIntPtr)source.Length,
};
return Ffi.tokenize_line(input).ToLine(source);
}
}
public static unsafe bool CompileFromString(string inputString, out string compiledString)
{
if (String.IsNullOrEmpty(inputString))
{
compiledString = String.Empty;
return false;
}
if (!EnsureLibLoaded())
{
compiledString = String.Empty;
return false;
}
fixed (char* ptrString = inputString)
{
var input = new slice_ref_uint16_t
{
ptr = (ushort*)ptrString,
len = (UIntPtr)inputString.Length,
};
var result = Ffi.compile_from_string(input);
try
{
if ((ulong)result.len < 1)
{
compiledString = String.Empty;
return false;
}
compiledString = result.AsString();
return true;
}
finally
{
result.Drop();
}
}
}
private static string ExtractNativeLibrary(string libName)
{
string destinationPath = Path.Combine(Path.GetTempPath(), libName);
Assembly assembly = Assembly.GetExecutingAssembly();
using (Stream stream = assembly.GetManifestResourceStream(libName))
{
if (stream == null)
{
L.Error(
$"{libName} not found. This means it was not embedded in the mod. Please contact the mod author!"
);
return "";
}
fixed (char* ptrString = inputString)
try
{
var input = new slice_ref_uint16_t
using (FileStream fileStream = new FileStream(destinationPath, FileMode.Create))
{
ptr = (ushort*)ptrString,
len = (UIntPtr)inputString.Length,
};
var result = Ffi.compile_from_string(input);
try
{
if ((ulong)result.len < 1)
{
compiledString = String.Empty;
return false;
}
compiledString = result.AsString();
return true;
}
finally
{
result.Drop();
stream.CopyTo(fileStream);
}
return destinationPath;
}
catch (IOException e)
{
L.Warning($"Could not overwrite {libName} (it might be in use): {e.Message}");
return "";
}
}
}

51
csharp_mod/Plugin.cs
View File

@@ -1,9 +1,6 @@
using System.IO;
using System.Reflection;
using System.Text.RegularExpressions;
using BepInEx;
using HarmonyLib;
using StationeersIC10Editor;
namespace Slang
{
@@ -44,6 +41,8 @@ namespace Slang
public const string PluginGuid = "com.biddydev.slang";
public const string PluginName = "Slang";
private Harmony? _harmony;
private static Regex? _slangSourceCheck = null;
private static Regex SlangSourceCheck
@@ -89,44 +88,28 @@ namespace Slang
private void Awake()
{
L.SetLogger(Logger);
this._harmony = new Harmony(PluginGuid);
L.Info("slang loaded");
if (ExtractNativeDll(Ffi.RustLib))
// If we failed to load the compiler, bail from the rest of the patches. It won't matter,
// as the compiler itself has failed to load.
if (!Marshal.Init())
{
var harmony = new Harmony(PluginGuid);
harmony.PatchAll();
CodeFormatters.RegisterFormatter("slang", () => new SlangFormatter(), true);
return;
}
this._harmony.PatchAll();
}
private bool ExtractNativeDll(string fileName)
private void OnDestroy()
{
string destinationPath = Path.Combine(Path.GetDirectoryName(Info.Location), fileName);
Assembly assembly = Assembly.GetExecutingAssembly();
using (Stream stream = assembly.GetManifestResourceStream(fileName))
if (Marshal.Destroy())
{
if (stream == null)
{
L.Error(
$"{Ffi.RustLib} not found. This means it was not embedded in the mod. Please contact the mod author!"
);
return false;
}
try
{
using (FileStream fileStream = new FileStream(destinationPath, FileMode.Create))
{
stream.CopyTo(fileStream);
}
return true;
}
catch (IOException e)
{
L.Warning($"Could not overwrite {fileName} (it might be in use): {e.Message}");
return false;
}
L.Info("FFI references cleaned up.");
}
if (this._harmony is not null)
{
this._harmony.UnpatchSelf();
}
}
}

261
rust_compiler/libs/tokenizer/src/lib.rs
View File

@@ -56,7 +56,7 @@ impl<'a> Tokenizer<'a> {
Ok(Self {
reader,
line: 1,
column: 1,
column: 0, // Start at 0 so first char becomes 1
char_buffer: [0],
returned_eof: false,
string_buffer: String::new(),
@@ -71,7 +71,7 @@ impl<'a> From<String> for Tokenizer<'a> {
Self {
reader,
line: 1,
column: 1,
column: 0,
char_buffer: [0],
returned_eof: false,
string_buffer: String::new(),
@@ -84,7 +84,7 @@ impl<'a> From<&'a str> for Tokenizer<'a> {
Self {
reader: BufReader::new(Box::new(Cursor::new(value)) as Box<dyn Tokenize>),
char_buffer: [0],
column: 1,
column: 0,
line: 1,
returned_eof: false,
string_buffer: String::new(),
@@ -93,12 +93,6 @@ impl<'a> From<&'a str> for Tokenizer<'a> {
}
impl<'a> Tokenizer<'a> {
/// Consumes the tokenizer and returns the next token in the stream
/// If there are no more tokens in the stream, this function returns None
/// If there is an error reading the stream, this function returns an error
///
/// # Important
/// This function will increment the line and column counters
fn next_char(&mut self) -> Result<Option<char>, Error> {
let bytes_read = self.reader.read(&mut self.char_buffer)?;
@@ -106,7 +100,6 @@ impl<'a> Tokenizer<'a> {
return Ok(None);
}
// Safety: The buffer is guaranteed to have 1 value as it is initialized with a size of 1
let c = self.char_buffer[0] as char;
if c == '\n' {
self.line += 1;
@@ -119,30 +112,17 @@ impl<'a> Tokenizer<'a> {
Ok(Some(c))
}
/// Peeks the next character in the stream without consuming it
///
/// # Important
/// This does not increment the line or column counters
fn peek_next_char(&mut self) -> Result<Option<char>, Error> {
let current_pos = self.reader.stream_position()?;
let to_return = if self.reader.read(&mut self.char_buffer)? == 0 {
None
} else {
self.reader.seek(SeekFrom::Start(current_pos))?;
// Safety: The buffer is guaranteed to have 1 value as it is initialized with a size of 1
Some(self.char_buffer[0] as char)
};
Ok(to_return)
}
/// Skips the current line in the stream.
/// Useful for skipping comments or empty lines
///
/// # Important
/// This function will increment the line and column counters
fn skip_line(&mut self) -> Result<(), Error> {
while let Some(next_char) = self.next_char()? {
if next_char == '\n' {
@@ -152,40 +132,50 @@ impl<'a> Tokenizer<'a> {
Ok(())
}
/// Consumes the tokenizer and returns the next token in the stream
/// If there are no more tokens in the stream, this function returns None
pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
self.string_buffer.clear();
while let Some(next_char) = self.next_char()? {
// skip whitespace
if next_char.is_whitespace() {
self.string_buffer.clear();
continue;
}
// skip comments
if next_char == '/' && self.peek_next_char()? == Some('/') {
self.skip_line()?;
self.string_buffer.clear();
continue;
}
// Capture start position before delegating
let start_line = self.line;
let start_col = self.column;
match next_char {
// numbers
'0'..='9' => {
return self.tokenize_number(next_char).map(Some);
return self
.tokenize_number(next_char, start_line, start_col)
.map(Some);
}
'"' | '\'' => {
return self
.tokenize_string(next_char, start_line, start_col)
.map(Some);
}
// strings
'"' | '\'' => return self.tokenize_string(next_char).map(Some),
// symbols excluding `"` and `'`
char if !char.is_alphanumeric() && char != '"' && char != '\'' => {
return self.tokenize_symbol(next_char).map(Some);
return self
.tokenize_symbol(next_char, start_line, start_col)
.map(Some);
}
// keywords and identifiers
char if char.is_alphabetic() => {
return self.tokenize_keyword_or_identifier(next_char).map(Some);
return self
.tokenize_keyword_or_identifier(next_char, start_line, start_col)
.map(Some);
}
_ => {
return Err(Error::UnknownSymbolError(
next_char,
self.line,
self.column,
start_line,
start_col,
std::mem::take(&mut self.string_buffer),
));
}
@@ -204,13 +194,10 @@ impl<'a> Tokenizer<'a> {
}
}
/// Peeks the next token in the stream without consuming it
/// If there are no more tokens in the stream, this function returns None
pub fn peek_next(&mut self) -> Result<Option<Token>, Error> {
let current_pos = self.reader.stream_position()?;
let column = self.column;
let line = self.line;
let token = self.next_token()?;
self.reader.seek(SeekFrom::Start(current_pos))?;
self.column = column;
@@ -218,22 +205,26 @@ impl<'a> Tokenizer<'a> {
Ok(token)
}
/// Tokenizes a symbol
fn tokenize_symbol(&mut self, first_symbol: char) -> Result<Token, Error> {
/// Helper macro to create a symbol token
// Updated helper functions to accept start_line and start_col
fn tokenize_symbol(
&mut self,
first_symbol: char,
line: usize,
col: usize,
) -> Result<Token, Error> {
macro_rules! symbol {
($symbol:ident) => {
Ok(Token::new(
TokenType::Symbol(Symbol::$symbol),
self.line,
self.column,
line,
col,
Some(std::mem::take(&mut self.string_buffer)),
))
};
}
match first_symbol {
// single character symbols
'(' => symbol!(LParen),
')' => symbol!(RParen),
'{' => symbol!(LBrace),
@@ -246,42 +237,34 @@ impl<'a> Tokenizer<'a> {
'+' => symbol!(Plus),
'-' => symbol!(Minus),
'/' => symbol!(Slash),
'.' => symbol!(Dot),
'^' => symbol!(Caret),
'%' => symbol!(Percent),
// multi-character symbols
'<' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(LessThanOrEqual)
}
'<' => symbol!(LessThan),
'>' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(GreaterThanOrEqual)
}
'>' => symbol!(GreaterThan),
'=' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(Equal)
}
'=' => symbol!(Assign),
'!' if self.peek_next_char()? == Some('=') => {
self.next_char()?;
symbol!(NotEqual)
}
'!' => symbol!(LogicalNot),
'*' if self.peek_next_char()? == Some('*') => {
self.next_char()?;
symbol!(Exp)
}
'*' => symbol!(Asterisk),
'&' if self.peek_next_char()? == Some('&') => {
self.next_char()?;
symbol!(LogicalAnd)
@@ -290,45 +273,39 @@ impl<'a> Tokenizer<'a> {
self.next_char()?;
symbol!(LogicalOr)
}
_ => Err(Error::UnknownSymbolError(
first_symbol,
self.line,
self.column,
line,
col,
std::mem::take(&mut self.string_buffer),
)),
}
}
/// Tokenizes a number literal. Also handles temperatures with a suffix of `c`, `f`, or `k`.
fn tokenize_number(&mut self, first_char: char) -> Result<Token, Error> {
fn tokenize_number(
&mut self,
first_char: char,
line: usize,
col: usize,
) -> Result<Token, Error> {
let mut primary = String::with_capacity(16);
let mut decimal: Option<String> = None;
let mut reading_decimal = false;
let column = self.column;
let line = self.line;
primary.push(first_char);
while let Some(next_char) = self.peek_next_char()? {
if next_char.is_whitespace() {
break;
}
if next_char == '.' {
reading_decimal = true;
self.next_char()?;
continue;
}
// support underscores in numbers for readability
if next_char == '_' {
self.next_char()?;
continue;
}
// This is for the times when we have a number followed by a symbol (like a semicolon or =)
if !next_char.is_numeric() {
break;
}
@@ -343,33 +320,21 @@ impl<'a> Tokenizer<'a> {
let number: Number = if let Some(decimal) = decimal {
let decimal_scale = decimal.len() as u32;
let number = format!("{}{}", primary, decimal)
.parse::<i128>()
.map_err(|e| {
Error::NumberParseError(
e,
self.line,
self.column,
std::mem::take(&mut self.string_buffer),
)
})?;
let number_str = format!("{}{}", primary, decimal);
let number = number_str.parse::<i128>().map_err(|e| {
Error::NumberParseError(e, line, col, std::mem::take(&mut self.string_buffer))
})?;
Number::Decimal(
Decimal::try_from_i128_with_scale(number, decimal_scale).map_err(|e| {
Error::DecimalParseError(
e,
line,
column,
std::mem::take(&mut self.string_buffer),
)
Error::DecimalParseError(e, line, col, std::mem::take(&mut self.string_buffer))
})?,
)
} else {
Number::Integer(primary.parse().map_err(|e| {
Error::NumberParseError(e, line, column, std::mem::take(&mut self.string_buffer))
Error::NumberParseError(e, line, col, std::mem::take(&mut self.string_buffer))
})?)
};
// check if the next char is a temperature suffix
if let Some(next_char) = self.peek_next_char()? {
let temperature = match next_char {
'c' => Temperature::Celsius(number),
@@ -379,7 +344,7 @@ impl<'a> Tokenizer<'a> {
return Ok(Token::new(
TokenType::Number(number),
line,
column,
col,
Some(std::mem::take(&mut self.string_buffer)),
));
}
@@ -390,74 +355,65 @@ impl<'a> Tokenizer<'a> {
Ok(Token::new(
TokenType::Number(temperature),
line,
column,
col,
Some(std::mem::take(&mut self.string_buffer)),
))
} else {
Ok(Token::new(
TokenType::Number(number),
line,
column,
col,
Some(std::mem::take(&mut self.string_buffer)),
))
}
}
/// Tokenizes a string literal
fn tokenize_string(&mut self, beginning_quote: char) -> Result<Token, Error> {
fn tokenize_string(
&mut self,
beginning_quote: char,
line: usize,
col: usize,
) -> Result<Token, Error> {
let mut buffer = String::with_capacity(16);
let column = self.column;
let line = self.line;
while let Some(next_char) = self.next_char()? {
if next_char == beginning_quote {
break;
}
buffer.push(next_char);
}
Ok(Token::new(
TokenType::String(buffer),
line,
column,
col,
Some(std::mem::take(&mut self.string_buffer)),
))
}
/// Tokenizes a keyword or an identifier. Also handles boolean literals
fn tokenize_keyword_or_identifier(&mut self, first_char: char) -> Result<Token, Error> {
fn tokenize_keyword_or_identifier(
&mut self,
first_char: char,
line: usize,
col: usize,
) -> Result<Token, Error> {
macro_rules! keyword {
($keyword:ident) => {{
return Ok(Token::new(
TokenType::Keyword(Keyword::$keyword),
self.line,
self.column,
line,
col,
Some(std::mem::take(&mut self.string_buffer)),
));
}};
}
/// Helper macro to check if the next character is whitespace or not alphanumeric
macro_rules! next_ws {
() => {
matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || !x.is_alphanumeric()) || self.peek_next_char()?.is_none()
};
() => { matches!(self.peek_next_char()?, Some(x) if x.is_whitespace() || !x.is_alphanumeric()) || self.peek_next_char()?.is_none() };
}
let mut buffer = String::with_capacity(16);
let line = self.line;
let column = self.column;
let mut looped_char = Some(first_char);
while let Some(next_char) = looped_char {
if next_char.is_whitespace() {
break;
}
if !next_char.is_alphanumeric() {
if next_char.is_whitespace() || !next_char.is_alphanumeric() {
break;
}
buffer.push(next_char);
@@ -474,51 +430,47 @@ impl<'a> Tokenizer<'a> {
"break" if next_ws!() => keyword!(Break),
"while" if next_ws!() => keyword!(While),
"continue" if next_ws!() => keyword!(Continue),
// boolean literals
"true" if next_ws!() => {
return Ok(Token::new(
TokenType::Boolean(true),
self.line,
self.column,
line,
col,
Some(std::mem::take(&mut self.string_buffer)),
));
}
"false" if next_ws!() => {
return Ok(Token::new(
TokenType::Boolean(false),
self.line,
self.column,
line,
col,
Some(std::mem::take(&mut self.string_buffer)),
));
}
// if the next character is whitespace or not alphanumeric, then we have an identifier
// this is because keywords are checked first
val if next_ws!() => {
return Ok(Token::new(
TokenType::Identifier(val.to_string()),
line,
column,
col,
Some(std::mem::take(&mut self.string_buffer)),
));
}
_ => {}
}
looped_char = self.next_char()?;
}
Err(Error::UnknownKeywordOrIdentifierError(
buffer,
line,
column,
col,
std::mem::take(&mut self.string_buffer),
))
}
}
// ... Iterator and TokenizerBuffer implementations remain unchanged ...
// They just call the methods above which now use the passed-in start coordinates.
impl<'a> Iterator for Tokenizer<'a> {
type Item = Result<Token, Error>;
fn next(&mut self) -> Option<Self::Item> {
match self.next_token() {
Ok(Some(tok)) => Some(Ok(tok)),
@@ -542,38 +494,26 @@ impl<'a> TokenizerBuffer<'a> {
history: VecDeque::with_capacity(128),
}
}
/// Reads the next token from the tokenizer, pushing the value to the back of the history
/// and returning the token
pub fn next_token(&mut self) -> Result<Option<Token>, Error> {
if let Some(token) = self.buffer.pop_front() {
self.history.push_back(token.clone());
return Ok(Some(token));
}
let token = self.tokenizer.next_token()?;
if let Some(ref token) = token {
self.history.push_back(token.clone());
}
Ok(token)
}
/// Peeks the next token in the stream without adding to the history stack
pub fn peek(&mut self) -> Result<Option<Token>, Error> {
if let Some(token) = self.buffer.front() {
return Ok(Some(token.clone()));
}
let token = self.tokenizer.peek_next()?;
Ok(token)
}
fn seek_from_current(&mut self, seek_to: i64) -> Result<(), Error> {
use Ordering::*;
// if seek_to > 0 then we need to check if the buffer has enough tokens to pop, otherwise we need to read from the tokenizer
// if seek_to < 0 then we need to pop from the history and push to the front of the buffer. If not enough, then we throw (we reached the front of the history)
// if seek_to == 0 then we don't need to do anything
match seek_to.cmp(&0) {
Greater => {
let mut tokens = Vec::with_capacity(seek_to as usize);
@@ -606,18 +546,13 @@ impl<'a> TokenizerBuffer<'a> {
}
_ => {}
}
Ok(())
}
/// Adds to or removes from the History stack, allowing the user to move back and forth in the stream
pub fn seek(&mut self, from: SeekFrom) -> Result<(), Error> {
match from {
SeekFrom::Current(seek_to) => self.seek_from_current(seek_to)?,
SeekFrom::End(_) => unimplemented!("SeekFrom::End will not be implemented"),
SeekFrom::Start(_) => unimplemented!("SeekFrom::Start will not be implemented"),
_ => unimplemented!("SeekFrom::End/Start not implemented"),
}
Ok(())
}
}
@@ -682,7 +617,7 @@ mod tests {
assert_eq!(char, Some('f'));
assert_eq!(tokenizer.line, 1);
assert_eq!(tokenizer.column, 2);
assert_eq!(tokenizer.column, 1);
Ok(())
}
@@ -695,7 +630,7 @@ mod tests {
assert_eq!(char, Some('\n'));
assert_eq!(tokenizer.line, 1);
assert_eq!(tokenizer.column, 1);
assert_eq!(tokenizer.column, 0);
let char = tokenizer.next_char()?;
assert_eq!(char, Some('\n'));
@@ -1010,4 +945,36 @@ mod tests {
Ok(())
}
#[test]
fn test_identifier_has_correct_length() -> Result<()> {
let mut tokenizer = Tokenizer::from("hello");
assert_eq!(
tokenizer.next_token()?,
Some(Token {
token_type: TokenType::Identifier("hello".into()),
original_string: Some("hello".into()),
column: 1,
line: 1
})
);
Ok(())
}
#[test]
fn test_keyword_token_has_correct_length() -> Result<()> {
let mut tokenizer = Tokenizer::from("while");
assert_eq!(
tokenizer.next_token()?,
Some(Token {
token_type: TokenType::Keyword(Keyword::While),
original_string: Some("while".into()),
column: 1,
line: 1
})
);
Ok(())
}
}

43
rust_compiler/src/lib.rs
View File

@@ -7,11 +7,24 @@ use tokenizer::{token::TokenType, Error as TokenizerError, Tokenizer};
#[derive_ReprC]
#[repr(C)]
pub struct FfiToken {
pub text: safer_ffi::String,
pub tooltip: safer_ffi::String,
pub error: safer_ffi::String,
pub status: safer_ffi::String,
pub column: i32,
pub length: i32,
pub token_kind: u32,
}
fn map_token_kind(t: &TokenType) -> u32 {
use TokenType::*;
match t {
Keyword(_) => 1,
Identifier(_) => 2,
Number(_) => 3,
String(_) => 4,
Boolean(_) => 5,
Symbol(_) => 6,
_ => 0,
}
}
/// C# handles strings as UTF16. We do NOT want to allocate that memory in C# because
@@ -49,29 +62,29 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<Ff
for token in tokenizer {
match token {
Err(TokenizerError::NumberParseError(_, _, col, ref original))
| Err(TokenizerError::UnknownSymbolError(_, _, col, ref original))
| Err(TokenizerError::DecimalParseError(_, _, col, ref original))
| Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col, ref original)) => {
Err(TokenizerError::NumberParseError(_, _, col, ref str))
| Err(TokenizerError::UnknownSymbolError(_, _, col, ref str))
| Err(TokenizerError::DecimalParseError(_, _, col, ref str))
| Err(TokenizerError::UnknownKeywordOrIdentifierError(_, _, col, ref str)) => {
tokens.push(FfiToken {
column: col as i32,
text: original.to_string().into(),
column: col as i32 - 1,
tooltip: "".into(),
length: str.len() as i32,
token_kind: 0,
// Safety: it's okay to unwrap the err here because we are matching on the `Err` variant
error: token.unwrap_err().to_string().into(),
status: "".into(),
});
}
Err(_) => return safer_ffi::Vec::EMPTY,
Ok(token) if !matches!(token.token_type, TokenType::EOF) => tokens.push(FfiToken {
text: token
.original_string
.unwrap_or(token.token_type.to_string())
.into(),
tooltip: "".into(),
error: "".into(),
status: "".into(),
column: token.column as i32,
length: token
.original_string
.map(|s| s.len() as i32)
.unwrap_or_default(),
token_kind: map_token_kind(&token.token_type),
column: token.column as i32 - 1,
}),
_ => {}
}