working in-game error diagnostics. memory access violation bug present. Need to debug

rust_compiler/libs/parser/src/lib.rs

@@ -31,16 +31,16 @@ quick_error! {
             source(err)
         }
         UnexpectedToken(span: Span, token: Token) {
-            display("Unexpected token: {:?}", token)
+            display("Unexpected token: {}", token.token_type)
         }
         DuplicateIdentifier(span: Span, token: Token) {
-            display("Duplicate identifier: {:?}", token)
+            display("Duplicate identifier: {}", token.token_type)
         }
         InvalidSyntax(span: Span, reason: String) {
-            display("Invalid syntax: {:?}, Reason: {}", span, reason)
+            display("Invalid syntax: {}", reason)
         }
         UnsupportedKeyword(span: Span, token: Token) {
-            display("Unsupported keyword: {:?}", token)
+            display("Unsupported keyword: {}", token.token_type)
         }
         UnexpectedEOF {
             display("Unexpected EOF")

rust_compiler/libs/tokenizer/src/lib.rs

@@ -19,18 +19,18 @@ quick_error! {
             source(err)
         }
         NumberParseError(err: std::num::ParseIntError, line: usize, column: usize, original: String) {
-            display("Number Parse Error: {}\nLine: {}, Column: {}", err, line, column)
+            display("Number Parse Error: {}", err)
             source(err)
         }
         DecimalParseError(err: rust_decimal::Error, line: usize, column: usize, original: String) {
-            display("Decimal Parse Error: {}\nLine: {}, Column: {}", err, line, column)
+            display("Decimal Parse Error: {}", err)
             source(err)
         }
         UnknownSymbolError(char: char, line: usize, column: usize, original: String) {
-            display("Unknown Symbol: {}\nLine: {}, Column: {}", char, line, column)
+            display("Unknown Symbol: {}", char)
         }
         UnknownKeywordOrIdentifierError(val: String, line: usize, column: usize, original: String) {
-            display("Unknown Keyword or Identifier: {}\nLine: {}, Column: {}", val, line, column)
+            display("Unknown Keyword or Identifier: {}", val)
         }
     }
 }

rust_compiler/src/ffi/mod.rs

@@ -106,8 +106,6 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<Ff
 
     let mut tokens = Vec::new();
 
-    // Error reporting is handled in `diagnose_source`. We only care about successful tokens here
-    // for syntax highlighting
     for token in tokenizer {
         if matches!(
             token,
@@ -119,7 +117,24 @@ pub fn tokenize_line(input: safer_ffi::slice::Ref<'_, u16>) -> safer_ffi::Vec<Ff
             continue;
         }
         match token {
-            Err(_) => {}
+            Err(ref e) => {
+                use tokenizer::Error::*;
+                let (err_str, col, og) = match e {
+                    NumberParseError(_, _, col, og)
+                    | DecimalParseError(_, _, col, og)
+                    | UnknownSymbolError(_, _, col, og)
+                    | UnknownKeywordOrIdentifierError(_, _, col, og) => (e.to_string(), col, og),
+                    _ => continue,
+                };
+
+                tokens.push(FfiToken {
+                    column: *col as i32,
+                    error: err_str.into(),
+                    tooltip: "".into(),
+                    length: og.len() as i32,
+                    token_kind: 0,
+                })
+            }
             Ok(Token {
                 column,
                 original_string,