First pass getting a logos tokenizer up and running

2025-12-08 21:06:42 -07:00
parent b6123219f8
commit 6afeec6da2
3 changed files with 207 additions and 1 deletions
--- a/rust_compiler/libs/tokenizer/Cargo.toml
+++ b/rust_compiler/libs/tokenizer/Cargo.toml
@@ -8,6 +8,7 @@ rust_decimal = { workspace = true }
 quick-error = { workspace = true }
 lsp-types = { workspace = true }
 helpers = { path = "../helpers" }
+logos = "0.16"

 [dev-dependencies]
 anyhow = { version = "^1" }