This commit is contained in:
2025-11-28 03:17:23 -07:00
parent 611f0f6f71
commit 6b69cc1459
8 changed files with 344 additions and 51 deletions

View File

@@ -447,6 +447,14 @@ impl Tokenizer {
}
}
impl Iterator for Tokenizer {
type Item = Result<Token, Error>;
fn next(&mut self) -> Option<Self::Item> {
todo!()
}
}
pub struct TokenizerBuffer {
tokenizer: Tokenizer,
buffer: VecDeque<Token>,