janis/SeaLang
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

-- and ++ tokens

Browse source
This commit is contained in:
janis 2025-09-19 13:47:49 +02:00
parent 735b60f3cb
commit f65aa8d921
Signed by: janis
SSH key fingerprint: SHA256:bB1qbbqmDXZNT0KKD5c2Dfjg53JGhj7B3CFcLIzSqq8
1 changed files with 33 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
crates/lexer/src/lib.rs
View file

@ -202,7 +202,9 @@ tokens!(pub Token: {
Bang => "!", Bang => "!",
Tilde => "~", Tilde => "~",
Plus => "+", Plus => "+",
PlusPlus => "++",
Minus => "-", Minus => "-",
MinusMinus => "--",
Star => "*", Star => "*",
Slash => "/", Slash => "/",
Percent => "%", Percent => "%",
@ -443,9 +445,10 @@ impl LexemeParser {
} }
} }
// but what if...? tree! // but what if...? prefix-tree!
mod tree { mod trie {
use super::Token; use super::Token;
use std::collections::BTreeMap;
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct Node { struct Node {
@ -496,7 +499,7 @@ mod tree {
fn search_tree(tree: &Tree, mut tokens: impl Iterator<Item = char>) -> Option<Token> { fn search_tree(tree: &Tree, mut tokens: impl Iterator<Item = char>) -> Option<Token> {
let mut current = tree.root.as_ref().unwrap(); let mut current = tree.root.as_ref().unwrap();
let mut p = 0; let mut p = None;
loop { loop {
let Some(ch) = tokens.next() else { let Some(ch) = tokens.next() else {
@ -507,7 +510,7 @@ mod tree {
break; break;
} }
p = match current.keys.binary_search(&ch) { let n = match current.keys.binary_search(&ch) {
Ok(p) => p, Ok(p) => p,
Err(p) => { Err(p) => {
eprintln!("No match for {ch} in {:?} (p={p})", current.keys); eprintln!("No match for {ch} in {:?} (p={p})", current.keys);
@ -515,14 +518,17 @@ mod tree {
} }
}; };
current = match current.edges.get(p) { current = match current.edges.get(n) {
Some(Some(node)) => node, Some(Some(node)) => node,
_ => break, _ => {
p = Some(n);
break;
}
}; };
} }
eprintln!("current: {:?}", current); eprintln!("current: {:?}", current);
current.values.get(p).copied().flatten() current.values.get(p?).copied().flatten()
} }
#[cfg(test)] #[cfg(test)]
@ -531,10 +537,28 @@ mod tree {
#[test] #[test]
fn test_tree() { fn test_tree() {
use werkzeug::iter::AdvanceWhile;
let tree = build_tree(); let tree = build_tree();
eprintln!("Tree: {tree:?}"); eprintln!("Tree: {tree:?}");
let tokens = "fn let void"; let mut tokens = "fn let void+++(++bool)".chars();
assert_eq!(search_tree(&tree, tokens.chars()), Some(Token::Fn)); assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Fn));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Let));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Void));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::PlusPlus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Plus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::OpenParens));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::PlusPlus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Bool));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::CloseParens));
assert_eq!(search_tree(&tree, &mut tokens), None);
} }
} }
} }