janis/SeaLang
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

-- and ++ tokens

Browse source
This commit is contained in:
janis 2025-09-19 13:47:49 +02:00
parent 735b60f3cb
commit f65aa8d921
Signed by: janis
SSH key fingerprint: SHA256:bB1qbbqmDXZNT0KKD5c2Dfjg53JGhj7B3CFcLIzSqq8
1 changed files with 33 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
crates/lexer/src/lib.rs
View file

@ -202,7 +202,9 @@ tokens!(pub Token: {
Bang => "!",
Tilde => "~",
Plus => "+",
PlusPlus => "++",
Minus => "-",
MinusMinus => "--",
Star => "*",
Slash => "/",
Percent => "%",
@ -443,9 +445,10 @@ impl LexemeParser {
}
}
// but what if...? tree!
mod tree {
// but what if...? prefix-tree!
mod trie {
use super::Token;
use std::collections::BTreeMap;
#[derive(Debug, Default)]
struct Node {
@ -496,7 +499,7 @@ mod tree {
fn search_tree(tree: &Tree, mut tokens: impl Iterator<Item = char>) -> Option<Token> {
let mut current = tree.root.as_ref().unwrap();
let mut p = 0;
let mut p = None;
loop {
let Some(ch) = tokens.next() else {
@ -507,7 +510,7 @@ mod tree {
break;
}
p = match current.keys.binary_search(&ch) {
let n = match current.keys.binary_search(&ch) {
Ok(p) => p,
Err(p) => {
eprintln!("No match for {ch} in {:?} (p={p})", current.keys);
@ -515,14 +518,17 @@ mod tree {
}
};
current = match current.edges.get(p) {
current = match current.edges.get(n) {
Some(Some(node)) => node,
_ => break,
_ => {
p = Some(n);
break;
}
};
}
eprintln!("current: {:?}", current);
current.values.get(p).copied().flatten()
current.values.get(p?).copied().flatten()
}
#[cfg(test)]
@ -531,10 +537,28 @@ mod tree {
#[test]
fn test_tree() {
use werkzeug::iter::AdvanceWhile;
let tree = build_tree();
eprintln!("Tree: {tree:?}");
let tokens = "fn let void";
assert_eq!(search_tree(&tree, tokens.chars()), Some(Token::Fn));
let mut tokens = "fn let void+++(++bool)".chars();
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Fn));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Let));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Void));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::PlusPlus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Plus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::OpenParens));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::PlusPlus));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::Bool));
tokens.advance_while(|&c| crate::is_things::is_whitespace(c));
assert_eq!(search_tree(&tree, &mut tokens), Some(Token::CloseParens));
assert_eq!(search_tree(&tree, &mut tokens), None);
}
}
}