diff --git a/lang/tests/tokens.rs b/lang/tests/tokens.rs index 57546d0..6a6310f 100644 --- a/lang/tests/tokens.rs +++ b/lang/tests/tokens.rs @@ -126,6 +126,20 @@ fn main() { Lexeme(15, ""), ][..]); + eprint!("Initializing tokeniser.. "); + tokeniser_init(c"tests/tokens/identifier.l".as_ptr()); + eprintln!("ok."); + + assert_eq!(&collect_tokens()[..], &[ + Lexeme(30, "this-is-an-ident"), + Lexeme(30, "another_ident123"), + Lexeme(30, "_underscore_test"), + Lexeme(30, "mixedCASEIdent"), + Lexeme(30, "number12345"), + Lexeme(30, "____"), + Lexeme(30, "_"), + ]); + eprintln!("Finished tokenising."); } }