diff --git a/lang/tests/tokens.rs b/lang/tests/tokens.rs index 6a6310f..1cf89a8 100644 --- a/lang/tests/tokens.rs +++ b/lang/tests/tokens.rs @@ -138,8 +138,28 @@ fn main() { Lexeme(30, "number12345"), Lexeme(30, "____"), Lexeme(30, "_"), + Lexeme(17, ""), Lexeme(30, "leading-minus"), + Lexeme(30, "trailing-minus-"), ]); + eprint!("Initializing tokeniser.. "); + tokeniser_init(c"tests/tokens/function.l".as_ptr()); + eprintln!("ok."); + + assert_eq!(&collect_tokens()[..], &[ + Lexeme(4, ""), + Lexeme(30, "my-function"), + Lexeme(19, ""), + Lexeme(18, ""), + Lexeme(12, ""), + Lexeme(11, ""), + Lexeme(21, ""), + Lexeme(5, ""), + Lexeme(10, ""), + Lexeme(23, ""), + Lexeme(20, ""), + ]); + eprintln!("Finished tokenising."); } } diff --git a/lang/tests/tokens/identifier.l b/lang/tests/tokens/identifier.l index 59e27a2..9682057 100644 --- a/lang/tests/tokens/identifier.l +++ b/lang/tests/tokens/identifier.l @@ -5,3 +5,5 @@ mixedCASEIdent number12345 ____ _ +-leading-minus +trailing-minus-