diff --git a/lang/src/tokeniser.asm b/lang/src/tokeniser.asm index 46fb908..ea17760 100644 --- a/lang/src/tokeniser.asm +++ b/lang/src/tokeniser.asm @@ -323,6 +323,7 @@ is_ident: push r13 push r14 push rdi + mov rax, [rel cursor] mov r12, [rel buffer] mov r13, [rel buffer_len] @@ -393,7 +394,7 @@ is_number: mov dil, [r12 + r14] call is_whitespace test rax, rax - je .number + jne .number cmp dil, '_' je .loop_next call is_numeric @@ -572,4 +573,5 @@ find_lexeme: .is_number: mov rdi, [rsp + 8] mov qword [rdi], TOKEN_NUMBER + mov [rdi + 16], rax jmp .epilogue diff --git a/lang/tests/tokens.rs b/lang/tests/tokens.rs index 1cf89a8..724187f 100644 --- a/lang/tests/tokens.rs +++ b/lang/tests/tokens.rs @@ -158,6 +158,18 @@ fn main() { Lexeme(10, ""), Lexeme(23, ""), Lexeme(20, ""), + ]); + + eprint!("Initializing tokeniser.. "); + tokeniser_init(c"tests/tokens/number.l".as_ptr()); + eprintln!("ok."); + + assert_eq!(&collect_tokens()[..], &[ + Lexeme(31, "1234"), + Lexeme(31, "123_345_"), + Lexeme(31, "1234____56"), + Lexeme(31, "1"), + Lexeme(31, "0"), ]); eprintln!("Finished tokenising."); diff --git a/lang/tests/tokens/number.l b/lang/tests/tokens/number.l new file mode 100644 index 0000000..2a655ee --- /dev/null +++ b/lang/tests/tokens/number.l @@ -0,0 +1,5 @@ +1234 +123_345_ +1234____56 +1 +0