fix tokeniser for: void

lang/src/tokeniser.asm

@@ -318,14 +318,11 @@ is_number:
     cmp r14, r13
     jge .number
     mov dil, [r12 + r14]
-    call is_whitespace
-    test rax, rax
-    jne .number
     cmp dil, '_'
     je .loop_next
     call is_numeric
     test rax, rax
-    je .not_number
+    je .number
 .loop_next:
     inc r14
     jmp .loop

lang/src/tokeniser.inc

@@ -102,7 +102,7 @@ LEXEME_LENS:
     dq LEX_VOID_len
 
 align 8
-NUM_LEXEMES: dq 30
+NUM_LEXEMES: dq 31
 
     LEX_NOT_A_LEXEME db "<not a lexeme>", 0
     LEX_LET db "let"

lang/tests/ast.rs

@@ -7,7 +7,7 @@ unsafe extern "C" {
     unsafe fn tokeniser_init_buf(bytes: *const u8, len: usize) -> ();
 }
 
-use util::defs::{parse_expr, Ast, AstNode};
+use util::defs::{Ast, AstNode, parse_expr, parse_func};
 
 fn main() {
     unsafe {
@@ -18,19 +18,28 @@ fn main() {
     let src = b"3 + 4";
 
     unsafe {
+        // tokeniser_init_buf(src.as_ptr(), src.len());
+        // let mut ast = Ast {
+        //     nodes: util::vec::Vec::new(),
+        // };
+        // let expr_id = parse_expr(&mut ast);
+        // println!("Parsed expression with ID: {}", expr_id);
+        // println!("{:#}", &ast);
+
+        let src = b"fn main() -> void { return 1 + 2; }";
         tokeniser_init_buf(src.as_ptr(), src.len());
         let mut ast = Ast {
             nodes: util::vec::Vec::new(),
         };
-        let expr_id = parse_expr(&mut ast);
+        let expr_id = parse_func(&mut ast);
-        println!("Parsed expression with ID: {}", expr_id);
+        println!("Parsed function with ID: {}", expr_id);
         println!("{:#}", &ast);
     }
 }
 
 impl std::fmt::Display for AstNode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        use util::defs::{BinaryExpr, AST_BINARY_OP, AST_NUMBER};
+        use util::defs::{AST_BINARY_OP, AST_NUMBER, BinaryExpr};
         match self.kind as u32 {
             AST_NUMBER => {
                 write!(f, "Number({})", self.data as usize)

lang/tests/shared/shared.rs

@@ -253,11 +253,7 @@ pub mod vec {
                     cmp_trampoline::<T, F>,
                     &raw mut cmp as *mut F as *mut (),
                 );
-                if vacant {
+                if vacant { Err(index) } else { Ok(index) }
-                    Err(index)
-                } else {
-                    Ok(index)
-                }
             }
         }
 

lang/tests/tokens.rs

@@ -239,6 +239,30 @@ fn main() {
             &[Lexeme(32, "3"), Lexeme(16, "+"), Lexeme(32, "4")],
         );
 
+        eprint!("Initializing tokeniser.. ");
+        let src = b"fn main() -> void { return 1 + 2; }";
+        tokeniser_init_buf(src.as_ptr(), src.len());
+        eprintln!("ok.");
+
+        assert_eq!(
+            &collect_tokens()[..],
+            &[
+                Lexeme(4, "fn"),
+                Lexeme(31, "main"),
+                Lexeme(19, "("),
+                Lexeme(18, ")"),
+                Lexeme(12, "->"),
+                Lexeme(30, "void"),
+                Lexeme(21, "{"),
+                Lexeme(5, "return"),
+                Lexeme(32, "1"),
+                Lexeme(16, "+"),
+                Lexeme(32, "2"),
+                Lexeme(23, ";"),
+                Lexeme(20, "}"),
+            ],
+        );
+
         eprintln!("Finished tokenising.");
     }
 }

lang/tests/vec.rs

@@ -1,7 +1,7 @@
 #[path = "shared/shared.rs"]
 mod util;
 
-use util::{ffi::*, vec::Vec, BlobVec};
+use util::{BlobVec, ffi::*, vec::Vec};
 
 fn main() {
     static mut DROPS: usize = 1;