tests for alloc,vec,tokeniser

lang/libcompiler/src/lib.rs

@@ -21,6 +21,15 @@ pub struct FFISlice {
     pub len: usize,
 }
 
+impl Default for FFISlice {
+    fn default() -> Self {
+        Self {
+            ptr: core::ptr::dangling::<u8>(),
+            len: 0,
+        }
+    }
+}
+
 #[repr(transparent)]
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
 pub struct MaybeFFISlice {
@@ -49,6 +58,13 @@ impl FFISlice {
         unsafe { core::slice::from_raw_parts(self.ptr.cast(), self.len) }
     }
 
+    /// # Safety
+    /// The caller must ensure that the slice is valid for type T, and lasts for 'a.
+    pub unsafe fn as_slice_mut_unchecked<'a, T: Sized>(self) -> &'a mut [T] {
+        // SAFETY: The caller ensures that the FFISlice is valid for type T.
+        unsafe { core::slice::from_raw_parts_mut(self.ptr.cast_mut().cast(), self.len) }
+    }
+
     /// # Safety
     /// The caller ensures that the slice is valid byte slice.
     /// Namely, the pointer must be well-aligned and point to `len` bytes, and
@@ -73,8 +89,7 @@ pub mod vec {
     impl Default for BlobVec {
         fn default() -> Self {
             Self {
-                data: core::ptr::null_mut(),
+                slice: FFISlice::default(),
-                len: 0,
                 cap: 0,
                 elem_size: 0,
                 drop: None,
@@ -85,6 +100,8 @@ pub mod vec {
     unsafe impl Send for BlobVec {}
     unsafe impl Sync for BlobVec {}
 
+    use crate::FFISlice;
+
     use super::ffi::*;
 
     #[repr(transparent)]
@@ -115,13 +132,7 @@ pub mod vec {
         }
 
         pub fn new_with(capacity: usize) -> Self {
-            let mut vec = BlobVec {
+            let mut vec = BlobVec::default();
-                data: core::ptr::null_mut(),
-                len: 0,
-                cap: 0,
-                elem_size: 0,
-                drop: None,
-            };
 
             unsafe extern "C" fn drop_fn<T>(ptr: *mut ()) {
                 unsafe {
@@ -144,14 +155,23 @@ pub mod vec {
             }
         }
 
+        pub fn leak<'a>(self) -> &'a mut [T] {
+            assert_eq!(self.vec.elem_size, core::mem::size_of::<T>());
+            unsafe {
+                let slice = self.vec.slice.as_slice_mut_unchecked();
+                core::mem::forget(self);
+                slice
+            }
+        }
+
         pub fn as_slice(&self) -> &[T] {
             assert_eq!(self.vec.elem_size, core::mem::size_of::<T>());
-            unsafe { core::slice::from_raw_parts(self.vec.data as *const T, self.vec.len) }
+            unsafe { self.vec.slice.as_slice_unchecked() }
         }
 
         pub fn as_slice_mut(&mut self) -> &mut [T] {
             assert_eq!(self.vec.elem_size, core::mem::size_of::<T>());
-            unsafe { core::slice::from_raw_parts_mut(self.vec.data as *mut T, self.vec.len) }
+            unsafe { self.vec.slice.as_slice_mut_unchecked() }
         }
 
         pub fn extend(&mut self, elements: Box<[T]>) {
@@ -170,7 +190,7 @@ pub mod vec {
         }
 
         pub fn insert(&mut self, value: T, index: usize) {
-            if index > self.vec.len {
+            if index > self.vec.slice.len {
                 return;
             }
             let value = core::mem::ManuallyDrop::new(value);
@@ -197,11 +217,11 @@ pub mod vec {
         }
 
         pub fn pop(&mut self) -> Option<T> {
-            if self.vec.len == 0 {
+            if self.vec.slice.len == 0 {
                 return None;
             }
             unsafe {
-                let ptr = vec_get(&mut self.vec, self.vec.len - 1) as *mut T;
+                let ptr = vec_get(&mut self.vec, self.vec.slice.len - 1) as *mut T;
                 let value = ptr.read();
                 vec_pop(&mut self.vec);
                 Some(value)
@@ -209,7 +229,7 @@ pub mod vec {
         }
 
         pub fn get(&self, index: usize) -> Option<&T> {
-            if index >= self.vec.len {
+            if index >= self.vec.slice.len {
                 return None;
             }
             unsafe {
@@ -219,7 +239,7 @@ pub mod vec {
         }
 
         pub fn get_mut(&mut self, index: usize) -> Option<&mut T> {
-            if index >= self.vec.len {
+            if index >= self.vec.slice.len {
                 return None;
             }
             unsafe {
@@ -229,7 +249,7 @@ pub mod vec {
         }
 
         pub fn remove(&mut self, index: usize) {
-            if index >= self.vec.len {
+            if index >= self.vec.slice.len {
                 return;
             }
             unsafe {
@@ -238,11 +258,11 @@ pub mod vec {
         }
 
         pub fn len(&self) -> usize {
-            self.vec.len
+            self.vec.slice.len
         }
 
         pub fn is_empty(&self) -> bool {
-            self.vec.len == 0
+            self.vec.slice.len == 0
         }
 
         pub fn position<F>(&self, elem: &T, mut cmp: F) -> Option<usize>
@@ -536,7 +556,7 @@ mod display {
 
     impl core::fmt::Display for ffi::Type {
         fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-            use ffi::{
+            use crate::ffi::{
                 TYPE_BOOL, TYPE_F32, TYPE_F64, TYPE_I8, TYPE_I16, TYPE_I32, TYPE_I64, TYPE_ISIZE,
                 TYPE_POINTER, TYPE_STR, TYPE_U8, TYPE_U16, TYPE_U32, TYPE_U64, TYPE_USIZE,
                 TYPE_VOID,
@@ -621,6 +641,10 @@ mod display {
 
 pub use display::{Displayed, DisplayedSliceExt};
 
+#[expect(
+    clippy::derivable_impls,
+    reason = "Struct is defined in auto-generated file"
+)]
 impl Default for ffi::Ast {
     fn default() -> Self {
         ffi::Ast {

lang/libcompiler/src/tests/mod.rs

@@ -0,0 +1,775 @@
+#[inline(never)]
+fn __do_panic() -> ! {
+    panic!("Called panic from external code.");
+}
+
+#[unsafe(no_mangle)]
+extern "C" fn panic_impl() -> ! {
+    __do_panic()
+}
+
+mod alloc {
+    use crate::ffi::bump_alloc;
+
+    pub struct BumpAllocator;
+    unsafe impl std::alloc::Allocator for BumpAllocator {
+        fn allocate(
+            &self,
+            layout: std::alloc::Layout,
+        ) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
+            unsafe {
+                let ptr = bump_alloc(layout.size(), layout.align());
+                if ptr.is_null() {
+                    Err(std::alloc::AllocError)
+                } else {
+                    Ok(std::ptr::NonNull::slice_from_raw_parts(
+                        std::ptr::NonNull::new_unchecked(ptr),
+                        layout.size(),
+                    ))
+                }
+            }
+        }
+
+        unsafe fn deallocate(&self, _ptr: std::ptr::NonNull<u8>, _layout: std::alloc::Layout) {
+            // Bump allocator does not deallocate individual allocations
+        }
+    }
+
+    #[test]
+    fn box_neq() {
+        let a = Box::new_in(42u32, BumpAllocator);
+        let b = Box::new_in(42u32, BumpAllocator);
+        let c = Box::new_in(52u32, BumpAllocator);
+
+        eprintln!("a: {a}, b: {b}, c: {c}");
+
+        assert_ne!(Box::as_ptr(&a), Box::as_ptr(&b));
+        assert_eq!(*a, *b);
+    }
+
+    #[test]
+    fn box_big() {
+        struct BigType {
+            data: [u8; 0x1010],
+        }
+
+        let a = Box::new_in(42u32, BumpAllocator);
+        let mut big = Box::new_in(BigType { data: [0; 0x1010] }, BumpAllocator);
+        assert_ne!(Box::as_ptr(&big) as *const (), Box::as_ptr(&a) as *const ());
+        big.data[47] = 123;
+        assert_eq!(big.data[47], 123);
+    }
+
+    #[test]
+    fn align() {
+        #[repr(align(256))]
+        struct AlignedType {
+            #[allow(dead_code)]
+            data: [u8; 512],
+        }
+        let aligned = Box::new_in(AlignedType { data: [0; 512] }, BumpAllocator);
+        assert_eq!(
+            (Box::as_ptr(&aligned) as usize) % 256,
+            0,
+            "Aligned allocation should be aligned to 256 bytes"
+        );
+    }
+}
+
+mod int2str {
+    use crate::ffi::int_to_str2;
+    use crate::ffi::str_to_int;
+
+    #[test]
+    fn to_str() {
+        let value = 1234567890i64;
+        let mut buffer = [0u8; 32];
+        unsafe {
+            let slice = int_to_str2(0, buffer.as_mut_ptr(), buffer.len(), 10);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {}, String: {}", 0, s);
+            assert_eq!(s, format!("{}", 0));
+
+            let slice = int_to_str2(value, buffer.as_mut_ptr(), buffer.len(), 10);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {}, String: {}", value, s);
+            assert_eq!(s, format!("{value}"));
+
+            let slice = int_to_str2(0 - value, buffer.as_mut_ptr(), buffer.len(), 10);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {}, String: {}", 0 - value, s);
+            assert_eq!(s, format!("{}", 0 - value));
+
+            let slice = int_to_str2(value, buffer.as_mut_ptr(), buffer.len(), 16);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {:x}, String: {}", value, s);
+            assert_eq!(s, format!("{value:x}"));
+
+            let slice = int_to_str2(value, buffer.as_mut_ptr(), buffer.len(), 8);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {:o}, String: {}", value, s);
+            assert_eq!(s, format!("{value:o}"));
+
+            let value = 235i64;
+            let slice = int_to_str2(value, buffer.as_mut_ptr(), buffer.len(), 2);
+            let s = slice.as_str_unchecked();
+            println!("Integer: {:b}, String: {}", value, s);
+            assert_eq!(s, format!("{value:b}"));
+        }
+    }
+
+    #[test]
+    fn from_str() {
+        unsafe {
+            let s = "1234567890";
+            let parsed = str_to_int(s.as_ptr(), s.len(), 10);
+            println!("String: {}, Integer: {}", s, parsed);
+            assert_eq!(parsed, 1234567890i64);
+
+            let s = "499602d2";
+            let parsed = str_to_int(s.as_ptr(), s.len(), 16);
+            println!("String: {}, Integer: {}", s, parsed);
+            assert_eq!(parsed, 1234567890i64);
+
+            let s = "11145401322";
+            let parsed = str_to_int(s.as_ptr(), s.len(), 8);
+            println!("String: {}, Integer: {}", s, parsed);
+            assert_eq!(parsed, 1234567890i64);
+            let s = "11101011";
+            let parsed = str_to_int(s.as_ptr(), s.len(), 2);
+            println!("String: {}, Integer: {}", s, parsed);
+            assert_eq!(parsed, 235i64);
+
+            let s = "9999999999999999999999999999999999999999";
+            let parsed = str_to_int(s.as_ptr(), s.len(), 10);
+            println!("String: {}, Integer: {}", s, parsed);
+            assert_eq!(parsed, i64::MAX);
+        }
+    }
+}
+
+mod vec {
+    use crate::{
+        ffi::{BlobVec, vec_drop_last, vec_get, vec_init, vec_push},
+        vec::Vec,
+    };
+
+    static mut DROPS: usize = 1;
+    fn get_drops() -> usize {
+        unsafe { (&raw const DROPS).read() }
+    }
+    unsafe extern "C" fn update_drops(f: impl FnOnce(&mut usize)) {
+        unsafe {
+            let drops = &raw mut DROPS;
+            f(&mut *drops);
+        }
+    }
+
+    extern "C" fn drop_element(ptr: *mut ()) {
+        unsafe {
+            update_drops(|drops| {
+                *drops *= ptr.cast::<u32>().read() as usize;
+            });
+        }
+    }
+
+    fn as_slice<T>(vec: &BlobVec) -> &[T] {
+        assert_eq!(vec.elem_size, core::mem::size_of::<T>());
+        unsafe { vec.slice.as_slice_unchecked() }
+    }
+
+    #[test]
+    fn push_pop() {
+        let mut vec = BlobVec::default();
+        unsafe {
+            vec_init(&mut vec, 4, Some(drop_element));
+            assert_eq!(vec.slice.len, 0);
+            let mut value = 2;
+            vec_push(&mut vec, &raw const value as _);
+            assert_eq!(vec.slice.len, 1);
+            assert_eq!(as_slice::<u32>(&vec), &[2]);
+            let retrieved = *(vec_get(&mut vec, 0) as *mut u32);
+            assert_eq!(retrieved, 2);
+            assert_eq!(get_drops(), 1);
+            vec_drop_last(&mut vec);
+            assert_eq!(vec.slice.len, 0);
+            assert_eq!(get_drops(), 2);
+            value = 3;
+            vec_push(&mut vec, &raw const value as _);
+            assert_eq!(as_slice::<u32>(&vec), &[3]);
+            value = 5;
+            vec_push(&mut vec, &raw const value as _);
+            assert_eq!(as_slice::<u32>(&vec), &[3, 5]);
+            assert_eq!(vec.slice.len, 2);
+            vec_drop_last(&mut vec);
+            vec_drop_last(&mut vec);
+            assert_eq!(get_drops(), 2 * 3 * 5);
+        }
+    }
+
+    #[test]
+    fn vec_impl() {
+        let mut vec = Vec::<u32>::new_with(100);
+        assert_eq!(vec.len(), 0);
+        vec.push(10);
+        vec.push(20);
+        vec.push(30);
+        assert_eq!(vec.len(), 3);
+        assert_eq!(vec.get(0), Some(&10));
+        assert_eq!(vec.get(1), Some(&20));
+        assert_eq!(vec.get(2), Some(&30));
+        assert_eq!(vec.pop(), Some(30));
+        assert_eq!(vec.len(), 2);
+        vec.remove(0);
+        assert_eq!(vec.len(), 1);
+        assert_eq!(vec.get(0), Some(&20));
+        vec.push(40);
+        vec.push(50);
+    }
+
+    #[test]
+    fn vec_extend() {
+        let mut vec = Vec::<u32>::new_with(100);
+        vec.extend(Box::new([10, 20, 30, 40, 50]));
+        assert_eq!(vec.len(), 5);
+        assert_eq!(vec.get(0), Some(&10));
+        assert_eq!(vec.as_slice(), &[10, 20, 30, 40, 50]);
+    }
+
+    #[test]
+    fn vec_insert() {
+        let mut vec = Vec::<u32>::new_with(100);
+        vec.extend(Box::new([10, 20, 40, 50]));
+        vec.insert(30, 2);
+        assert_eq!(vec.as_slice(), &[10, 20, 30, 40, 50]);
+    }
+
+    #[test]
+    fn vec_binary_search() {
+        let mut vec = Vec::<u32>::new_with(100);
+        vec.extend(Box::new([20, 30, 40, 50]));
+
+        let cmp = |a: &u32, b: &u32| match a.cmp(b) {
+            core::cmp::Ordering::Less => -1,
+            core::cmp::Ordering::Equal => 0,
+            core::cmp::Ordering::Greater => 1,
+        };
+        assert_eq!(vec.binary_search_by(&35, cmp), Err(2));
+        assert_eq!(vec.binary_search_by(&25, cmp), Err(1));
+        assert_eq!(vec.binary_search_by(&30, cmp), Ok(1));
+        assert_eq!(vec.binary_search_by(&5, cmp), Err(0));
+        assert_eq!(vec.binary_search_by(&55, cmp), Err(4));
+
+        _ = vec.insert_sorted(35, cmp);
+        assert_eq!(vec.as_slice(), &[20, 30, 35, 40, 50]);
+    }
+
+    #[test]
+    fn vec_binary_serach_empty() {
+        let cmp = |a: &u32, b: &u32| match a.cmp(b) {
+            core::cmp::Ordering::Less => -1,
+            core::cmp::Ordering::Equal => 0,
+            core::cmp::Ordering::Greater => 1,
+        };
+
+        let mut vec = Vec::<u32>::new_with(100);
+
+        assert_eq!(vec.len(), 0);
+        _ = vec.insert_sorted(5, cmp);
+        assert_eq!(vec.len(), 1);
+        assert_eq!(vec.as_slice(), &[5]);
+
+        _ = vec.insert_sorted(2, cmp);
+        assert_eq!(vec.len(), 2);
+        assert_eq!(vec.as_slice(), &[2, 5]);
+
+        _ = vec.insert_sorted(7, cmp);
+        assert_eq!(vec.len(), 3);
+        assert_eq!(vec.as_slice(), &[2, 5, 7]);
+    }
+
+    #[test]
+    fn vec_insert_many() {
+        let mut vec = Vec::<u32>::new_with(100);
+        vec.extend(Box::new([10, 20, 30, 40, 50]));
+        assert_eq!(vec.as_slice(), &[10, 20, 30, 40, 50]);
+        vec.insert_many(2, Box::new([22, 23, 24]));
+        assert_eq!(vec.as_slice(), &[10, 20, 22, 23, 24, 30, 40, 50]);
+    }
+
+    #[test]
+    fn vec_position() {
+        let mut vec = Vec::<u32>::new_with(100);
+        vec.extend(Box::new([10, 20, 30, 40, 50]));
+
+        assert_eq!(
+            vec.position(&40, |a, b| {
+                eprintln!("Comparing {} and {}", a, b);
+                a == b
+            }),
+            Some(3)
+        );
+    }
+}
+
+mod tokens {
+    use crate::ffi::{RawLexeme, find_lexeme, tokeniser_init_buf};
+
+    fn collect_tokens() -> Vec<Lexeme> {
+        let mut lexemes = Vec::new();
+        unsafe {
+            while let Some(lexeme) = find_lexeme().into_lexeme() {
+                lexemes.push(lexeme);
+            }
+        }
+
+        lexemes
+    }
+
+    #[derive(Debug)]
+    struct Lexeme(u8, &'static str);
+
+    impl PartialEq for Lexeme {
+        fn eq(&self, other: &Self) -> bool {
+            use crate::ffi::{TOKEN_IDENT, TOKEN_NUMBER};
+            match self.0 {
+                // Identifiers and numbers compare both token and lexeme
+                TOKEN_IDENT | TOKEN_NUMBER => self.0 == other.0 && self.1 == other.1,
+                _ => self.0 == other.0,
+            }
+        }
+    }
+
+    impl Eq for Lexeme {}
+
+    trait AsLexeme {
+        fn into_lexeme(self) -> Option<Lexeme>;
+    }
+
+    impl AsLexeme for RawLexeme {
+        fn into_lexeme(self) -> Option<Lexeme> {
+            let Self { token, slice } = self;
+            let slice = unsafe { slice.as_str_unchecked() };
+            match token {
+                1.. => Some(Lexeme(token, slice)),
+                _ => None,
+            }
+        }
+    }
+
+    fn init_tokeniser(s: &str) {
+        unsafe {
+            tokeniser_init_buf(s.as_ptr(), s.len());
+        }
+    }
+
+    macro_rules! token {
+        ($token:ident) => {
+            Lexeme(crate::ffi::$token, "")
+        };
+        ({$token:ident: $lexeme:expr}) => {
+            Lexeme(crate::ffi::$token, $lexeme)
+        };
+    }
+
+    macro_rules! tokens {
+        [$( $token:tt ),* $(,)?] => {
+            [
+                $( token!($token) ),*
+            ]
+        };
+    }
+
+    #[test]
+    fn keywords() {
+        init_tokeniser(
+            r#"
+fn let if else fn continue loop break return while for match switch as
+i32      bool false true void
+        return usize isize f32 f64
+i8 u8 i16 u16 i32 u32 i64 u64
+"#,
+        );
+
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                TOKEN_FN,
+                TOKEN_LET,
+                TOKEN_IF,
+                TOKEN_ELSE,
+                TOKEN_FN,
+                TOKEN_CONTINUE,
+                TOKEN_LOOP,
+                TOKEN_BREAK,
+                TOKEN_RETURN,
+                TOKEN_WHILE,
+                TOKEN_FOR,
+                TOKEN_MATCH,
+                TOKEN_SWITCH,
+                TOKEN_AS,
+                TOKEN_I32,
+                TOKEN_BOOL,
+                TOKEN_FALSE,
+                TOKEN_TRUE,
+                TOKEN_VOID,
+                TOKEN_RETURN,
+                TOKEN_USIZE,
+                TOKEN_ISIZE,
+                TOKEN_F32,
+                TOKEN_F64,
+                TOKEN_I8,
+                TOKEN_U8,
+                TOKEN_I16,
+                TOKEN_U16,
+                TOKEN_I32,
+                TOKEN_U32,
+                TOKEN_I64,
+                TOKEN_U64,
+            ]
+        );
+    }
+
+    #[test]
+    fn delimiters() {
+        init_tokeniser("()[]{},->;:=");
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                TOKEN_LPARENS,
+                TOKEN_RPARENS,
+                TOKEN_LBRACKET,
+                TOKEN_RBRACKET,
+                TOKEN_LBRACE,
+                TOKEN_RBRACE,
+                TOKEN_COMMA,
+                TOKEN_ARROW,
+                TOKEN_SEMI,
+                TOKEN_COLON,
+                TOKEN_EQUALS,
+            ]
+        );
+    }
+
+    #[test]
+    fn identifiers() {
+        init_tokeniser(
+            r#"
+this-is-an-ident
+another_ident123
+_underscore_test
+mixedCASEIdent
+number12345
+____
+_
+-leading-minus
+trailing-minus-
+"#,
+        );
+
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                {TOKEN_IDENT: "this-is-an-ident"},
+                {TOKEN_IDENT: "another_ident123"},
+                {TOKEN_IDENT: "_underscore_test"},
+                {TOKEN_IDENT: "mixedCASEIdent"},
+                {TOKEN_IDENT: "number12345"},
+                {TOKEN_IDENT: "____"},
+                {TOKEN_IDENT: "_"},
+                TOKEN_MINUS,
+                {TOKEN_IDENT: "leading-minus"},
+                {TOKEN_IDENT: "trailing-minus-"},
+
+            ],
+        );
+    }
+
+    #[test]
+    fn simple_function() {
+        init_tokeniser(
+            "fn my-function() -> bool {
+  return false;
+}",
+        );
+
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                TOKEN_FN,
+                {TOKEN_IDENT: "my-function"},
+                TOKEN_LPARENS,
+                TOKEN_RPARENS,
+                TOKEN_ARROW,
+                TOKEN_BOOL,
+                TOKEN_LBRACE,
+                TOKEN_RETURN,
+                TOKEN_FALSE,
+                TOKEN_SEMI,
+                TOKEN_RBRACE,
+            ],
+        );
+    }
+
+    #[test]
+    fn simple_function_commented() {
+        init_tokeniser(
+            "// This is a comment line
+fn my-function() -> bool {
+  // This function always returns false
+  return false;
+}
+",
+        );
+
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                TOKEN_COMMENT,
+                TOKEN_FN,
+                {TOKEN_IDENT: "my-function"},
+                TOKEN_LPARENS,
+                TOKEN_RPARENS,
+                TOKEN_ARROW,
+                TOKEN_BOOL,
+                TOKEN_LBRACE,
+                TOKEN_COMMENT,
+                TOKEN_RETURN,
+                TOKEN_FALSE,
+                TOKEN_SEMI,
+                TOKEN_RBRACE,
+            ],
+        );
+    }
+
+    #[test]
+    fn numbers() {
+        init_tokeniser(
+            "
+1234
+123_345_
+1234____56
+1
+0",
+        );
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                {TOKEN_NUMBER: "1234"},
+                {TOKEN_NUMBER: "123_345_"},
+                {TOKEN_NUMBER: "1234____56"},
+                {TOKEN_NUMBER: "1"},
+                {TOKEN_NUMBER: "0"},
+            ],
+        );
+    }
+
+    #[test]
+    fn strings() {
+        init_tokeniser(
+            r#"
+
+"this is a string"
+"another
+string
+spanning multiple
+   lines"
+"string with a \"quoted\" word"
+"a"
+""
+"#,
+        );
+
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                {TOKEN_STRING: "this is a string"},
+                {TOKEN_STRING: "another\nstring\nspanning multiple\n   lines"},
+                {TOKEN_STRING: "string with a \\\"quoted\\\" word"},
+                {TOKEN_STRING: "a"},
+                {TOKEN_STRING: ""},
+            ]
+        );
+    }
+
+    #[test]
+    fn complex_tokens() {
+        init_tokeniser("<<<=<a == b = c ||| &||&&|&");
+        assert_eq!(
+            collect_tokens().as_slice(),
+            &tokens![
+                TOKEN_LESSLESS,
+                TOKEN_LEQ,
+                TOKEN_LT,
+                {TOKEN_IDENT: "a"},
+                TOKEN_EQEQ,
+                {TOKEN_IDENT: "b"},
+                TOKEN_EQUALS,
+                {TOKEN_IDENT: "c"},
+                TOKEN_PIPE2,
+                TOKEN_PIPE,
+                TOKEN_AMP,
+                TOKEN_PIPE2,
+                TOKEN_AMP2,
+                TOKEN_PIPE,
+                TOKEN_AMP,
+            ]
+        );
+    }
+}
+
+mod ast {
+    use crate::ffi::Ast;
+
+    macro_rules! ast_node {
+        (@expr $ast:expr, $($expr:tt)*) => {};
+        (@expr $ast:expr, Num($expr:tt)) => {
+            {
+                let num_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_NUMBER,
+                    data: $expr as _,
+                    extra: 0,
+                    span: 0,
+                });
+                num_id
+            }
+        };
+        (@expr $ast:expr, $($a:tt)* + $($b:tt)*) => {
+            {
+                let left_id = ast_node!(@expr $ast, $($a)*);
+                let right_id = ast_node!(@expr $ast, $($b)*);
+                let binop = Box::new_in(
+                    crate::ffi::AstBinaryOp {
+                        op: crate::ffi::TOKEN_PLUS,
+                        left: left_id as u64,
+                        right: right_id as u64,
+                    },
+                    super::alloc::BumpAllocator,
+                );
+                let binop_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_BINARY_OP,
+                    data: Box::into_raw(binop.into()) as _,
+                    extra: 0,
+                    span: 0,
+                });
+                binop_id
+            }
+        };
+        (@stmt $ast:expr, $($expr:tt)*) => {};
+        // (@stmt $ast:expr, let $name:ident: $ty:tt = $($expr:tt)*) => {};
+        (@stmt $ast:expr, return $($expr:tt)*) => {
+            {
+                let expr_id = ast_node!(@expr $ast, $expr);
+                let return_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_RETURN,
+                    data: expr_id as _,
+                    extra: 0,
+                    span: 0,
+                });
+                return_id
+            }
+        };
+        ($ast:expr, Fn{name: $name:expr, args: [$($arg_name:expr => $arg:tt),*], ret: $ret:tt, body: $body:tt}) => {
+            {
+                #[allow(unused_mut)]
+                let mut args = Vec::<usize>::with_capacity(128);
+                $(
+                    args.push(ast_node!($ast, Arg{$arg_name => $arg}));
+                )*
+                    let args = args.leak();
+                let func = Box::new_in(
+                    crate::ffi::AstFunction {
+                        name: $name.as_ptr(),
+                        name_len: $name.len(),
+                        args: args.as_ptr().cast(),
+                        args_len: args.len(),
+                        return_type: crate::ffi::Type $ret,
+                        body: ast_node!($ast, Block $body) as u64,
+                    },
+                    super::alloc::BumpAllocator,
+                );
+
+                let func_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_FUNCTION,
+                    data: Box::into_raw(func.into()) as _,
+                    extra: 0,
+                    span: 0,
+                });
+
+                func_id
+            }
+        };
+        ($ast:expr, Block[$(($($stmt:tt)*)),* $(,)?]) => {
+            {
+            #[allow(unused_mut)]
+                let mut stmts = Vec::with_capacity_in(128, super::alloc::BumpAllocator);
+                $(
+                    stmts.push(ast_node!($ast, $($stmt)*));
+                )*
+                let stmts = stmts.leak();
+                let block_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_BLOCK,
+                    data: stmts.as_ptr() as _,
+                    extra: stmts.len(),
+                    span: 0,
+                });
+                block_id
+            }
+        };
+        ($ast:expr, Arg{$name:expr => $ty:tt}) => {
+            {
+                let arg = Box::new_in(
+                    crate::ffi::AstArgument {
+                        name: $name.as_ptr(),
+                        name_len: $name.len(),
+                        arg_type: crate::ffi::Type $ty,
+                    },
+                    super::alloc::BumpAllocator,
+                );
+                let arg_id = $ast.nodes.len();
+                $ast.nodes.push(crate::ffi::AstNode {
+                    kind: crate::ffi::AST_ARG,
+                    data: Box::into_raw(arg.into()) as _,
+                    extra: 0,
+                    span: 0,
+                });
+
+                arg_id
+            }
+        };
+        ($ast:expr, File[$($tag:ident $then:tt),* $(,)?]) => {
+            let mut gdecls = Vec::with_capacity_in(128, super::alloc::BumpAllocator);
+            $(
+                gdecls.push(ast_node!($ast, $tag $then));
+            )*
+        };
+    }
+
+    macro_rules! AST {
+        ($($ast:tt)*) => {{
+            let mut _ast = Ast::default();
+            ast_node!(_ast, $($ast)*);
+            _ast
+        }};
+    }
+
+    fn asdf() {
+        AST!(
+            File[
+                Fn {
+                    name: "main",
+                    args: ["a" => {kind: crate::ffi::TYPE_I32, data: 0}],
+                    ret: { kind: 1, data: 0 },
+                    body: []
+                },
+            ]
+        );
+    }
+
+    fn parse_and_print() {}
+}

lang/src/vec.asm

@@ -35,8 +35,7 @@ global vec_tests
 ;; Byte vector structure
 ;; start-structs
 ;; struct BlobVec {
-;; data: *mut (),
+;; slice: FFISlice,
-;; len: usize,
 ;; cap: usize,
 ;; elem_size: usize,
 ;; drop: Option<unsafe extern "C" fn(*mut ())>,