From 26c37e9dd3aa00af5f3b4fb9b6be7b767ad2d965 Mon Sep 17 00:00:00 2001
From: janis <janis@nirgendwo.xyz>
Date: Tue, 28 Oct 2025 16:48:13 +0100
Subject: [PATCH] test identifiers

---
 lang/tests/tokens.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/lang/tests/tokens.rs b/lang/tests/tokens.rs
index 57546d0..6a6310f 100644
--- a/lang/tests/tokens.rs
+++ b/lang/tests/tokens.rs
@@ -126,6 +126,20 @@ fn main() {
             Lexeme(15, ""),
         ][..]);
 
+        eprint!("Initializing tokeniser.. ");
+        tokeniser_init(c"tests/tokens/identifier.l".as_ptr());
+        eprintln!("ok.");
+
+        assert_eq!(&collect_tokens()[..], &[
+            Lexeme(30, "this-is-an-ident"),
+            Lexeme(30, "another_ident123"),
+            Lexeme(30, "_underscore_test"),
+            Lexeme(30, "mixedCASEIdent"),
+            Lexeme(30, "number12345"),
+            Lexeme(30, "____"),
+            Lexeme(30, "_"),
+        ]);
+
         eprintln!("Finished tokenising.");
     }
 }