idk man..

.gitignore

@@ -1,2 +1,3 @@
 /target
 /Cargo.lock
+/.direnv/

Cargo.toml

@@ -1,3 +1,9 @@
+[workspace]
+resolver = "3"
+members = [
+    "crates/lexer"
+]
+
 [package]
 name = "compiler"
 version = "0.1.0"
@@ -16,5 +22,16 @@ paste = "1.0.15"
 petgraph = "0.6.5"
 thiserror = "1.0.63"
 unicode-xid = "0.2.4"
+tracing = "0.1.41"
 
-werkzeug = { path = "../../rust/werkzeug" }
+werkzeug = { path = "../../rust/werkzeug" }
+
+
+[workspace.dependencies]
+unicode-xid = "0.2.4"
+tracing = "0.1.41"
+thiserror = "1.0.63"
+itertools = "0.13.0"
+
+werkzeug = { path = "../../rust/werkzeug" }
+trie = { path = "../../rust/trie" }

crates/lexer/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "lexer"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+tracing = { workspace = true }
+werkzeug = { workspace = true }
+thiserror = { workspace = true }
+itertools = { workspace = true }
+trie = { workspace = true }
+unicode-xid = { workspace = true }

crates/lexer/src/complex_tokens.rs

@@ -0,0 +1,394 @@
+use crate::{Source, Token, is_things};
+use itertools::Itertools;
+use werkzeug::iter::{FallibleMapIter, NextIf};
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum Error {
+    #[error("{0}")]
+    StringError(String),
+    #[error("Exp part of floating constant had no digits.")]
+    FloatingConstantExpPartNoDigit,
+    #[error("constant cannot start with leading underscore '_'.")]
+    NumericalConstantDigitLeadingUnderscore,
+    #[error("Expected digit here for constant.")]
+    NumericalConstantDigitNoDigit,
+    #[error("Expected digit here for integer constant.")]
+    IntegralTypeExpectedDigit,
+    #[error("Floating constant has invalid trailing type.")]
+    FloatingConstantInvalidTrailingType,
+    #[error("Invalid token.")]
+    InvalidToken,
+    #[error("Identifier starts with invalid character.")]
+    ExpectedIdStartForIdentifier,
+    #[error("Unknown suffix in constant.")]
+    NumericalConstantUnknownSuffix,
+}
+
+type Result<T> = core::result::Result<T, Error>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Radix {
+    Hex,
+    Bin,
+    Dec,
+    Oct,
+}
+
+impl Radix {
+    #[allow(unused)]
+    /// must be called with one of `['b','x','d','o']`
+    unsafe fn from_char_unchecked(c: char) -> Self {
+        match c.to_ascii_lowercase() {
+            'o' => Self::Oct,
+            'b' => Self::Bin,
+            'x' => Self::Hex,
+            'd' => Self::Dec,
+            _ => unreachable!(),
+        }
+    }
+    fn from_char(c: char) -> Option<Self> {
+        match c.to_ascii_lowercase() {
+            'o' => Some(Self::Oct),
+            'b' => Some(Self::Bin),
+            'x' => Some(Self::Hex),
+            'd' => Some(Self::Dec),
+            _ => None,
+        }
+    }
+
+    #[allow(unused)]
+    pub fn radix(self) -> u8 {
+        match self {
+            Radix::Hex => 16,
+            Radix::Bin => 2,
+            Radix::Oct => 8,
+            Radix::Dec => 10,
+        }
+    }
+    fn to_token(self) -> Token {
+        match self {
+            Radix::Hex => Token::IntegerHexConstant,
+            Radix::Bin => Token::IntegerBinConstant,
+            Radix::Oct => Token::IntegerOctConstant,
+            Radix::Dec => Token::IntegerConstant,
+        }
+    }
+    pub fn from_token(token: Token) -> Option<Self> {
+        match token {
+            Token::IntegerHexConstant => Some(Radix::Hex),
+            Token::IntegerBinConstant => Some(Radix::Bin),
+            Token::IntegerOctConstant => Some(Radix::Oct),
+            Token::IntegerConstant => Some(Radix::Dec),
+            _ => None,
+        }
+    }
+    pub fn map_digit(self, c: char) -> u8 {
+        match self {
+            Radix::Hex => match c {
+                '0'..='9' => c as u8 - b'0',
+                'a'..='f' => 10 + c as u8 - b'a',
+                'A'..='F' => 10 + c as u8 - b'A',
+                _ => unreachable!(),
+            },
+            Radix::Bin => match c {
+                '0'..='1' => c as u8 - b'0',
+                _ => unreachable!(),
+            },
+            Radix::Dec => match c {
+                '0'..='9' => c as u8 - b'0',
+                _ => unreachable!(),
+            },
+            Radix::Oct => match c {
+                '0'..='7' => c as u8 - b'0',
+                _ => unreachable!(),
+            },
+        }
+    }
+    pub fn folding_method(self) -> fn(u64, char) -> u64 {
+        match self {
+            Radix::Hex => {
+                fn fold(acc: u64, c: char) -> u64 {
+                    let digit = match c {
+                        '0'..='9' => c as u8 - b'0',
+                        'a'..='f' => c as u8 - b'a',
+                        'A'..='F' => c as u8 - b'A',
+                        _ => unreachable!(),
+                    };
+                    acc * 16 + digit as u64
+                }
+                fold
+            }
+            Radix::Bin => {
+                fn fold(acc: u64, c: char) -> u64 {
+                    let digit = match c {
+                        '0'..='1' => c as u8 - b'0',
+                        _ => unreachable!(),
+                    };
+                    acc * 2 + digit as u64
+                }
+                fold
+            }
+            Radix::Dec => {
+                fn fold(acc: u64, c: char) -> u64 {
+                    let digit = match c {
+                        '0'..='9' => c as u8 - b'0',
+                        _ => unreachable!(),
+                    };
+                    acc * 10 + digit as u64
+                }
+                fold
+            }
+            Radix::Oct => {
+                fn fold(acc: u64, c: char) -> u64 {
+                    let digit = match c {
+                        '0'..='7' => c as u8 - b'0',
+                        _ => unreachable!(),
+                    };
+                    acc * 8 + digit as u64
+                }
+                fold
+            }
+        }
+    }
+    pub fn is_digit(self) -> fn(char) -> bool {
+        match self {
+            Radix::Hex => crate::is_things::is_hex_digit,
+            Radix::Bin => crate::is_things::is_bin_digit,
+            Radix::Oct => crate::is_things::is_oct_digit,
+            Radix::Dec => crate::is_things::is_digit,
+        }
+    }
+}
+
+// where DIGIT is defined by radix:
+// DIGITS <-
+//     if allow_leading_underscore: `_`* DIGIT (DIGIT|`_`)*
+//     else: DIGIT (DIGIT|`_`)*
+fn parse_digit_part(
+    source: &mut Source,
+    allow_leading_underscore: bool,
+    radix: Radix,
+) -> Result<()> {
+    let is_digit = radix.is_digit();
+
+    if allow_leading_underscore {
+        let _underscore = source.take_while_ref(|&c| c == '_').count();
+    }
+    let _need_digit = source.next_if(|&c| is_digit(c)).ok_or_else(|| {
+        if source.peek() == Some(&'_') {
+            Error::NumericalConstantDigitLeadingUnderscore
+        } else {
+            Error::NumericalConstantDigitNoDigit
+        }
+    })?;
+    let _rest = source.take_while_ref(|&c| is_digit(c) || c == '_').count();
+
+    Ok(())
+}
+
+// IntegralType <-
+//     ( 'u' | 'i' ) DIGITS+
+fn try_parse_integral_type(source: &mut Source) -> Result<Option<()>> {
+    if !source.next_if(|&c| c == 'u' || c == 'i').is_some() {
+        return Ok(None);
+    }
+
+    if source
+        .take_while_ref(|&c| crate::is_things::is_digit(c))
+        .count()
+        <= 0
+    {
+        return Err(Error::IntegralTypeExpectedDigit);
+    };
+
+    Ok(Some(()))
+}
+
+// returns `Err(E)` if it failed to parse.
+// returns `Ok(None)` if no exp part was found.
+// returns `Ok(Some(()))` if an exp part was found and parsed.
+//
+// EXP_PART <-
+//     (`e`|`E`) (`-`|`+`)? DEC_DIGITS
+fn try_parse_exp_part(source: &mut Source) -> Result<Option<()>> {
+    if source.next_if(|&c| c.to_ascii_lowercase() == 'e').is_some() {
+        let _sign = source.next_if(|&c| c == '-' || c == '+');
+        if source
+            .take_while_ref(|&c| crate::is_things::is_digit(c))
+            .count()
+            .lt(&1)
+        {
+            // need digits following exp notation
+            Err(Error::FloatingConstantExpPartNoDigit)
+        } else {
+            Ok(Some(()))
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+// CONSTANT <-
+//    DEC_DIGITS IntegralType?
+//    `0x` HEX_DIGITS IntegralType?
+//    `0b` BIN_DIGITS IntegralType?
+//    `0o` OCT_DIGITS IntegralType?
+//    DEC_DIGITS FloatingType?
+//    `.` DEC_DIGITS EXP_PART? FloatingType?
+//    DEC_DIGITS `.` DEC_DIGITS? EXP_PART? FloatingType?
+fn parse_constant_inner(source: &mut Source) -> Result<Token> {
+    let zero = source.next_if(|&c| c == '0').is_some();
+
+    let radix = zero
+        .then(|| source.next_if_map(|c| Radix::from_char(c)))
+        .flatten();
+
+    if let Some(radix) = radix {
+        parse_digit_part(source, false, radix)?;
+        if source.peek().map(|&c| c == 'u' || c == 'i') == Some(true) {
+            try_parse_integral_type(source)?;
+        }
+        return Ok(radix.to_token());
+    }
+
+    // if zero: `_`* DIGIT (DIGIT|`_`)*
+    // else: DIGIT (DIGIT|`_`)*
+    _ = match parse_digit_part(source, zero, Radix::Dec) {
+        Ok(_) => Ok(()),
+        Err(Error::NumericalConstantDigitNoDigit) if zero => Ok(()),
+        Err(e) => Err(e),
+    }?;
+
+    if let Some(_) = source.try_map_iter_if(|source| try_parse_integral_type(source))? {
+        return Ok(Token::IntegerConstant);
+    }
+
+    let dot = source.next_if(|&c| c == '.').is_some();
+
+    if dot {
+        parse_digit_part(source, false, Radix::Dec)?;
+    }
+
+    // parse exp notation
+    let exp = try_parse_exp_part(source)?.is_some();
+
+    // trailing FloatingType?
+    let floating = if source.next_if(|&c| c == 'f').is_some() {
+        let digits = source.next_tuple::<(char, char)>();
+        if !(digits == Some(('6', '4')) || digits == Some(('3', '2'))) {
+            // need either f64 or f32 here!
+            return Err(Error::FloatingConstantInvalidTrailingType);
+        }
+        true
+    } else {
+        false
+    };
+
+    let token = match (dot, exp, floating) {
+        (false, false, false) => Token::IntegerConstant,
+        (true, false, _) => Token::DotFloatingConstant,
+        (true, true, _) => Token::DotFloatingExpConstant,
+        (false, true, _) => Token::FloatingExpConstant,
+        (false, _, _) => Token::FloatingConstant,
+    };
+
+    Ok(token)
+}
+
+pub(crate) fn parse_constant(source: &mut Source) -> Result<Token> {
+    let constant = parse_constant_inner(source)?;
+    // char following a constant must not be id_continue
+    if source
+        .peek()
+        .map(|&c| crate::is_things::is_id_continue(c))
+        .unwrap_or(false)
+    {
+        return Err(Error::NumericalConstantUnknownSuffix);
+    }
+
+    Ok(constant)
+}
+
+pub(crate) fn parse_string_or_char_constant(source: &mut Source) -> Result<Token> {
+    let quote = source
+        .next_if(|&c| c == '"' || c == '\'')
+        .ok_or(Error::InvalidToken)?;
+
+    let is_char = quote == '\'';
+
+    let mut escaped = false;
+    let mut closed = false;
+
+    while let Some(c) = source.next() {
+        if escaped {
+            // accept any escaped char
+            escaped = false;
+            continue;
+        }
+        if c == '\\' {
+            escaped = true;
+            continue;
+        }
+        if c == quote {
+            closed = true;
+            break;
+        }
+    }
+
+    if !closed {
+        return Err(Error::StringError("Unterminated string/char.".into()));
+    }
+
+    if is_char {
+        Ok(Token::CharConstant)
+    } else {
+        Ok(Token::StringConstant)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    fn make_source(s: &'_ str) -> Source<'_> {
+        s.chars().peekable().into()
+    }
+
+    #[test]
+    fn parse_constant_number() {
+        assert_eq!(
+            parse_constant(&mut make_source("0x1A3F_u32")),
+            Ok(Token::IntegerHexConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("13f32")),
+            Ok(Token::FloatingConstant)
+        );
+
+        assert_eq!(
+            parse_constant(&mut make_source("0b1011_0010i16")),
+            Ok(Token::IntegerBinConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("0o755u8")),
+            Ok(Token::IntegerOctConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("42i64")),
+            Ok(Token::IntegerConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("3.14f64")),
+            Ok(Token::DotFloatingConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("2.71828e0f32")),
+            Ok(Token::DotFloatingExpConstant)
+        );
+        assert_eq!(
+            parse_constant(&mut make_source("22e23")),
+            Ok(Token::FloatingExpConstant)
+        );
+    }
+}

crates/lexer/src/lib.rs

@@ -0,0 +1,686 @@
+#![feature(slice_swap_unchecked, iter_collect_into)]
+
+mod is_things {
+    /// True if `c` is considered a whitespace according to Rust language definition.
+    /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
+    /// for definitions of these classes.
+    pub fn is_whitespace(c: char) -> bool {
+        // This is Pattern_White_Space.
+        //
+        // Note that this set is stable (ie, it doesn't change with different
+        // Unicode versions), so it's ok to just hard-code the values.
+
+        matches!(
+            c,
+            // Usual ASCII suspects
+            '\u{0009}'   // \t
+        | '\u{000A}' // \n
+        | '\u{000B}' // vertical tab
+        | '\u{000C}' // form feed
+        | '\u{000D}' // \r
+        | '\u{0020}' // space
+
+    // NEXT LINE from latin1
+        | '\u{0085}'
+
+    // Bidi markers
+        | '\u{200E}' // LEFT-TO-RIGHT MARK
+        | '\u{200F}' // RIGHT-TO-LEFT MARK
+
+    // Dedicated whitespace characters from Unicode
+        | '\u{2028}' // LINE SEPARATOR
+        | '\u{2029}' // PARAGRAPH SEPARATOR
+        )
+    }
+
+    /// True if `c` is valid as a first character of an identifier.
+    /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+    /// a formal definition of valid identifier name.
+    pub fn is_id_start(c: char) -> bool {
+        // This is XID_Start OR '_' (which formally is not a XID_Start).
+        c == '_' || c == '-' || unicode_xid::UnicodeXID::is_xid_start(c)
+    }
+
+    /// True if `c` is valid as a non-first character of an identifier.
+    /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
+    /// a formal definition of valid identifier name.
+    pub fn is_id_continue(c: char) -> bool {
+        unicode_xid::UnicodeXID::is_xid_continue(c) || c == '-'
+    }
+
+    /// The passed string is lexically an identifier.
+    pub fn is_ident(string: &str) -> bool {
+        let mut chars = string.chars();
+        if let Some(start) = chars.next() {
+            is_id_start(start) && chars.all(is_id_continue)
+        } else {
+            false
+        }
+    }
+
+    #[expect(dead_code)]
+    pub fn is_digit(ch: char) -> bool {
+        ('0'..='9').contains(&ch)
+    }
+
+    #[expect(dead_code)]
+    pub fn is_bin_digit(ch: char) -> bool {
+        ch == '0' || ch == '1'
+    }
+
+    #[expect(dead_code)]
+    pub fn is_nonzero_digit(ch: char) -> bool {
+        ('1'..='9').contains(&ch)
+    }
+
+    #[expect(dead_code)]
+    pub fn is_oct_digit(ch: char) -> bool {
+        ('0'..='7').contains(&ch)
+    }
+
+    #[expect(dead_code)]
+    pub fn is_hex_digit(ch: char) -> bool {
+        ('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
+    }
+}
+
+macro_rules! tokens {
+    ($vis:vis $ty_name:ident:
+        {
+            $($name2:ident),*
+        },
+        {
+            $($name:ident => $lexeme:literal),*
+        }) => {
+
+            #[allow(dead_code)]
+            #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+            $vis enum $ty_name {
+                $($name,
+                )*
+                $($name2,)*
+            }
+
+            impl ::core::fmt::Display for $ty_name {
+                fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+                    match self {
+                        $(Self::$name => write!(f, "{}", $lexeme),)*
+                        $(Self::$name2 => write!(f, "<{}>", stringify!($name2))),*
+                    }
+                }
+            }
+
+            #[allow(dead_code)]
+            impl $ty_name {
+                $vis fn lexeme(&self) -> Option<&'static str> {
+                    match self {
+                        $(Self::$name => Some($lexeme),)*
+                        $(Self::$name2 => None),*
+                    }
+                }
+
+                /// returns the number of chars in this lexeme
+                $vis fn lexeme_len(&self) -> usize {
+                    self.lexeme().map(|lexeme|lexeme.chars().count()).unwrap_or(0)
+                }
+
+                /// returns the number of chars in this lexeme
+                $vis fn lexeme_len_utf8(&self) -> usize {
+                    self.lexeme().map(|lexeme|lexeme.len()).unwrap_or(0)
+                }
+
+                $vis fn maybe_ident(&self) -> bool {
+                    self.lexeme().map(|lexeme| crate::is_things::is_ident(lexeme)).unwrap_or(false)
+                }
+
+                $vis fn lexemes() -> &'static [(Self, &'static str)] {
+                    &[
+                        $((Self::$name, $lexeme)),*
+                    ]
+                }
+            }
+        };
+}
+
+tokens!(pub Token: {
+    Eof,
+    ParseError,
+    // Marker Token for any Comment
+    Comment,
+    DocComment,
+        // Marker Token for any pre-processing directive
+    CharConstant,
+    IntegerConstant,
+    IntegerHexConstant,
+    IntegerBinConstant,
+    IntegerOctConstant,
+    FloatingConstant,
+    FloatingExpConstant,
+    DotFloatingConstant,
+    DotFloatingExpConstant,
+    StringConstant,
+    Ident
+},
+    // Lexical Tokens:
+    {
+        SlashSlash => "//",
+        SlashSlashSlash => "///",
+        // SlashStar => "/*",
+        // SlashStarStar => "/**",
+        //StarSlash => "*/",
+        // Punctuation:
+        OpenParens => "(",
+        CloseParens => ")",
+        OpenBrace => "{",
+        CloseBrace => "}",
+        OpenSquareBracket => "[",
+        CloseSquareBracket => "]",
+        Semi => ";",
+        Comma => ",",
+        Elipsis3 => "...",
+        Elipsis2 => "..",
+        Colon => ":",
+        Equal => "=",
+        // Keywords:
+        Void => "void",
+        Bool => "bool",
+        F32 => "f32",
+        F64 => "f64",
+        ISize => "isize",
+        USize => "usize",
+        Const => "const",
+        Volatile => "volatile",
+        Noalias => "noalias",
+        Fn => "fn",
+        Let => "let",
+        Var => "var",
+        If => "if",
+        As => "as",
+        Else => "else",
+        Return => "return",
+        Struct => "struct",
+        Type => "type",
+        Union => "union",
+        Enum => "enum",
+        Packed => "packed",
+        Extern => "extern",
+        Pub => "pub",
+        // Operators
+        Dot => ".",
+        MinusGreater => "->",
+        Bang => "!",
+        Tilde => "~",
+        Plus => "+",
+        PlusPlus => "++",
+        Minus => "-",
+        MinusMinus => "--",
+        Star => "*",
+        Slash => "/",
+        Percent => "%",
+        Less => "<",
+        Greater => ">",
+        LessEqual => "<=",
+        GreaterEqual => ">=",
+        EqualEqual => "==",
+        BangEqual => "!=",
+        PipePipe => "||",
+        AmpersandAmpersand => "&&",
+        Ampersand => "&",
+        Caret => "^",
+        Pipe => "|",
+        LessLess => "<<",
+        GreaterGreater => ">>",
+        Question => "?",
+        PlusEqual => "+=",
+        MinusEqual => "-=",
+        StarEqual => "*=",
+        SlashEqual => "/=",
+        PercentEqual => "%=",
+        AmpersandEqual => "&=",
+        PipeEqual => "|=",
+        CaretEqual => "^=",
+        LessLessEqual => "<<=",
+        GreaterGreaterEqual => ">>="
+    });
+
+impl Token {
+    pub fn is_assignment_op(self) -> bool {
+        match self {
+            Token::PlusEqual
+            | Token::MinusEqual
+            | Token::StarEqual
+            | Token::SlashEqual
+            | Token::PercentEqual
+            | Token::PipeEqual
+            | Token::CaretEqual
+            | Token::AmpersandEqual
+            | Token::LessLessEqual
+            | Token::GreaterGreaterEqual
+            | Token::Equal => true,
+            _ => false,
+        }
+    }
+    pub fn is_unary_op(self) -> bool {
+        match self {
+            Token::Plus | Token::Minus | Token::Star | Token::Ampersand | Token::Bang => true,
+            _ => false,
+        }
+    }
+    pub fn is_binary_op(self) -> bool {
+        match self {
+            Token::Star
+            | Token::Slash
+            | Token::Percent
+            | Token::Pipe
+            | Token::Ampersand
+            | Token::Caret
+            | Token::Plus
+            | Token::Minus
+            | Token::PipePipe
+            | Token::AmpersandAmpersand
+            | Token::BangEqual
+            | Token::EqualEqual
+            | Token::Less
+            | Token::Greater
+            | Token::LessEqual
+            | Token::GreaterEqual
+            | Token::LessLess
+            | Token::GreaterGreater => true,
+            _ => false,
+        }
+    }
+}
+
+/// A list of lexemes used by the `LexemeParser`.
+/// `lexemes` contains every token that has a defined lexeme, such as `fn`, `f32`, `const`, etc.
+/// The `LexemeList` keeps track of two offsets into the `lexemes` array,
+/// splitting it into three windows:
+/// - [0, start_candidates) - tokens that are still being considered for parsing
+/// - [start_candidates, end_candidates) - the tokens which this lexeme matches
+/// - [end_candidates, len) - tokens that have been filtered out and are no longer considered
+/// On each iteration of the parsing loop, the remaining tokens are matched
+/// against the next character and, if they match completely, are swapped into
+/// the candidates window, or swapped to the end if they don't.
+struct LexemeList {
+    lexemes: Box<[Token]>,
+    start_candidates: usize,
+    end_candidates: usize,
+    filtered: Vec<(usize, FilterResult)>,
+}
+
+enum FilterResult {
+    Remove,
+    Candidate,
+}
+
+impl LexemeList {
+    fn new() -> Self {
+        let lexemes = Token::lexemes()
+            .iter()
+            .map(|(tok, _)| tok.clone())
+            .collect::<Box<_>>();
+
+        Self {
+            start_candidates: lexemes.len(),
+            end_candidates: lexemes.len(),
+            lexemes,
+            filtered: Vec::new(),
+        }
+    }
+
+    fn clear(&mut self) {
+        self.start_candidates = self.lexemes.len();
+        self.end_candidates = self.lexemes.len();
+    }
+
+    fn remaining(&self) -> &[Token] {
+        &self.lexemes[0..self.start_candidates]
+    }
+
+    fn candidates(&self) -> &[Token] {
+        &self.lexemes[self.start_candidates..self.end_candidates]
+    }
+
+    fn step(&mut self, ch: char, pos: usize) {
+        // smartly reuse allocation for `filtered`
+        // truly one of the premature optimizations.
+        // but it just feels good, innit?
+        let mut filtered = core::mem::take(&mut self.filtered);
+
+        self.remaining()
+            .iter()
+            .enumerate()
+            .filter_map(|(i, tok)| {
+                let bytes = tok.lexeme().unwrap().as_bytes();
+                // SAFETY: all tokens in `self.remaining()` are lexical tokens, and
+                // they are all valid ascii
+                let c = unsafe {
+                    // TODO: maybe keep a list of `Char<'_>`s around in order to
+                    // support fully utf8 tokens?
+                    char::from_u32_unchecked(bytes[pos] as u32)
+                };
+                match c == ch {
+                    false => Some((i, FilterResult::Remove)),
+                    true if bytes.len() <= pos + 1 => Some((i, FilterResult::Candidate)),
+                    true => None,
+                }
+            })
+            .collect_into(&mut filtered);
+
+        // iterate in reverse order so that we can safely swap elements
+        // drain here so that we can possibly reuse the `filtered` Vec allcoation
+        filtered.drain(..).rev().for_each(|(i, f)| {
+            match f {
+                // for candidates, swap the candidate with the last remaining
+                // token, then dec `start_candidates`
+                FilterResult::Candidate => {
+                    // SAFETY: we know that `i` and `self.start_candidates - 1`
+                    // are both valid indices: `self.start_candidates` starts at
+                    // the end and each time it is decremented, one more element
+                    // is removed from the front, so that as long as an element
+                    // is remaining, `self.start_candidates` is always greater
+                    // than 0.
+                    // the order of the remaining elements is not meaningfully
+                    // impacted because we only ever swap with elements after
+                    // `i`, and `i` is the greatest index we will touch.
+                    unsafe {
+                        self.lexemes.swap_unchecked(i, self.start_candidates - 1);
+                        self.start_candidates = self.start_candidates.saturating_sub(1);
+                    }
+                }
+                // for removes, swap the last candidate with the last remainign
+                // token, then swap the remove with the last candidate, then dec
+                // `end_candidates` and `start_candidates`
+                FilterResult::Remove => {
+                    unsafe {
+                        // in the case that `start_candidates` ==
+                        // `end_candidates`, no swap happens and that's fine.
+                        // remove this: v
+                        //           [a,b,c][d,e,f][g,h,i]
+                        // swap these:    ^      ^
+                        //           [a,b,f][d,e,c][g,h,i]
+                        // swap these:  ^        ^
+                        //           [a,c,f][d,e,b][g,h,i]
+                        // decrement both counters:
+                        //           [a,c][f,d,e][b,g,h,i]
+                        self.lexemes
+                            .swap_unchecked(self.start_candidates - 1, self.end_candidates - 1);
+                        self.lexemes.swap_unchecked(i, self.end_candidates - 1);
+                        self.start_candidates = self.start_candidates.saturating_sub(1);
+                        self.end_candidates = self.end_candidates.saturating_sub(1);
+                    }
+                }
+            }
+        });
+
+        // replace `filtered`
+        self.filtered = filtered;
+    }
+}
+
+/// Helper type for parsing tokens that have a defined lexeme, such as `fn`,
+/// `f32`, `const`, etc. Tokens with variable lexemes, such as primitive
+/// integral types, constants or identifiers are not parsed by this.
+pub struct LexemeParser {
+    lexemes: LexemeList,
+    len: usize,
+}
+
+impl LexemeParser {
+    pub fn new() -> Self {
+        Self {
+            lexemes: LexemeList::new(),
+            len: 0,
+        }
+    }
+
+    pub fn parse(&mut self, mut tokens: impl Iterator<Item = char>) -> Option<Token> {
+        self.lexemes.clear();
+        loop {
+            let Some(ch) = tokens.next() else {
+                break;
+            };
+
+            if crate::is_things::is_whitespace(ch) {
+                break;
+            }
+
+            self.lexemes.step(ch, self.len);
+            if self.lexemes.remaining().is_empty() {
+                break;
+            }
+        }
+        self.lexemes.candidates().last().copied()
+    }
+}
+
+use itertools::Itertools;
+use trie::Tree;
+
+#[derive(Debug, Clone, Copy)]
+struct CountingIterator<I: Iterator> {
+    iter: I,
+    count: usize,
+}
+
+impl<I: Iterator> From<I> for CountingIterator<I> {
+    fn from(iter: I) -> Self {
+        Self { iter, count: 0 }
+    }
+}
+
+impl<I: Iterator<Item = char>> Iterator for CountingIterator<I> {
+    type Item = I::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.iter.next().inspect(|c| self.count += c.len_utf8())
+    }
+}
+
+impl<I: Iterator> CountingIterator<I> {
+    pub(crate) fn offset(&self) -> usize {
+        self.count
+    }
+}
+
+impl<I: Iterator> core::ops::Deref for CountingIterator<I> {
+    type Target = I;
+
+    fn deref(&self) -> &Self::Target {
+        &self.iter
+    }
+}
+
+impl<I: Iterator> core::ops::DerefMut for CountingIterator<I> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.iter
+    }
+}
+
+type Source<'a> = CountingIterator<core::iter::Peekable<core::str::Chars<'a>>>;
+
+pub struct TokenIterator<'a> {
+    trie: Tree<char, Token>,
+    source: &'a str,
+    offset: usize,
+}
+
+impl<'a> TokenIterator<'a> {
+    pub fn new(source: &'a str) -> Self {
+        let mut trie = Tree::new();
+
+        for (token, token_str) in Token::lexemes() {
+            trie.insert(token_str.chars(), *token);
+        }
+
+        Self {
+            trie,
+            source,
+            offset: 0,
+        }
+    }
+
+    fn peekable_source(&self) -> Source<'a> {
+        CountingIterator::from(self.source[self.offset..].chars().peekable())
+    }
+
+    fn parse(&mut self) -> Option<Token> {
+        let mut iter = CountingIterator::from(self.source[self.offset..].chars());
+
+        match self.trie.get_closest(&mut iter) {
+            Some(token) => {
+                // skip the peeked item
+                self.offset += token.lexeme_len();
+                Some(*token)
+            }
+            None => None,
+        }
+    }
+
+    fn skip_whitespaces(&mut self) -> usize {
+        self.skip_while(is_things::is_whitespace)
+    }
+
+    fn skip(&mut self, mut n: usize) -> usize {
+        self.skip_while(|_| {
+            n -= 1;
+            n > 0
+        })
+    }
+
+    fn skip_while(&mut self, mut pred: impl FnMut(char) -> bool) -> usize {
+        let mut count = 0;
+        loop {
+            let Some(c) = self.source[self.offset..].chars().next() else {
+                break;
+            };
+
+            if pred(c) {
+                self.offset += c.len_utf8();
+                count += c.len_utf8();
+                continue;
+            } else {
+                break;
+            }
+        }
+        count
+    }
+}
+
+impl<'a> Iterator for TokenIterator<'a> {
+    type Item = (Token, &'a str);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // skip whitespace
+        self.skip_whitespaces();
+
+        let start = self.offset;
+
+        let mut source = self.peekable_source();
+        let mut cursor = self.peekable_source();
+        let token = match cursor.next() {
+            Some('0'..='9') => {
+                let token = complex_tokens::parse_constant(&mut source).ok()?;
+                self.offset += source.offset();
+
+                Some((token, &self.source[start..self.offset]))
+            }
+            Some('.') if cursor.next().map_or(false, is_things::is_digit) => {
+                let token = complex_tokens::parse_constant(&mut source).ok()?;
+                self.offset += source.offset();
+
+                Some((token, &self.source[start..self.offset]))
+            }
+            Some('\'' | '"') => {
+                let token = complex_tokens::parse_string_or_char_constant(&mut source).ok()?;
+                self.offset += source.offset();
+
+                Some((token, &self.source[start..self.offset]))
+            }
+            _ => match self.parse().map(|tok| match tok {
+                Token::SlashSlash => {
+                    self.skip_while(|c| c == '\n');
+                    (Token::Comment)
+                }
+                Token::SlashSlashSlash => {
+                    self.skip_while(|c| c == '\n');
+                    (Token::DocComment)
+                }
+                _ => tok,
+            }) {
+                Some(tok) => {
+                    if tok.maybe_ident() && self.skip_while(|c| is_things::is_id_continue(c)) > 0 {
+                        Some((Token::Ident, &self.source[start..self.offset]))
+                    } else {
+                        Some((tok, &self.source[start..self.offset]))
+                    }
+                }
+                None => {
+                    if self
+                        .peekable_source()
+                        .next()
+                        .map_or(false, |c| is_things::is_id_start(c))
+                    {
+                        self.skip(1);
+                        self.skip_while(|c| is_things::is_id_continue(c));
+                        Some((Token::Ident, &self.source[start..self.offset]))
+                    } else {
+                        None
+                    }
+                }
+            },
+        };
+
+        token
+    }
+}
+
+mod complex_tokens;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_iterator() {
+        let tokens = "fn let void+++(++bool)";
+        let mut lexer = TokenIterator::new(&tokens);
+        assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
+        assert_eq!(lexer.next(), Some((Token::Let, "let")));
+        assert_eq!(lexer.next(), Some((Token::Void, "void")));
+        assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
+        assert_eq!(lexer.next(), Some((Token::Plus, "+")));
+        assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
+        assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
+        assert_eq!(lexer.next(), Some((Token::Bool, "bool")));
+        assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
+        assert_eq!(lexer.next(), None);
+    }
+
+    #[test]
+    fn complex_iterator() {
+        let tokens = "fn my-function(x: i32, y: f32) -> f32 { return x + y; }";
+        let mut lexer = TokenIterator::new(&tokens);
+        assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "my-function")));
+        assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "x")));
+        assert_eq!(lexer.next(), Some((Token::Colon, ":")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "i32")));
+        assert_eq!(lexer.next(), Some((Token::Comma, ",")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "y")));
+        assert_eq!(lexer.next(), Some((Token::Colon, ":")));
+        assert_eq!(lexer.next(), Some((Token::F32, "f32")));
+        assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
+        assert_eq!(lexer.next(), Some((Token::MinusGreater, "->")));
+        assert_eq!(lexer.next(), Some((Token::F32, "f32")));
+        assert_eq!(lexer.next(), Some((Token::OpenBrace, "{")));
+        assert_eq!(lexer.next(), Some((Token::Return, "return")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "x")));
+        assert_eq!(lexer.next(), Some((Token::Plus, "+")));
+        assert_eq!(lexer.next(), Some((Token::Ident, "y")));
+        assert_eq!(lexer.next(), Some((Token::Semi, ";")));
+        assert_eq!(lexer.next(), Some((Token::CloseBrace, "}")));
+        assert_eq!(lexer.next(), None);
+    }
+}

flake.lock

@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1753939845,
-        "narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=",
+        "lastModified": 1757745802,
+        "narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "94def634a20494ee057c76998843c015909d6311",
+        "rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1",
         "type": "github"
       },
       "original": {
@@ -62,11 +62,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1754102567,
-        "narHash": "sha256-d6nZ+1e4VDqW6VAwfx9EAUDJdPxSwqwGiuli32FEgoE=",
+        "lastModified": 1757989933,
+        "narHash": "sha256-9cpKYWWPCFhgwQTww8S94rTXgg8Q8ydFv9fXM6I8xQM=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "08ff39bf869cadca3102b39824f4c7025186b7dc",
+        "rev": "8249aa3442fb9b45e615a35f39eca2fe5510d7c3",
         "type": "github"
       },
       "original": {

src/ast2/biunification.rs

@@ -5,30 +5,43 @@
 // Visitor pattern has lots of unused arguments
 #![allow(unused_variables)]
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+
+use crate::ast2::tag::{AstNode, AstNodeExt};
 
 use super::{Ast, Index, intern, visitor::AstVisitorTrait};
 
 type Id = u32;
 
-enum Type {
-    Reified(intern::Index),
-    Variable(Id),
+trait TypeVariance {
+    type T;
+    type Opposite;
+}
+
+#[derive(Debug, Clone)]
+enum TypeHead<T: TypeVariance> {
+    Real(intern::Index),
+    Function { args: Vec<T::T>, ret: T::Opposite },
 }
 
 /// Variance of a type parameter or constraint.
 /// A function of type `A -> B` is covariant in `B` and contravariant in `A`.
-/// This means that a type `T` may be substituted for `A` if `T` is a subtype of `A`, but
-/// a type `T` may only be substituted for `B` if `T` is a supertype of `B`.
+/// This means that a type `T` may be substituted for `A` if `T` is a subtype of
+/// `A`, that is, every `T` is also an `A`,
+/// but a type `T` may only be substituted for `B` if `T` is a supertype of `B`,
+/// that is, every `B` is also a `T`.
 ///
 /// Namely, in a type system with `int` and `nat <: int`, for a function `f: int
 /// -> int` in the expression `let u: int = 3; let t: nat = f(u);`, `u` may
 /// safely be used as an argument to `f` because `nat <: int`, but `f(u`)` may
 /// not be assigned to `t` because `int <: nat` is not true.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
 enum Variance {
-    #[doc(alias = "Positive")]
+    /// A Positive, or union relationship between types.
+    /// used in value-places.
     Covariant,
-    #[doc(alias = "Negative")]
+    /// A Negative, or intersection relationship between types.
+    /// used in use-places.
     Contravariant,
 }
 
@@ -37,27 +50,81 @@ struct Value(Id);
 #[derive(Debug, Clone, Copy)]
 struct Use(Id);
 
+impl TypeVariance for Value {
+    type T = Value;
+    type Opposite = Use;
+}
+
+impl TypeVariance for Use {
+    type T = Use;
+    type Opposite = Value;
+}
+
 /// Typechecking error.
 #[derive(Debug, Clone, thiserror::Error)]
 enum Error {
     #[error("Unimplemented feature")]
     Unimplemented,
+    #[error("{0}")]
+    StringError(String),
 }
 
 type Result<T> = std::result::Result<T, Error>;
 
 struct Bindings {
-    inner: HashMap<super::Index, Value>,
+    next_id: Id,
+    inner: HashMap<super::Index, Id>,
+    bounds: HashSet<(Id, Id, Variance)>,
+    types: HashMap<Id, intern::Index>,
+}
+
+impl Bindings {
+    fn new() -> Self {
+        Bindings {
+            next_id: 1,
+            inner: HashMap::new(),
+            bounds: HashSet::new(),
+            types: HashMap::new(),
+        }
+    }
+
+    fn new_id(&mut self) -> Id {
+        let id = self.next_id;
+        self.next_id += 1;
+        id
+    }
+
+    fn get_or_create(&mut self, idx: super::Index) -> Id {
+        self.inner.get(&idx).copied().unwrap_or_else(|| {
+            let id = self.new_id();
+            self.inner.insert(idx, id);
+            id
+        })
+    }
+
+    /// retrieves the type Id for the given ast node.
+    fn get(&self, idx: super::Index) -> Option<Id> {
+        self.inner.get(&idx).copied()
+    }
+
+    /// inserts a proper type for `id`.
+    fn insert_type(&mut self, id: Id, ty: intern::Index) {
+        self.types.insert(id, ty);
+    }
 }
 
 struct TypeChecker<'a> {
     pool: &'a mut intern::InternPool,
+    bindings: Bindings,
 }
 
 // Core
 impl TypeChecker<'_> {
     pub fn new(pool: &mut intern::InternPool) -> TypeChecker {
-        TypeChecker { pool }
+        TypeChecker {
+            pool,
+            bindings: Bindings::new(),
+        }
     }
 
     fn var(&mut self) -> (Value, Use) {
@@ -73,6 +140,33 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
 
     const UNIMPL: Self::Error = Error::Unimplemented;
 
+    fn visit_interned_type_impl(
+        &mut self,
+        ast: &'a Ast,
+        idx: Index,
+        intern: intern::Index,
+    ) -> std::result::Result<Self::Value, Self::Error> {
+        let id = self.bindings.get_or_create(idx);
+        match self.pool.get_key(intern) {
+            intern::Key::SimpleType {
+                ty: intern::SimpleType::ComptimeInt,
+            } => {
+                // This is a type variable.
+            }
+            intern::Key::SimpleType { .. }
+            | intern::Key::PointerType { .. }
+            | intern::Key::ArrayType { .. }
+            | intern::Key::FunctionType { .. }
+            | intern::Key::StructType { .. } => {
+                // This is a real type.
+                self.bindings.insert_type(id, intern);
+            }
+            _ => unreachable!(),
+        }
+
+        Ok(Value(id))
+    }
+
     fn visit_constant_impl(
         &mut self,
         ast: &'a Ast,
@@ -80,8 +174,30 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
         ty: Index,
         value: intern::Index,
     ) -> std::result::Result<Self::Value, Self::Error> {
-        // constants may be of type `comptime_int`, which is a special type that
-        // cannot exist at runtime.
+        // get type from the pool
+
+        let AstNode::InternedType { intern } = ast.get_ast_node(ty) else {
+            panic!(
+                "Expected an interned type node, got {:?}",
+                ast.get_ast_node(ty)
+            );
+        };
+
+        match self.pool.get_key(intern) {
+            intern::Key::SimpleType {
+                ty: intern::SimpleType::ComptimeInt,
+            } => {
+                // This is a type variable.
+            }
+            intern::Key::SimpleType { .. }
+            | intern::Key::PointerType { .. }
+            | intern::Key::ArrayType { .. }
+            | intern::Key::FunctionType { .. }
+            | intern::Key::StructType { .. } => {
+                // This is a real type.
+            }
+            _ => unreachable!(),
+        }
 
         todo!()
     }

src/ast2/intern.rs

@@ -15,7 +15,7 @@ use crate::{
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[repr(u8)]
 pub enum SimpleType {
-    F32 = 0,
+    F32,
     F64,
     Bool,
     Void,
@@ -28,11 +28,14 @@ pub enum SimpleType {
     /// Bottom type: this is the subtype of all types, and it can coerce into a value of any type.
     /// Akin to Rust's `!`.
     Bottom,
+    UInt(u16),
+    SInt(u16),
 }
 
-impl From<u8> for SimpleType {
-    fn from(value: u8) -> Self {
-        match value {
+impl From<u32> for SimpleType {
+    fn from(value: u32) -> Self {
+        let [discriminant, bits] = *crate::common::u32_as_u16_slice(&value);
+        match discriminant {
             0 => Self::F32,
             1 => Self::F64,
             2 => Self::Bool,
@@ -40,28 +43,49 @@ impl From<u8> for SimpleType {
             4 => Self::USize,
             5 => Self::ISize,
             6 => Self::ComptimeInt,
+            7 => Self::Top,
+            8 => Self::Bottom,
+            9 => Self::UInt(bits),
+            10 => Self::SInt(bits),
             _ => panic!("{value} is not a simple type"),
         }
     }
 }
 
+impl From<SimpleType> for u32 {
+    fn from(value: SimpleType) -> Self {
+        match value {
+            SimpleType::F32 => crate::common::u32_from_u16_slice(&[0, 0]),
+            SimpleType::F64 => crate::common::u32_from_u16_slice(&[1, 0]),
+            SimpleType::Bool => crate::common::u32_from_u16_slice(&[2, 0]),
+            SimpleType::Void => crate::common::u32_from_u16_slice(&[3, 0]),
+            SimpleType::USize => crate::common::u32_from_u16_slice(&[4, 0]),
+            SimpleType::ISize => crate::common::u32_from_u16_slice(&[5, 0]),
+            SimpleType::ComptimeInt => crate::common::u32_from_u16_slice(&[6, 0]),
+            SimpleType::Top => crate::common::u32_from_u16_slice(&[7, 0]),
+            SimpleType::Bottom => crate::common::u32_from_u16_slice(&[8, 0]),
+            SimpleType::UInt(bits) => crate::common::u32_from_u16_slice(&[9, bits]),
+            SimpleType::SInt(bits) => crate::common::u32_from_u16_slice(&[10, bits]),
+        }
+    }
+}
+
 impl Display for SimpleType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            match self {
-                SimpleType::F32 => "f32",
-                SimpleType::F64 => "f64",
-                SimpleType::Bool => "bool",
-                SimpleType::Void => "void",
-                SimpleType::USize => "usize",
-                SimpleType::ISize => "isize",
-                SimpleType::ComptimeInt => "comptime_int",
-                SimpleType::Top => "⊤",
-                SimpleType::Bottom => "⊥",
-            }
-        )
+        let fmt: std::borrow::Cow<str> = match self {
+            SimpleType::F32 => "f32".into(),
+            SimpleType::F64 => "f64".into(),
+            SimpleType::Bool => "bool".into(),
+            SimpleType::Void => "void".into(),
+            SimpleType::USize => "usize".into(),
+            SimpleType::ISize => "isize".into(),
+            SimpleType::ComptimeInt => "comptime_int".into(),
+            SimpleType::Top => "⊤".into(),
+            SimpleType::Bottom => "⊥".into(),
+            SimpleType::UInt(bits) => format!("u{bits}").into(),
+            SimpleType::SInt(bits) => format!("i{bits}").into(),
+        };
+        write!(f, "{fmt}",)
     }
 }
 
@@ -78,8 +102,6 @@ pub enum Tag {
     F64,
     PositiveInt,
     NegativeInt,
-    UIntType,
-    SIntType,
     SimpleType,
     PointerType,
     ArrayType,
@@ -88,9 +110,9 @@ pub enum Tag {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-struct Item {
-    tag: Tag,
-    index: u32,
+pub(super) struct Item {
+    pub(super) tag: Tag,
+    pub(super) index: u32,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -126,12 +148,6 @@ pub enum Key<'a> {
     NegativeInt {
         bigint: BigInt,
     },
-    UIntType {
-        bit_width: u16,
-    },
-    SIntType {
-        bit_width: u16,
-    },
     SimpleType {
         ty: SimpleType,
     },
@@ -178,8 +194,6 @@ impl Display for KeyDisplay<'_> {
             Key::F64 { bits } => write!(f, "{bits}")?,
             Key::PositiveInt { ref bigint } => write!(f, "{bigint}")?,
             Key::NegativeInt { ref bigint } => write!(f, "{bigint}")?,
-            Key::UIntType { bit_width } => write!(f, "u{bit_width}")?,
-            Key::SIntType { bit_width } => write!(f, "i{bit_width}")?,
             Key::SimpleType { ty } => write!(f, "{ty}")?,
             Key::PointerType { pointee, flags } => {
                 write!(f, "*{flags}{}", self.ip.display_key(pointee))?
@@ -238,8 +252,6 @@ impl Hash for Key<'_> {
             Key::F64 { bits } => ordered_float::OrderedFloat(*bits).hash(state),
             Key::PositiveInt { bigint } => bigint.hash(state),
             Key::NegativeInt { bigint } => bigint.hash(state),
-            Key::UIntType { bit_width: bits } => bits.hash(state),
-            Key::SIntType { bit_width: bits } => bits.hash(state),
             Key::SimpleType { ty } => ty.hash(state),
             Key::PointerType { pointee, flags } => (pointee, flags).hash(state),
             Key::ArrayType {
@@ -544,20 +556,20 @@ static_keys!(
     ISIZE => Key::SimpleType {ty: SimpleType::ISize,},
     VOID => Key::SimpleType {ty: SimpleType::Void,},
     COMPTIME_INT => Key::SimpleType {ty: SimpleType::ComptimeInt,},
-    I1 => Key::SIntType { bit_width: 1 },
-    U1 => Key::UIntType { bit_width: 1 },
-    I0 => Key::SIntType { bit_width: 0 },
-    U0 => Key::UIntType { bit_width: 0 },
-    I8 => Key::SIntType { bit_width: 8 },
-    U8 => Key::UIntType { bit_width: 8 },
-    I16 => Key::SIntType { bit_width: 16 },
-    U16 => Key::UIntType { bit_width: 16 },
-    I32 => Key::SIntType { bit_width: 32 },
-    U32 => Key::UIntType { bit_width: 32 },
-    I64 => Key::SIntType { bit_width: 64 },
-    U64 => Key::UIntType { bit_width: 64 },
-    I128 => Key::SIntType { bit_width: 128 },
-    U128 => Key::UIntType { bit_width: 128 },
+    I0 => Key::SimpleType { ty: SimpleType::SInt(0) },
+    U0 => Key::SimpleType { ty: SimpleType::UInt(0) },
+    I1 => Key::SimpleType { ty: SimpleType::SInt(1) },
+    U1 => Key::SimpleType { ty: SimpleType::UInt(1) },
+    I8 => Key::SimpleType { ty: SimpleType::SInt(8) },
+    U8 => Key::SimpleType { ty: SimpleType::UInt(8) },
+    I16 => Key::SimpleType { ty: SimpleType::SInt(16) },
+    U16 => Key::SimpleType { ty: SimpleType::UInt(16) },
+    I32 => Key::SimpleType { ty: SimpleType::SInt(32) },
+    U32 => Key::SimpleType { ty: SimpleType::UInt(32) },
+    I64 => Key::SimpleType { ty: SimpleType::SInt(64) },
+    U64 => Key::SimpleType { ty: SimpleType::UInt(64) },
+    I128 => Key::SimpleType { ty: SimpleType::SInt(128) },
+    U128 => Key::SimpleType { ty: SimpleType::UInt(128) },
     TRUE => Key::TrueValue,
     FALSE => Key::FalseValue,
     EMPTY_STRING => Key::String { str: "" },
@@ -606,41 +618,51 @@ impl InternPool {
             ty: SimpleType::ISize,
         })
     }
+    // Assumes the type is present in the pool.
+    fn get_simple_type_unchecked(&self, ty: SimpleType) -> Index {
+        self.get_assume_present(&Key::SimpleType { ty })
+    }
     pub fn get_u0_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 0 })
+        self.get_simple_type_unchecked(SimpleType::UInt(0))
     }
     pub fn get_i0_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 0 })
+        self.get_simple_type_unchecked(SimpleType::SInt(0))
     }
     pub fn get_u1_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 1 })
+        self.get_simple_type_unchecked(SimpleType::UInt(1))
     }
     pub fn get_i1_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 1 })
+        self.get_simple_type_unchecked(SimpleType::SInt(1))
     }
     pub fn get_u8_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 8 })
+        self.get_simple_type_unchecked(SimpleType::UInt(8))
     }
     pub fn get_i8_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 8 })
+        self.get_simple_type_unchecked(SimpleType::SInt(8))
     }
     pub fn get_u16_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 16 })
+        self.get_simple_type_unchecked(SimpleType::UInt(16))
     }
     pub fn get_i16_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 16 })
+        self.get_simple_type_unchecked(SimpleType::SInt(16))
     }
     pub fn get_u32_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 32 })
+        self.get_simple_type_unchecked(SimpleType::UInt(32))
     }
     pub fn get_i32_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 32 })
+        self.get_simple_type_unchecked(SimpleType::SInt(32))
     }
     pub fn get_u64_type(&self) -> Index {
-        self.get_assume_present(&Key::UIntType { bit_width: 64 })
+        self.get_simple_type_unchecked(SimpleType::UInt(64))
     }
     pub fn get_i64_type(&self) -> Index {
-        self.get_assume_present(&Key::SIntType { bit_width: 64 })
+        self.get_simple_type_unchecked(SimpleType::SInt(64))
+    }
+    pub fn get_u128_type(&self) -> Index {
+        self.get_simple_type_unchecked(SimpleType::UInt(128))
+    }
+    pub fn get_i128_type(&self) -> Index {
+        self.get_simple_type_unchecked(SimpleType::SInt(128))
     }
 
     pub fn get_top_type(&self) -> Index {
@@ -700,8 +722,9 @@ impl InternPool {
                 | Key::SimpleType {
                     ty: SimpleType::ISize,
                 }
-                | Key::SIntType { .. }
-                | Key::UIntType { .. },
+                | Key::SimpleType {
+                    ty: SimpleType::SInt(_) | SimpleType::UInt(_),
+                },
             ) => Some(rhs),
             (
                 Key::SimpleType {
@@ -716,8 +739,9 @@ impl InternPool {
                 | Key::SimpleType {
                     ty: SimpleType::ISize,
                 }
-                | Key::SIntType { .. }
-                | Key::UIntType { .. },
+                | Key::SimpleType {
+                    ty: SimpleType::SInt(_) | SimpleType::UInt(_),
+                },
                 Key::SimpleType {
                     ty: SimpleType::ComptimeInt,
                 },
@@ -745,15 +769,6 @@ impl InternPool {
     pub fn to_mir_type(&self, index: Index, _ptr_size: TypeInfo) -> crate::mir::Type {
         use crate::mir::Type;
         match self.get_key(index) {
-            Key::UIntType { bit_width: bits } => {
-                let bits = bits as u32;
-
-                Type::from_bitsize_int(bits)
-            }
-            Key::SIntType { bit_width: bits } => {
-                let bits = bits as u32;
-                Type::from_bitsize_int(bits)
-            }
             Key::SimpleType { ty } => match ty {
                 SimpleType::F32 => Type::SinglePrecision,
                 SimpleType::F64 => Type::DoublePrecision,
@@ -762,6 +777,9 @@ impl InternPool {
                     todo!("void can't be turned into a mir type")
                 }
                 SimpleType::ISize | SimpleType::USize => Type::QWord,
+                SimpleType::UInt(bits) | SimpleType::SInt(bits) => {
+                    Type::from_bitsize_int(bits as u32)
+                }
                 SimpleType::Top | SimpleType::Bottom | SimpleType::ComptimeInt => {
                     panic!("{ty} can't be turned into a mir type")
                 }
@@ -782,11 +800,9 @@ impl InternPool {
 
     pub fn is_type_signed(&self, index: Index, _ptr_size: TypeInfo) -> bool {
         match self.get_key(index) {
-            Key::UIntType { .. } => false,
-            Key::SIntType { .. } => true,
             Key::SimpleType { ty } => match ty {
-                SimpleType::USize => false,
-                SimpleType::ISize => true,
+                SimpleType::USize | SimpleType::UInt(_) => false,
+                SimpleType::ISize | SimpleType::SInt(_) => true,
                 _ => false,
             },
             Key::PointerType { .. } => false,
@@ -799,23 +815,17 @@ impl InternPool {
 
     pub fn size_of_type(&self, index: Index, ptr_size: TypeInfo) -> TypeInfo {
         match self.get_key(index) {
-            Key::UIntType { bit_width: bits } => {
-                let bits = bits as u32;
-                TypeInfo {
-                    bitsize: bits,
-                    bitalign: bits.next_multiple_of(8).next_power_of_two(),
-                    signed: false,
-                }
-            }
-            Key::SIntType { bit_width: bits } => {
-                let bits = bits as u32;
-                TypeInfo {
-                    bitsize: bits,
-                    bitalign: bits.next_multiple_of(8).next_power_of_two(),
-                    signed: true,
-                }
-            }
             Key::SimpleType { ty } => match ty {
+                SimpleType::SInt(bits) => TypeInfo {
+                    bitsize: bits as u32,
+                    bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
+                    signed: true,
+                },
+                SimpleType::UInt(bits) => TypeInfo {
+                    bitsize: bits as u32,
+                    bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
+                    signed: false,
+                },
                 SimpleType::F32 => TypeInfo {
                     bitsize: 32,
                     bitalign: 32,
@@ -889,9 +899,13 @@ impl InternPool {
             crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
             crate::ast::Type::Integer(i) => self.get_or_insert({
                 if i.signed {
-                    Key::SIntType { bit_width: i.bits }
+                    Key::SimpleType {
+                        ty: SimpleType::SInt(i.bits),
+                    }
                 } else {
-                    Key::UIntType { bit_width: i.bits }
+                    Key::SimpleType {
+                        ty: SimpleType::UInt(i.bits),
+                    }
                 }
             }),
             crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@@ -930,9 +944,13 @@ impl InternPool {
             crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
             crate::ast::Type::Integer(i) => self.get_assume_present(&{
                 if i.signed {
-                    Key::SIntType { bit_width: i.bits }
+                    Key::SimpleType {
+                        ty: SimpleType::SInt(i.bits),
+                    }
                 } else {
-                    Key::UIntType { bit_width: i.bits }
+                    Key::SimpleType {
+                        ty: SimpleType::UInt(i.bits),
+                    }
                 }
             }),
             crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@@ -968,11 +986,11 @@ impl InternPool {
     pub fn as_ast1_type(&self, pointer_bits: u16, index: Index) -> crate::ast::Type {
         use crate::ast::Type;
         match self.get_key(index) {
-            Key::UIntType { bit_width: bits } => Type::Integer(IntegralType::new(false, bits)),
-            Key::SIntType { bit_width: bits } => Type::Integer(IntegralType::new(true, bits)),
             Key::SimpleType { ty } => match ty {
                 SimpleType::F32 => Type::Floating(crate::ast::FloatingType::Binary32),
                 SimpleType::F64 => Type::Floating(crate::ast::FloatingType::Binary64),
+                SimpleType::SInt(bits) => Type::Integer(IntegralType::new(true, bits)),
+                SimpleType::UInt(bits) => Type::Integer(IntegralType::new(false, bits)),
                 SimpleType::Bool => Type::Bool,
                 SimpleType::Void => Type::Void,
                 SimpleType::USize => Type::Integer(IntegralType::new(false, pointer_bits)),
@@ -1098,9 +1116,7 @@ impl InternPool {
                 self.create_item(Tag::NegativeInt, i)
             }
 
-            Key::UIntType { bit_width: bits } => self.create_item(Tag::UIntType, bits as u32),
-            Key::SIntType { bit_width: bits } => self.create_item(Tag::SIntType, bits as u32),
-            Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty as u8 as u32),
+            Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty.into()),
             Key::PointerType { pointee, flags } => {
                 let flags = flags.pack();
                 let i = self.extend_words([pointee.index() as u32, flags as u32]);
@@ -1230,17 +1246,11 @@ impl InternPool {
                 let bigint = BigInt::from_biguint(Sign::Plus, data);
                 Key::PositiveInt { bigint }
             }
-            Tag::SIntType => Key::SIntType {
-                bit_width: item.index as u16,
-            },
-            Tag::UIntType => Key::UIntType {
-                bit_width: item.index as u16,
-            },
             Tag::SimpleType => {
-                let ty = item.idx() as u8;
+                let ty = item.idx() as u32;
 
                 Key::SimpleType {
-                    ty: unsafe { core::mem::transmute::<u8, SimpleType>(ty) },
+                    ty: SimpleType::from(ty),
                 }
             }
             Tag::PointerType => {
@@ -1337,8 +1347,12 @@ impl InternPool {
 
     pub fn get_int_type(&mut self, signed: bool, bits: u16) -> Index {
         let key = match signed {
-            true => Key::SIntType { bit_width: bits },
-            false => Key::UIntType { bit_width: bits },
+            true => Key::SimpleType {
+                ty: SimpleType::SInt(bits),
+            },
+            false => Key::SimpleType {
+                ty: SimpleType::UInt(bits),
+            },
         };
 
         self.get_or_insert(key)
@@ -1535,7 +1549,7 @@ impl InternPool {
         ((index.index() as u32) < self.len()).then_some(index)
     }
 
-    fn get_item(&self, index: Index) -> Option<Item> {
+    pub(super) fn get_item(&self, index: Index) -> Option<Item> {
         self.check_bounds(index).map(|i| Item {
             tag: self.tags[i.index()],
             index: self.indices[i.index()],
@@ -1554,8 +1568,6 @@ impl InternPool {
                 | Key::ArrayType { .. }
                 | Key::PointerType { .. }
                 | Key::SimpleType { .. }
-                | Key::SIntType { .. }
-                | Key::UIntType { .. }
                 | Key::StructType { .. }
         )
     }

src/ast2/internable.rs

@@ -0,0 +1,34 @@
+use super::*;
+use core::hash::Hash;
+use std::hash::Hasher;
+
+// Types implementing this trait can be stored in the internpool.
+trait KeyTrait: Hash + Eq {
+    const TAG: Tag;
+    fn serialise(self, pool: &mut InternPool);
+    fn deserialise(index: Index, pool: &mut InternPool) -> Self;
+}
+
+impl KeyTrait for String {
+    const TAG: Tag = Tag::String;
+    fn serialise(self, pool: &mut InternPool) {
+        todo!()
+    }
+
+    fn deserialise(index: Index, pool: &mut InternPool) -> Self {
+        // let mut hasher = std::hash::DefaultHasher::new();
+        // core::any::TypeId::of::<Self>().hash(&mut hasher);
+        // let tag = hasher.finish() as u32;
+        let item = pool.get_item(index).unwrap();
+        assert_eq!(item.tag, Self::TAG);
+
+        let start = pool.words[item.idx()] as usize;
+        let len = pool.words[item.idx() + 1] as usize;
+        let str = unsafe {
+            let bytes = &pool.strings[start..start + len];
+            std::str::from_utf8_unchecked(bytes)
+        };
+
+        str.to_owned()
+    }
+}

src/ast2/mod.rs

@@ -2134,36 +2134,6 @@ where
     let ty_key = ip.get_key(ty);
     let signed = ip.is_type_signed(ty, AMD64_POINTER_TYPE_INFO);
     match ty_key {
-        intern::Key::SIntType { bit_width: bits } | intern::Key::UIntType { bit_width: bits } => {
-            let ty = IntegralType::new(signed, bits);
-            match ip.get_key(val) {
-                intern::Key::SIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                    bits: bits as _,
-                    ty,
-                }),
-                intern::Key::UIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                    bits: bits as _,
-                    ty,
-                }),
-                intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                    bits: bits as _,
-                    ty,
-                }),
-                intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                    bits: bits as _,
-                    ty,
-                }),
-                intern::Key::PositiveInt { bigint } => {
-                    ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
-                }
-                intern::Key::NegativeInt { bigint } => {
-                    ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
-                }
-                _ => {
-                    unreachable!()
-                }
-            }
-        }
         intern::Key::SimpleType { ty } => match ty {
             intern::SimpleType::F32 => match ip.get_key(val) {
                 intern::Key::F32 { bits } => {
@@ -2256,6 +2226,40 @@ where
             intern::SimpleType::Top | intern::SimpleType::Bottom => {
                 unreachable!()
             }
+            intern::SimpleType::UInt(bits) | intern::SimpleType::SInt(bits) => {
+                let ty = IntegralType::new(signed, bits);
+                match ip.get_key(val) {
+                    intern::Key::SIntSmall { bits } => {
+                        ComptimeNumber::Integral(ComptimeInt::Native {
+                            bits: bits as _,
+                            ty,
+                        })
+                    }
+                    intern::Key::UIntSmall { bits } => {
+                        ComptimeNumber::Integral(ComptimeInt::Native {
+                            bits: bits as _,
+                            ty,
+                        })
+                    }
+                    intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                        bits: bits as _,
+                        ty,
+                    }),
+                    intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                        bits: bits as _,
+                        ty,
+                    }),
+                    intern::Key::PositiveInt { bigint } => {
+                        ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
+                    }
+                    intern::Key::NegativeInt { bigint } => {
+                        ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                }
+            }
         },
         _ => {
             unreachable!()

src/common.rs

@@ -72,9 +72,7 @@ pub fn is_oct_digit(ch: char) -> bool {
 }
 
 pub fn is_hex_digit(ch: char) -> bool {
-    ('0'..='9').contains(&ch)
-        || ('a'..='f').contains(&ch)
-        || ('A'..='F').contains(&ch)
+    ('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
 }
 
 /// Trait for only yielding the next item in the Iterator if it tests true for some predicate
@@ -176,3 +174,19 @@ pub fn from_lo_hi_dwords(lo: u32, hi: u32) -> u64 {
 pub fn into_lo_hi_dwords(qword: u64) -> (u32, u32) {
     (qword as u32, (qword >> 32) as u32)
 }
+
+pub fn u32_as_u16_slice(value: &u32) -> &[u16; 2] {
+    // SAFETY: This is safe because u32 is guaranteed to be 4 bytes and
+    // we are creating a slice of 2 u16s which is also 4 bytes.
+    unsafe { &*(value as *const u32 as *const [u16; 2]) }
+}
+
+// we can't transform any &[u16; 2] into a &u32 because of alignment guarantees
+pub fn u32_from_u16_slice(slice: &[u16; 2]) -> u32 {
+    let mut out = 0u32;
+    unsafe {
+        core::ptr::copy_nonoverlapping(slice.as_ptr(), &raw mut out as *mut u16, 2);
+    }
+
+    out
+}