.gitignore

@@ -1,3 +1,2 @@
 /target
 /Cargo.lock
-/.direnv/

Cargo.toml

@@ -1,9 +1,3 @@
-[workspace]
-resolver = "3"
-members = [
-    "crates/lexer"
-]
-
 [package]
 name = "compiler"
 version = "0.1.0"
@@ -22,16 +16,5 @@ paste = "1.0.15"
 petgraph = "0.6.5"
 thiserror = "1.0.63"
 unicode-xid = "0.2.4"
-tracing = "0.1.41"
 
-werkzeug = { path = "../../rust/werkzeug" }
+werkzeug = { path = "../../rust/werkzeug" }
-
-
-[workspace.dependencies]
-unicode-xid = "0.2.4"
-tracing = "0.1.41"
-thiserror = "1.0.63"
-itertools = "0.13.0"
-
-werkzeug = { path = "../../rust/werkzeug" }
-trie = { path = "../../rust/trie" }

crates/lexer/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "lexer"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-tracing = { workspace = true }
-werkzeug = { workspace = true }
-thiserror = { workspace = true }
-itertools = { workspace = true }
-trie = { workspace = true }
-unicode-xid = { workspace = true }

crates/lexer/src/complex_tokens.rs

@@ -1,394 +0,0 @@
-use crate::{Source, Token, is_things};
-use itertools::Itertools;
-use werkzeug::iter::{FallibleMapIter, NextIf};
-
-#[derive(Debug, thiserror::Error, PartialEq, Eq)]
-pub enum Error {
-    #[error("{0}")]
-    StringError(String),
-    #[error("Exp part of floating constant had no digits.")]
-    FloatingConstantExpPartNoDigit,
-    #[error("constant cannot start with leading underscore '_'.")]
-    NumericalConstantDigitLeadingUnderscore,
-    #[error("Expected digit here for constant.")]
-    NumericalConstantDigitNoDigit,
-    #[error("Expected digit here for integer constant.")]
-    IntegralTypeExpectedDigit,
-    #[error("Floating constant has invalid trailing type.")]
-    FloatingConstantInvalidTrailingType,
-    #[error("Invalid token.")]
-    InvalidToken,
-    #[error("Identifier starts with invalid character.")]
-    ExpectedIdStartForIdentifier,
-    #[error("Unknown suffix in constant.")]
-    NumericalConstantUnknownSuffix,
-}
-
-type Result<T> = core::result::Result<T, Error>;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Radix {
-    Hex,
-    Bin,
-    Dec,
-    Oct,
-}
-
-impl Radix {
-    #[allow(unused)]
-    /// must be called with one of `['b','x','d','o']`
-    unsafe fn from_char_unchecked(c: char) -> Self {
-        match c.to_ascii_lowercase() {
-            'o' => Self::Oct,
-            'b' => Self::Bin,
-            'x' => Self::Hex,
-            'd' => Self::Dec,
-            _ => unreachable!(),
-        }
-    }
-    fn from_char(c: char) -> Option<Self> {
-        match c.to_ascii_lowercase() {
-            'o' => Some(Self::Oct),
-            'b' => Some(Self::Bin),
-            'x' => Some(Self::Hex),
-            'd' => Some(Self::Dec),
-            _ => None,
-        }
-    }
-
-    #[allow(unused)]
-    pub fn radix(self) -> u8 {
-        match self {
-            Radix::Hex => 16,
-            Radix::Bin => 2,
-            Radix::Oct => 8,
-            Radix::Dec => 10,
-        }
-    }
-    fn to_token(self) -> Token {
-        match self {
-            Radix::Hex => Token::IntegerHexConstant,
-            Radix::Bin => Token::IntegerBinConstant,
-            Radix::Oct => Token::IntegerOctConstant,
-            Radix::Dec => Token::IntegerConstant,
-        }
-    }
-    pub fn from_token(token: Token) -> Option<Self> {
-        match token {
-            Token::IntegerHexConstant => Some(Radix::Hex),
-            Token::IntegerBinConstant => Some(Radix::Bin),
-            Token::IntegerOctConstant => Some(Radix::Oct),
-            Token::IntegerConstant => Some(Radix::Dec),
-            _ => None,
-        }
-    }
-    pub fn map_digit(self, c: char) -> u8 {
-        match self {
-            Radix::Hex => match c {
-                '0'..='9' => c as u8 - b'0',
-                'a'..='f' => 10 + c as u8 - b'a',
-                'A'..='F' => 10 + c as u8 - b'A',
-                _ => unreachable!(),
-            },
-            Radix::Bin => match c {
-                '0'..='1' => c as u8 - b'0',
-                _ => unreachable!(),
-            },
-            Radix::Dec => match c {
-                '0'..='9' => c as u8 - b'0',
-                _ => unreachable!(),
-            },
-            Radix::Oct => match c {
-                '0'..='7' => c as u8 - b'0',
-                _ => unreachable!(),
-            },
-        }
-    }
-    pub fn folding_method(self) -> fn(u64, char) -> u64 {
-        match self {
-            Radix::Hex => {
-                fn fold(acc: u64, c: char) -> u64 {
-                    let digit = match c {
-                        '0'..='9' => c as u8 - b'0',
-                        'a'..='f' => c as u8 - b'a',
-                        'A'..='F' => c as u8 - b'A',
-                        _ => unreachable!(),
-                    };
-                    acc * 16 + digit as u64
-                }
-                fold
-            }
-            Radix::Bin => {
-                fn fold(acc: u64, c: char) -> u64 {
-                    let digit = match c {
-                        '0'..='1' => c as u8 - b'0',
-                        _ => unreachable!(),
-                    };
-                    acc * 2 + digit as u64
-                }
-                fold
-            }
-            Radix::Dec => {
-                fn fold(acc: u64, c: char) -> u64 {
-                    let digit = match c {
-                        '0'..='9' => c as u8 - b'0',
-                        _ => unreachable!(),
-                    };
-                    acc * 10 + digit as u64
-                }
-                fold
-            }
-            Radix::Oct => {
-                fn fold(acc: u64, c: char) -> u64 {
-                    let digit = match c {
-                        '0'..='7' => c as u8 - b'0',
-                        _ => unreachable!(),
-                    };
-                    acc * 8 + digit as u64
-                }
-                fold
-            }
-        }
-    }
-    pub fn is_digit(self) -> fn(char) -> bool {
-        match self {
-            Radix::Hex => crate::is_things::is_hex_digit,
-            Radix::Bin => crate::is_things::is_bin_digit,
-            Radix::Oct => crate::is_things::is_oct_digit,
-            Radix::Dec => crate::is_things::is_digit,
-        }
-    }
-}
-
-// where DIGIT is defined by radix:
-// DIGITS <-
-//     if allow_leading_underscore: `_`* DIGIT (DIGIT|`_`)*
-//     else: DIGIT (DIGIT|`_`)*
-fn parse_digit_part(
-    source: &mut Source,
-    allow_leading_underscore: bool,
-    radix: Radix,
-) -> Result<()> {
-    let is_digit = radix.is_digit();
-
-    if allow_leading_underscore {
-        let _underscore = source.take_while_ref(|&c| c == '_').count();
-    }
-    let _need_digit = source.next_if(|&c| is_digit(c)).ok_or_else(|| {
-        if source.peek() == Some(&'_') {
-            Error::NumericalConstantDigitLeadingUnderscore
-        } else {
-            Error::NumericalConstantDigitNoDigit
-        }
-    })?;
-    let _rest = source.take_while_ref(|&c| is_digit(c) || c == '_').count();
-
-    Ok(())
-}
-
-// IntegralType <-
-//     ( 'u' | 'i' ) DIGITS+
-fn try_parse_integral_type(source: &mut Source) -> Result<Option<()>> {
-    if !source.next_if(|&c| c == 'u' || c == 'i').is_some() {
-        return Ok(None);
-    }
-
-    if source
-        .take_while_ref(|&c| crate::is_things::is_digit(c))
-        .count()
-        <= 0
-    {
-        return Err(Error::IntegralTypeExpectedDigit);
-    };
-
-    Ok(Some(()))
-}
-
-// returns `Err(E)` if it failed to parse.
-// returns `Ok(None)` if no exp part was found.
-// returns `Ok(Some(()))` if an exp part was found and parsed.
-//
-// EXP_PART <-
-//     (`e`|`E`) (`-`|`+`)? DEC_DIGITS
-fn try_parse_exp_part(source: &mut Source) -> Result<Option<()>> {
-    if source.next_if(|&c| c.to_ascii_lowercase() == 'e').is_some() {
-        let _sign = source.next_if(|&c| c == '-' || c == '+');
-        if source
-            .take_while_ref(|&c| crate::is_things::is_digit(c))
-            .count()
-            .lt(&1)
-        {
-            // need digits following exp notation
-            Err(Error::FloatingConstantExpPartNoDigit)
-        } else {
-            Ok(Some(()))
-        }
-    } else {
-        Ok(None)
-    }
-}
-
-// CONSTANT <-
-//    DEC_DIGITS IntegralType?
-//    `0x` HEX_DIGITS IntegralType?
-//    `0b` BIN_DIGITS IntegralType?
-//    `0o` OCT_DIGITS IntegralType?
-//    DEC_DIGITS FloatingType?
-//    `.` DEC_DIGITS EXP_PART? FloatingType?
-//    DEC_DIGITS `.` DEC_DIGITS? EXP_PART? FloatingType?
-fn parse_constant_inner(source: &mut Source) -> Result<Token> {
-    let zero = source.next_if(|&c| c == '0').is_some();
-
-    let radix = zero
-        .then(|| source.next_if_map(|c| Radix::from_char(c)))
-        .flatten();
-
-    if let Some(radix) = radix {
-        parse_digit_part(source, false, radix)?;
-        if source.peek().map(|&c| c == 'u' || c == 'i') == Some(true) {
-            try_parse_integral_type(source)?;
-        }
-        return Ok(radix.to_token());
-    }
-
-    // if zero: `_`* DIGIT (DIGIT|`_`)*
-    // else: DIGIT (DIGIT|`_`)*
-    _ = match parse_digit_part(source, zero, Radix::Dec) {
-        Ok(_) => Ok(()),
-        Err(Error::NumericalConstantDigitNoDigit) if zero => Ok(()),
-        Err(e) => Err(e),
-    }?;
-
-    if let Some(_) = source.try_map_iter_if(|source| try_parse_integral_type(source))? {
-        return Ok(Token::IntegerConstant);
-    }
-
-    let dot = source.next_if(|&c| c == '.').is_some();
-
-    if dot {
-        parse_digit_part(source, false, Radix::Dec)?;
-    }
-
-    // parse exp notation
-    let exp = try_parse_exp_part(source)?.is_some();
-
-    // trailing FloatingType?
-    let floating = if source.next_if(|&c| c == 'f').is_some() {
-        let digits = source.next_tuple::<(char, char)>();
-        if !(digits == Some(('6', '4')) || digits == Some(('3', '2'))) {
-            // need either f64 or f32 here!
-            return Err(Error::FloatingConstantInvalidTrailingType);
-        }
-        true
-    } else {
-        false
-    };
-
-    let token = match (dot, exp, floating) {
-        (false, false, false) => Token::IntegerConstant,
-        (true, false, _) => Token::DotFloatingConstant,
-        (true, true, _) => Token::DotFloatingExpConstant,
-        (false, true, _) => Token::FloatingExpConstant,
-        (false, _, _) => Token::FloatingConstant,
-    };
-
-    Ok(token)
-}
-
-pub(crate) fn parse_constant(source: &mut Source) -> Result<Token> {
-    let constant = parse_constant_inner(source)?;
-    // char following a constant must not be id_continue
-    if source
-        .peek()
-        .map(|&c| crate::is_things::is_id_continue(c))
-        .unwrap_or(false)
-    {
-        return Err(Error::NumericalConstantUnknownSuffix);
-    }
-
-    Ok(constant)
-}
-
-pub(crate) fn parse_string_or_char_constant(source: &mut Source) -> Result<Token> {
-    let quote = source
-        .next_if(|&c| c == '"' || c == '\'')
-        .ok_or(Error::InvalidToken)?;
-
-    let is_char = quote == '\'';
-
-    let mut escaped = false;
-    let mut closed = false;
-
-    while let Some(c) = source.next() {
-        if escaped {
-            // accept any escaped char
-            escaped = false;
-            continue;
-        }
-        if c == '\\' {
-            escaped = true;
-            continue;
-        }
-        if c == quote {
-            closed = true;
-            break;
-        }
-    }
-
-    if !closed {
-        return Err(Error::StringError("Unterminated string/char.".into()));
-    }
-
-    if is_char {
-        Ok(Token::CharConstant)
-    } else {
-        Ok(Token::StringConstant)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    fn make_source(s: &'_ str) -> Source<'_> {
-        s.chars().peekable().into()
-    }
-
-    #[test]
-    fn parse_constant_number() {
-        assert_eq!(
-            parse_constant(&mut make_source("0x1A3F_u32")),
-            Ok(Token::IntegerHexConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("13f32")),
-            Ok(Token::FloatingConstant)
-        );
-
-        assert_eq!(
-            parse_constant(&mut make_source("0b1011_0010i16")),
-            Ok(Token::IntegerBinConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("0o755u8")),
-            Ok(Token::IntegerOctConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("42i64")),
-            Ok(Token::IntegerConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("3.14f64")),
-            Ok(Token::DotFloatingConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("2.71828e0f32")),
-            Ok(Token::DotFloatingExpConstant)
-        );
-        assert_eq!(
-            parse_constant(&mut make_source("22e23")),
-            Ok(Token::FloatingExpConstant)
-        );
-    }
-}

crates/lexer/src/lib.rs

@@ -1,686 +0,0 @@
-#![feature(slice_swap_unchecked, iter_collect_into)]
-
-mod is_things {
-    /// True if `c` is considered a whitespace according to Rust language definition.
-    /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
-    /// for definitions of these classes.
-    pub fn is_whitespace(c: char) -> bool {
-        // This is Pattern_White_Space.
-        //
-        // Note that this set is stable (ie, it doesn't change with different
-        // Unicode versions), so it's ok to just hard-code the values.
-
-        matches!(
-            c,
-            // Usual ASCII suspects
-            '\u{0009}'   // \t
-        | '\u{000A}' // \n
-        | '\u{000B}' // vertical tab
-        | '\u{000C}' // form feed
-        | '\u{000D}' // \r
-        | '\u{0020}' // space
-
-    // NEXT LINE from latin1
-        | '\u{0085}'
-
-    // Bidi markers
-        | '\u{200E}' // LEFT-TO-RIGHT MARK
-        | '\u{200F}' // RIGHT-TO-LEFT MARK
-
-    // Dedicated whitespace characters from Unicode
-        | '\u{2028}' // LINE SEPARATOR
-        | '\u{2029}' // PARAGRAPH SEPARATOR
-        )
-    }
-
-    /// True if `c` is valid as a first character of an identifier.
-    /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
-    /// a formal definition of valid identifier name.
-    pub fn is_id_start(c: char) -> bool {
-        // This is XID_Start OR '_' (which formally is not a XID_Start).
-        c == '_' || c == '-' || unicode_xid::UnicodeXID::is_xid_start(c)
-    }
-
-    /// True if `c` is valid as a non-first character of an identifier.
-    /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
-    /// a formal definition of valid identifier name.
-    pub fn is_id_continue(c: char) -> bool {
-        unicode_xid::UnicodeXID::is_xid_continue(c) || c == '-'
-    }
-
-    /// The passed string is lexically an identifier.
-    pub fn is_ident(string: &str) -> bool {
-        let mut chars = string.chars();
-        if let Some(start) = chars.next() {
-            is_id_start(start) && chars.all(is_id_continue)
-        } else {
-            false
-        }
-    }
-
-    #[expect(dead_code)]
-    pub fn is_digit(ch: char) -> bool {
-        ('0'..='9').contains(&ch)
-    }
-
-    #[expect(dead_code)]
-    pub fn is_bin_digit(ch: char) -> bool {
-        ch == '0' || ch == '1'
-    }
-
-    #[expect(dead_code)]
-    pub fn is_nonzero_digit(ch: char) -> bool {
-        ('1'..='9').contains(&ch)
-    }
-
-    #[expect(dead_code)]
-    pub fn is_oct_digit(ch: char) -> bool {
-        ('0'..='7').contains(&ch)
-    }
-
-    #[expect(dead_code)]
-    pub fn is_hex_digit(ch: char) -> bool {
-        ('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
-    }
-}
-
-macro_rules! tokens {
-    ($vis:vis $ty_name:ident:
-        {
-            $($name2:ident),*
-        },
-        {
-            $($name:ident => $lexeme:literal),*
-        }) => {
-
-            #[allow(dead_code)]
-            #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
-            $vis enum $ty_name {
-                $($name,
-                )*
-                $($name2,)*
-            }
-
-            impl ::core::fmt::Display for $ty_name {
-                fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
-                    match self {
-                        $(Self::$name => write!(f, "{}", $lexeme),)*
-                        $(Self::$name2 => write!(f, "<{}>", stringify!($name2))),*
-                    }
-                }
-            }
-
-            #[allow(dead_code)]
-            impl $ty_name {
-                $vis fn lexeme(&self) -> Option<&'static str> {
-                    match self {
-                        $(Self::$name => Some($lexeme),)*
-                        $(Self::$name2 => None),*
-                    }
-                }
-
-                /// returns the number of chars in this lexeme
-                $vis fn lexeme_len(&self) -> usize {
-                    self.lexeme().map(|lexeme|lexeme.chars().count()).unwrap_or(0)
-                }
-
-                /// returns the number of chars in this lexeme
-                $vis fn lexeme_len_utf8(&self) -> usize {
-                    self.lexeme().map(|lexeme|lexeme.len()).unwrap_or(0)
-                }
-
-                $vis fn maybe_ident(&self) -> bool {
-                    self.lexeme().map(|lexeme| crate::is_things::is_ident(lexeme)).unwrap_or(false)
-                }
-
-                $vis fn lexemes() -> &'static [(Self, &'static str)] {
-                    &[
-                        $((Self::$name, $lexeme)),*
-                    ]
-                }
-            }
-        };
-}
-
-tokens!(pub Token: {
-    Eof,
-    ParseError,
-    // Marker Token for any Comment
-    Comment,
-    DocComment,
-        // Marker Token for any pre-processing directive
-    CharConstant,
-    IntegerConstant,
-    IntegerHexConstant,
-    IntegerBinConstant,
-    IntegerOctConstant,
-    FloatingConstant,
-    FloatingExpConstant,
-    DotFloatingConstant,
-    DotFloatingExpConstant,
-    StringConstant,
-    Ident
-},
-    // Lexical Tokens:
-    {
-        SlashSlash => "//",
-        SlashSlashSlash => "///",
-        // SlashStar => "/*",
-        // SlashStarStar => "/**",
-        //StarSlash => "*/",
-        // Punctuation:
-        OpenParens => "(",
-        CloseParens => ")",
-        OpenBrace => "{",
-        CloseBrace => "}",
-        OpenSquareBracket => "[",
-        CloseSquareBracket => "]",
-        Semi => ";",
-        Comma => ",",
-        Elipsis3 => "...",
-        Elipsis2 => "..",
-        Colon => ":",
-        Equal => "=",
-        // Keywords:
-        Void => "void",
-        Bool => "bool",
-        F32 => "f32",
-        F64 => "f64",
-        ISize => "isize",
-        USize => "usize",
-        Const => "const",
-        Volatile => "volatile",
-        Noalias => "noalias",
-        Fn => "fn",
-        Let => "let",
-        Var => "var",
-        If => "if",
-        As => "as",
-        Else => "else",
-        Return => "return",
-        Struct => "struct",
-        Type => "type",
-        Union => "union",
-        Enum => "enum",
-        Packed => "packed",
-        Extern => "extern",
-        Pub => "pub",
-        // Operators
-        Dot => ".",
-        MinusGreater => "->",
-        Bang => "!",
-        Tilde => "~",
-        Plus => "+",
-        PlusPlus => "++",
-        Minus => "-",
-        MinusMinus => "--",
-        Star => "*",
-        Slash => "/",
-        Percent => "%",
-        Less => "<",
-        Greater => ">",
-        LessEqual => "<=",
-        GreaterEqual => ">=",
-        EqualEqual => "==",
-        BangEqual => "!=",
-        PipePipe => "||",
-        AmpersandAmpersand => "&&",
-        Ampersand => "&",
-        Caret => "^",
-        Pipe => "|",
-        LessLess => "<<",
-        GreaterGreater => ">>",
-        Question => "?",
-        PlusEqual => "+=",
-        MinusEqual => "-=",
-        StarEqual => "*=",
-        SlashEqual => "/=",
-        PercentEqual => "%=",
-        AmpersandEqual => "&=",
-        PipeEqual => "|=",
-        CaretEqual => "^=",
-        LessLessEqual => "<<=",
-        GreaterGreaterEqual => ">>="
-    });
-
-impl Token {
-    pub fn is_assignment_op(self) -> bool {
-        match self {
-            Token::PlusEqual
-            | Token::MinusEqual
-            | Token::StarEqual
-            | Token::SlashEqual
-            | Token::PercentEqual
-            | Token::PipeEqual
-            | Token::CaretEqual
-            | Token::AmpersandEqual
-            | Token::LessLessEqual
-            | Token::GreaterGreaterEqual
-            | Token::Equal => true,
-            _ => false,
-        }
-    }
-    pub fn is_unary_op(self) -> bool {
-        match self {
-            Token::Plus | Token::Minus | Token::Star | Token::Ampersand | Token::Bang => true,
-            _ => false,
-        }
-    }
-    pub fn is_binary_op(self) -> bool {
-        match self {
-            Token::Star
-            | Token::Slash
-            | Token::Percent
-            | Token::Pipe
-            | Token::Ampersand
-            | Token::Caret
-            | Token::Plus
-            | Token::Minus
-            | Token::PipePipe
-            | Token::AmpersandAmpersand
-            | Token::BangEqual
-            | Token::EqualEqual
-            | Token::Less
-            | Token::Greater
-            | Token::LessEqual
-            | Token::GreaterEqual
-            | Token::LessLess
-            | Token::GreaterGreater => true,
-            _ => false,
-        }
-    }
-}
-
-/// A list of lexemes used by the `LexemeParser`.
-/// `lexemes` contains every token that has a defined lexeme, such as `fn`, `f32`, `const`, etc.
-/// The `LexemeList` keeps track of two offsets into the `lexemes` array,
-/// splitting it into three windows:
-/// - [0, start_candidates) - tokens that are still being considered for parsing
-/// - [start_candidates, end_candidates) - the tokens which this lexeme matches
-/// - [end_candidates, len) - tokens that have been filtered out and are no longer considered
-/// On each iteration of the parsing loop, the remaining tokens are matched
-/// against the next character and, if they match completely, are swapped into
-/// the candidates window, or swapped to the end if they don't.
-struct LexemeList {
-    lexemes: Box<[Token]>,
-    start_candidates: usize,
-    end_candidates: usize,
-    filtered: Vec<(usize, FilterResult)>,
-}
-
-enum FilterResult {
-    Remove,
-    Candidate,
-}
-
-impl LexemeList {
-    fn new() -> Self {
-        let lexemes = Token::lexemes()
-            .iter()
-            .map(|(tok, _)| tok.clone())
-            .collect::<Box<_>>();
-
-        Self {
-            start_candidates: lexemes.len(),
-            end_candidates: lexemes.len(),
-            lexemes,
-            filtered: Vec::new(),
-        }
-    }
-
-    fn clear(&mut self) {
-        self.start_candidates = self.lexemes.len();
-        self.end_candidates = self.lexemes.len();
-    }
-
-    fn remaining(&self) -> &[Token] {
-        &self.lexemes[0..self.start_candidates]
-    }
-
-    fn candidates(&self) -> &[Token] {
-        &self.lexemes[self.start_candidates..self.end_candidates]
-    }
-
-    fn step(&mut self, ch: char, pos: usize) {
-        // smartly reuse allocation for `filtered`
-        // truly one of the premature optimizations.
-        // but it just feels good, innit?
-        let mut filtered = core::mem::take(&mut self.filtered);
-
-        self.remaining()
-            .iter()
-            .enumerate()
-            .filter_map(|(i, tok)| {
-                let bytes = tok.lexeme().unwrap().as_bytes();
-                // SAFETY: all tokens in `self.remaining()` are lexical tokens, and
-                // they are all valid ascii
-                let c = unsafe {
-                    // TODO: maybe keep a list of `Char<'_>`s around in order to
-                    // support fully utf8 tokens?
-                    char::from_u32_unchecked(bytes[pos] as u32)
-                };
-                match c == ch {
-                    false => Some((i, FilterResult::Remove)),
-                    true if bytes.len() <= pos + 1 => Some((i, FilterResult::Candidate)),
-                    true => None,
-                }
-            })
-            .collect_into(&mut filtered);
-
-        // iterate in reverse order so that we can safely swap elements
-        // drain here so that we can possibly reuse the `filtered` Vec allcoation
-        filtered.drain(..).rev().for_each(|(i, f)| {
-            match f {
-                // for candidates, swap the candidate with the last remaining
-                // token, then dec `start_candidates`
-                FilterResult::Candidate => {
-                    // SAFETY: we know that `i` and `self.start_candidates - 1`
-                    // are both valid indices: `self.start_candidates` starts at
-                    // the end and each time it is decremented, one more element
-                    // is removed from the front, so that as long as an element
-                    // is remaining, `self.start_candidates` is always greater
-                    // than 0.
-                    // the order of the remaining elements is not meaningfully
-                    // impacted because we only ever swap with elements after
-                    // `i`, and `i` is the greatest index we will touch.
-                    unsafe {
-                        self.lexemes.swap_unchecked(i, self.start_candidates - 1);
-                        self.start_candidates = self.start_candidates.saturating_sub(1);
-                    }
-                }
-                // for removes, swap the last candidate with the last remainign
-                // token, then swap the remove with the last candidate, then dec
-                // `end_candidates` and `start_candidates`
-                FilterResult::Remove => {
-                    unsafe {
-                        // in the case that `start_candidates` ==
-                        // `end_candidates`, no swap happens and that's fine.
-                        // remove this: v
-                        //           [a,b,c][d,e,f][g,h,i]
-                        // swap these:    ^      ^
-                        //           [a,b,f][d,e,c][g,h,i]
-                        // swap these:  ^        ^
-                        //           [a,c,f][d,e,b][g,h,i]
-                        // decrement both counters:
-                        //           [a,c][f,d,e][b,g,h,i]
-                        self.lexemes
-                            .swap_unchecked(self.start_candidates - 1, self.end_candidates - 1);
-                        self.lexemes.swap_unchecked(i, self.end_candidates - 1);
-                        self.start_candidates = self.start_candidates.saturating_sub(1);
-                        self.end_candidates = self.end_candidates.saturating_sub(1);
-                    }
-                }
-            }
-        });
-
-        // replace `filtered`
-        self.filtered = filtered;
-    }
-}
-
-/// Helper type for parsing tokens that have a defined lexeme, such as `fn`,
-/// `f32`, `const`, etc. Tokens with variable lexemes, such as primitive
-/// integral types, constants or identifiers are not parsed by this.
-pub struct LexemeParser {
-    lexemes: LexemeList,
-    len: usize,
-}
-
-impl LexemeParser {
-    pub fn new() -> Self {
-        Self {
-            lexemes: LexemeList::new(),
-            len: 0,
-        }
-    }
-
-    pub fn parse(&mut self, mut tokens: impl Iterator<Item = char>) -> Option<Token> {
-        self.lexemes.clear();
-        loop {
-            let Some(ch) = tokens.next() else {
-                break;
-            };
-
-            if crate::is_things::is_whitespace(ch) {
-                break;
-            }
-
-            self.lexemes.step(ch, self.len);
-            if self.lexemes.remaining().is_empty() {
-                break;
-            }
-        }
-        self.lexemes.candidates().last().copied()
-    }
-}
-
-use itertools::Itertools;
-use trie::Tree;
-
-#[derive(Debug, Clone, Copy)]
-struct CountingIterator<I: Iterator> {
-    iter: I,
-    count: usize,
-}
-
-impl<I: Iterator> From<I> for CountingIterator<I> {
-    fn from(iter: I) -> Self {
-        Self { iter, count: 0 }
-    }
-}
-
-impl<I: Iterator<Item = char>> Iterator for CountingIterator<I> {
-    type Item = I::Item;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.iter.next().inspect(|c| self.count += c.len_utf8())
-    }
-}
-
-impl<I: Iterator> CountingIterator<I> {
-    pub(crate) fn offset(&self) -> usize {
-        self.count
-    }
-}
-
-impl<I: Iterator> core::ops::Deref for CountingIterator<I> {
-    type Target = I;
-
-    fn deref(&self) -> &Self::Target {
-        &self.iter
-    }
-}
-
-impl<I: Iterator> core::ops::DerefMut for CountingIterator<I> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.iter
-    }
-}
-
-type Source<'a> = CountingIterator<core::iter::Peekable<core::str::Chars<'a>>>;
-
-pub struct TokenIterator<'a> {
-    trie: Tree<char, Token>,
-    source: &'a str,
-    offset: usize,
-}
-
-impl<'a> TokenIterator<'a> {
-    pub fn new(source: &'a str) -> Self {
-        let mut trie = Tree::new();
-
-        for (token, token_str) in Token::lexemes() {
-            trie.insert(token_str.chars(), *token);
-        }
-
-        Self {
-            trie,
-            source,
-            offset: 0,
-        }
-    }
-
-    fn peekable_source(&self) -> Source<'a> {
-        CountingIterator::from(self.source[self.offset..].chars().peekable())
-    }
-
-    fn parse(&mut self) -> Option<Token> {
-        let mut iter = CountingIterator::from(self.source[self.offset..].chars());
-
-        match self.trie.get_closest(&mut iter) {
-            Some(token) => {
-                // skip the peeked item
-                self.offset += token.lexeme_len();
-                Some(*token)
-            }
-            None => None,
-        }
-    }
-
-    fn skip_whitespaces(&mut self) -> usize {
-        self.skip_while(is_things::is_whitespace)
-    }
-
-    fn skip(&mut self, mut n: usize) -> usize {
-        self.skip_while(|_| {
-            n -= 1;
-            n > 0
-        })
-    }
-
-    fn skip_while(&mut self, mut pred: impl FnMut(char) -> bool) -> usize {
-        let mut count = 0;
-        loop {
-            let Some(c) = self.source[self.offset..].chars().next() else {
-                break;
-            };
-
-            if pred(c) {
-                self.offset += c.len_utf8();
-                count += c.len_utf8();
-                continue;
-            } else {
-                break;
-            }
-        }
-        count
-    }
-}
-
-impl<'a> Iterator for TokenIterator<'a> {
-    type Item = (Token, &'a str);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // skip whitespace
-        self.skip_whitespaces();
-
-        let start = self.offset;
-
-        let mut source = self.peekable_source();
-        let mut cursor = self.peekable_source();
-        let token = match cursor.next() {
-            Some('0'..='9') => {
-                let token = complex_tokens::parse_constant(&mut source).ok()?;
-                self.offset += source.offset();
-
-                Some((token, &self.source[start..self.offset]))
-            }
-            Some('.') if cursor.next().map_or(false, is_things::is_digit) => {
-                let token = complex_tokens::parse_constant(&mut source).ok()?;
-                self.offset += source.offset();
-
-                Some((token, &self.source[start..self.offset]))
-            }
-            Some('\'' | '"') => {
-                let token = complex_tokens::parse_string_or_char_constant(&mut source).ok()?;
-                self.offset += source.offset();
-
-                Some((token, &self.source[start..self.offset]))
-            }
-            _ => match self.parse().map(|tok| match tok {
-                Token::SlashSlash => {
-                    self.skip_while(|c| c == '\n');
-                    (Token::Comment)
-                }
-                Token::SlashSlashSlash => {
-                    self.skip_while(|c| c == '\n');
-                    (Token::DocComment)
-                }
-                _ => tok,
-            }) {
-                Some(tok) => {
-                    if tok.maybe_ident() && self.skip_while(|c| is_things::is_id_continue(c)) > 0 {
-                        Some((Token::Ident, &self.source[start..self.offset]))
-                    } else {
-                        Some((tok, &self.source[start..self.offset]))
-                    }
-                }
-                None => {
-                    if self
-                        .peekable_source()
-                        .next()
-                        .map_or(false, |c| is_things::is_id_start(c))
-                    {
-                        self.skip(1);
-                        self.skip_while(|c| is_things::is_id_continue(c));
-                        Some((Token::Ident, &self.source[start..self.offset]))
-                    } else {
-                        None
-                    }
-                }
-            },
-        };
-
-        token
-    }
-}
-
-mod complex_tokens;
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_iterator() {
-        let tokens = "fn let void+++(++bool)";
-        let mut lexer = TokenIterator::new(&tokens);
-        assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
-        assert_eq!(lexer.next(), Some((Token::Let, "let")));
-        assert_eq!(lexer.next(), Some((Token::Void, "void")));
-        assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
-        assert_eq!(lexer.next(), Some((Token::Plus, "+")));
-        assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
-        assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
-        assert_eq!(lexer.next(), Some((Token::Bool, "bool")));
-        assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
-        assert_eq!(lexer.next(), None);
-    }
-
-    #[test]
-    fn complex_iterator() {
-        let tokens = "fn my-function(x: i32, y: f32) -> f32 { return x + y; }";
-        let mut lexer = TokenIterator::new(&tokens);
-        assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "my-function")));
-        assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "x")));
-        assert_eq!(lexer.next(), Some((Token::Colon, ":")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "i32")));
-        assert_eq!(lexer.next(), Some((Token::Comma, ",")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "y")));
-        assert_eq!(lexer.next(), Some((Token::Colon, ":")));
-        assert_eq!(lexer.next(), Some((Token::F32, "f32")));
-        assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
-        assert_eq!(lexer.next(), Some((Token::MinusGreater, "->")));
-        assert_eq!(lexer.next(), Some((Token::F32, "f32")));
-        assert_eq!(lexer.next(), Some((Token::OpenBrace, "{")));
-        assert_eq!(lexer.next(), Some((Token::Return, "return")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "x")));
-        assert_eq!(lexer.next(), Some((Token::Plus, "+")));
-        assert_eq!(lexer.next(), Some((Token::Ident, "y")));
-        assert_eq!(lexer.next(), Some((Token::Semi, ";")));
-        assert_eq!(lexer.next(), Some((Token::CloseBrace, "}")));
-        assert_eq!(lexer.next(), None);
-    }
-}

flake.lock

@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1757745802,
+        "lastModified": 1753939845,
-        "narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=",
+        "narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1",
+        "rev": "94def634a20494ee057c76998843c015909d6311",
         "type": "github"
       },
       "original": {
@@ -62,11 +62,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1757989933,
+        "lastModified": 1754102567,
-        "narHash": "sha256-9cpKYWWPCFhgwQTww8S94rTXgg8Q8ydFv9fXM6I8xQM=",
+        "narHash": "sha256-d6nZ+1e4VDqW6VAwfx9EAUDJdPxSwqwGiuli32FEgoE=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "8249aa3442fb9b45e615a35f39eca2fe5510d7c3",
+        "rev": "08ff39bf869cadca3102b39824f4c7025186b7dc",
         "type": "github"
       },
       "original": {

src/ast2/biunification.rs

@@ -5,43 +5,30 @@
 // Visitor pattern has lots of unused arguments
 #![allow(unused_variables)]
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
-
-use crate::ast2::tag::{AstNode, AstNodeExt};
 
 use super::{Ast, Index, intern, visitor::AstVisitorTrait};
 
 type Id = u32;
 
-trait TypeVariance {
+enum Type {
-    type T;
+    Reified(intern::Index),
-    type Opposite;
+    Variable(Id),
-}
-
-#[derive(Debug, Clone)]
-enum TypeHead<T: TypeVariance> {
-    Real(intern::Index),
-    Function { args: Vec<T::T>, ret: T::Opposite },
 }
 
 /// Variance of a type parameter or constraint.
 /// A function of type `A -> B` is covariant in `B` and contravariant in `A`.
-/// This means that a type `T` may be substituted for `A` if `T` is a subtype of
+/// This means that a type `T` may be substituted for `A` if `T` is a subtype of `A`, but
-/// `A`, that is, every `T` is also an `A`,
+/// a type `T` may only be substituted for `B` if `T` is a supertype of `B`.
-/// but a type `T` may only be substituted for `B` if `T` is a supertype of `B`,
-/// that is, every `B` is also a `T`.
 ///
 /// Namely, in a type system with `int` and `nat <: int`, for a function `f: int
 /// -> int` in the expression `let u: int = 3; let t: nat = f(u);`, `u` may
 /// safely be used as an argument to `f` because `nat <: int`, but `f(u`)` may
 /// not be assigned to `t` because `int <: nat` is not true.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
 enum Variance {
-    /// A Positive, or union relationship between types.
+    #[doc(alias = "Positive")]
-    /// used in value-places.
     Covariant,
-    /// A Negative, or intersection relationship between types.
+    #[doc(alias = "Negative")]
-    /// used in use-places.
     Contravariant,
 }
 
@@ -50,81 +37,27 @@ struct Value(Id);
 #[derive(Debug, Clone, Copy)]
 struct Use(Id);
 
-impl TypeVariance for Value {
-    type T = Value;
-    type Opposite = Use;
-}
-
-impl TypeVariance for Use {
-    type T = Use;
-    type Opposite = Value;
-}
-
 /// Typechecking error.
 #[derive(Debug, Clone, thiserror::Error)]
 enum Error {
     #[error("Unimplemented feature")]
     Unimplemented,
-    #[error("{0}")]
-    StringError(String),
 }
 
 type Result<T> = std::result::Result<T, Error>;
 
 struct Bindings {
-    next_id: Id,
+    inner: HashMap<super::Index, Value>,
-    inner: HashMap<super::Index, Id>,
-    bounds: HashSet<(Id, Id, Variance)>,
-    types: HashMap<Id, intern::Index>,
-}
-
-impl Bindings {
-    fn new() -> Self {
-        Bindings {
-            next_id: 1,
-            inner: HashMap::new(),
-            bounds: HashSet::new(),
-            types: HashMap::new(),
-        }
-    }
-
-    fn new_id(&mut self) -> Id {
-        let id = self.next_id;
-        self.next_id += 1;
-        id
-    }
-
-    fn get_or_create(&mut self, idx: super::Index) -> Id {
-        self.inner.get(&idx).copied().unwrap_or_else(|| {
-            let id = self.new_id();
-            self.inner.insert(idx, id);
-            id
-        })
-    }
-
-    /// retrieves the type Id for the given ast node.
-    fn get(&self, idx: super::Index) -> Option<Id> {
-        self.inner.get(&idx).copied()
-    }
-
-    /// inserts a proper type for `id`.
-    fn insert_type(&mut self, id: Id, ty: intern::Index) {
-        self.types.insert(id, ty);
-    }
 }
 
 struct TypeChecker<'a> {
     pool: &'a mut intern::InternPool,
-    bindings: Bindings,
 }
 
 // Core
 impl TypeChecker<'_> {
     pub fn new(pool: &mut intern::InternPool) -> TypeChecker {
-        TypeChecker {
+        TypeChecker { pool }
-            pool,
-            bindings: Bindings::new(),
-        }
     }
 
     fn var(&mut self) -> (Value, Use) {
@@ -140,33 +73,6 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
 
     const UNIMPL: Self::Error = Error::Unimplemented;
 
-    fn visit_interned_type_impl(
-        &mut self,
-        ast: &'a Ast,
-        idx: Index,
-        intern: intern::Index,
-    ) -> std::result::Result<Self::Value, Self::Error> {
-        let id = self.bindings.get_or_create(idx);
-        match self.pool.get_key(intern) {
-            intern::Key::SimpleType {
-                ty: intern::SimpleType::ComptimeInt,
-            } => {
-                // This is a type variable.
-            }
-            intern::Key::SimpleType { .. }
-            | intern::Key::PointerType { .. }
-            | intern::Key::ArrayType { .. }
-            | intern::Key::FunctionType { .. }
-            | intern::Key::StructType { .. } => {
-                // This is a real type.
-                self.bindings.insert_type(id, intern);
-            }
-            _ => unreachable!(),
-        }
-
-        Ok(Value(id))
-    }
-
     fn visit_constant_impl(
         &mut self,
         ast: &'a Ast,
@@ -174,30 +80,8 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
         ty: Index,
         value: intern::Index,
     ) -> std::result::Result<Self::Value, Self::Error> {
-        // get type from the pool
+        // constants may be of type `comptime_int`, which is a special type that
-
+        // cannot exist at runtime.
-        let AstNode::InternedType { intern } = ast.get_ast_node(ty) else {
-            panic!(
-                "Expected an interned type node, got {:?}",
-                ast.get_ast_node(ty)
-            );
-        };
-
-        match self.pool.get_key(intern) {
-            intern::Key::SimpleType {
-                ty: intern::SimpleType::ComptimeInt,
-            } => {
-                // This is a type variable.
-            }
-            intern::Key::SimpleType { .. }
-            | intern::Key::PointerType { .. }
-            | intern::Key::ArrayType { .. }
-            | intern::Key::FunctionType { .. }
-            | intern::Key::StructType { .. } => {
-                // This is a real type.
-            }
-            _ => unreachable!(),
-        }
 
         todo!()
     }

src/ast2/intern.rs

@@ -15,7 +15,7 @@ use crate::{
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[repr(u8)]
 pub enum SimpleType {
-    F32,
+    F32 = 0,
     F64,
     Bool,
     Void,
@@ -28,14 +28,11 @@ pub enum SimpleType {
     /// Bottom type: this is the subtype of all types, and it can coerce into a value of any type.
     /// Akin to Rust's `!`.
     Bottom,
-    UInt(u16),
-    SInt(u16),
 }
 
-impl From<u32> for SimpleType {
+impl From<u8> for SimpleType {
-    fn from(value: u32) -> Self {
+    fn from(value: u8) -> Self {
-        let [discriminant, bits] = *crate::common::u32_as_u16_slice(&value);
+        match value {
-        match discriminant {
             0 => Self::F32,
             1 => Self::F64,
             2 => Self::Bool,
@@ -43,49 +40,28 @@ impl From<u32> for SimpleType {
             4 => Self::USize,
             5 => Self::ISize,
             6 => Self::ComptimeInt,
-            7 => Self::Top,
-            8 => Self::Bottom,
-            9 => Self::UInt(bits),
-            10 => Self::SInt(bits),
             _ => panic!("{value} is not a simple type"),
         }
     }
 }
 
-impl From<SimpleType> for u32 {
-    fn from(value: SimpleType) -> Self {
-        match value {
-            SimpleType::F32 => crate::common::u32_from_u16_slice(&[0, 0]),
-            SimpleType::F64 => crate::common::u32_from_u16_slice(&[1, 0]),
-            SimpleType::Bool => crate::common::u32_from_u16_slice(&[2, 0]),
-            SimpleType::Void => crate::common::u32_from_u16_slice(&[3, 0]),
-            SimpleType::USize => crate::common::u32_from_u16_slice(&[4, 0]),
-            SimpleType::ISize => crate::common::u32_from_u16_slice(&[5, 0]),
-            SimpleType::ComptimeInt => crate::common::u32_from_u16_slice(&[6, 0]),
-            SimpleType::Top => crate::common::u32_from_u16_slice(&[7, 0]),
-            SimpleType::Bottom => crate::common::u32_from_u16_slice(&[8, 0]),
-            SimpleType::UInt(bits) => crate::common::u32_from_u16_slice(&[9, bits]),
-            SimpleType::SInt(bits) => crate::common::u32_from_u16_slice(&[10, bits]),
-        }
-    }
-}
-
 impl Display for SimpleType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let fmt: std::borrow::Cow<str> = match self {
+        write!(
-            SimpleType::F32 => "f32".into(),
+            f,
-            SimpleType::F64 => "f64".into(),
+            "{}",
-            SimpleType::Bool => "bool".into(),
+            match self {
-            SimpleType::Void => "void".into(),
+                SimpleType::F32 => "f32",
-            SimpleType::USize => "usize".into(),
+                SimpleType::F64 => "f64",
-            SimpleType::ISize => "isize".into(),
+                SimpleType::Bool => "bool",
-            SimpleType::ComptimeInt => "comptime_int".into(),
+                SimpleType::Void => "void",
-            SimpleType::Top => "⊤".into(),
+                SimpleType::USize => "usize",
-            SimpleType::Bottom => "⊥".into(),
+                SimpleType::ISize => "isize",
-            SimpleType::UInt(bits) => format!("u{bits}").into(),
+                SimpleType::ComptimeInt => "comptime_int",
-            SimpleType::SInt(bits) => format!("i{bits}").into(),
+                SimpleType::Top => "⊤",
-        };
+                SimpleType::Bottom => "⊥",
-        write!(f, "{fmt}",)
+            }
+        )
     }
 }
 
@@ -102,6 +78,8 @@ pub enum Tag {
     F64,
     PositiveInt,
     NegativeInt,
+    UIntType,
+    SIntType,
     SimpleType,
     PointerType,
     ArrayType,
@@ -110,9 +88,9 @@ pub enum Tag {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub(super) struct Item {
+struct Item {
-    pub(super) tag: Tag,
+    tag: Tag,
-    pub(super) index: u32,
+    index: u32,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -148,6 +126,12 @@ pub enum Key<'a> {
     NegativeInt {
         bigint: BigInt,
     },
+    UIntType {
+        bit_width: u16,
+    },
+    SIntType {
+        bit_width: u16,
+    },
     SimpleType {
         ty: SimpleType,
     },
@@ -194,6 +178,8 @@ impl Display for KeyDisplay<'_> {
             Key::F64 { bits } => write!(f, "{bits}")?,
             Key::PositiveInt { ref bigint } => write!(f, "{bigint}")?,
             Key::NegativeInt { ref bigint } => write!(f, "{bigint}")?,
+            Key::UIntType { bit_width } => write!(f, "u{bit_width}")?,
+            Key::SIntType { bit_width } => write!(f, "i{bit_width}")?,
             Key::SimpleType { ty } => write!(f, "{ty}")?,
             Key::PointerType { pointee, flags } => {
                 write!(f, "*{flags}{}", self.ip.display_key(pointee))?
@@ -252,6 +238,8 @@ impl Hash for Key<'_> {
             Key::F64 { bits } => ordered_float::OrderedFloat(*bits).hash(state),
             Key::PositiveInt { bigint } => bigint.hash(state),
             Key::NegativeInt { bigint } => bigint.hash(state),
+            Key::UIntType { bit_width: bits } => bits.hash(state),
+            Key::SIntType { bit_width: bits } => bits.hash(state),
             Key::SimpleType { ty } => ty.hash(state),
             Key::PointerType { pointee, flags } => (pointee, flags).hash(state),
             Key::ArrayType {
@@ -556,20 +544,20 @@ static_keys!(
     ISIZE => Key::SimpleType {ty: SimpleType::ISize,},
     VOID => Key::SimpleType {ty: SimpleType::Void,},
     COMPTIME_INT => Key::SimpleType {ty: SimpleType::ComptimeInt,},
-    I0 => Key::SimpleType { ty: SimpleType::SInt(0) },
+    I1 => Key::SIntType { bit_width: 1 },
-    U0 => Key::SimpleType { ty: SimpleType::UInt(0) },
+    U1 => Key::UIntType { bit_width: 1 },
-    I1 => Key::SimpleType { ty: SimpleType::SInt(1) },
+    I0 => Key::SIntType { bit_width: 0 },
-    U1 => Key::SimpleType { ty: SimpleType::UInt(1) },
+    U0 => Key::UIntType { bit_width: 0 },
-    I8 => Key::SimpleType { ty: SimpleType::SInt(8) },
+    I8 => Key::SIntType { bit_width: 8 },
-    U8 => Key::SimpleType { ty: SimpleType::UInt(8) },
+    U8 => Key::UIntType { bit_width: 8 },
-    I16 => Key::SimpleType { ty: SimpleType::SInt(16) },
+    I16 => Key::SIntType { bit_width: 16 },
-    U16 => Key::SimpleType { ty: SimpleType::UInt(16) },
+    U16 => Key::UIntType { bit_width: 16 },
-    I32 => Key::SimpleType { ty: SimpleType::SInt(32) },
+    I32 => Key::SIntType { bit_width: 32 },
-    U32 => Key::SimpleType { ty: SimpleType::UInt(32) },
+    U32 => Key::UIntType { bit_width: 32 },
-    I64 => Key::SimpleType { ty: SimpleType::SInt(64) },
+    I64 => Key::SIntType { bit_width: 64 },
-    U64 => Key::SimpleType { ty: SimpleType::UInt(64) },
+    U64 => Key::UIntType { bit_width: 64 },
-    I128 => Key::SimpleType { ty: SimpleType::SInt(128) },
+    I128 => Key::SIntType { bit_width: 128 },
-    U128 => Key::SimpleType { ty: SimpleType::UInt(128) },
+    U128 => Key::UIntType { bit_width: 128 },
     TRUE => Key::TrueValue,
     FALSE => Key::FalseValue,
     EMPTY_STRING => Key::String { str: "" },
@@ -618,51 +606,41 @@ impl InternPool {
             ty: SimpleType::ISize,
         })
     }
-    // Assumes the type is present in the pool.
-    fn get_simple_type_unchecked(&self, ty: SimpleType) -> Index {
-        self.get_assume_present(&Key::SimpleType { ty })
-    }
     pub fn get_u0_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(0))
+        self.get_assume_present(&Key::UIntType { bit_width: 0 })
     }
     pub fn get_i0_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(0))
+        self.get_assume_present(&Key::SIntType { bit_width: 0 })
     }
     pub fn get_u1_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(1))
+        self.get_assume_present(&Key::UIntType { bit_width: 1 })
     }
     pub fn get_i1_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(1))
+        self.get_assume_present(&Key::SIntType { bit_width: 1 })
     }
     pub fn get_u8_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(8))
+        self.get_assume_present(&Key::UIntType { bit_width: 8 })
     }
     pub fn get_i8_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(8))
+        self.get_assume_present(&Key::SIntType { bit_width: 8 })
     }
     pub fn get_u16_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(16))
+        self.get_assume_present(&Key::UIntType { bit_width: 16 })
     }
     pub fn get_i16_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(16))
+        self.get_assume_present(&Key::SIntType { bit_width: 16 })
     }
     pub fn get_u32_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(32))
+        self.get_assume_present(&Key::UIntType { bit_width: 32 })
     }
     pub fn get_i32_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(32))
+        self.get_assume_present(&Key::SIntType { bit_width: 32 })
     }
     pub fn get_u64_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(64))
+        self.get_assume_present(&Key::UIntType { bit_width: 64 })
     }
     pub fn get_i64_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(64))
+        self.get_assume_present(&Key::SIntType { bit_width: 64 })
-    }
-    pub fn get_u128_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::UInt(128))
-    }
-    pub fn get_i128_type(&self) -> Index {
-        self.get_simple_type_unchecked(SimpleType::SInt(128))
     }
 
     pub fn get_top_type(&self) -> Index {
@@ -722,9 +700,8 @@ impl InternPool {
                 | Key::SimpleType {
                     ty: SimpleType::ISize,
                 }
-                | Key::SimpleType {
+                | Key::SIntType { .. }
-                    ty: SimpleType::SInt(_) | SimpleType::UInt(_),
+                | Key::UIntType { .. },
-                },
             ) => Some(rhs),
             (
                 Key::SimpleType {
@@ -739,9 +716,8 @@ impl InternPool {
                 | Key::SimpleType {
                     ty: SimpleType::ISize,
                 }
-                | Key::SimpleType {
+                | Key::SIntType { .. }
-                    ty: SimpleType::SInt(_) | SimpleType::UInt(_),
+                | Key::UIntType { .. },
-                },
                 Key::SimpleType {
                     ty: SimpleType::ComptimeInt,
                 },
@@ -769,6 +745,15 @@ impl InternPool {
     pub fn to_mir_type(&self, index: Index, _ptr_size: TypeInfo) -> crate::mir::Type {
         use crate::mir::Type;
         match self.get_key(index) {
+            Key::UIntType { bit_width: bits } => {
+                let bits = bits as u32;
+
+                Type::from_bitsize_int(bits)
+            }
+            Key::SIntType { bit_width: bits } => {
+                let bits = bits as u32;
+                Type::from_bitsize_int(bits)
+            }
             Key::SimpleType { ty } => match ty {
                 SimpleType::F32 => Type::SinglePrecision,
                 SimpleType::F64 => Type::DoublePrecision,
@@ -777,9 +762,6 @@ impl InternPool {
                     todo!("void can't be turned into a mir type")
                 }
                 SimpleType::ISize | SimpleType::USize => Type::QWord,
-                SimpleType::UInt(bits) | SimpleType::SInt(bits) => {
-                    Type::from_bitsize_int(bits as u32)
-                }
                 SimpleType::Top | SimpleType::Bottom | SimpleType::ComptimeInt => {
                     panic!("{ty} can't be turned into a mir type")
                 }
@@ -800,9 +782,11 @@ impl InternPool {
 
     pub fn is_type_signed(&self, index: Index, _ptr_size: TypeInfo) -> bool {
         match self.get_key(index) {
+            Key::UIntType { .. } => false,
+            Key::SIntType { .. } => true,
             Key::SimpleType { ty } => match ty {
-                SimpleType::USize | SimpleType::UInt(_) => false,
+                SimpleType::USize => false,
-                SimpleType::ISize | SimpleType::SInt(_) => true,
+                SimpleType::ISize => true,
                 _ => false,
             },
             Key::PointerType { .. } => false,
@@ -815,17 +799,23 @@ impl InternPool {
 
     pub fn size_of_type(&self, index: Index, ptr_size: TypeInfo) -> TypeInfo {
         match self.get_key(index) {
-            Key::SimpleType { ty } => match ty {
+            Key::UIntType { bit_width: bits } => {
-                SimpleType::SInt(bits) => TypeInfo {
+                let bits = bits as u32;
-                    bitsize: bits as u32,
+                TypeInfo {
-                    bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
+                    bitsize: bits,
-                    signed: true,
+                    bitalign: bits.next_multiple_of(8).next_power_of_two(),
-                },
-                SimpleType::UInt(bits) => TypeInfo {
-                    bitsize: bits as u32,
-                    bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
                     signed: false,
-                },
+                }
+            }
+            Key::SIntType { bit_width: bits } => {
+                let bits = bits as u32;
+                TypeInfo {
+                    bitsize: bits,
+                    bitalign: bits.next_multiple_of(8).next_power_of_two(),
+                    signed: true,
+                }
+            }
+            Key::SimpleType { ty } => match ty {
                 SimpleType::F32 => TypeInfo {
                     bitsize: 32,
                     bitalign: 32,
@@ -899,13 +889,9 @@ impl InternPool {
             crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
             crate::ast::Type::Integer(i) => self.get_or_insert({
                 if i.signed {
-                    Key::SimpleType {
+                    Key::SIntType { bit_width: i.bits }
-                        ty: SimpleType::SInt(i.bits),
-                    }
                 } else {
-                    Key::SimpleType {
+                    Key::UIntType { bit_width: i.bits }
-                        ty: SimpleType::UInt(i.bits),
-                    }
                 }
             }),
             crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@@ -944,13 +930,9 @@ impl InternPool {
             crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
             crate::ast::Type::Integer(i) => self.get_assume_present(&{
                 if i.signed {
-                    Key::SimpleType {
+                    Key::SIntType { bit_width: i.bits }
-                        ty: SimpleType::SInt(i.bits),
-                    }
                 } else {
-                    Key::SimpleType {
+                    Key::UIntType { bit_width: i.bits }
-                        ty: SimpleType::UInt(i.bits),
-                    }
                 }
             }),
             crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@@ -986,11 +968,11 @@ impl InternPool {
     pub fn as_ast1_type(&self, pointer_bits: u16, index: Index) -> crate::ast::Type {
         use crate::ast::Type;
         match self.get_key(index) {
+            Key::UIntType { bit_width: bits } => Type::Integer(IntegralType::new(false, bits)),
+            Key::SIntType { bit_width: bits } => Type::Integer(IntegralType::new(true, bits)),
             Key::SimpleType { ty } => match ty {
                 SimpleType::F32 => Type::Floating(crate::ast::FloatingType::Binary32),
                 SimpleType::F64 => Type::Floating(crate::ast::FloatingType::Binary64),
-                SimpleType::SInt(bits) => Type::Integer(IntegralType::new(true, bits)),
-                SimpleType::UInt(bits) => Type::Integer(IntegralType::new(false, bits)),
                 SimpleType::Bool => Type::Bool,
                 SimpleType::Void => Type::Void,
                 SimpleType::USize => Type::Integer(IntegralType::new(false, pointer_bits)),
@@ -1116,7 +1098,9 @@ impl InternPool {
                 self.create_item(Tag::NegativeInt, i)
             }
 
-            Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty.into()),
+            Key::UIntType { bit_width: bits } => self.create_item(Tag::UIntType, bits as u32),
+            Key::SIntType { bit_width: bits } => self.create_item(Tag::SIntType, bits as u32),
+            Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty as u8 as u32),
             Key::PointerType { pointee, flags } => {
                 let flags = flags.pack();
                 let i = self.extend_words([pointee.index() as u32, flags as u32]);
@@ -1246,11 +1230,17 @@ impl InternPool {
                 let bigint = BigInt::from_biguint(Sign::Plus, data);
                 Key::PositiveInt { bigint }
             }
+            Tag::SIntType => Key::SIntType {
+                bit_width: item.index as u16,
+            },
+            Tag::UIntType => Key::UIntType {
+                bit_width: item.index as u16,
+            },
             Tag::SimpleType => {
-                let ty = item.idx() as u32;
+                let ty = item.idx() as u8;
 
                 Key::SimpleType {
-                    ty: SimpleType::from(ty),
+                    ty: unsafe { core::mem::transmute::<u8, SimpleType>(ty) },
                 }
             }
             Tag::PointerType => {
@@ -1347,12 +1337,8 @@ impl InternPool {
 
     pub fn get_int_type(&mut self, signed: bool, bits: u16) -> Index {
         let key = match signed {
-            true => Key::SimpleType {
+            true => Key::SIntType { bit_width: bits },
-                ty: SimpleType::SInt(bits),
+            false => Key::UIntType { bit_width: bits },
-            },
-            false => Key::SimpleType {
-                ty: SimpleType::UInt(bits),
-            },
         };
 
         self.get_or_insert(key)
@@ -1549,7 +1535,7 @@ impl InternPool {
         ((index.index() as u32) < self.len()).then_some(index)
     }
 
-    pub(super) fn get_item(&self, index: Index) -> Option<Item> {
+    fn get_item(&self, index: Index) -> Option<Item> {
         self.check_bounds(index).map(|i| Item {
             tag: self.tags[i.index()],
             index: self.indices[i.index()],
@@ -1568,6 +1554,8 @@ impl InternPool {
                 | Key::ArrayType { .. }
                 | Key::PointerType { .. }
                 | Key::SimpleType { .. }
+                | Key::SIntType { .. }
+                | Key::UIntType { .. }
                 | Key::StructType { .. }
         )
     }

src/ast2/internable.rs

@@ -1,34 +0,0 @@
-use super::*;
-use core::hash::Hash;
-use std::hash::Hasher;
-
-// Types implementing this trait can be stored in the internpool.
-trait KeyTrait: Hash + Eq {
-    const TAG: Tag;
-    fn serialise(self, pool: &mut InternPool);
-    fn deserialise(index: Index, pool: &mut InternPool) -> Self;
-}
-
-impl KeyTrait for String {
-    const TAG: Tag = Tag::String;
-    fn serialise(self, pool: &mut InternPool) {
-        todo!()
-    }
-
-    fn deserialise(index: Index, pool: &mut InternPool) -> Self {
-        // let mut hasher = std::hash::DefaultHasher::new();
-        // core::any::TypeId::of::<Self>().hash(&mut hasher);
-        // let tag = hasher.finish() as u32;
-        let item = pool.get_item(index).unwrap();
-        assert_eq!(item.tag, Self::TAG);
-
-        let start = pool.words[item.idx()] as usize;
-        let len = pool.words[item.idx() + 1] as usize;
-        let str = unsafe {
-            let bytes = &pool.strings[start..start + len];
-            std::str::from_utf8_unchecked(bytes)
-        };
-
-        str.to_owned()
-    }
-}

src/ast2/mod.rs

@@ -2134,6 +2134,36 @@ where
     let ty_key = ip.get_key(ty);
     let signed = ip.is_type_signed(ty, AMD64_POINTER_TYPE_INFO);
     match ty_key {
+        intern::Key::SIntType { bit_width: bits } | intern::Key::UIntType { bit_width: bits } => {
+            let ty = IntegralType::new(signed, bits);
+            match ip.get_key(val) {
+                intern::Key::SIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                    bits: bits as _,
+                    ty,
+                }),
+                intern::Key::UIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                    bits: bits as _,
+                    ty,
+                }),
+                intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                    bits: bits as _,
+                    ty,
+                }),
+                intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
+                    bits: bits as _,
+                    ty,
+                }),
+                intern::Key::PositiveInt { bigint } => {
+                    ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
+                }
+                intern::Key::NegativeInt { bigint } => {
+                    ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
+                }
+                _ => {
+                    unreachable!()
+                }
+            }
+        }
         intern::Key::SimpleType { ty } => match ty {
             intern::SimpleType::F32 => match ip.get_key(val) {
                 intern::Key::F32 { bits } => {
@@ -2226,40 +2256,6 @@ where
             intern::SimpleType::Top | intern::SimpleType::Bottom => {
                 unreachable!()
             }
-            intern::SimpleType::UInt(bits) | intern::SimpleType::SInt(bits) => {
-                let ty = IntegralType::new(signed, bits);
-                match ip.get_key(val) {
-                    intern::Key::SIntSmall { bits } => {
-                        ComptimeNumber::Integral(ComptimeInt::Native {
-                            bits: bits as _,
-                            ty,
-                        })
-                    }
-                    intern::Key::UIntSmall { bits } => {
-                        ComptimeNumber::Integral(ComptimeInt::Native {
-                            bits: bits as _,
-                            ty,
-                        })
-                    }
-                    intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                        bits: bits as _,
-                        ty,
-                    }),
-                    intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
-                        bits: bits as _,
-                        ty,
-                    }),
-                    intern::Key::PositiveInt { bigint } => {
-                        ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
-                    }
-                    intern::Key::NegativeInt { bigint } => {
-                        ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
-                    }
-                    _ => {
-                        unreachable!()
-                    }
-                }
-            }
         },
         _ => {
             unreachable!()

src/common.rs

@@ -72,7 +72,9 @@ pub fn is_oct_digit(ch: char) -> bool {
 }
 
 pub fn is_hex_digit(ch: char) -> bool {
-    ('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
+    ('0'..='9').contains(&ch)
+        || ('a'..='f').contains(&ch)
+        || ('A'..='F').contains(&ch)
 }
 
 /// Trait for only yielding the next item in the Iterator if it tests true for some predicate
@@ -174,19 +176,3 @@ pub fn from_lo_hi_dwords(lo: u32, hi: u32) -> u64 {
 pub fn into_lo_hi_dwords(qword: u64) -> (u32, u32) {
     (qword as u32, (qword >> 32) as u32)
 }
-
-pub fn u32_as_u16_slice(value: &u32) -> &[u16; 2] {
-    // SAFETY: This is safe because u32 is guaranteed to be 4 bytes and
-    // we are creating a slice of 2 u16s which is also 4 bytes.
-    unsafe { &*(value as *const u32 as *const [u16; 2]) }
-}
-
-// we can't transform any &[u16; 2] into a &u32 because of alignment guarantees
-pub fn u32_from_u16_slice(slice: &[u16; 2]) -> u32 {
-    let mut out = 0u32;
-    unsafe {
-        core::ptr::copy_nonoverlapping(slice.as_ptr(), &raw mut out as *mut u16, 2);
-    }
-
-    out
-}