janis/SeaLang
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: janis:2293b514cf3531987a9ff627697b0fc72c679920
Branches Tags
janis:main
janis:chumsky
janis:bottom-up-value-place
..
compare: janis:d124ae2b599351c77a4eb5551b811804f0733466
Branches Tags
janis:chumsky
janis:bottom-up-value-place
janis:main

No commits in common. "2293b514cf3531987a9ff627697b0fc72c679920" and "d124ae2b599351c77a4eb5551b811804f0733466" have entirely different histories.
2293b514cf ... d124ae2b59

11 changed files with 167 additions and 1457 deletions
Show stats Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,3 +1,2 @@
/target
/Cargo.lock
/.direnv/

19
Cargo.toml
View file

@ -1,9 +1,3 @@
[workspace]
resolver = "3"
members = [
"crates/lexer"
]
[package]
name = "compiler"
version = "0.1.0"
@ -22,16 +16,5 @@ paste = "1.0.15"
petgraph = "0.6.5"
thiserror = "1.0.63"
unicode-xid = "0.2.4"
tracing = "0.1.41"
werkzeug = { path = "../../rust/werkzeug" }
[workspace.dependencies]
unicode-xid = "0.2.4"
tracing = "0.1.41"
thiserror = "1.0.63"
itertools = "0.13.0"
werkzeug = { path = "../../rust/werkzeug" }
trie = { path = "../../rust/trie" }
werkzeug = { path = "../../rust/werkzeug" }

12
crates/lexer/Cargo.toml
View file

@ -1,12 +0,0 @@
[package]
name = "lexer"
version = "0.1.0"
edition = "2024"
[dependencies]
tracing = { workspace = true }
werkzeug = { workspace = true }
thiserror = { workspace = true }
itertools = { workspace = true }
trie = { workspace = true }
unicode-xid = { workspace = true }

394
crates/lexer/src/complex_tokens.rs
View file

@ -1,394 +0,0 @@
use crate::{Source, Token, is_things};
use itertools::Itertools;
use werkzeug::iter::{FallibleMapIter, NextIf};
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum Error {
#[error("{0}")]
StringError(String),
#[error("Exp part of floating constant had no digits.")]
FloatingConstantExpPartNoDigit,
#[error("constant cannot start with leading underscore '_'.")]
NumericalConstantDigitLeadingUnderscore,
#[error("Expected digit here for constant.")]
NumericalConstantDigitNoDigit,
#[error("Expected digit here for integer constant.")]
IntegralTypeExpectedDigit,
#[error("Floating constant has invalid trailing type.")]
FloatingConstantInvalidTrailingType,
#[error("Invalid token.")]
InvalidToken,
#[error("Identifier starts with invalid character.")]
ExpectedIdStartForIdentifier,
#[error("Unknown suffix in constant.")]
NumericalConstantUnknownSuffix,
}
type Result<T> = core::result::Result<T, Error>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Radix {
Hex,
Bin,
Dec,
Oct,
}
impl Radix {
#[allow(unused)]
/// must be called with one of `['b','x','d','o']`
unsafe fn from_char_unchecked(c: char) -> Self {
match c.to_ascii_lowercase() {
'o' => Self::Oct,
'b' => Self::Bin,
'x' => Self::Hex,
'd' => Self::Dec,
_ => unreachable!(),
}
}
fn from_char(c: char) -> Option<Self> {
match c.to_ascii_lowercase() {
'o' => Some(Self::Oct),
'b' => Some(Self::Bin),
'x' => Some(Self::Hex),
'd' => Some(Self::Dec),
_ => None,
}
}
#[allow(unused)]
pub fn radix(self) -> u8 {
match self {
Radix::Hex => 16,
Radix::Bin => 2,
Radix::Oct => 8,
Radix::Dec => 10,
}
}
fn to_token(self) -> Token {
match self {
Radix::Hex => Token::IntegerHexConstant,
Radix::Bin => Token::IntegerBinConstant,
Radix::Oct => Token::IntegerOctConstant,
Radix::Dec => Token::IntegerConstant,
}
}
pub fn from_token(token: Token) -> Option<Self> {
match token {
Token::IntegerHexConstant => Some(Radix::Hex),
Token::IntegerBinConstant => Some(Radix::Bin),
Token::IntegerOctConstant => Some(Radix::Oct),
Token::IntegerConstant => Some(Radix::Dec),
_ => None,
}
}
pub fn map_digit(self, c: char) -> u8 {
match self {
Radix::Hex => match c {
'0'..='9' => c as u8 - b'0',
'a'..='f' => 10 + c as u8 - b'a',
'A'..='F' => 10 + c as u8 - b'A',
_ => unreachable!(),
},
Radix::Bin => match c {
'0'..='1' => c as u8 - b'0',
_ => unreachable!(),
},
Radix::Dec => match c {
'0'..='9' => c as u8 - b'0',
_ => unreachable!(),
},
Radix::Oct => match c {
'0'..='7' => c as u8 - b'0',
_ => unreachable!(),
},
}
}
pub fn folding_method(self) -> fn(u64, char) -> u64 {
match self {
Radix::Hex => {
fn fold(acc: u64, c: char) -> u64 {
let digit = match c {
'0'..='9' => c as u8 - b'0',
'a'..='f' => c as u8 - b'a',
'A'..='F' => c as u8 - b'A',
_ => unreachable!(),
};
acc * 16 + digit as u64
}
fold
}
Radix::Bin => {
fn fold(acc: u64, c: char) -> u64 {
let digit = match c {
'0'..='1' => c as u8 - b'0',
_ => unreachable!(),
};
acc * 2 + digit as u64
}
fold
}
Radix::Dec => {
fn fold(acc: u64, c: char) -> u64 {
let digit = match c {
'0'..='9' => c as u8 - b'0',
_ => unreachable!(),
};
acc * 10 + digit as u64
}
fold
}
Radix::Oct => {
fn fold(acc: u64, c: char) -> u64 {
let digit = match c {
'0'..='7' => c as u8 - b'0',
_ => unreachable!(),
};
acc * 8 + digit as u64
}
fold
}
}
}
pub fn is_digit(self) -> fn(char) -> bool {
match self {
Radix::Hex => crate::is_things::is_hex_digit,
Radix::Bin => crate::is_things::is_bin_digit,
Radix::Oct => crate::is_things::is_oct_digit,
Radix::Dec => crate::is_things::is_digit,
}
}
}
// where DIGIT is defined by radix:
// DIGITS <-
// if allow_leading_underscore: `_`* DIGIT (DIGIT|`_`)*
// else: DIGIT (DIGIT|`_`)*
fn parse_digit_part(
source: &mut Source,
allow_leading_underscore: bool,
radix: Radix,
) -> Result<()> {
let is_digit = radix.is_digit();
if allow_leading_underscore {
let _underscore = source.take_while_ref(|&c| c == '_').count();
}
let _need_digit = source.next_if(|&c| is_digit(c)).ok_or_else(|| {
if source.peek() == Some(&'_') {
Error::NumericalConstantDigitLeadingUnderscore
} else {
Error::NumericalConstantDigitNoDigit
}
})?;
let _rest = source.take_while_ref(|&c| is_digit(c) || c == '_').count();
Ok(())
}
// IntegralType <-
// ( 'u' | 'i' ) DIGITS+
fn try_parse_integral_type(source: &mut Source) -> Result<Option<()>> {
if !source.next_if(|&c| c == 'u' || c == 'i').is_some() {
return Ok(None);
}
if source
.take_while_ref(|&c| crate::is_things::is_digit(c))
.count()
<= 0
{
return Err(Error::IntegralTypeExpectedDigit);
};
Ok(Some(()))
}
// returns `Err(E)` if it failed to parse.
// returns `Ok(None)` if no exp part was found.
// returns `Ok(Some(()))` if an exp part was found and parsed.
//
// EXP_PART <-
// (`e`|`E`) (`-`|`+`)? DEC_DIGITS
fn try_parse_exp_part(source: &mut Source) -> Result<Option<()>> {
if source.next_if(|&c| c.to_ascii_lowercase() == 'e').is_some() {
let _sign = source.next_if(|&c| c == '-' || c == '+');
if source
.take_while_ref(|&c| crate::is_things::is_digit(c))
.count()
.lt(&1)
{
// need digits following exp notation
Err(Error::FloatingConstantExpPartNoDigit)
} else {
Ok(Some(()))
}
} else {
Ok(None)
}
}
// CONSTANT <-
// DEC_DIGITS IntegralType?
// `0x` HEX_DIGITS IntegralType?
// `0b` BIN_DIGITS IntegralType?
// `0o` OCT_DIGITS IntegralType?
// DEC_DIGITS FloatingType?
// `.` DEC_DIGITS EXP_PART? FloatingType?
// DEC_DIGITS `.` DEC_DIGITS? EXP_PART? FloatingType?
fn parse_constant_inner(source: &mut Source) -> Result<Token> {
let zero = source.next_if(|&c| c == '0').is_some();
let radix = zero
.then(|| source.next_if_map(|c| Radix::from_char(c)))
.flatten();
if let Some(radix) = radix {
parse_digit_part(source, false, radix)?;
if source.peek().map(|&c| c == 'u' || c == 'i') == Some(true) {
try_parse_integral_type(source)?;
}
return Ok(radix.to_token());
}
// if zero: `_`* DIGIT (DIGIT|`_`)*
// else: DIGIT (DIGIT|`_`)*
_ = match parse_digit_part(source, zero, Radix::Dec) {
Ok(_) => Ok(()),
Err(Error::NumericalConstantDigitNoDigit) if zero => Ok(()),
Err(e) => Err(e),
}?;
if let Some(_) = source.try_map_iter_if(|source| try_parse_integral_type(source))? {
return Ok(Token::IntegerConstant);
}
let dot = source.next_if(|&c| c == '.').is_some();
if dot {
parse_digit_part(source, false, Radix::Dec)?;
}
// parse exp notation
let exp = try_parse_exp_part(source)?.is_some();
// trailing FloatingType?
let floating = if source.next_if(|&c| c == 'f').is_some() {
let digits = source.next_tuple::<(char, char)>();
if !(digits == Some(('6', '4')) || digits == Some(('3', '2'))) {
// need either f64 or f32 here!
return Err(Error::FloatingConstantInvalidTrailingType);
}
true
} else {
false
};
let token = match (dot, exp, floating) {
(false, false, false) => Token::IntegerConstant,
(true, false, _) => Token::DotFloatingConstant,
(true, true, _) => Token::DotFloatingExpConstant,
(false, true, _) => Token::FloatingExpConstant,
(false, _, _) => Token::FloatingConstant,
};
Ok(token)
}
pub(crate) fn parse_constant(source: &mut Source) -> Result<Token> {
let constant = parse_constant_inner(source)?;
// char following a constant must not be id_continue
if source
.peek()
.map(|&c| crate::is_things::is_id_continue(c))
.unwrap_or(false)
{
return Err(Error::NumericalConstantUnknownSuffix);
}
Ok(constant)
}
pub(crate) fn parse_string_or_char_constant(source: &mut Source) -> Result<Token> {
let quote = source
.next_if(|&c| c == '"' || c == '\'')
.ok_or(Error::InvalidToken)?;
let is_char = quote == '\'';
let mut escaped = false;
let mut closed = false;
while let Some(c) = source.next() {
if escaped {
// accept any escaped char
escaped = false;
continue;
}
if c == '\\' {
escaped = true;
continue;
}
if c == quote {
closed = true;
break;
}
}
if !closed {
return Err(Error::StringError("Unterminated string/char.".into()));
}
if is_char {
Ok(Token::CharConstant)
} else {
Ok(Token::StringConstant)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_source(s: &'_ str) -> Source<'_> {
s.chars().peekable().into()
}
#[test]
fn parse_constant_number() {
assert_eq!(
parse_constant(&mut make_source("0x1A3F_u32")),
Ok(Token::IntegerHexConstant)
);
assert_eq!(
parse_constant(&mut make_source("13f32")),
Ok(Token::FloatingConstant)
);
assert_eq!(
parse_constant(&mut make_source("0b1011_0010i16")),
Ok(Token::IntegerBinConstant)
);
assert_eq!(
parse_constant(&mut make_source("0o755u8")),
Ok(Token::IntegerOctConstant)
);
assert_eq!(
parse_constant(&mut make_source("42i64")),
Ok(Token::IntegerConstant)
);
assert_eq!(
parse_constant(&mut make_source("3.14f64")),
Ok(Token::DotFloatingConstant)
);
assert_eq!(
parse_constant(&mut make_source("2.71828e0f32")),
Ok(Token::DotFloatingExpConstant)
);
assert_eq!(
parse_constant(&mut make_source("22e23")),
Ok(Token::FloatingExpConstant)
);
}
}

686
crates/lexer/src/lib.rs
View file

@ -1,686 +0,0 @@
#![feature(slice_swap_unchecked, iter_collect_into)]
mod is_things {
/// True if `c` is considered a whitespace according to Rust language definition.
/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
/// for definitions of these classes.
pub fn is_whitespace(c: char) -> bool {
// This is Pattern_White_Space.
//
// Note that this set is stable (ie, it doesn't change with different
// Unicode versions), so it's ok to just hard-code the values.
matches!(
c,
// Usual ASCII suspects
'\u{0009}' // \t
| '\u{000A}' // \n
| '\u{000B}' // vertical tab
| '\u{000C}' // form feed
| '\u{000D}' // \r
| '\u{0020}' // space
// NEXT LINE from latin1
| '\u{0085}'
// Bidi markers
| '\u{200E}' // LEFT-TO-RIGHT MARK
| '\u{200F}' // RIGHT-TO-LEFT MARK
// Dedicated whitespace characters from Unicode
| '\u{2028}' // LINE SEPARATOR
| '\u{2029}' // PARAGRAPH SEPARATOR
)
}
/// True if `c` is valid as a first character of an identifier.
/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
/// a formal definition of valid identifier name.
pub fn is_id_start(c: char) -> bool {
// This is XID_Start OR '_' (which formally is not a XID_Start).
c == '_' || c == '-' || unicode_xid::UnicodeXID::is_xid_start(c)
}
/// True if `c` is valid as a non-first character of an identifier.
/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
/// a formal definition of valid identifier name.
pub fn is_id_continue(c: char) -> bool {
unicode_xid::UnicodeXID::is_xid_continue(c) || c == '-'
}
/// The passed string is lexically an identifier.
pub fn is_ident(string: &str) -> bool {
let mut chars = string.chars();
if let Some(start) = chars.next() {
is_id_start(start) && chars.all(is_id_continue)
} else {
false
}
}
#[expect(dead_code)]
pub fn is_digit(ch: char) -> bool {
('0'..='9').contains(&ch)
}
#[expect(dead_code)]
pub fn is_bin_digit(ch: char) -> bool {
ch == '0' || ch == '1'
}
#[expect(dead_code)]
pub fn is_nonzero_digit(ch: char) -> bool {
('1'..='9').contains(&ch)
}
#[expect(dead_code)]
pub fn is_oct_digit(ch: char) -> bool {
('0'..='7').contains(&ch)
}
#[expect(dead_code)]
pub fn is_hex_digit(ch: char) -> bool {
('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
}
}
macro_rules! tokens {
($vis:vis $ty_name:ident:
{
$($name2:ident),*
},
{
$($name:ident => $lexeme:literal),*
}) => {
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
$vis enum $ty_name {
$($name,
)*
$($name2,)*
}
impl ::core::fmt::Display for $ty_name {
fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
match self {
$(Self::$name => write!(f, "{}", $lexeme),)*
$(Self::$name2 => write!(f, "<{}>", stringify!($name2))),*
}
}
}
#[allow(dead_code)]
impl $ty_name {
$vis fn lexeme(&self) -> Option<&'static str> {
match self {
$(Self::$name => Some($lexeme),)*
$(Self::$name2 => None),*
}
}
/// returns the number of chars in this lexeme
$vis fn lexeme_len(&self) -> usize {
self.lexeme().map(|lexeme|lexeme.chars().count()).unwrap_or(0)
}
/// returns the number of chars in this lexeme
$vis fn lexeme_len_utf8(&self) -> usize {
self.lexeme().map(|lexeme|lexeme.len()).unwrap_or(0)
}
$vis fn maybe_ident(&self) -> bool {
self.lexeme().map(|lexeme| crate::is_things::is_ident(lexeme)).unwrap_or(false)
}
$vis fn lexemes() -> &'static [(Self, &'static str)] {
&[
$((Self::$name, $lexeme)),*
]
}
}
};
}
tokens!(pub Token: {
Eof,
ParseError,
// Marker Token for any Comment
Comment,
DocComment,
// Marker Token for any pre-processing directive
CharConstant,
IntegerConstant,
IntegerHexConstant,
IntegerBinConstant,
IntegerOctConstant,
FloatingConstant,
FloatingExpConstant,
DotFloatingConstant,
DotFloatingExpConstant,
StringConstant,
Ident
},
// Lexical Tokens:
{
SlashSlash => "//",
SlashSlashSlash => "///",
// SlashStar => "/*",
// SlashStarStar => "/**",
//StarSlash => "*/",
// Punctuation:
OpenParens => "(",
CloseParens => ")",
OpenBrace => "{",
CloseBrace => "}",
OpenSquareBracket => "[",
CloseSquareBracket => "]",
Semi => ";",
Comma => ",",
Elipsis3 => "...",
Elipsis2 => "..",
Colon => ":",
Equal => "=",
// Keywords:
Void => "void",
Bool => "bool",
F32 => "f32",
F64 => "f64",
ISize => "isize",
USize => "usize",
Const => "const",
Volatile => "volatile",
Noalias => "noalias",
Fn => "fn",
Let => "let",
Var => "var",
If => "if",
As => "as",
Else => "else",
Return => "return",
Struct => "struct",
Type => "type",
Union => "union",
Enum => "enum",
Packed => "packed",
Extern => "extern",
Pub => "pub",
// Operators
Dot => ".",
MinusGreater => "->",
Bang => "!",
Tilde => "~",
Plus => "+",
PlusPlus => "++",
Minus => "-",
MinusMinus => "--",
Star => "*",
Slash => "/",
Percent => "%",
Less => "<",
Greater => ">",
LessEqual => "<=",
GreaterEqual => ">=",
EqualEqual => "==",
BangEqual => "!=",
PipePipe => "||",
AmpersandAmpersand => "&&",
Ampersand => "&",
Caret => "^",
Pipe => "|",
LessLess => "<<",
GreaterGreater => ">>",
Question => "?",
PlusEqual => "+=",
MinusEqual => "-=",
StarEqual => "*=",
SlashEqual => "/=",
PercentEqual => "%=",
AmpersandEqual => "&=",
PipeEqual => "|=",
CaretEqual => "^=",
LessLessEqual => "<<=",
GreaterGreaterEqual => ">>="
});
impl Token {
pub fn is_assignment_op(self) -> bool {
match self {
Token::PlusEqual
| Token::MinusEqual
| Token::StarEqual
| Token::SlashEqual
| Token::PercentEqual
| Token::PipeEqual
| Token::CaretEqual
| Token::AmpersandEqual
| Token::LessLessEqual
| Token::GreaterGreaterEqual
| Token::Equal => true,
_ => false,
}
}
pub fn is_unary_op(self) -> bool {
match self {
Token::Plus | Token::Minus | Token::Star | Token::Ampersand | Token::Bang => true,
_ => false,
}
}
pub fn is_binary_op(self) -> bool {
match self {
Token::Star
| Token::Slash
| Token::Percent
| Token::Pipe
| Token::Ampersand
| Token::Caret
| Token::Plus
| Token::Minus
| Token::PipePipe
| Token::AmpersandAmpersand
| Token::BangEqual
| Token::EqualEqual
| Token::Less
| Token::Greater
| Token::LessEqual
| Token::GreaterEqual
| Token::LessLess
| Token::GreaterGreater => true,
_ => false,
}
}
}
/// A list of lexemes used by the `LexemeParser`.
/// `lexemes` contains every token that has a defined lexeme, such as `fn`, `f32`, `const`, etc.
/// The `LexemeList` keeps track of two offsets into the `lexemes` array,
/// splitting it into three windows:
/// - [0, start_candidates) - tokens that are still being considered for parsing
/// - [start_candidates, end_candidates) - the tokens which this lexeme matches
/// - [end_candidates, len) - tokens that have been filtered out and are no longer considered
/// On each iteration of the parsing loop, the remaining tokens are matched
/// against the next character and, if they match completely, are swapped into
/// the candidates window, or swapped to the end if they don't.
struct LexemeList {
lexemes: Box<[Token]>,
start_candidates: usize,
end_candidates: usize,
filtered: Vec<(usize, FilterResult)>,
}
enum FilterResult {
Remove,
Candidate,
}
impl LexemeList {
fn new() -> Self {
let lexemes = Token::lexemes()
.iter()
.map(|(tok, _)| tok.clone())
.collect::<Box<_>>();
Self {
start_candidates: lexemes.len(),
end_candidates: lexemes.len(),
lexemes,
filtered: Vec::new(),
}
}
fn clear(&mut self) {
self.start_candidates = self.lexemes.len();
self.end_candidates = self.lexemes.len();
}
fn remaining(&self) -> &[Token] {
&self.lexemes[0..self.start_candidates]
}
fn candidates(&self) -> &[Token] {
&self.lexemes[self.start_candidates..self.end_candidates]
}
fn step(&mut self, ch: char, pos: usize) {
// smartly reuse allocation for `filtered`
// truly one of the premature optimizations.
// but it just feels good, innit?
let mut filtered = core::mem::take(&mut self.filtered);
self.remaining()
.iter()
.enumerate()
.filter_map(|(i, tok)| {
let bytes = tok.lexeme().unwrap().as_bytes();
// SAFETY: all tokens in `self.remaining()` are lexical tokens, and
// they are all valid ascii
let c = unsafe {
// TODO: maybe keep a list of `Char<'_>`s around in order to
// support fully utf8 tokens?
char::from_u32_unchecked(bytes[pos] as u32)
};
match c == ch {
false => Some((i, FilterResult::Remove)),
true if bytes.len() <= pos + 1 => Some((i, FilterResult::Candidate)),
true => None,
}
})
.collect_into(&mut filtered);
// iterate in reverse order so that we can safely swap elements
// drain here so that we can possibly reuse the `filtered` Vec allcoation
filtered.drain(..).rev().for_each(|(i, f)| {
match f {
// for candidates, swap the candidate with the last remaining
// token, then dec `start_candidates`
FilterResult::Candidate => {
// SAFETY: we know that `i` and `self.start_candidates - 1`
// are both valid indices: `self.start_candidates` starts at
// the end and each time it is decremented, one more element
// is removed from the front, so that as long as an element
// is remaining, `self.start_candidates` is always greater
// than 0.
// the order of the remaining elements is not meaningfully
// impacted because we only ever swap with elements after
// `i`, and `i` is the greatest index we will touch.
unsafe {
self.lexemes.swap_unchecked(i, self.start_candidates - 1);
self.start_candidates = self.start_candidates.saturating_sub(1);
}
}
// for removes, swap the last candidate with the last remainign
// token, then swap the remove with the last candidate, then dec
// `end_candidates` and `start_candidates`
FilterResult::Remove => {
unsafe {
// in the case that `start_candidates` ==
// `end_candidates`, no swap happens and that's fine.
// remove this: v
// [a,b,c][d,e,f][g,h,i]
// swap these: ^ ^
// [a,b,f][d,e,c][g,h,i]
// swap these: ^ ^
// [a,c,f][d,e,b][g,h,i]
// decrement both counters:
// [a,c][f,d,e][b,g,h,i]
self.lexemes
.swap_unchecked(self.start_candidates - 1, self.end_candidates - 1);
self.lexemes.swap_unchecked(i, self.end_candidates - 1);
self.start_candidates = self.start_candidates.saturating_sub(1);
self.end_candidates = self.end_candidates.saturating_sub(1);
}
}
}
});
// replace `filtered`
self.filtered = filtered;
}
}
/// Helper type for parsing tokens that have a defined lexeme, such as `fn`,
/// `f32`, `const`, etc. Tokens with variable lexemes, such as primitive
/// integral types, constants or identifiers are not parsed by this.
pub struct LexemeParser {
lexemes: LexemeList,
len: usize,
}
impl LexemeParser {
pub fn new() -> Self {
Self {
lexemes: LexemeList::new(),
len: 0,
}
}
pub fn parse(&mut self, mut tokens: impl Iterator<Item = char>) -> Option<Token> {
self.lexemes.clear();
loop {
let Some(ch) = tokens.next() else {
break;
};
if crate::is_things::is_whitespace(ch) {
break;
}
self.lexemes.step(ch, self.len);
if self.lexemes.remaining().is_empty() {
break;
}
}
self.lexemes.candidates().last().copied()
}
}
use itertools::Itertools;
use trie::Tree;
#[derive(Debug, Clone, Copy)]
struct CountingIterator<I: Iterator> {
iter: I,
count: usize,
}
impl<I: Iterator> From<I> for CountingIterator<I> {
fn from(iter: I) -> Self {
Self { iter, count: 0 }
}
}
impl<I: Iterator<Item = char>> Iterator for CountingIterator<I> {
type Item = I::Item;
fn next(&mut self) -> Option<Self::Item> {
self.iter.next().inspect(|c| self.count += c.len_utf8())
}
}
impl<I: Iterator> CountingIterator<I> {
pub(crate) fn offset(&self) -> usize {
self.count
}
}
impl<I: Iterator> core::ops::Deref for CountingIterator<I> {
type Target = I;
fn deref(&self) -> &Self::Target {
&self.iter
}
}
impl<I: Iterator> core::ops::DerefMut for CountingIterator<I> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.iter
}
}
type Source<'a> = CountingIterator<core::iter::Peekable<core::str::Chars<'a>>>;
pub struct TokenIterator<'a> {
trie: Tree<char, Token>,
source: &'a str,
offset: usize,
}
impl<'a> TokenIterator<'a> {
pub fn new(source: &'a str) -> Self {
let mut trie = Tree::new();
for (token, token_str) in Token::lexemes() {
trie.insert(token_str.chars(), *token);
}
Self {
trie,
source,
offset: 0,
}
}
fn peekable_source(&self) -> Source<'a> {
CountingIterator::from(self.source[self.offset..].chars().peekable())
}
fn parse(&mut self) -> Option<Token> {
let mut iter = CountingIterator::from(self.source[self.offset..].chars());
match self.trie.get_closest(&mut iter) {
Some(token) => {
// skip the peeked item
self.offset += token.lexeme_len();
Some(*token)
}
None => None,
}
}
fn skip_whitespaces(&mut self) -> usize {
self.skip_while(is_things::is_whitespace)
}
fn skip(&mut self, mut n: usize) -> usize {
self.skip_while(|_| {
n -= 1;
n > 0
})
}
fn skip_while(&mut self, mut pred: impl FnMut(char) -> bool) -> usize {
let mut count = 0;
loop {
let Some(c) = self.source[self.offset..].chars().next() else {
break;
};
if pred(c) {
self.offset += c.len_utf8();
count += c.len_utf8();
continue;
} else {
break;
}
}
count
}
}
impl<'a> Iterator for TokenIterator<'a> {
type Item = (Token, &'a str);
fn next(&mut self) -> Option<Self::Item> {
// skip whitespace
self.skip_whitespaces();
let start = self.offset;
let mut source = self.peekable_source();
let mut cursor = self.peekable_source();
let token = match cursor.next() {
Some('0'..='9') => {
let token = complex_tokens::parse_constant(&mut source).ok()?;
self.offset += source.offset();
Some((token, &self.source[start..self.offset]))
}
Some('.') if cursor.next().map_or(false, is_things::is_digit) => {
let token = complex_tokens::parse_constant(&mut source).ok()?;
self.offset += source.offset();
Some((token, &self.source[start..self.offset]))
}
Some('\'' | '"') => {
let token = complex_tokens::parse_string_or_char_constant(&mut source).ok()?;
self.offset += source.offset();
Some((token, &self.source[start..self.offset]))
}
_ => match self.parse().map(|tok| match tok {
Token::SlashSlash => {
self.skip_while(|c| c == '\n');
(Token::Comment)
}
Token::SlashSlashSlash => {
self.skip_while(|c| c == '\n');
(Token::DocComment)
}
_ => tok,
}) {
Some(tok) => {
if tok.maybe_ident() && self.skip_while(|c| is_things::is_id_continue(c)) > 0 {
Some((Token::Ident, &self.source[start..self.offset]))
} else {
Some((tok, &self.source[start..self.offset]))
}
}
None => {
if self
.peekable_source()
.next()
.map_or(false, |c| is_things::is_id_start(c))
{
self.skip(1);
self.skip_while(|c| is_things::is_id_continue(c));
Some((Token::Ident, &self.source[start..self.offset]))
} else {
None
}
}
},
};
token
}
}
mod complex_tokens;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_iterator() {
let tokens = "fn let void+++(++bool)";
let mut lexer = TokenIterator::new(&tokens);
assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
assert_eq!(lexer.next(), Some((Token::Let, "let")));
assert_eq!(lexer.next(), Some((Token::Void, "void")));
assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
assert_eq!(lexer.next(), Some((Token::Plus, "+")));
assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
assert_eq!(lexer.next(), Some((Token::PlusPlus, "++")));
assert_eq!(lexer.next(), Some((Token::Bool, "bool")));
assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
assert_eq!(lexer.next(), None);
}
#[test]
fn complex_iterator() {
let tokens = "fn my-function(x: i32, y: f32) -> f32 { return x + y; }";
let mut lexer = TokenIterator::new(&tokens);
assert_eq!(lexer.next(), Some((Token::Fn, "fn")));
assert_eq!(lexer.next(), Some((Token::Ident, "my-function")));
assert_eq!(lexer.next(), Some((Token::OpenParens, "(")));
assert_eq!(lexer.next(), Some((Token::Ident, "x")));
assert_eq!(lexer.next(), Some((Token::Colon, ":")));
assert_eq!(lexer.next(), Some((Token::Ident, "i32")));
assert_eq!(lexer.next(), Some((Token::Comma, ",")));
assert_eq!(lexer.next(), Some((Token::Ident, "y")));
assert_eq!(lexer.next(), Some((Token::Colon, ":")));
assert_eq!(lexer.next(), Some((Token::F32, "f32")));
assert_eq!(lexer.next(), Some((Token::CloseParens, ")")));
assert_eq!(lexer.next(), Some((Token::MinusGreater, "->")));
assert_eq!(lexer.next(), Some((Token::F32, "f32")));
assert_eq!(lexer.next(), Some((Token::OpenBrace, "{")));
assert_eq!(lexer.next(), Some((Token::Return, "return")));
assert_eq!(lexer.next(), Some((Token::Ident, "x")));
assert_eq!(lexer.next(), Some((Token::Plus, "+")));
assert_eq!(lexer.next(), Some((Token::Ident, "y")));
assert_eq!(lexer.next(), Some((Token::Semi, ";")));
assert_eq!(lexer.next(), Some((Token::CloseBrace, "}")));
assert_eq!(lexer.next(), None);
}
}

12
flake.lock
View file

@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1757745802,
"narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=",
"lastModified": 1753939845,
"narHash": "sha256-K2ViRJfdVGE8tpJejs8Qpvvejks1+A4GQej/lBk5y7I=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1",
"rev": "94def634a20494ee057c76998843c015909d6311",
"type": "github"
},
"original": {
@ -62,11 +62,11 @@
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1757989933,
"narHash": "sha256-9cpKYWWPCFhgwQTww8S94rTXgg8Q8ydFv9fXM6I8xQM=",
"lastModified": 1754102567,
"narHash": "sha256-d6nZ+1e4VDqW6VAwfx9EAUDJdPxSwqwGiuli32FEgoE=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "8249aa3442fb9b45e615a35f39eca2fe5510d7c3",
"rev": "08ff39bf869cadca3102b39824f4c7025186b7dc",
"type": "github"
},
"original": {

140
src/ast2/biunification.rs
View file

@ -5,43 +5,30 @@
// Visitor pattern has lots of unused arguments
#![allow(unused_variables)]
use std::collections::{HashMap, HashSet};
use crate::ast2::tag::{AstNode, AstNodeExt};
use std::collections::HashMap;
use super::{Ast, Index, intern, visitor::AstVisitorTrait};
type Id = u32;
trait TypeVariance {
type T;
type Opposite;
}
#[derive(Debug, Clone)]
enum TypeHead<T: TypeVariance> {
Real(intern::Index),
Function { args: Vec<T::T>, ret: T::Opposite },
enum Type {
Reified(intern::Index),
Variable(Id),
}
/// Variance of a type parameter or constraint.
/// A function of type `A -> B` is covariant in `B` and contravariant in `A`.
/// This means that a type `T` may be substituted for `A` if `T` is a subtype of
/// `A`, that is, every `T` is also an `A`,
/// but a type `T` may only be substituted for `B` if `T` is a supertype of `B`,
/// that is, every `B` is also a `T`.
/// This means that a type `T` may be substituted for `A` if `T` is a subtype of `A`, but
/// a type `T` may only be substituted for `B` if `T` is a supertype of `B`.
///
/// Namely, in a type system with `int` and `nat <: int`, for a function `f: int
/// -> int` in the expression `let u: int = 3; let t: nat = f(u);`, `u` may
/// safely be used as an argument to `f` because `nat <: int`, but `f(u`)` may
/// not be assigned to `t` because `int <: nat` is not true.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
enum Variance {
/// A Positive, or union relationship between types.
/// used in value-places.
#[doc(alias = "Positive")]
Covariant,
/// A Negative, or intersection relationship between types.
/// used in use-places.
#[doc(alias = "Negative")]
Contravariant,
}
@ -50,81 +37,27 @@ struct Value(Id);
#[derive(Debug, Clone, Copy)]
struct Use(Id);
impl TypeVariance for Value {
type T = Value;
type Opposite = Use;
}
impl TypeVariance for Use {
type T = Use;
type Opposite = Value;
}
/// Typechecking error.
#[derive(Debug, Clone, thiserror::Error)]
enum Error {
#[error("Unimplemented feature")]
Unimplemented,
#[error("{0}")]
StringError(String),
}
type Result<T> = std::result::Result<T, Error>;
struct Bindings {
next_id: Id,
inner: HashMap<super::Index, Id>,
bounds: HashSet<(Id, Id, Variance)>,
types: HashMap<Id, intern::Index>,
}
impl Bindings {
fn new() -> Self {
Bindings {
next_id: 1,
inner: HashMap::new(),
bounds: HashSet::new(),
types: HashMap::new(),
}
}
fn new_id(&mut self) -> Id {
let id = self.next_id;
self.next_id += 1;
id
}
fn get_or_create(&mut self, idx: super::Index) -> Id {
self.inner.get(&idx).copied().unwrap_or_else(|| {
let id = self.new_id();
self.inner.insert(idx, id);
id
})
}
/// retrieves the type Id for the given ast node.
fn get(&self, idx: super::Index) -> Option<Id> {
self.inner.get(&idx).copied()
}
/// inserts a proper type for `id`.
fn insert_type(&mut self, id: Id, ty: intern::Index) {
self.types.insert(id, ty);
}
inner: HashMap<super::Index, Value>,
}
struct TypeChecker<'a> {
pool: &'a mut intern::InternPool,
bindings: Bindings,
}
// Core
impl TypeChecker<'_> {
pub fn new(pool: &mut intern::InternPool) -> TypeChecker {
TypeChecker {
pool,
bindings: Bindings::new(),
}
TypeChecker { pool }
}
fn var(&mut self) -> (Value, Use) {
@ -140,33 +73,6 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
const UNIMPL: Self::Error = Error::Unimplemented;
fn visit_interned_type_impl(
&mut self,
ast: &'a Ast,
idx: Index,
intern: intern::Index,
) -> std::result::Result<Self::Value, Self::Error> {
let id = self.bindings.get_or_create(idx);
match self.pool.get_key(intern) {
intern::Key::SimpleType {
ty: intern::SimpleType::ComptimeInt,
} => {
// This is a type variable.
}
intern::Key::SimpleType { .. }
| intern::Key::PointerType { .. }
| intern::Key::ArrayType { .. }
| intern::Key::FunctionType { .. }
| intern::Key::StructType { .. } => {
// This is a real type.
self.bindings.insert_type(id, intern);
}
_ => unreachable!(),
}
Ok(Value(id))
}
fn visit_constant_impl(
&mut self,
ast: &'a Ast,
@ -174,30 +80,8 @@ impl<'a> AstVisitorTrait<&'a Ast> for TypeChecker<'_> {
ty: Index,
value: intern::Index,
) -> std::result::Result<Self::Value, Self::Error> {
// get type from the pool
let AstNode::InternedType { intern } = ast.get_ast_node(ty) else {
panic!(
"Expected an interned type node, got {:?}",
ast.get_ast_node(ty)
);
};
match self.pool.get_key(intern) {
intern::Key::SimpleType {
ty: intern::SimpleType::ComptimeInt,
} => {
// This is a type variable.
}
intern::Key::SimpleType { .. }
| intern::Key::PointerType { .. }
| intern::Key::ArrayType { .. }
| intern::Key::FunctionType { .. }
| intern::Key::StructType { .. } => {
// This is a real type.
}
_ => unreachable!(),
}
// constants may be of type `comptime_int`, which is a special type that
// cannot exist at runtime.
todo!()
}

242
src/ast2/intern.rs
View file

@ -15,7 +15,7 @@ use crate::{
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum SimpleType {
F32,
F32 = 0,
F64,
Bool,
Void,
@ -28,14 +28,11 @@ pub enum SimpleType {
/// Bottom type: this is the subtype of all types, and it can coerce into a value of any type.
/// Akin to Rust's `!`.
Bottom,
UInt(u16),
SInt(u16),
}
impl From<u32> for SimpleType {
fn from(value: u32) -> Self {
let [discriminant, bits] = *crate::common::u32_as_u16_slice(&value);
match discriminant {
impl From<u8> for SimpleType {
fn from(value: u8) -> Self {
match value {
0 => Self::F32,
1 => Self::F64,
2 => Self::Bool,
@ -43,49 +40,28 @@ impl From<u32> for SimpleType {
4 => Self::USize,
5 => Self::ISize,
6 => Self::ComptimeInt,
7 => Self::Top,
8 => Self::Bottom,
9 => Self::UInt(bits),
10 => Self::SInt(bits),
_ => panic!("{value} is not a simple type"),
}
}
}
impl From<SimpleType> for u32 {
fn from(value: SimpleType) -> Self {
match value {
SimpleType::F32 => crate::common::u32_from_u16_slice(&[0, 0]),
SimpleType::F64 => crate::common::u32_from_u16_slice(&[1, 0]),
SimpleType::Bool => crate::common::u32_from_u16_slice(&[2, 0]),
SimpleType::Void => crate::common::u32_from_u16_slice(&[3, 0]),
SimpleType::USize => crate::common::u32_from_u16_slice(&[4, 0]),
SimpleType::ISize => crate::common::u32_from_u16_slice(&[5, 0]),
SimpleType::ComptimeInt => crate::common::u32_from_u16_slice(&[6, 0]),
SimpleType::Top => crate::common::u32_from_u16_slice(&[7, 0]),
SimpleType::Bottom => crate::common::u32_from_u16_slice(&[8, 0]),
SimpleType::UInt(bits) => crate::common::u32_from_u16_slice(&[9, bits]),
SimpleType::SInt(bits) => crate::common::u32_from_u16_slice(&[10, bits]),
}
}
}
impl Display for SimpleType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let fmt: std::borrow::Cow<str> = match self {
SimpleType::F32 => "f32".into(),
SimpleType::F64 => "f64".into(),
SimpleType::Bool => "bool".into(),
SimpleType::Void => "void".into(),
SimpleType::USize => "usize".into(),
SimpleType::ISize => "isize".into(),
SimpleType::ComptimeInt => "comptime_int".into(),
SimpleType::Top => "⊤".into(),
SimpleType::Bottom => "⊥".into(),
SimpleType::UInt(bits) => format!("u{bits}").into(),
SimpleType::SInt(bits) => format!("i{bits}").into(),
};
write!(f, "{fmt}",)
write!(
f,
"{}",
match self {
SimpleType::F32 => "f32",
SimpleType::F64 => "f64",
SimpleType::Bool => "bool",
SimpleType::Void => "void",
SimpleType::USize => "usize",
SimpleType::ISize => "isize",
SimpleType::ComptimeInt => "comptime_int",
SimpleType::Top => "⊤",
SimpleType::Bottom => "⊥",
}
)
}
}
@ -102,6 +78,8 @@ pub enum Tag {
F64,
PositiveInt,
NegativeInt,
UIntType,
SIntType,
SimpleType,
PointerType,
ArrayType,
@ -110,9 +88,9 @@ pub enum Tag {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub(super) struct Item {
pub(super) tag: Tag,
pub(super) index: u32,
struct Item {
tag: Tag,
index: u32,
}
#[derive(Debug, Clone, PartialEq)]
@ -148,6 +126,12 @@ pub enum Key<'a> {
NegativeInt {
bigint: BigInt,
},
UIntType {
bit_width: u16,
},
SIntType {
bit_width: u16,
},
SimpleType {
ty: SimpleType,
},
@ -194,6 +178,8 @@ impl Display for KeyDisplay<'_> {
Key::F64 { bits } => write!(f, "{bits}")?,
Key::PositiveInt { ref bigint } => write!(f, "{bigint}")?,
Key::NegativeInt { ref bigint } => write!(f, "{bigint}")?,
Key::UIntType { bit_width } => write!(f, "u{bit_width}")?,
Key::SIntType { bit_width } => write!(f, "i{bit_width}")?,
Key::SimpleType { ty } => write!(f, "{ty}")?,
Key::PointerType { pointee, flags } => {
write!(f, "*{flags}{}", self.ip.display_key(pointee))?
@ -252,6 +238,8 @@ impl Hash for Key<'_> {
Key::F64 { bits } => ordered_float::OrderedFloat(*bits).hash(state),
Key::PositiveInt { bigint } => bigint.hash(state),
Key::NegativeInt { bigint } => bigint.hash(state),
Key::UIntType { bit_width: bits } => bits.hash(state),
Key::SIntType { bit_width: bits } => bits.hash(state),
Key::SimpleType { ty } => ty.hash(state),
Key::PointerType { pointee, flags } => (pointee, flags).hash(state),
Key::ArrayType {
@ -556,20 +544,20 @@ static_keys!(
ISIZE => Key::SimpleType {ty: SimpleType::ISize,},
VOID => Key::SimpleType {ty: SimpleType::Void,},
COMPTIME_INT => Key::SimpleType {ty: SimpleType::ComptimeInt,},
I0 => Key::SimpleType { ty: SimpleType::SInt(0) },
U0 => Key::SimpleType { ty: SimpleType::UInt(0) },
I1 => Key::SimpleType { ty: SimpleType::SInt(1) },
U1 => Key::SimpleType { ty: SimpleType::UInt(1) },
I8 => Key::SimpleType { ty: SimpleType::SInt(8) },
U8 => Key::SimpleType { ty: SimpleType::UInt(8) },
I16 => Key::SimpleType { ty: SimpleType::SInt(16) },
U16 => Key::SimpleType { ty: SimpleType::UInt(16) },
I32 => Key::SimpleType { ty: SimpleType::SInt(32) },
U32 => Key::SimpleType { ty: SimpleType::UInt(32) },
I64 => Key::SimpleType { ty: SimpleType::SInt(64) },
U64 => Key::SimpleType { ty: SimpleType::UInt(64) },
I128 => Key::SimpleType { ty: SimpleType::SInt(128) },
U128 => Key::SimpleType { ty: SimpleType::UInt(128) },
I1 => Key::SIntType { bit_width: 1 },
U1 => Key::UIntType { bit_width: 1 },
I0 => Key::SIntType { bit_width: 0 },
U0 => Key::UIntType { bit_width: 0 },
I8 => Key::SIntType { bit_width: 8 },
U8 => Key::UIntType { bit_width: 8 },
I16 => Key::SIntType { bit_width: 16 },
U16 => Key::UIntType { bit_width: 16 },
I32 => Key::SIntType { bit_width: 32 },
U32 => Key::UIntType { bit_width: 32 },
I64 => Key::SIntType { bit_width: 64 },
U64 => Key::UIntType { bit_width: 64 },
I128 => Key::SIntType { bit_width: 128 },
U128 => Key::UIntType { bit_width: 128 },
TRUE => Key::TrueValue,
FALSE => Key::FalseValue,
EMPTY_STRING => Key::String { str: "" },
@ -618,51 +606,41 @@ impl InternPool {
ty: SimpleType::ISize,
})
}
// Assumes the type is present in the pool.
fn get_simple_type_unchecked(&self, ty: SimpleType) -> Index {
self.get_assume_present(&Key::SimpleType { ty })
}
pub fn get_u0_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(0))
self.get_assume_present(&Key::UIntType { bit_width: 0 })
}
pub fn get_i0_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(0))
self.get_assume_present(&Key::SIntType { bit_width: 0 })
}
pub fn get_u1_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(1))
self.get_assume_present(&Key::UIntType { bit_width: 1 })
}
pub fn get_i1_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(1))
self.get_assume_present(&Key::SIntType { bit_width: 1 })
}
pub fn get_u8_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(8))
self.get_assume_present(&Key::UIntType { bit_width: 8 })
}
pub fn get_i8_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(8))
self.get_assume_present(&Key::SIntType { bit_width: 8 })
}
pub fn get_u16_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(16))
self.get_assume_present(&Key::UIntType { bit_width: 16 })
}
pub fn get_i16_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(16))
self.get_assume_present(&Key::SIntType { bit_width: 16 })
}
pub fn get_u32_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(32))
self.get_assume_present(&Key::UIntType { bit_width: 32 })
}
pub fn get_i32_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(32))
self.get_assume_present(&Key::SIntType { bit_width: 32 })
}
pub fn get_u64_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(64))
self.get_assume_present(&Key::UIntType { bit_width: 64 })
}
pub fn get_i64_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(64))
}
pub fn get_u128_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::UInt(128))
}
pub fn get_i128_type(&self) -> Index {
self.get_simple_type_unchecked(SimpleType::SInt(128))
self.get_assume_present(&Key::SIntType { bit_width: 64 })
}
pub fn get_top_type(&self) -> Index {
@ -722,9 +700,8 @@ impl InternPool {
| Key::SimpleType {
ty: SimpleType::ISize,
}
| Key::SimpleType {
ty: SimpleType::SInt(_) | SimpleType::UInt(_),
},
| Key::SIntType { .. }
| Key::UIntType { .. },
) => Some(rhs),
(
Key::SimpleType {
@ -739,9 +716,8 @@ impl InternPool {
| Key::SimpleType {
ty: SimpleType::ISize,
}
| Key::SimpleType {
ty: SimpleType::SInt(_) | SimpleType::UInt(_),
},
| Key::SIntType { .. }
| Key::UIntType { .. },
Key::SimpleType {
ty: SimpleType::ComptimeInt,
},
@ -769,6 +745,15 @@ impl InternPool {
pub fn to_mir_type(&self, index: Index, _ptr_size: TypeInfo) -> crate::mir::Type {
use crate::mir::Type;
match self.get_key(index) {
Key::UIntType { bit_width: bits } => {
let bits = bits as u32;
Type::from_bitsize_int(bits)
}
Key::SIntType { bit_width: bits } => {
let bits = bits as u32;
Type::from_bitsize_int(bits)
}
Key::SimpleType { ty } => match ty {
SimpleType::F32 => Type::SinglePrecision,
SimpleType::F64 => Type::DoublePrecision,
@ -777,9 +762,6 @@ impl InternPool {
todo!("void can't be turned into a mir type")
}
SimpleType::ISize | SimpleType::USize => Type::QWord,
SimpleType::UInt(bits) | SimpleType::SInt(bits) => {
Type::from_bitsize_int(bits as u32)
}
SimpleType::Top | SimpleType::Bottom | SimpleType::ComptimeInt => {
panic!("{ty} can't be turned into a mir type")
}
@ -800,9 +782,11 @@ impl InternPool {
pub fn is_type_signed(&self, index: Index, _ptr_size: TypeInfo) -> bool {
match self.get_key(index) {
Key::UIntType { .. } => false,
Key::SIntType { .. } => true,
Key::SimpleType { ty } => match ty {
SimpleType::USize | SimpleType::UInt(_) => false,
SimpleType::ISize | SimpleType::SInt(_) => true,
SimpleType::USize => false,
SimpleType::ISize => true,
_ => false,
},
Key::PointerType { .. } => false,
@ -815,17 +799,23 @@ impl InternPool {
pub fn size_of_type(&self, index: Index, ptr_size: TypeInfo) -> TypeInfo {
match self.get_key(index) {
Key::SimpleType { ty } => match ty {
SimpleType::SInt(bits) => TypeInfo {
bitsize: bits as u32,
bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
signed: true,
},
SimpleType::UInt(bits) => TypeInfo {
bitsize: bits as u32,
bitalign: (bits as u32).next_multiple_of(8).next_power_of_two(),
Key::UIntType { bit_width: bits } => {
let bits = bits as u32;
TypeInfo {
bitsize: bits,
bitalign: bits.next_multiple_of(8).next_power_of_two(),
signed: false,
},
}
}
Key::SIntType { bit_width: bits } => {
let bits = bits as u32;
TypeInfo {
bitsize: bits,
bitalign: bits.next_multiple_of(8).next_power_of_two(),
signed: true,
}
}
Key::SimpleType { ty } => match ty {
SimpleType::F32 => TypeInfo {
bitsize: 32,
bitalign: 32,
@ -899,13 +889,9 @@ impl InternPool {
crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
crate::ast::Type::Integer(i) => self.get_or_insert({
if i.signed {
Key::SimpleType {
ty: SimpleType::SInt(i.bits),
}
Key::SIntType { bit_width: i.bits }
} else {
Key::SimpleType {
ty: SimpleType::UInt(i.bits),
}
Key::UIntType { bit_width: i.bits }
}
}),
crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@ -944,13 +930,9 @@ impl InternPool {
crate::ast::Type::ComptimeNumber => self.get_comptime_int_type(),
crate::ast::Type::Integer(i) => self.get_assume_present(&{
if i.signed {
Key::SimpleType {
ty: SimpleType::SInt(i.bits),
}
Key::SIntType { bit_width: i.bits }
} else {
Key::SimpleType {
ty: SimpleType::UInt(i.bits),
}
Key::UIntType { bit_width: i.bits }
}
}),
crate::ast::Type::Floating(crate::ast::FloatingType::Binary32) => self.get_f32_type(),
@ -986,11 +968,11 @@ impl InternPool {
pub fn as_ast1_type(&self, pointer_bits: u16, index: Index) -> crate::ast::Type {
use crate::ast::Type;
match self.get_key(index) {
Key::UIntType { bit_width: bits } => Type::Integer(IntegralType::new(false, bits)),
Key::SIntType { bit_width: bits } => Type::Integer(IntegralType::new(true, bits)),
Key::SimpleType { ty } => match ty {
SimpleType::F32 => Type::Floating(crate::ast::FloatingType::Binary32),
SimpleType::F64 => Type::Floating(crate::ast::FloatingType::Binary64),
SimpleType::SInt(bits) => Type::Integer(IntegralType::new(true, bits)),
SimpleType::UInt(bits) => Type::Integer(IntegralType::new(false, bits)),
SimpleType::Bool => Type::Bool,
SimpleType::Void => Type::Void,
SimpleType::USize => Type::Integer(IntegralType::new(false, pointer_bits)),
@ -1116,7 +1098,9 @@ impl InternPool {
self.create_item(Tag::NegativeInt, i)
}
Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty.into()),
Key::UIntType { bit_width: bits } => self.create_item(Tag::UIntType, bits as u32),
Key::SIntType { bit_width: bits } => self.create_item(Tag::SIntType, bits as u32),
Key::SimpleType { ty } => self.create_item(Tag::SimpleType, ty as u8 as u32),
Key::PointerType { pointee, flags } => {
let flags = flags.pack();
let i = self.extend_words([pointee.index() as u32, flags as u32]);
@ -1246,11 +1230,17 @@ impl InternPool {
let bigint = BigInt::from_biguint(Sign::Plus, data);
Key::PositiveInt { bigint }
}
Tag::SIntType => Key::SIntType {
bit_width: item.index as u16,
},
Tag::UIntType => Key::UIntType {
bit_width: item.index as u16,
},
Tag::SimpleType => {
let ty = item.idx() as u32;
let ty = item.idx() as u8;
Key::SimpleType {
ty: SimpleType::from(ty),
ty: unsafe { core::mem::transmute::<u8, SimpleType>(ty) },
}
}
Tag::PointerType => {
@ -1347,12 +1337,8 @@ impl InternPool {
pub fn get_int_type(&mut self, signed: bool, bits: u16) -> Index {
let key = match signed {
true => Key::SimpleType {
ty: SimpleType::SInt(bits),
},
false => Key::SimpleType {
ty: SimpleType::UInt(bits),
},
true => Key::SIntType { bit_width: bits },
false => Key::UIntType { bit_width: bits },
};
self.get_or_insert(key)
@ -1549,7 +1535,7 @@ impl InternPool {
((index.index() as u32) < self.len()).then_some(index)
}
pub(super) fn get_item(&self, index: Index) -> Option<Item> {
fn get_item(&self, index: Index) -> Option<Item> {
self.check_bounds(index).map(|i| Item {
tag: self.tags[i.index()],
index: self.indices[i.index()],
@ -1568,6 +1554,8 @@ impl InternPool {
| Key::ArrayType { .. }
| Key::PointerType { .. }
| Key::SimpleType { .. }
| Key::SIntType { .. }
| Key::UIntType { .. }
| Key::StructType { .. }
)
}

34
src/ast2/internable.rs
View file

@ -1,34 +0,0 @@
use super::*;
use core::hash::Hash;
use std::hash::Hasher;
// Types implementing this trait can be stored in the internpool.
trait KeyTrait: Hash + Eq {
const TAG: Tag;
fn serialise(self, pool: &mut InternPool);
fn deserialise(index: Index, pool: &mut InternPool) -> Self;
}
impl KeyTrait for String {
const TAG: Tag = Tag::String;
fn serialise(self, pool: &mut InternPool) {
todo!()
}
fn deserialise(index: Index, pool: &mut InternPool) -> Self {
// let mut hasher = std::hash::DefaultHasher::new();
// core::any::TypeId::of::<Self>().hash(&mut hasher);
// let tag = hasher.finish() as u32;
let item = pool.get_item(index).unwrap();
assert_eq!(item.tag, Self::TAG);
let start = pool.words[item.idx()] as usize;
let len = pool.words[item.idx() + 1] as usize;
let str = unsafe {
let bytes = &pool.strings[start..start + len];
std::str::from_utf8_unchecked(bytes)
};
str.to_owned()
}
}

64
src/ast2/mod.rs
View file

@ -2134,6 +2134,36 @@ where
let ty_key = ip.get_key(ty);
let signed = ip.is_type_signed(ty, AMD64_POINTER_TYPE_INFO);
match ty_key {
intern::Key::SIntType { bit_width: bits } | intern::Key::UIntType { bit_width: bits } => {
let ty = IntegralType::new(signed, bits);
match ip.get_key(val) {
intern::Key::SIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::UIntSmall { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::PositiveInt { bigint } => {
ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
}
intern::Key::NegativeInt { bigint } => {
ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
}
_ => {
unreachable!()
}
}
}
intern::Key::SimpleType { ty } => match ty {
intern::SimpleType::F32 => match ip.get_key(val) {
intern::Key::F32 { bits } => {
@ -2226,40 +2256,6 @@ where
intern::SimpleType::Top | intern::SimpleType::Bottom => {
unreachable!()
}
intern::SimpleType::UInt(bits) | intern::SimpleType::SInt(bits) => {
let ty = IntegralType::new(signed, bits);
match ip.get_key(val) {
intern::Key::SIntSmall { bits } => {
ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
})
}
intern::Key::UIntSmall { bits } => {
ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
})
}
intern::Key::SInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::UInt64 { bits } => ComptimeNumber::Integral(ComptimeInt::Native {
bits: bits as _,
ty,
}),
intern::Key::PositiveInt { bigint } => {
ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
}
intern::Key::NegativeInt { bigint } => {
ComptimeNumber::Integral(ComptimeInt::BigInt { bits: bigint, ty })
}
_ => {
unreachable!()
}
}
}
},
_ => {
unreachable!()

20
src/common.rs
View file

@ -72,7 +72,9 @@ pub fn is_oct_digit(ch: char) -> bool {
}
pub fn is_hex_digit(ch: char) -> bool {
('0'..='9').contains(&ch) || ('a'..='f').contains(&ch) || ('A'..='F').contains(&ch)
('0'..='9').contains(&ch)
|| ('a'..='f').contains(&ch)
|| ('A'..='F').contains(&ch)
}
/// Trait for only yielding the next item in the Iterator if it tests true for some predicate
@ -174,19 +176,3 @@ pub fn from_lo_hi_dwords(lo: u32, hi: u32) -> u64 {
pub fn into_lo_hi_dwords(qword: u64) -> (u32, u32) {
(qword as u32, (qword >> 32) as u32)
}
pub fn u32_as_u16_slice(value: &u32) -> &[u16; 2] {
// SAFETY: This is safe because u32 is guaranteed to be 4 bytes and
// we are creating a slice of 2 u16s which is also 4 bytes.
unsafe { &*(value as *const u32 as *const [u16; 2]) }
}
// we can't transform any &[u16; 2] into a &u32 because of alignment guarantees
pub fn u32_from_u16_slice(slice: &[u16; 2]) -> u32 {
let mut out = 0u32;
unsafe {
core::ptr::copy_nonoverlapping(slice.as_ptr(), &raw mut out as *mut u16, 2);
}
out
}