#![feature(extract_if, iter_advance_by)]
#![allow(dead_code, unused_macros)]

pub mod ast;
pub mod common;
pub mod lexer;
pub mod parser;
pub mod tokens;

pub fn tokenize<'a>(
    bytes: &'a [u8],
) -> Result<lexer::Tokenizer<'a>, (lexer::Tokenizer<'a>, Vec<lexer::TokenizeError>)> {
    lexer::Tokenizer::new_with_errors(bytes)
}