stuff..

crates/lexer/src/lib.rs

@@ -195,6 +195,7 @@ tokens!(pub Token: {
         I32 => "i32",
         I64 => "i64",
         Const => "const",
+        Mutable => "mut",
         Volatile => "volatile",
         Noalias => "noalias",
         Fn => "fn",
@@ -211,6 +212,7 @@ tokens!(pub Token: {
         Packed => "packed",
         Extern => "extern",
         Pub => "pub",
+        Module => "mod",
         // Operators
         Dot => ".",
         MinusGreater => "->",
@@ -305,6 +307,7 @@ use std::{
 
 use trie::Tree;
 
+#[derive(Debug, Clone)]
 pub struct TokenItem<'a> {
     pub token: Token,
     pub lexeme: &'a str,
@@ -584,10 +587,13 @@ where
 {
     iter: &'a mut I,
     cache: Queue<'b, T>,
-    cursor: usize,
+    peeking_cursor: usize,
     _marker: PhantomData<Marker>,
 }
 
+pub type ReborrowingPeekingIterator<'a, 'b, I, T> = ReborrowingIterator<'a, 'b, I, T, Peeking>;
+pub type ReborrowingConsumingIterator<'a, 'b, I, T> = ReborrowingIterator<'a, 'b, I, T, Consuming>;
+
 impl<'a, 'b, I, T, Marker> ReborrowingIterator<'a, 'b, I, T, Marker>
 where
     I: Iterator<Item = T>,
@@ -596,7 +602,7 @@ where
         Self {
             iter,
             cache: Queue::Owned(VecDeque::new()),
-            cursor: 0,
+            peeking_cursor: 0,
             _marker: PhantomData,
         }
     }
@@ -605,7 +611,7 @@ where
         ReborrowingIterator {
             iter: self.iter,
             cache: self.cache,
-            cursor: 0,
+            peeking_cursor: 0,
             _marker: PhantomData,
         }
     }
@@ -614,7 +620,7 @@ where
         ReborrowingIterator {
             iter: self.iter,
             cache: self.cache,
-            cursor: 0,
+            peeking_cursor: 0,
             _marker: PhantomData,
         }
     }
@@ -623,7 +629,7 @@ where
         ReborrowingIterator {
             iter: self.iter,
             cache: self.cache.borrowed(),
-            cursor: 0,
+            peeking_cursor: 0,
             _marker: PhantomData,
         }
     }
@@ -632,30 +638,27 @@ where
         ReborrowingIterator {
             iter: self.iter,
             cache: self.cache.borrowed(),
-            cursor: 0,
+            peeking_cursor: 0,
             _marker: PhantomData,
         }
     }
+}
 
-    pub fn borrow_consuming_at_cursor(
+impl<'a, 'b, I, T> ReborrowingIterator<'a, 'b, I, T, Consuming>
-        &'_ mut self,
+where
-    ) -> ReborrowingIterator<'_, '_, I, T, Consuming> {
+    I: Iterator<Item = T>,
-        _ = self.cache.drain(0..self.cursor);
+{
-        ReborrowingIterator {
+    pub fn expect_one_of<Ts: IntoIterator<Item = T>>(&mut self, candidates: Ts) -> Option<T>
-            iter: self.iter,
+    where
-            cache: self.cache.borrowed(),
+        T: Eq,
-            cursor: 0,
+    {
-            _marker: PhantomData,
+        let mut candidates = candidates.into_iter();
-        }
-    }
 
-    pub fn reborrow_consuming_at_cursor(mut self) -> ReborrowingIterator<'a, 'b, I, T, Consuming> {
+        let token = self.next()?;
-        _ = self.cache.drain(0..self.cursor);
+        if candidates.any(|cand| cand == token) {
-        ReborrowingIterator {
+            Some(token)
-            iter: self.iter,
+        } else {
-            cache: self.cache,
+            None
-            cursor: 0,
-            _marker: PhantomData,
         }
     }
 }
@@ -671,33 +674,87 @@ where
     }
 }
 
+impl<'a, 'b, I, T> Iterator for ReborrowingIterator<'a, 'b, I, T, Peeking>
+where
+    I: Iterator<Item = T>,
+    T: Copy,
+{
+    type Item = T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.peek_next().copied()
+    }
+}
+
 impl<'a, 'b, I, T> ReborrowingIterator<'a, 'b, I, T, Peeking>
 where
     I: Iterator<Item = T>,
 {
-    pub fn peek(&mut self) -> Option<&T> {
+    pub fn peek_next(&mut self) -> Option<&T> {
-        if self.cursor >= self.cache.len() {
+        if self.peeking_cursor >= self.cache.len() {
             if let Some(item) = self.iter.next() {
-                self.cursor += 1;
+                self.peeking_cursor += 1;
                 Some(self.cache.push_back_mut(item))
             } else {
                 None
             }
         } else {
-            let item = self.cache.get(self.cursor)?;
+            let item = self.cache.get(self.peeking_cursor)?;
-            self.cursor += 1;
+            self.peeking_cursor += 1;
             Some(item)
         }
     }
 
+    pub fn drain_peeked(&mut self) -> impl Iterator<Item = T> + '_ {
+        let drained = self.cache.drain(0..self.peeking_cursor);
+        self.peeking_cursor = 0;
+        drained
+    }
+
     pub fn skip(&mut self, n: usize) {
-        let cached = self.cache.len() - self.cursor;
+        let cached = self.cache.len() - self.peeking_cursor;
-        self.cursor.saturating_add(n);
+        self.peeking_cursor = self.peeking_cursor.saturating_add(n);
         if n > cached {
             // need to pull from the underlying iterator
             let surplus = n - cached;
             self.cache.extend(self.iter.take(surplus));
-            self.cursor += n;
+            self.peeking_cursor += n;
+        }
+    }
+
+    pub fn borrow_consuming_at_cursor(
+        &'_ mut self,
+    ) -> ReborrowingIterator<'_, '_, I, T, Consuming> {
+        _ = self.drain_peeked();
+        ReborrowingIterator {
+            iter: self.iter,
+            cache: self.cache.borrowed(),
+            peeking_cursor: 0,
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn reborrow_consuming_at_cursor(mut self) -> ReborrowingIterator<'a, 'b, I, T, Consuming> {
+        _ = self.drain_peeked();
+        ReborrowingIterator {
+            iter: self.iter,
+            cache: self.cache,
+            peeking_cursor: 0,
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn peek_one_of<Ts: IntoIterator<Item = T>>(&mut self, candidates: Ts) -> Option<&T>
+    where
+        T: Eq,
+    {
+        let mut candidates = candidates.into_iter();
+
+        let token = self.peek_next()?;
+        if candidates.any(|cand| &cand == token) {
+            Some(token)
+        } else {
+            None
         }
     }
 }