feat: implement IR lexer

crates/jcc/src/ast/parse.rs

@@ -780,6 +780,12 @@ impl<'a, 'ctx> Parser<'a, 'ctx> {
     // Auxiliary Methods
     // ---------------------------------------------------------------------------
 
+    #[inline]
+    fn intern_span(&mut self, span: Span) -> Ident {
+        self.interner
+            .intern(self.file.slice(span).expect("expected span to be valid"))
+    }
+
     #[inline]
     fn push_const(&mut self, constant: Constant, span: Span) -> Expr {
         self.result.ast.expr.push(ExprData::new(
@@ -789,6 +795,15 @@ impl<'a, 'ctx> Parser<'a, 'ctx> {
         ))
     }
 
+    #[inline]
+    fn build_func_type(&mut self, params_slice: DeclSlice, ret: Ty<'ctx>) -> Ty<'ctx> {
+        let mut params = Vec::with_capacity(self.result.ast.decls.len_of(params_slice));
+        self.result.ast.decls[params_slice].iter().for_each(|d| {
+            params.push(self.result.ast.decl[*d].ty);
+        });
+        self.tys.func(ret, params)
+    }
+
     #[inline]
     fn parse_literal_or_emit<T: std::str::FromStr>(&mut self, text: &str, span: Span) -> Option<T> {
         text.parse().ok().or_else(|| {
@@ -799,21 +814,6 @@ impl<'a, 'ctx> Parser<'a, 'ctx> {
         })
     }
 
-    #[inline]
-    fn intern_span(&mut self, span: Span) -> Ident {
-        self.interner
-            .intern(self.file.slice(span).expect("expected span to be valid"))
-    }
-
-    #[inline]
-    fn build_func_type(&mut self, params_slice: DeclSlice, ret: Ty<'ctx>) -> Ty<'ctx> {
-        let mut params = Vec::with_capacity(self.result.ast.decls.len_of(params_slice));
-        self.result.ast.decls[params_slice].iter().for_each(|d| {
-            params.push(self.result.ast.decl[*d].ty);
-        });
-        self.tys.func(ret, params)
-    }
-
     // ---------------------------------------------------------------------------
     // Lexer Helpers
     // ---------------------------------------------------------------------------

crates/jcc/src/testutil.rs

@@ -1,5 +1,5 @@
 use jcc_backend::codemap::{byte::BytePos, file::SourceFile};
 
 pub fn file(src: &str) -> SourceFile {
-    SourceFile::from_source(BytePos::ZERO, "<test>", format!("{src}\n"))
+    SourceFile::from_source(BytePos::ZERO, "<test>", src.to_owned())
 }

crates/jcc/src/token/lex.rs

@@ -130,13 +130,13 @@ impl<'a> Lexer<'a> {
     }
 
     #[inline]
-    fn match1(&mut self, ch: char, kind: TokenKind, fallback: TokenKind) -> Token {
+    fn match1(&mut self, ch: char, kind: TokenKind, fall: TokenKind) -> Token {
         match self.chars.peek() {
             Some((_, ch2)) if *ch2 == ch => {
                 self.chars.next();
                 Token::new(kind, self.span(2))
             }
-            _ => Token::new(fallback, self.span(1)),
+            _ => Token::new(fall, self.span(1)),
         }
     }
 
@@ -145,7 +145,7 @@ impl<'a> Lexer<'a> {
         &mut self,
         pair1: (char, TokenKind),
         pair2: (char, TokenKind),
-        fallback: TokenKind,
+        fall: TokenKind,
     ) -> Token {
         let (kind, len) = match self.chars.peek() {
             Some((_, c)) if *c == pair1.0 => {
@@ -156,7 +156,7 @@ impl<'a> Lexer<'a> {
                 self.chars.next();
                 (pair2.1, 2)
             }
-            _ => (fallback, 1),
+            _ => (fall, 1),
         };
         Token::new(kind, self.span(len))
     }
@@ -167,7 +167,7 @@ impl<'a> Lexer<'a> {
         kind1: (char, TokenKind),
         kind2: (char, TokenKind),
         kind3: (char, TokenKind),
-        fallback: TokenKind,
+        fall: TokenKind,
     ) -> Token {
         match self.chars.peek() {
             Some((_, c)) if *c == kind1.0 => {
@@ -178,7 +178,7 @@ impl<'a> Lexer<'a> {
                 self.chars.next();
                 self.match1(kind3.0, kind3.1, kind2.1)
             }
-            _ => Token::new(fallback, self.span(1)),
+            _ => Token::new(fall, self.span(1)),
         }
     }
 
@@ -332,7 +332,7 @@ impl<'a> Lexer<'a> {
         self.chars
             .peek()
             .map(|(idx, _)| *idx)
-            .unwrap_or(self.pos + 1)
+            .unwrap_or(self.file.source().len())
     }
 }
 

crates/jcc_arena/src/intern.rs

@@ -256,7 +256,7 @@ mod tests {
     }
 
     #[test]
-    fn test_identity_and_deduplication() {
+    fn identity_and_deduplication() {
         let arena = InternArena::new();
         let v1 = arena.intern(SimpleValue(42)).unwrap();
         let v2 = arena.intern(SimpleValue(42)).unwrap();
@@ -269,7 +269,7 @@ mod tests {
     }
 
     #[test]
-    fn test_empty_vectors_deduplication() {
+    fn empty_vectors_deduplication() {
         let ctx = TyCtx::new();
         let arr1 = ctx.arrow(vec![]);
         let arr2 = ctx.arrow(vec![]);
@@ -279,7 +279,7 @@ mod tests {
     }
 
     #[test]
-    fn test_recursive_structures() {
+    fn recursive_structures() {
         let ctx = TyCtx::new();
         let int = ctx.int();
 
@@ -300,7 +300,7 @@ mod tests {
     }
 
     #[test]
-    fn test_stress_reallocation() {
+    fn stress_reallocation() {
         let arena = InternArena::new();
 
         let mut refs = Vec::new();

crates/jcc_arena/src/lib.rs

@@ -220,7 +220,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_zst() {
+    fn zst() {
         struct Zst;
         let arena = Arena::new();
         let _a = arena.alloc(Zst).unwrap();
@@ -229,7 +229,7 @@ mod tests {
     }
 
     #[test]
-    fn test_basic_alloc() {
+    fn basic_alloc() {
         let arena = Arena::new();
         let a = arena.alloc(10).unwrap();
         let b = arena.alloc(20).unwrap();
@@ -240,14 +240,14 @@ mod tests {
     }
 
     #[test]
-    fn test_alloc_slice_basic() {
+    fn alloc_slice_basic() {
         let arena = Arena::new();
         let slice = arena.alloc_extend(vec![1, 2, 3]).unwrap();
         assert_eq!(slice, &[1, 2, 3]);
     }
 
     #[test]
-    fn test_persistence_across_chunks() {
+    fn persistence_across_chunks() {
         let arena = Arena::with_capacity(2);
         let a = arena.alloc(1).unwrap();
         let b = arena.alloc(2).unwrap();
@@ -258,7 +258,7 @@ mod tests {
     }
 
     #[test]
-    fn test_alloc_extend_overflow_handling() {
+    fn alloc_extend_overflow_handling() {
         struct LyingIter {
             val: i32,
             count: i32,

crates/jcc_backend/src/ir/mod.rs

@@ -1,10 +1,14 @@
 pub mod analysis;
 pub mod builder;
 pub mod inst;
+pub mod parser;
 pub mod pretty;
 pub mod term;
 pub mod ty;
 
+#[cfg(test)]
+mod testutil;
+
 use jcc_codemap::span::Span;
 use jcc_entity::{entity_impl, EntitySet, PrimaryMap};
 

crates/jcc_backend/src/ir/parser/lex.rs

@@ -0,0 +1,163 @@
+use crate::ir::parser::token::{Token, TokenKind};
+
+use jcc_codemap::{file::SourceFile, span::Span, Diagnostic, IntoDiagnostic, Issue, Label};
+
+use std::{iter::FusedIterator, iter::Peekable, str::CharIndices};
+
+// ---------------------------------------------------------------------------
+// Lexer
+// ---------------------------------------------------------------------------
+
+pub struct Lexer<'a> {
+    pos: usize,
+    file: &'a SourceFile,
+    chars: Peekable<CharIndices<'a>>,
+}
+
+impl<'a> FusedIterator for Lexer<'a> {}
+impl<'a> Iterator for Lexer<'a> {
+    type Item = Result<Token, Issue<LexerIssue>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while let Some((_, c)) = self.chars.peek() {
+            if !c.is_whitespace() {
+                break;
+            }
+            self.chars.next();
+        }
+        let (idx, c) = self.chars.next()?;
+        self.pos = idx;
+        Some(match c {
+            c if c.is_ascii_digit() => Ok(self.number()),
+            c if c.is_ascii_alphabetic() => self.word(TokenKind::BlockRef),
+            '@' => self.word(TokenKind::AtIdent),
+            '%' => self.word(TokenKind::ValueRef),
+            '#' => self.word(TokenKind::ParamIndex),
+            '-' => Ok(self.match1('>', TokenKind::Arrow, TokenKind::Minus)),
+            ',' => Ok(Token::new(TokenKind::Comma, self.file.single(self.pos))),
+            ':' => Ok(Token::new(TokenKind::Colon, self.file.single(self.pos))),
+            '=' => Ok(Token::new(TokenKind::Assign, self.file.single(self.pos))),
+            '{' => Ok(Token::new(TokenKind::LBrace, self.file.single(self.pos))),
+            '}' => Ok(Token::new(TokenKind::RBrace, self.file.single(self.pos))),
+            '[' => Ok(Token::new(TokenKind::LBrack, self.file.single(self.pos))),
+            ']' => Ok(Token::new(TokenKind::RBrack, self.file.single(self.pos))),
+            '(' => Ok(Token::new(TokenKind::LParen, self.file.single(self.pos))),
+            ')' => Ok(Token::new(TokenKind::RParen, self.file.single(self.pos))),
+            _ => Err(Issue {
+                kind: LexerIssue::UnexpectedChar,
+                span: self.file.single(self.pos),
+            }),
+        })
+    }
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new(file: &'a SourceFile) -> Self {
+        Self {
+            file,
+            pos: 0,
+            chars: file.source().char_indices().peekable(),
+        }
+    }
+
+    /// Creates a global span of `len` bytes starting at the current position.
+    #[inline]
+    fn span(&self, len: usize) -> Span {
+        self.file.span(self.pos, self.pos + len).unwrap()
+    }
+
+    #[inline]
+    fn match1(&mut self, ch: char, kind: TokenKind, fall: TokenKind) -> Token {
+        match self.chars.peek() {
+            Some((_, ch2)) if *ch2 == ch => {
+                self.chars.next();
+                Token::new(kind, self.span(2))
+            }
+            _ => Token::new(fall, self.span(1)),
+        }
+    }
+
+    fn word(&mut self, kind: TokenKind) -> Result<Token, Issue<LexerIssue>> {
+        let end = self.next_while(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.');
+        let span = self.file.span(self.pos, end).unwrap();
+        let word = &self.file.source()[self.pos..end];
+        match TokenKind::from_keyword(word) {
+            None => Ok(Token::new(kind, span)),
+            Some(kind) => Ok(Token::new(kind, span)),
+        }
+    }
+
+    fn number(&mut self) -> Token {
+        let mut kind = TokenKind::Integer;
+        let mut end = self.next_while(|c| c.is_ascii_digit());
+        if matches!(self.chars.peek(), Some((_, '.'))) {
+            self.chars.next();
+            kind = TokenKind::Float;
+            end = self.next_while(|c| c.is_ascii_digit());
+        }
+        if matches!(self.chars.peek(), Some((_, 'e' | 'E'))) {
+            self.chars.next();
+            if matches!(self.chars.peek(), Some((_, '+' | '-'))) {
+                self.chars.next();
+            }
+            end = self.next_while(|c| c.is_ascii_digit());
+            kind = TokenKind::Float;
+        }
+        Token::new(kind, self.file.span(self.pos, end).unwrap())
+    }
+
+    #[inline]
+    fn next_while<F>(&mut self, mut predicate: F) -> usize
+    where
+        F: FnMut(char) -> bool,
+    {
+        while let Some(&(_, c)) = self.chars.peek() {
+            if !predicate(c) {
+                break;
+            }
+            self.chars.next();
+        }
+        self.chars
+            .peek()
+            .map(|(idx, _)| *idx)
+            .unwrap_or(self.file.source().len())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// LexerIssue
+// ---------------------------------------------------------------------------
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LexerIssue {
+    UnexpectedChar,
+    InvalidEntityRef,
+    UnknownIdentifier,
+    UnknownAtIdentifier,
+}
+
+impl IntoDiagnostic for LexerIssue {
+    fn into_diagnostic(self, span: Span) -> Diagnostic {
+        let (msg, note) = match self {
+            LexerIssue::UnexpectedChar => (
+                "unexpected character",
+                "this character is not part of the IR syntax",
+            ),
+            LexerIssue::InvalidEntityRef => (
+                "invalid entity reference",
+                "expected decimal digits after '%' or '#'",
+            ),
+            LexerIssue::UnknownIdentifier => (
+                "unknown identifier",
+                "this identifier is not a recognized IR keyword",
+            ),
+            LexerIssue::UnknownAtIdentifier => (
+                "unknown '@' identifier",
+                "expected '@global<N>' or '@function<N>'",
+            ),
+        };
+        Diagnostic::error()
+            .with_label(Label::primary(span).with_message(msg))
+            .with_note(note)
+    }
+}