1 files changed, 93 insertions, 17 deletions

diff --git a/matrix-lang/src/lex.rs b/matrix-lang/src/lex.rs
index 82545c4..08cd7ed 100644
--- a/matrix-lang/src/lex.rs
+++ b/matrix-lang/src/lex.rs
@@ -152,6 +152,9 @@ pub enum TokenData {
 
     // eof
     Eof,
+
+    // comment
+    Comment,
 }
 
 #[derive(Debug, PartialEq)]
@@ -171,6 +174,18 @@ pub struct Lexer {
     byte_len: usize,
 }
 
+impl Default for Token {
+    fn default() -> Self {
+        Self {
+            data: TokenData::Eof,
+            pos: Position {row: 0, col: 0},
+            str: "".into(),
+            bidx: 0,
+            blen: 0,
+        }
+    }
+}
+
 trait IsIdent {
     fn is_initial_ident(&self) -> bool;
     fn is_ident(&self) -> bool;
@@ -471,7 +486,34 @@ impl Lexer {
         Err(error!("invalid number '{buf}'"))
     }
 
-    fn read_token(&mut self, ignore_newlines: bool) -> Result<Token> {
+    fn lex_comment(&mut self) -> TokenData {
+        loop {
+            match self.peek() {
+                '\0' | '\n' => break,
+                _ => self.next()
+            };
+        }
+        TokenData::Comment
+    }
+
+    fn lex_block_comment(&mut self) -> TokenData {
+        loop {
+            match self.peek() {
+                '\0' => break,
+                '*' => {
+                    self.next();
+                    if self.peek() == '/' {
+                        self.next();
+                        break;
+                    }
+                }
+                _ => {self.next();}
+            };
+        }
+        TokenData::Comment
+    }
+
+    fn read_token_impl(&mut self, ignore_newlines: bool) -> Result<Token> {
         use TokenData as T;
 
         self.skip_whitespace(ignore_newlines);
@@ -519,6 +561,14 @@ impl Lexer {
                     '=' => {
                         self.next();
                         T::AssignDivide
+                    },
+                    '/' => {
+                        self.next();
+                        self.lex_comment()
+                    },
+                    '*' => {
+                        self.next();
+                        self.lex_block_comment()
                     }
                     _ => T::Divide
                 }
@@ -731,9 +781,38 @@ impl Lexer {
         })
     }
 
+    fn save_state(&self) -> (Position, usize, usize) {
+        (self.pos, self.index, self.byte_len)
+    }
+
+    fn restore_state(&mut self, state: (Position, usize, usize)) {
+        self.pos = state.0;
+        self.index = state.1;
+        self.byte_len = state.2;
+    }
+
+    pub fn read_token(
+        &mut self,
+        peek: bool, // reset state to undo read
+        keep_comments: bool, // dont ignore comment tokens
+        significant_newlines: bool // newlines become semicolons
+    ) -> Result<Token> {
+        let state = self.save_state();
+        loop {
+            let token = self.read_token_impl(!significant_newlines)?;
+            if !keep_comments && token.data == TokenData::Comment {
+                continue;
+            }
+            if peek {
+                self.restore_state(state);
+            }
+            return Ok(token)
+        }
+    }
+
     pub fn next_token(&mut self) -> Result<Token> {
         let pos = self.pos;
-        match self.read_token(true) {
+        match self.read_token(false, false, false) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
@@ -741,37 +820,34 @@ impl Lexer {
 
     pub fn next_token_nl(&mut self) -> Result<Token> {
         let pos = self.pos;
-        match self.read_token(false) {
+        match self.read_token(false, false, true) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
 
     pub fn peek_token(&mut self) -> Result<Token> {
-        let idx = self.index;
         let pos = self.pos;
-        let bidx = self.byte_len;
-        let token = self.read_token(true);
-        self.index = idx;
-        self.pos = pos;
-        self.byte_len = bidx;
-        match token {
+        match self.read_token(true, false, false) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
 
     pub fn peek_token_nl(&mut self) -> Result<Token> {
-        let idx = self.index;
         let pos = self.pos;
-        let bidx = self.byte_len;
-        let token = self.read_token(false);
-        self.index = idx;
-        self.pos = pos;
-        self.byte_len = bidx;
-        match token {
+        match self.read_token(true, false, true) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
+
+    pub fn next_token_cmt(&mut self) -> Result<Token> {
+        let pos = self.pos;
+        match self.read_token(false, true, false) {
+            Ok(token) => Ok(token),
+            Err(e) => Err(e.pos(pos)),
+        }
+    }
+
 }