comments

matrix-bin/src/helper.rs

@@ -27,7 +27,7 @@ impl Validator for MatrixHelper {
 
         loop {
             use TokenData as T;
-            let token = match lexer.next_token() {
+            let token = match lexer.next_token_cmt() {
                 Ok(t) if t.data == T::Eof => break,
                 Ok(t) => t,
                 Err(err) => return Ok(ValidationResult::Invalid(Some(format!("{err}"))))
@@ -64,6 +64,15 @@ impl Validator for MatrixHelper {
                         Some(T::Try) => {},
                         _ => return unmatched!(token, T::Try)
                     },
+                T::Comment => {
+                    let str = token.str.as_str();
+                    let l = str.len();
+                    if l >= 2 && &str[0..2] == "/*" {
+                        if &str[l-2..l] != "*/" {
+                            return Ok(ValidationResult::Incomplete)
+                        }
+                    }
+                }
                 _ => {}
             };
         }
@@ -160,7 +169,7 @@ impl Highlighter for MatrixHelper {
         let mut buf = String::new();
         let mut last = 0;
         loop {
-            let token = match lexer.next_token() {
+            let token = match lexer.next_token_cmt() {
                 Ok(t) if t.data == T::Eof => break,
                 Ok(t) => t,
                 Err(_) => break
@@ -216,6 +225,7 @@ impl Highlighter for MatrixHelper {
                     T::Try |
                     T::Catch
                     => "\x1b[38;2;203;166;247m",
+                T::Comment => "\x1b[38;2;108;112;134m",
                 _ => {
                     buf += &token.str;
                     continue;

matrix-lang/src/lex.rs

@@ -152,6 +152,9 @@ pub enum TokenData {
 
     // eof
     Eof,
+
+    // comment
+    Comment,
 }
 
 #[derive(Debug, PartialEq)]
@@ -171,6 +174,18 @@ pub struct Lexer {
     byte_len: usize,
 }
 
+impl Default for Token {
+    fn default() -> Self {
+        Self {
+            data: TokenData::Eof,
+            pos: Position {row: 0, col: 0},
+            str: "".into(),
+            bidx: 0,
+            blen: 0,
+        }
+    }
+}
+
 trait IsIdent {
     fn is_initial_ident(&self) -> bool;
     fn is_ident(&self) -> bool;
@@ -471,7 +486,34 @@ impl Lexer {
         Err(error!("invalid number '{buf}'"))
     }
 
-    fn read_token(&mut self, ignore_newlines: bool) -> Result<Token> {
+    fn lex_comment(&mut self) -> TokenData {
+        loop {
+            match self.peek() {
+                '\0' | '\n' => break,
+                _ => self.next()
+            };
+        }
+        TokenData::Comment
+    }
+
+    fn lex_block_comment(&mut self) -> TokenData {
+        loop {
+            match self.peek() {
+                '\0' => break,
+                '*' => {
+                    self.next();
+                    if self.peek() == '/' {
+                        self.next();
+                        break;
+                    }
+                }
+                _ => {self.next();}
+            };
+        }
+        TokenData::Comment
+    }
+
+    fn read_token_impl(&mut self, ignore_newlines: bool) -> Result<Token> {
         use TokenData as T;
 
         self.skip_whitespace(ignore_newlines);
@@ -519,6 +561,14 @@ impl Lexer {
                     '=' => {
                         self.next();
                         T::AssignDivide
+                    },
+                    '/' => {
+                        self.next();
+                        self.lex_comment()
+                    },
+                    '*' => {
+                        self.next();
+                        self.lex_block_comment()
                     }
                     _ => T::Divide
                 }
@@ -731,9 +781,38 @@ impl Lexer {
         })
     }
 
+    fn save_state(&self) -> (Position, usize, usize) {
+        (self.pos, self.index, self.byte_len)
+    }
+
+    fn restore_state(&mut self, state: (Position, usize, usize)) {
+        self.pos = state.0;
+        self.index = state.1;
+        self.byte_len = state.2;
+    }
+
+    pub fn read_token(
+        &mut self,
+        peek: bool, // reset state to undo read
+        keep_comments: bool, // dont ignore comment tokens
+        significant_newlines: bool // newlines become semicolons
+    ) -> Result<Token> {
+        let state = self.save_state();
+        loop {
+            let token = self.read_token_impl(!significant_newlines)?;
+            if !keep_comments && token.data == TokenData::Comment {
+                continue;
+            }
+            if peek {
+                self.restore_state(state);
+            }
+            return Ok(token)
+        }
+    }
+
     pub fn next_token(&mut self) -> Result<Token> {
         let pos = self.pos;
-        match self.read_token(true) {
+        match self.read_token(false, false, false) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
@@ -741,37 +820,34 @@ impl Lexer {
 
     pub fn next_token_nl(&mut self) -> Result<Token> {
         let pos = self.pos;
-        match self.read_token(false) {
+        match self.read_token(false, false, true) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
 
     pub fn peek_token(&mut self) -> Result<Token> {
-        let idx = self.index;
         let pos = self.pos;
-        let bidx = self.byte_len;
-        let token = self.read_token(true);
-        self.index = idx;
-        self.pos = pos;
-        self.byte_len = bidx;
-        match token {
+        match self.read_token(true, false, false) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
 
     pub fn peek_token_nl(&mut self) -> Result<Token> {
-        let idx = self.index;
         let pos = self.pos;
-        let bidx = self.byte_len;
-        let token = self.read_token(false);
-        self.index = idx;
-        self.pos = pos;
-        self.byte_len = bidx;
-        match token {
+        match self.read_token(true, false, true) {
             Ok(token) => Ok(token),
             Err(e) => Err(e.pos(pos)),
         }
     }
+
+    pub fn next_token_cmt(&mut self) -> Result<Token> {
+        let pos = self.pos;
+        match self.read_token(false, true, false) {
+            Ok(token) => Ok(token),
+            Err(e) => Err(e.pos(pos)),
+        }
+    }
+
 }