diff --git a/matrix-bin/src/helper.rs b/matrix-bin/src/helper.rs index df40946..a452218 100644 --- a/matrix-bin/src/helper.rs +++ b/matrix-bin/src/helper.rs @@ -27,7 +27,7 @@ impl Validator for MatrixHelper { loop { use TokenData as T; - let token = match lexer.next_token() { + let token = match lexer.next_token_cmt() { Ok(t) if t.data == T::Eof => break, Ok(t) => t, Err(err) => return Ok(ValidationResult::Invalid(Some(format!("{err}")))) @@ -64,6 +64,15 @@ impl Validator for MatrixHelper { Some(T::Try) => {}, _ => return unmatched!(token, T::Try) }, + T::Comment => { + let str = token.str.as_str(); + let l = str.len(); + if l >= 2 && &str[0..2] == "/*" { + if &str[l-2..l] != "*/" { + return Ok(ValidationResult::Incomplete) + } + } + } _ => {} }; } @@ -160,7 +169,7 @@ impl Highlighter for MatrixHelper { let mut buf = String::new(); let mut last = 0; loop { - let token = match lexer.next_token() { + let token = match lexer.next_token_cmt() { Ok(t) if t.data == T::Eof => break, Ok(t) => t, Err(_) => break @@ -216,6 +225,7 @@ impl Highlighter for MatrixHelper { T::Try | T::Catch => "\x1b[38;2;203;166;247m", + T::Comment => "\x1b[38;2;108;112;134m", _ => { buf += &token.str; continue; diff --git a/matrix-lang/src/lex.rs b/matrix-lang/src/lex.rs index 82545c4..08cd7ed 100644 --- a/matrix-lang/src/lex.rs +++ b/matrix-lang/src/lex.rs @@ -152,6 +152,9 @@ pub enum TokenData { // eof Eof, + + // comment + Comment, } #[derive(Debug, PartialEq)] @@ -171,6 +174,18 @@ pub struct Lexer { byte_len: usize, } +impl Default for Token { + fn default() -> Self { + Self { + data: TokenData::Eof, + pos: Position {row: 0, col: 0}, + str: "".into(), + bidx: 0, + blen: 0, + } + } +} + trait IsIdent { fn is_initial_ident(&self) -> bool; fn is_ident(&self) -> bool; @@ -471,7 +486,34 @@ impl Lexer { Err(error!("invalid number '{buf}'")) } - fn read_token(&mut self, ignore_newlines: bool) -> Result { + fn lex_comment(&mut self) -> TokenData { + loop { + match self.peek() { + '\0' | '\n' => break, + _ => self.next() + }; + } + TokenData::Comment + } + + fn lex_block_comment(&mut self) -> TokenData { + loop { + match self.peek() { + '\0' => break, + '*' => { + self.next(); + if self.peek() == '/' { + self.next(); + break; + } + } + _ => {self.next();} + }; + } + TokenData::Comment + } + + fn read_token_impl(&mut self, ignore_newlines: bool) -> Result { use TokenData as T; self.skip_whitespace(ignore_newlines); @@ -519,6 +561,14 @@ impl Lexer { '=' => { self.next(); T::AssignDivide + }, + '/' => { + self.next(); + self.lex_comment() + }, + '*' => { + self.next(); + self.lex_block_comment() } _ => T::Divide } @@ -731,9 +781,38 @@ impl Lexer { }) } + fn save_state(&self) -> (Position, usize, usize) { + (self.pos, self.index, self.byte_len) + } + + fn restore_state(&mut self, state: (Position, usize, usize)) { + self.pos = state.0; + self.index = state.1; + self.byte_len = state.2; + } + + pub fn read_token( + &mut self, + peek: bool, // reset state to undo read + keep_comments: bool, // dont ignore comment tokens + significant_newlines: bool // newlines become semicolons + ) -> Result { + let state = self.save_state(); + loop { + let token = self.read_token_impl(!significant_newlines)?; + if !keep_comments && token.data == TokenData::Comment { + continue; + } + if peek { + self.restore_state(state); + } + return Ok(token) + } + } + pub fn next_token(&mut self) -> Result { let pos = self.pos; - match self.read_token(true) { + match self.read_token(false, false, false) { Ok(token) => Ok(token), Err(e) => Err(e.pos(pos)), } @@ -741,37 +820,34 @@ impl Lexer { pub fn next_token_nl(&mut self) -> Result { let pos = self.pos; - match self.read_token(false) { + match self.read_token(false, false, true) { Ok(token) => Ok(token), Err(e) => Err(e.pos(pos)), } } pub fn peek_token(&mut self) -> Result { - let idx = self.index; let pos = self.pos; - let bidx = self.byte_len; - let token = self.read_token(true); - self.index = idx; - self.pos = pos; - self.byte_len = bidx; - match token { + match self.read_token(true, false, false) { Ok(token) => Ok(token), Err(e) => Err(e.pos(pos)), } } pub fn peek_token_nl(&mut self) -> Result { - let idx = self.index; let pos = self.pos; - let bidx = self.byte_len; - let token = self.read_token(false); - self.index = idx; - self.pos = pos; - self.byte_len = bidx; - match token { + match self.read_token(true, false, true) { Ok(token) => Ok(token), Err(e) => Err(e.pos(pos)), } } + + pub fn next_token_cmt(&mut self) -> Result { + let pos = self.pos; + match self.read_token(false, true, false) { + Ok(token) => Ok(token), + Err(e) => Err(e.pos(pos)), + } + } + }