diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs index d657ee1cd0..f676a183c0 100644 --- a/src/parser/event_parser/grammar.rs +++ b/src/parser/event_parser/grammar.rs @@ -4,7 +4,7 @@ use syntax_kinds::*; // Items // -pub fn file(p: &mut Parser) { +pub(crate) fn file(p: &mut Parser) { node(p, FILE, |p| { p.optional(SHEBANG); inner_attributes(p); diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs index 923d0bd5a1..f0d1d358b7 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/event_parser/parser.rs @@ -3,9 +3,15 @@ use super::{Event}; use super::super::is_insignificant; use syntax_kinds::{L_CURLY, R_CURLY, ERROR}; -pub struct Parser<'t> { +pub(crate) const EOF: SyntaxKind = SyntaxKind(10000); + + +pub(crate) struct Parser<'t> { + #[allow(unused)] text: &'t str, - non_ws_tokens: Vec<(Token, TextUnit)>, + #[allow(unused)] + start_offsets: Vec, + tokens: Vec, // non-whitespace tokens pos: usize, events: Vec, @@ -16,18 +22,21 @@ pub struct Parser<'t> { impl<'t> Parser<'t> { pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { - let mut non_ws_tokens = Vec::new(); + let mut tokens = Vec::new(); + let mut start_offsets = Vec::new(); let mut len = TextUnit::new(0); for &token in raw_tokens.iter() { if !is_insignificant(token.kind) { - non_ws_tokens.push((token, len)) + tokens.push(token); + start_offsets.push(len); } len += token.len; } Parser { text, - non_ws_tokens, + start_offsets, + tokens, pos: 0, events: Vec::new(), @@ -42,11 +51,11 @@ impl<'t> Parser<'t> { } pub(crate) fn is_eof(&self) -> bool { - if self.pos == self.non_ws_tokens.len() { + if self.pos == self.tokens.len() { return true } if let Some(limit) = self.curly_limit { - let token = self.non_ws_tokens[self.pos].0; + let token = self.tokens[self.pos]; return limit == self.curly_level && token.kind == R_CURLY; } false @@ -68,7 +77,7 @@ impl<'t> Parser<'t> { if self.is_eof() { return None; } - let token = self.non_ws_tokens[self.pos].0; + let token = self.tokens[self.pos]; Some(token.kind) } @@ -85,10 +94,10 @@ impl<'t> Parser<'t> { } pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool { - if self.non_ws_tokens[self.pos..].len() < kinds.len() { + if self.tokens[self.pos..].len() < kinds.len() { return false } - kinds.iter().zip(self.non_ws_tokens[self.pos..].iter().map(|&(t, _)| t.kind)) + kinds.iter().zip(self.tokens[self.pos..].iter().map(|t| t.kind)) .all(|(&k1, k2)| k1 == k2) }