Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-25 20:43:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

Simplify non-white token handling

Browse source
This commit is contained in:
Aleksey Kladov 2018-01-07 20:14:26 +03:00
parent bbf7e9bbd2
commit fc4d6cc298
2 changed files with 20 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/parser/event_parser/grammar.rs
View file

@ -4,7 +4,7 @@ use syntax_kinds::*;
// Items // // Items //
pub fn file(p: &mut Parser) { pub(crate) fn file(p: &mut Parser) {
node(p, FILE, |p| { node(p, FILE, |p| {
p.optional(SHEBANG); p.optional(SHEBANG);
inner_attributes(p); inner_attributes(p);

29
src/parser/event_parser/parser.rs
View file

@ -3,9 +3,15 @@ use super::{Event};
use super::super::is_insignificant; use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR}; use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
pub struct Parser<'t> { pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
pub(crate) struct Parser<'t> {
#[allow(unused)]
text: &'t str, text: &'t str,
non_ws_tokens: Vec<(Token, TextUnit)>, #[allow(unused)]
start_offsets: Vec<TextUnit>,
tokens: Vec<Token>, // non-whitespace tokens
pos: usize, pos: usize,
events: Vec<Event>, events: Vec<Event>,
@ -16,18 +22,21 @@ pub struct Parser<'t> {
impl<'t> Parser<'t> { impl<'t> Parser<'t> {
pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
let mut non_ws_tokens = Vec::new(); let mut tokens = Vec::new();
let mut start_offsets = Vec::new();
let mut len = TextUnit::new(0); let mut len = TextUnit::new(0);
for &token in raw_tokens.iter() { for &token in raw_tokens.iter() {
if !is_insignificant(token.kind) { if !is_insignificant(token.kind) {
non_ws_tokens.push((token, len)) tokens.push(token);
start_offsets.push(len);
} }
len += token.len; len += token.len;
} }
Parser { Parser {
text, text,
non_ws_tokens, start_offsets,
tokens,
pos: 0, pos: 0,
events: Vec::new(), events: Vec::new(),
@ -42,11 +51,11 @@ impl<'t> Parser<'t> {
} }
pub(crate) fn is_eof(&self) -> bool { pub(crate) fn is_eof(&self) -> bool {
if self.pos == self.non_ws_tokens.len() { if self.pos == self.tokens.len() {
return true return true
} }
if let Some(limit) = self.curly_limit { if let Some(limit) = self.curly_limit {
let token = self.non_ws_tokens[self.pos].0; let token = self.tokens[self.pos];
return limit == self.curly_level && token.kind == R_CURLY; return limit == self.curly_level && token.kind == R_CURLY;
} }
false false
@ -68,7 +77,7 @@ impl<'t> Parser<'t> {
if self.is_eof() { if self.is_eof() {
return None; return None;
} }
let token = self.non_ws_tokens[self.pos].0; let token = self.tokens[self.pos];
Some(token.kind) Some(token.kind)
} }
@ -85,10 +94,10 @@ impl<'t> Parser<'t> {
} }
pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool { pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool {
if self.non_ws_tokens[self.pos..].len() < kinds.len() { if self.tokens[self.pos..].len() < kinds.len() {
return false return false
} }
kinds.iter().zip(self.non_ws_tokens[self.pos..].iter().map(|&(t, _)| t.kind)) kinds.iter().zip(self.tokens[self.pos..].iter().map(|t| t.kind))
.all(|(&k1, k2)| k1 == k2) .all(|(&k1, k2)| k1 == k2)
} }