Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-25 20:43:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

Simplify non-white token handling

Browse source
This commit is contained in:
Aleksey Kladov 2018-01-07 20:14:26 +03:00
parent bbf7e9bbd2
commit fc4d6cc298
2 changed files with 20 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/parser/event_parser/grammar.rs
View file

@ -4,7 +4,7 @@ use syntax_kinds::*;
// Items //
pub fn file(p: &mut Parser) {
pub(crate) fn file(p: &mut Parser) {
node(p, FILE, |p| {
p.optional(SHEBANG);
inner_attributes(p);

29
src/parser/event_parser/parser.rs
View file

@ -3,9 +3,15 @@ use super::{Event};
use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
pub struct Parser<'t> {
pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
pub(crate) struct Parser<'t> {
#[allow(unused)]
text: &'t str,
non_ws_tokens: Vec<(Token, TextUnit)>,
#[allow(unused)]
start_offsets: Vec<TextUnit>,
tokens: Vec<Token>, // non-whitespace tokens
pos: usize,
events: Vec<Event>,
@ -16,18 +22,21 @@ pub struct Parser<'t> {
impl<'t> Parser<'t> {
pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
let mut non_ws_tokens = Vec::new();
let mut tokens = Vec::new();
let mut start_offsets = Vec::new();
let mut len = TextUnit::new(0);
for &token in raw_tokens.iter() {
if !is_insignificant(token.kind) {
non_ws_tokens.push((token, len))
tokens.push(token);
start_offsets.push(len);
}
len += token.len;
}
Parser {
text,
non_ws_tokens,
start_offsets,
tokens,
pos: 0,
events: Vec::new(),
@ -42,11 +51,11 @@ impl<'t> Parser<'t> {
}
pub(crate) fn is_eof(&self) -> bool {
if self.pos == self.non_ws_tokens.len() {
if self.pos == self.tokens.len() {
return true
}
if let Some(limit) = self.curly_limit {
let token = self.non_ws_tokens[self.pos].0;
let token = self.tokens[self.pos];
return limit == self.curly_level && token.kind == R_CURLY;
}
false
@ -68,7 +77,7 @@ impl<'t> Parser<'t> {
if self.is_eof() {
return None;
}
let token = self.non_ws_tokens[self.pos].0;
let token = self.tokens[self.pos];
Some(token.kind)
}
@ -85,10 +94,10 @@ impl<'t> Parser<'t> {
}
pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool {
if self.non_ws_tokens[self.pos..].len() < kinds.len() {
if self.tokens[self.pos..].len() < kinds.len() {
return false
}
kinds.iter().zip(self.non_ws_tokens[self.pos..].iter().map(|&(t, _)| t.kind))
kinds.iter().zip(self.tokens[self.pos..].iter().map(|t| t.kind))
.all(|(&k1, k2)| k1 == k2)
}