Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-11-15 01:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity

move whitespace handling to tree builder

Browse source
This commit is contained in:
Aleksey Kladov 2019-02-21 12:03:42 +03:00
parent c84561bb62
commit 79ce0fa8d7
4 changed files with 157 additions and 186 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
crates/ra_syntax/src/parsing.rs
View file

@ -9,11 +9,11 @@ mod grammar;
mod reparsing;
use crate::{
SyntaxKind, SmolStr, SyntaxError,
SyntaxKind, SyntaxError,
parsing::{
builder::GreenBuilder,
builder::TreeBuilder,
input::ParserInput,
event::EventProcessor,
event::process,
parser::Parser,
},
syntax_node::GreenNode,
@ -28,22 +28,24 @@ pub(crate) use self::reparsing::incremental_reparse;
pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
let tokens = tokenize(&text);
parse_with(GreenBuilder::default(), text, &tokens, grammar::root)
let tree_sink = TreeBuilder::new(text, &tokens);
parse_with(tree_sink, text, &tokens, grammar::root)
}
fn parse_with<S: TreeSink>(
tree_sink: S,
mut tree_sink: S,
text: &str,
tokens: &[Token],
f: fn(&mut Parser),
) -> S::Tree {
let mut events = {
let events = {
let input = ParserInput::new(text, &tokens);
let mut p = Parser::new(&input);
f(&mut p);
p.finish()
};
EventProcessor::new(tree_sink, text, tokens, &mut events).process().finish()
process(&mut tree_sink, events);
tree_sink.finish()
}
/// `TreeSink` abstracts details of a particular syntax tree implementation.
@ -51,14 +53,14 @@ trait TreeSink {
type Tree;
/// Adds new leaf to the current branch.
fn leaf(&mut self, kind: SyntaxKind, text: SmolStr);
fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8);
/// Start new branch and make it current.
fn start_branch(&mut self, kind: SyntaxKind);
fn start_branch(&mut self, kind: SyntaxKind, root: bool);
/// Finish current branch and restore previous
/// branch as current.
fn finish_branch(&mut self);
fn finish_branch(&mut self, root: bool);
fn error(&mut self, error: ParseError);

127
crates/ra_syntax/src/parsing/builder.rs
View file

@ -1,40 +1,63 @@
use crate::{
SmolStr, SyntaxKind, SyntaxError, SyntaxErrorKind, TextUnit,
parsing::{TreeSink, ParseError},
SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange,
SyntaxKind::{self, *},
parsing::{TreeSink, ParseError, Token},
syntax_node::{GreenNode, RaTypes},
};
use rowan::GreenNodeBuilder;
pub(crate) struct GreenBuilder {
pub(crate) struct TreeBuilder<'a> {
text: &'a str,
tokens: &'a [Token],
text_pos: TextUnit,
token_pos: usize,
errors: Vec<SyntaxError>,
inner: GreenNodeBuilder<RaTypes>,
}
impl Default for GreenBuilder {
fn default() -> GreenBuilder {
GreenBuilder {
text_pos: TextUnit::default(),
errors: Vec::new(),
inner: GreenNodeBuilder::new(),
}
}
}
impl TreeSink for GreenBuilder {
impl<'a> TreeSink for TreeBuilder<'a> {
type Tree = (GreenNode, Vec<SyntaxError>);
fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) {
self.text_pos += TextUnit::of_str(text.as_str());
self.inner.leaf(kind, text);
fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
self.eat_trivias();
let n_tokens = n_tokens as usize;
let len = self.tokens[self.token_pos..self.token_pos + n_tokens]
.iter()
.map(|it| it.len)
.sum::<TextUnit>();
self.do_leaf(kind, len, n_tokens);
}
fn start_branch(&mut self, kind: SyntaxKind) {
self.inner.start_internal(kind)
fn start_branch(&mut self, kind: SyntaxKind, root: bool) {
if root {
self.inner.start_internal(kind);
return;
}
let n_trivias =
self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
let mut trivia_end =
self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>();
let n_attached_trivias = {
let leading_trivias = leading_trivias.iter().rev().map(|it| {
let next_end = trivia_end - it.len;
let range = TextRange::from_to(next_end, trivia_end);
trivia_end = next_end;
(it.kind, &self.text[range])
});
n_attached_trivias(kind, leading_trivias)
};
self.eat_n_trivias(n_trivias - n_attached_trivias);
self.inner.start_internal(kind);
self.eat_n_trivias(n_attached_trivias);
}
fn finish_branch(&mut self) {
fn finish_branch(&mut self, root: bool) {
if root {
self.eat_trivias()
}
self.inner.finish_internal();
}
@ -47,3 +70,67 @@ impl TreeSink for GreenBuilder {
(self.inner.finish(), self.errors)
}
}
impl<'a> TreeBuilder<'a> {
pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> {
TreeBuilder {
text,
tokens,
text_pos: 0.into(),
token_pos: 0,
errors: Vec::new(),
inner: GreenNodeBuilder::new(),
}
}
fn eat_trivias(&mut self) {
while let Some(&token) = self.tokens.get(self.token_pos) {
if !token.kind.is_trivia() {
break;
}
self.do_leaf(token.kind, token.len, 1);
}
}
fn eat_n_trivias(&mut self, n: usize) {
for _ in 0..n {
let token = self.tokens[self.token_pos];
assert!(token.kind.is_trivia());
self.do_leaf(token.kind, token.len, 1);
}
}
fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) {
let range = TextRange::offset_len(self.text_pos, len);
let text: SmolStr = self.text[range].into();
self.text_pos += len;
self.token_pos += n_tokens;
self.inner.leaf(kind, text);
}
}
fn n_attached_trivias<'a>(
kind: SyntaxKind,
trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
) -> usize {
match kind {
CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF
| MODULE | NAMED_FIELD_DEF => {
let mut res = 0;
for (i, (kind, text)) in trivias.enumerate() {
match kind {
WHITESPACE => {
if text.contains("\n\n") {
break;
}
}
COMMENT => {
res = i + 1;
}
_ => (),
}
}
res
}
_ => 0,
}
}

189
crates/ra_syntax/src/parsing/event.rs
View file

@ -10,13 +10,8 @@
use std::mem;
use crate::{
SmolStr,
SyntaxKind::{self, *},
TextRange, TextUnit,
parsing::{
ParseError, TreeSink,
lexer::Token,
},
parsing::{ParseError, TreeSink},
};
/// `Parser` produces a flat list of `Event`s.
@ -88,160 +83,46 @@ impl Event {
}
}
pub(super) struct EventProcessor<'a, S: TreeSink> {
sink: S,
text_pos: TextUnit,
text: &'a str,
token_pos: usize,
tokens: &'a [Token],
events: &'a mut [Event],
}
/// Generate the syntax tree with the control of events.
pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec<Event>) {
let mut forward_parents = Vec::new();
impl<'a, S: TreeSink> EventProcessor<'a, S> {
pub(super) fn new(
sink: S,
text: &'a str,
tokens: &'a [Token],
events: &'a mut [Event],
) -> EventProcessor<'a, S> {
EventProcessor { sink, text_pos: 0.into(), text, token_pos: 0, tokens, events }
}
for i in 0..events.len() {
match mem::replace(&mut events[i], Event::tombstone()) {
Event::Start { kind: TOMBSTONE, .. } => (),
/// Generate the syntax tree with the control of events.
pub(crate) fn process(mut self) -> S {
let mut forward_parents = Vec::new();
Event::Start { kind, forward_parent } => {
// For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
// in the normal control flow, the parent-child relation: `A -> B -> C`,
// while with the magic forward_parent, it writes: `C <- B <- A`.
for i in 0..self.events.len() {
match mem::replace(&mut self.events[i], Event::tombstone()) {
Event::Start { kind: TOMBSTONE, .. } => (),
Event::Start { kind, forward_parent } => {
// For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
// in the normal control flow, the parent-child relation: `A -> B -> C`,
// while with the magic forward_parent, it writes: `C <- B <- A`.
// append `A` into parents.
forward_parents.push(kind);
let mut idx = i;
let mut fp = forward_parent;
while let Some(fwd) = fp {
idx += fwd as usize;
// append `A`'s forward_parent `B`
fp = match mem::replace(&mut self.events[idx], Event::tombstone()) {
Event::Start { kind, forward_parent } => {
forward_parents.push(kind);
forward_parent
}
_ => unreachable!(),
};
// append `B`'s forward_parent `C` in the next stage.
}
for kind in forward_parents.drain(..).rev() {
self.start(kind);
}
}
Event::Finish => {
let is_last = i == self.events.len() - 1;
self.finish(is_last);
}
Event::Token { kind, n_raw_tokens } => {
self.eat_trivias();
let n_raw_tokens = n_raw_tokens as usize;
let len = self.tokens[self.token_pos..self.token_pos + n_raw_tokens]
.iter()
.map(|it| it.len)
.sum::<TextUnit>();
self.leaf(kind, len, n_raw_tokens);
}
Event::Error { msg } => self.sink.error(msg),
}
}
self.sink
}
/// Add the node into syntax tree but discard the comments/whitespaces.
fn start(&mut self, kind: SyntaxKind) {
if kind == SOURCE_FILE {
self.sink.start_branch(kind);
return;
}
let n_trivias =
self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
let mut trivia_end =
self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>();
let n_attached_trivias = {
let leading_trivias = leading_trivias.iter().rev().map(|it| {
let next_end = trivia_end - it.len;
let range = TextRange::from_to(next_end, trivia_end);
trivia_end = next_end;
(it.kind, &self.text[range])
});
n_attached_trivias(kind, leading_trivias)
};
self.eat_n_trivias(n_trivias - n_attached_trivias);
self.sink.start_branch(kind);
self.eat_n_trivias(n_attached_trivias);
}
fn finish(&mut self, is_last: bool) {
if is_last {
self.eat_trivias()
}
self.sink.finish_branch();
}
fn eat_trivias(&mut self) {
while let Some(&token) = self.tokens.get(self.token_pos) {
if !token.kind.is_trivia() {
break;
}
self.leaf(token.kind, token.len, 1);
}
}
fn eat_n_trivias(&mut self, n: usize) {
for _ in 0..n {
let token = self.tokens[self.token_pos];
assert!(token.kind.is_trivia());
self.leaf(token.kind, token.len, 1);
}
}
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) {
let range = TextRange::offset_len(self.text_pos, len);
let text: SmolStr = self.text[range].into();
self.text_pos += len;
self.token_pos += n_tokens;
self.sink.leaf(kind, text);
}
}
fn n_attached_trivias<'a>(
kind: SyntaxKind,
trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
) -> usize {
match kind {
CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF
| MODULE | NAMED_FIELD_DEF => {
let mut res = 0;
for (i, (kind, text)) in trivias.enumerate() {
match kind {
WHITESPACE => {
if text.contains("\n\n") {
break;
// append `A` into parents.
forward_parents.push(kind);
let mut idx = i;
let mut fp = forward_parent;
while let Some(fwd) = fp {
idx += fwd as usize;
// append `A`'s forward_parent `B`
fp = match mem::replace(&mut events[idx], Event::tombstone()) {
Event::Start { kind, forward_parent } => {
forward_parents.push(kind);
forward_parent
}
}
COMMENT => {
res = i + 1;
}
_ => (),
_ => unreachable!(),
};
// append `B`'s forward_parent `C` in the next stage.
}
for (j, kind) in forward_parents.drain(..).rev().enumerate() {
let is_root_node = i == 0 && j == 0;
sink.start_branch(kind, is_root_node);
}
}
res
Event::Finish => sink.finish_branch(i == events.len() - 1),
Event::Token { kind, n_raw_tokens } => {
sink.leaf(kind, n_raw_tokens);
}
Event::Error { msg } => sink.error(msg),
}
_ => 0,
}
}

5
crates/ra_syntax/src/parsing/reparsing.rs
View file

@ -5,7 +5,7 @@ use crate::{
syntax_error::SyntaxError,
parsing::{
grammar, parse_with,
builder::GreenBuilder,
builder::TreeBuilder,
parser::Parser,
lexer::{tokenize, Token},
}
@ -61,7 +61,8 @@ fn reparse_block<'node>(
if !is_balanced(&tokens) {
return None;
}
let (green, new_errors) = parse_with(GreenBuilder::default(), &text, &tokens, reparser);
let tree_sink = TreeBuilder::new(&text, &tokens);
let (green, new_errors) = parse_with(tree_sink, &text, &tokens, reparser);
Some((node, green, new_errors))
}