From d0d05075ed52aa22dfec36b5a7b23e6a1a554496 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 19 Dec 2021 17:36:23 +0300 Subject: [PATCH 1/3] internal: replace TreeSink with a data structure The general theme of this is to make parser a better independent library. The specific thing we do here is replacing callback based TreeSink with a data structure. That is, rather than calling user-provided tree construction methods, the parser now spits out a very bare-bones tree, effectively a log of a DFS traversal. This makes the parser usable without any *specifc* tree sink, and allows us to, eg, move tests into this crate. Now, it's also true that this is a distinction without a difference, as the old and the new interface are equivalent in expressiveness. Still, this new thing seems somewhat simpler. But yeah, I admit I don't have a suuper strong motivation here, just a hunch that this is better. --- crates/mbe/src/syntax_bridge.rs | 17 ++++-- crates/mbe/src/tt_iter.rs | 49 +++++++-------- crates/parser/src/event.rs | 18 +++--- crates/parser/src/lib.rs | 38 ++++-------- crates/parser/src/parser.rs | 3 +- crates/parser/src/tree_traversal.rs | 67 +++++++++++++++++++++ crates/syntax/src/parsing.rs | 34 +++-------- crates/syntax/src/parsing/reparsing.rs | 8 +-- crates/syntax/src/parsing/text_tree_sink.rs | 44 ++++++++++---- crates/syntax/src/syntax_node.rs | 4 +- 10 files changed, 172 insertions(+), 110 deletions(-) create mode 100644 crates/parser/src/tree_traversal.rs diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 109842b0cd..39129b0305 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -1,6 +1,5 @@ //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`]. -use parser::{ParseError, TreeSink}; use rustc_hash::{FxHashMap, FxHashSet}; use syntax::{ ast::{self, make::tokens::doc_comment}, @@ -56,8 +55,18 @@ pub fn token_tree_to_syntax_node( _ => TokenBuffer::from_subtree(tt), }; let parser_tokens = to_parser_tokens(&buffer); + let tree_traversal = parser::parse(&parser_tokens, entry_point); let mut tree_sink = TtTreeSink::new(buffer.begin()); - parser::parse(&parser_tokens, &mut tree_sink, entry_point); + for event in tree_traversal.iter() { + match event { + parser::TraversalStep::Token { kind, n_raw_tokens } => { + tree_sink.token(kind, n_raw_tokens) + } + parser::TraversalStep::EnterNode { kind } => tree_sink.start_node(kind), + parser::TraversalStep::LeaveNode => tree_sink.finish_node(), + parser::TraversalStep::Error { msg } => tree_sink.error(msg.to_string()), + } + } if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -643,7 +652,7 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { &texts[idx..texts.len() - (1 - idx)] } -impl<'a> TreeSink for TtTreeSink<'a> { +impl<'a> TtTreeSink<'a> { fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { if kind == LIFETIME_IDENT { n_tokens = 2; @@ -741,7 +750,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { *self.roots.last_mut().unwrap() -= 1; } - fn error(&mut self, error: ParseError) { + fn error(&mut self, error: String) { self.inner.error(error, self.text_pos) } } diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index d05e84b0f0..632b591f65 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -3,9 +3,8 @@ use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint}; -use parser::TreeSink; use syntax::SyntaxKind; -use tt::buffer::{Cursor, TokenBuffer}; +use tt::buffer::TokenBuffer; macro_rules! err { () => { @@ -94,34 +93,28 @@ impl<'a> TtIter<'a> { &mut self, entry_point: ParserEntryPoint, ) -> ExpandResult> { - struct OffsetTokenSink<'a> { - cursor: Cursor<'a>, - error: bool, - } - - impl<'a> TreeSink for OffsetTokenSink<'a> { - fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == SyntaxKind::LIFETIME_IDENT { - n_tokens = 2; - } - for _ in 0..n_tokens { - self.cursor = self.cursor.bump_subtree(); - } - } - fn start_node(&mut self, _kind: SyntaxKind) {} - fn finish_node(&mut self) {} - fn error(&mut self, _error: parser::ParseError) { - self.error = true; - } - } - let buffer = TokenBuffer::from_tokens(self.inner.as_slice()); let parser_tokens = to_parser_tokens(&buffer); - let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; + let tree_traversal = parser::parse(&parser_tokens, entry_point); - parser::parse(&parser_tokens, &mut sink, entry_point); + let mut cursor = buffer.begin(); + let mut error = false; + for step in tree_traversal.iter() { + match step { + parser::TraversalStep::Token { kind, mut n_raw_tokens } => { + if kind == SyntaxKind::LIFETIME_IDENT { + n_raw_tokens = 2; + } + for _ in 0..n_raw_tokens { + cursor = cursor.bump_subtree(); + } + } + parser::TraversalStep::EnterNode { .. } | parser::TraversalStep::LeaveNode => (), + parser::TraversalStep::Error { .. } => error = true, + } + } - let mut err = if !sink.cursor.is_root() || sink.error { + let mut err = if !cursor.is_root() || error { Some(err!("expected {:?}", entry_point)) } else { None @@ -130,8 +123,8 @@ impl<'a> TtIter<'a> { let mut curr = buffer.begin(); let mut res = vec![]; - if sink.cursor.is_root() { - while curr != sink.cursor { + if cursor.is_root() { + while curr != cursor { if let Some(token) = curr.token_tree() { res.push(token); } diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs index 41b0328027..ca4c38f2e6 100644 --- a/crates/parser/src/event.rs +++ b/crates/parser/src/event.rs @@ -10,9 +10,8 @@ use std::mem; use crate::{ - ParseError, + tree_traversal::TreeTraversal, SyntaxKind::{self, *}, - TreeSink, }; /// `Parser` produces a flat list of `Event`s. @@ -77,7 +76,7 @@ pub(crate) enum Event { }, Error { - msg: ParseError, + msg: String, }, } @@ -88,7 +87,8 @@ impl Event { } /// Generate the syntax tree with the control of events. -pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { +pub(super) fn process(mut events: Vec) -> TreeTraversal { + let mut res = TreeTraversal::default(); let mut forward_parents = Vec::new(); for i in 0..events.len() { @@ -117,15 +117,17 @@ pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { for kind in forward_parents.drain(..).rev() { if kind != TOMBSTONE { - sink.start_node(kind); + res.enter_node(kind); } } } - Event::Finish => sink.finish_node(), + Event::Finish => res.leave_node(), Event::Token { kind, n_raw_tokens } => { - sink.token(kind, n_raw_tokens); + res.token(kind, n_raw_tokens); } - Event::Error { msg } => sink.error(msg), + Event::Error { msg } => res.error(msg), } } + + res } diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index dc02ae6e83..67bc7d9906 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -25,31 +25,19 @@ mod event; mod parser; mod grammar; mod tokens; +mod tree_traversal; #[cfg(test)] mod tests; pub(crate) use token_set::TokenSet; -pub use crate::{lexed_str::LexedStr, syntax_kind::SyntaxKind, tokens::Tokens}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ParseError(pub Box); - -/// `TreeSink` abstracts details of a particular syntax tree implementation. -pub trait TreeSink { - /// Adds new token to the current branch. - fn token(&mut self, kind: SyntaxKind, n_tokens: u8); - - /// Start new branch and make it current. - fn start_node(&mut self, kind: SyntaxKind); - - /// Finish current branch and restore previous - /// branch as current. - fn finish_node(&mut self); - - fn error(&mut self, error: ParseError); -} +pub use crate::{ + lexed_str::LexedStr, + syntax_kind::SyntaxKind, + tokens::Tokens, + tree_traversal::{TraversalStep, TreeTraversal}, +}; /// rust-analyzer parser allows you to choose one of the possible entry points. /// @@ -74,11 +62,11 @@ pub enum ParserEntryPoint { } /// Parse given tokens into the given sink as a rust file. -pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) { - parse(tokens, tree_sink, ParserEntryPoint::SourceFile); +pub fn parse_source_file(tokens: &Tokens) -> TreeTraversal { + parse(tokens, ParserEntryPoint::SourceFile) } -pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) { +pub fn parse(tokens: &Tokens, entry_point: ParserEntryPoint) -> TreeTraversal { let entry_point: fn(&'_ mut parser::Parser) = match entry_point { ParserEntryPoint::SourceFile => grammar::entry_points::source_file, ParserEntryPoint::Path => grammar::entry_points::path, @@ -99,7 +87,7 @@ pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserE let mut p = parser::Parser::new(tokens); entry_point(&mut p); let events = p.finish(); - event::process(tree_sink, events); + event::process(events) } /// A parsing function for a specific braced-block. @@ -119,11 +107,11 @@ impl Reparser { /// /// Tokens must start with `{`, end with `}` and form a valid brace /// sequence. - pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) { + pub fn parse(self, tokens: &Tokens) -> TreeTraversal { let Reparser(r) = self; let mut p = parser::Parser::new(tokens); r(&mut p); let events = p.finish(); - event::process(tree_sink, events); + event::process(events) } } diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 4c891108a6..4fc734f9c6 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -8,7 +8,6 @@ use limit::Limit; use crate::{ event::Event, tokens::Tokens, - ParseError, SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, TokenSet, T, }; @@ -196,7 +195,7 @@ impl<'t> Parser<'t> { /// structured errors with spans and notes, like rustc /// does. pub(crate) fn error>(&mut self, message: T) { - let msg = ParseError(Box::new(message.into())); + let msg = message.into(); self.push_event(Event::Error { msg }); } diff --git a/crates/parser/src/tree_traversal.rs b/crates/parser/src/tree_traversal.rs new file mode 100644 index 0000000000..4b3a64c85c --- /dev/null +++ b/crates/parser/src/tree_traversal.rs @@ -0,0 +1,67 @@ +//! TODO +use crate::SyntaxKind; + +/// Output of the parser. +#[derive(Default)] +pub struct TreeTraversal { + /// 32-bit encoding of events. If LSB is zero, then that's an index into the + /// error vector. Otherwise, it's one of the thee other variants, with data encoded as + /// + /// |16 bit kind|8 bit n_raw_tokens|4 bit tag|4 bit leftover| + /// + event: Vec, + error: Vec, +} + +pub enum TraversalStep<'a> { + Token { kind: SyntaxKind, n_raw_tokens: u8 }, + EnterNode { kind: SyntaxKind }, + LeaveNode, + Error { msg: &'a str }, +} + +impl TreeTraversal { + pub fn iter(&self) -> impl Iterator> { + self.event.iter().map(|&event| { + if event & 0b1 == 0 { + return TraversalStep::Error { msg: self.error[(event as usize) >> 1].as_str() }; + } + let tag = ((event & 0x0000_00F0) >> 4) as u8; + match tag { + 0 => { + let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into(); + let n_raw_tokens = ((event & 0x0000_FF00) >> 8) as u8; + TraversalStep::Token { kind, n_raw_tokens } + } + 1 => { + let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into(); + TraversalStep::EnterNode { kind } + } + 2 => TraversalStep::LeaveNode, + _ => unreachable!(), + } + }) + } + + pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { + let e = ((kind as u16 as u32) << 16) | ((n_tokens as u32) << 8) | (0 << 4) | 1; + self.event.push(e) + } + + pub(crate) fn enter_node(&mut self, kind: SyntaxKind) { + let e = ((kind as u16 as u32) << 16) | (1 << 4) | 1; + self.event.push(e) + } + + pub(crate) fn leave_node(&mut self) { + let e = 2 << 4 | 1; + self.event.push(e) + } + + pub(crate) fn error(&mut self, error: String) { + let idx = self.error.len(); + self.error.push(error); + let e = (idx as u32) << 1; + self.event.push(e); + } +} diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs index cba1ddde85..6721e5aa81 100644 --- a/crates/syntax/src/parsing.rs +++ b/crates/syntax/src/parsing.rs @@ -4,24 +4,18 @@ mod text_tree_sink; mod reparsing; -use parser::SyntaxKind; -use text_tree_sink::TextTreeSink; - -use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode}; +use crate::{ + parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode, +}; pub(crate) use crate::parsing::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let lexed = parser::LexedStr::new(text); let parser_tokens = lexed.to_tokens(); - - let mut tree_sink = TextTreeSink::new(lexed); - - parser::parse_source_file(&parser_tokens, &mut tree_sink); - - let (tree, parser_errors) = tree_sink.finish(); - - (tree, parser_errors) + let tree_traversal = parser::parse_source_file(&parser_tokens); + let (node, errors, _eof) = build_tree(lexed, tree_traversal, false); + (node, errors) } /// Returns `text` parsed as a `T` provided there are no parse errors. @@ -34,20 +28,12 @@ pub(crate) fn parse_text_as( return Err(()); } let parser_tokens = lexed.to_tokens(); + let tree_traversal = parser::parse(&parser_tokens, entry_point); + let (node, errors, eof) = build_tree(lexed, tree_traversal, true); - let mut tree_sink = TextTreeSink::new(lexed); - - // TextTreeSink assumes that there's at least some root node to which it can attach errors and - // tokens. We arbitrarily give it a SourceFile. - use parser::TreeSink; - tree_sink.start_node(SyntaxKind::SOURCE_FILE); - parser::parse(&parser_tokens, &mut tree_sink, entry_point); - tree_sink.finish_node(); - - let (tree, parser_errors, eof) = tree_sink.finish_eof(); - if !parser_errors.is_empty() || !eof { + if !errors.is_empty() || !eof { return Err(()); } - SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(()) + SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(()) } diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index e9567a838c..dca97a7f58 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -10,7 +10,7 @@ use parser::Reparser; use text_edit::Indel; use crate::{ - parsing::text_tree_sink::TextTreeSink, + parsing::text_tree_sink::build_tree, syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, SyntaxError, SyntaxKind::*, @@ -94,11 +94,9 @@ fn reparse_block( return None; } - let mut tree_sink = TextTreeSink::new(lexed); + let tree_traversal = reparser.parse(&parser_tokens); - reparser.parse(&parser_tokens, &mut tree_sink); - - let (green, new_parser_errors) = tree_sink.finish(); + let (green, new_parser_errors, _eof) = build_tree(lexed, tree_traversal, false); Some((node.replace_with(green), new_parser_errors, node.text_range())) } diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs index c9e7feb965..c435791746 100644 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ b/crates/syntax/src/parsing/text_tree_sink.rs @@ -2,7 +2,7 @@ use std::mem; -use parser::{LexedStr, ParseError, TreeSink}; +use parser::{LexedStr, TreeTraversal}; use crate::{ ast, @@ -12,6 +12,36 @@ use crate::{ SyntaxTreeBuilder, TextRange, }; +pub(crate) fn build_tree( + lexed: LexedStr<'_>, + tree_traversal: TreeTraversal, + synthetic_root: bool, +) -> (GreenNode, Vec, bool) { + let mut builder = TextTreeSink::new(lexed); + + if synthetic_root { + builder.start_node(SyntaxKind::SOURCE_FILE); + } + + for event in tree_traversal.iter() { + match event { + parser::TraversalStep::Token { kind, n_raw_tokens } => { + builder.token(kind, n_raw_tokens) + } + parser::TraversalStep::EnterNode { kind } => builder.start_node(kind), + parser::TraversalStep::LeaveNode => builder.finish_node(), + parser::TraversalStep::Error { msg } => { + let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap(); + builder.inner.error(msg.to_string(), text_pos); + } + } + } + if synthetic_root { + builder.finish_node() + } + builder.finish_eof() +} + /// Bridges the parser with our specific syntax tree representation. /// /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. @@ -28,7 +58,7 @@ enum State { PendingFinish, } -impl<'a> TreeSink for TextTreeSink<'a> { +impl<'a> TextTreeSink<'a> { fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => unreachable!(), @@ -70,11 +100,6 @@ impl<'a> TreeSink for TextTreeSink<'a> { State::Normal => (), } } - - fn error(&mut self, error: ParseError) { - let text_pos = self.lexed.text_start(self.pos).try_into().unwrap(); - self.inner.error(error, text_pos); - } } impl<'a> TextTreeSink<'a> { @@ -106,11 +131,6 @@ impl<'a> TextTreeSink<'a> { (node, errors, is_eof) } - pub(super) fn finish(self) -> (GreenNode, Vec) { - let (node, errors, _eof) = self.finish_eof(); - (node, errors) - } - fn eat_trivias(&mut self) { while self.pos < self.lexed.len() { let kind = self.lexed.kind(self.pos); diff --git a/crates/syntax/src/syntax_node.rs b/crates/syntax/src/syntax_node.rs index c95c76c0a8..b96f10c173 100644 --- a/crates/syntax/src/syntax_node.rs +++ b/crates/syntax/src/syntax_node.rs @@ -69,7 +69,7 @@ impl SyntaxTreeBuilder { self.inner.finish_node(); } - pub fn error(&mut self, error: parser::ParseError, text_pos: TextSize) { - self.errors.push(SyntaxError::new_at_offset(*error.0, text_pos)); + pub fn error(&mut self, error: String, text_pos: TextSize) { + self.errors.push(SyntaxError::new_at_offset(error, text_pos)); } } From 74de79b1daeefb4868ce34e03e84949d33d3dd1e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 25 Dec 2021 21:59:02 +0300 Subject: [PATCH 2/3] internal: rename --- crates/mbe/src/lib.rs | 2 +- crates/mbe/src/syntax_bridge.rs | 16 ++++---- ...to_parser_tokens.rs => to_parser_input.rs} | 4 +- crates/mbe/src/tt_iter.rs | 16 ++++---- crates/parser/src/event.rs | 6 +-- crates/parser/src/{tokens.rs => input.rs} | 22 +++++------ crates/parser/src/lexed_str.rs | 4 +- crates/parser/src/lib.rs | 26 ++++++++----- .../src/{tree_traversal.rs => output.rs} | 39 ++++++++++++------- crates/parser/src/parser.rs | 30 +++++++------- crates/syntax/src/parsing.rs | 12 +++--- crates/syntax/src/parsing/reparsing.rs | 2 +- crates/syntax/src/parsing/text_tree_sink.rs | 16 ++++---- 13 files changed, 106 insertions(+), 89 deletions(-) rename crates/mbe/src/{to_parser_tokens.rs => to_parser_input.rs} (97%) rename crates/parser/src/{tokens.rs => input.rs} (84%) rename crates/parser/src/{tree_traversal.rs => output.rs} (57%) diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 1a56878fdb..5e14a3fb59 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -10,7 +10,7 @@ mod parser; mod expander; mod syntax_bridge; mod tt_iter; -mod to_parser_tokens; +mod to_parser_input; #[cfg(test)] mod benchmark; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 39129b0305..f0c1f806ff 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -10,7 +10,7 @@ use syntax::{ use tt::buffer::{Cursor, TokenBuffer}; use crate::{ - to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap, + to_parser_input::to_parser_input, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap, }; /// Convert the syntax node to a `TokenTree` (what macro @@ -54,17 +54,17 @@ pub fn token_tree_to_syntax_node( } _ => TokenBuffer::from_subtree(tt), }; - let parser_tokens = to_parser_tokens(&buffer); - let tree_traversal = parser::parse(&parser_tokens, entry_point); + let parser_input = to_parser_input(&buffer); + let parser_output = parser::parse(&parser_input, entry_point); let mut tree_sink = TtTreeSink::new(buffer.begin()); - for event in tree_traversal.iter() { + for event in parser_output.iter() { match event { - parser::TraversalStep::Token { kind, n_raw_tokens } => { + parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => { tree_sink.token(kind, n_raw_tokens) } - parser::TraversalStep::EnterNode { kind } => tree_sink.start_node(kind), - parser::TraversalStep::LeaveNode => tree_sink.finish_node(), - parser::TraversalStep::Error { msg } => tree_sink.error(msg.to_string()), + parser::Step::Enter { kind } => tree_sink.start_node(kind), + parser::Step::Exit => tree_sink.finish_node(), + parser::Step::Error { msg } => tree_sink.error(msg.to_string()), } } if tree_sink.roots.len() != 1 { diff --git a/crates/mbe/src/to_parser_tokens.rs b/crates/mbe/src/to_parser_input.rs similarity index 97% rename from crates/mbe/src/to_parser_tokens.rs rename to crates/mbe/src/to_parser_input.rs index f419c78d46..6faa147218 100644 --- a/crates/mbe/src/to_parser_tokens.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -4,8 +4,8 @@ use syntax::{SyntaxKind, SyntaxKind::*, T}; use tt::buffer::TokenBuffer; -pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens { - let mut res = parser::Tokens::default(); +pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { + let mut res = parser::Input::default(); let mut current = buffer.begin(); diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index 632b591f65..2d2dbd8994 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -1,7 +1,7 @@ //! A "Parser" structure for token trees. We use this when parsing a declarative //! macro definition into a list of patterns and templates. -use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint}; +use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult, ParserEntryPoint}; use syntax::SyntaxKind; use tt::buffer::TokenBuffer; @@ -94,23 +94,23 @@ impl<'a> TtIter<'a> { entry_point: ParserEntryPoint, ) -> ExpandResult> { let buffer = TokenBuffer::from_tokens(self.inner.as_slice()); - let parser_tokens = to_parser_tokens(&buffer); - let tree_traversal = parser::parse(&parser_tokens, entry_point); + let parser_input = to_parser_input(&buffer); + let tree_traversal = parser::parse(&parser_input, entry_point); let mut cursor = buffer.begin(); let mut error = false; for step in tree_traversal.iter() { match step { - parser::TraversalStep::Token { kind, mut n_raw_tokens } => { + parser::Step::Token { kind, mut n_input_tokens } => { if kind == SyntaxKind::LIFETIME_IDENT { - n_raw_tokens = 2; + n_input_tokens = 2; } - for _ in 0..n_raw_tokens { + for _ in 0..n_input_tokens { cursor = cursor.bump_subtree(); } } - parser::TraversalStep::EnterNode { .. } | parser::TraversalStep::LeaveNode => (), - parser::TraversalStep::Error { .. } => error = true, + parser::Step::Enter { .. } | parser::Step::Exit => (), + parser::Step::Error { .. } => error = true, } } diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs index ca4c38f2e6..b0e70e7943 100644 --- a/crates/parser/src/event.rs +++ b/crates/parser/src/event.rs @@ -10,7 +10,7 @@ use std::mem; use crate::{ - tree_traversal::TreeTraversal, + output::Output, SyntaxKind::{self, *}, }; @@ -87,8 +87,8 @@ impl Event { } /// Generate the syntax tree with the control of events. -pub(super) fn process(mut events: Vec) -> TreeTraversal { - let mut res = TreeTraversal::default(); +pub(super) fn process(mut events: Vec) -> Output { + let mut res = Output::default(); let mut forward_parents = Vec::new(); for i in 0..events.len() { diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/input.rs similarity index 84% rename from crates/parser/src/tokens.rs rename to crates/parser/src/input.rs index 4fc2361add..9504bd4d9e 100644 --- a/crates/parser/src/tokens.rs +++ b/crates/parser/src/input.rs @@ -1,26 +1,26 @@ -//! Input for the parser -- a sequence of tokens. -//! -//! As of now, parser doesn't have access to the *text* of the tokens, and makes -//! decisions based solely on their classification. Unlike `LexerToken`, the -//! `Tokens` doesn't include whitespace and comments. +//! See [`Input`]. use crate::SyntaxKind; #[allow(non_camel_case_types)] type bits = u64; -/// Main input to the parser. +/// Input for the parser -- a sequence of tokens. /// -/// A sequence of tokens represented internally as a struct of arrays. +/// As of now, parser doesn't have access to the *text* of the tokens, and makes +/// decisions based solely on their classification. Unlike `LexerToken`, the +/// `Tokens` doesn't include whitespace and comments. Main input to the parser. +/// +/// Struct of arrays internally, but this shouldn't really matter. #[derive(Default)] -pub struct Tokens { +pub struct Input { kind: Vec, joint: Vec, contextual_kind: Vec, } /// `pub` impl used by callers to create `Tokens`. -impl Tokens { +impl Input { #[inline] pub fn push(&mut self, kind: SyntaxKind) { self.push_impl(kind, SyntaxKind::EOF) @@ -63,7 +63,7 @@ impl Tokens { } /// pub(crate) impl used by the parser to consume `Tokens`. -impl Tokens { +impl Input { pub(crate) fn kind(&self, idx: usize) -> SyntaxKind { self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF) } @@ -76,7 +76,7 @@ impl Tokens { } } -impl Tokens { +impl Input { fn bit_index(&self, n: usize) -> (usize, usize) { let idx = n / (bits::BITS as usize); let b_idx = n % (bits::BITS as usize); diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index b8936c3440..f17aae1d31 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -122,8 +122,8 @@ impl<'a> LexedStr<'a> { self.error.iter().map(|it| (it.token as usize, it.msg.as_str())) } - pub fn to_tokens(&self) -> crate::Tokens { - let mut res = crate::Tokens::default(); + pub fn to_input(&self) -> crate::Input { + let mut res = crate::Input::default(); let mut was_joint = false; for i in 0..self.len() { let kind = self.kind(i); diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 67bc7d9906..da78889f35 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -24,8 +24,8 @@ mod syntax_kind; mod event; mod parser; mod grammar; -mod tokens; -mod tree_traversal; +mod input; +mod output; #[cfg(test)] mod tests; @@ -33,10 +33,10 @@ mod tests; pub(crate) use token_set::TokenSet; pub use crate::{ + input::Input, lexed_str::LexedStr, + output::{Output, Step}, syntax_kind::SyntaxKind, - tokens::Tokens, - tree_traversal::{TraversalStep, TreeTraversal}, }; /// rust-analyzer parser allows you to choose one of the possible entry points. @@ -62,11 +62,19 @@ pub enum ParserEntryPoint { } /// Parse given tokens into the given sink as a rust file. -pub fn parse_source_file(tokens: &Tokens) -> TreeTraversal { - parse(tokens, ParserEntryPoint::SourceFile) +pub fn parse_source_file(inp: &Input) -> Output { + parse(inp, ParserEntryPoint::SourceFile) } -pub fn parse(tokens: &Tokens, entry_point: ParserEntryPoint) -> TreeTraversal { +/// Parses the given [`Input`] into [`Output`] assuming that the top-level +/// syntactic construct is the given [`ParserEntryPoint`]. +/// +/// Both input and output here are fairly abstract. The overall flow is that the +/// caller has some "real" tokens, converts them to [`Input`], parses them to +/// [`Output`], and then converts that into a "real" tree. The "real" tree is +/// made of "real" tokens, so this all hinges on rather tight coordination of +/// indices between the four stages. +pub fn parse(inp: &Input, entry_point: ParserEntryPoint) -> Output { let entry_point: fn(&'_ mut parser::Parser) = match entry_point { ParserEntryPoint::SourceFile => grammar::entry_points::source_file, ParserEntryPoint::Path => grammar::entry_points::path, @@ -84,7 +92,7 @@ pub fn parse(tokens: &Tokens, entry_point: ParserEntryPoint) -> TreeTraversal { ParserEntryPoint::Attr => grammar::entry_points::attr, }; - let mut p = parser::Parser::new(tokens); + let mut p = parser::Parser::new(inp); entry_point(&mut p); let events = p.finish(); event::process(events) @@ -107,7 +115,7 @@ impl Reparser { /// /// Tokens must start with `{`, end with `}` and form a valid brace /// sequence. - pub fn parse(self, tokens: &Tokens) -> TreeTraversal { + pub fn parse(self, tokens: &Input) -> Output { let Reparser(r) = self; let mut p = parser::Parser::new(tokens); r(&mut p); diff --git a/crates/parser/src/tree_traversal.rs b/crates/parser/src/output.rs similarity index 57% rename from crates/parser/src/tree_traversal.rs rename to crates/parser/src/output.rs index 4b3a64c85c..b613df029f 100644 --- a/crates/parser/src/tree_traversal.rs +++ b/crates/parser/src/output.rs @@ -1,43 +1,52 @@ -//! TODO +//! See [`Output`] + use crate::SyntaxKind; -/// Output of the parser. +/// Output of the parser -- a DFS traversal of a concrete syntax tree. +/// +/// Use the [`Output::iter`] method to iterate over traversal steps and consume +/// a syntax tree. +/// +/// In a sense, this is just a sequence of [`SyntaxKind`]-colored parenthesis +/// interspersed into the original [`crate::Input`]. The output is fundamentally +/// coordinated with the input and `n_input_tokens` refers to the number of +/// times [`crate::Input::push`] was called. #[derive(Default)] -pub struct TreeTraversal { +pub struct Output { /// 32-bit encoding of events. If LSB is zero, then that's an index into the /// error vector. Otherwise, it's one of the thee other variants, with data encoded as /// - /// |16 bit kind|8 bit n_raw_tokens|4 bit tag|4 bit leftover| + /// |16 bit kind|8 bit n_input_tokens|4 bit tag|4 bit leftover| /// event: Vec, error: Vec, } -pub enum TraversalStep<'a> { - Token { kind: SyntaxKind, n_raw_tokens: u8 }, - EnterNode { kind: SyntaxKind }, - LeaveNode, +pub enum Step<'a> { + Token { kind: SyntaxKind, n_input_tokens: u8 }, + Enter { kind: SyntaxKind }, + Exit, Error { msg: &'a str }, } -impl TreeTraversal { - pub fn iter(&self) -> impl Iterator> { +impl Output { + pub fn iter(&self) -> impl Iterator> { self.event.iter().map(|&event| { if event & 0b1 == 0 { - return TraversalStep::Error { msg: self.error[(event as usize) >> 1].as_str() }; + return Step::Error { msg: self.error[(event as usize) >> 1].as_str() }; } let tag = ((event & 0x0000_00F0) >> 4) as u8; match tag { 0 => { let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into(); - let n_raw_tokens = ((event & 0x0000_FF00) >> 8) as u8; - TraversalStep::Token { kind, n_raw_tokens } + let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8; + Step::Token { kind, n_input_tokens } } 1 => { let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into(); - TraversalStep::EnterNode { kind } + Step::Enter { kind } } - 2 => TraversalStep::LeaveNode, + 2 => Step::Exit, _ => unreachable!(), } }) diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 4fc734f9c6..d4aecf9b44 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -7,7 +7,7 @@ use limit::Limit; use crate::{ event::Event, - tokens::Tokens, + input::Input, SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, TokenSet, T, }; @@ -22,7 +22,7 @@ use crate::{ /// "start expression, consume number literal, /// finish expression". See `Event` docs for more. pub(crate) struct Parser<'t> { - tokens: &'t Tokens, + inp: &'t Input, pos: usize, events: Vec, steps: Cell, @@ -31,8 +31,8 @@ pub(crate) struct Parser<'t> { static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000); impl<'t> Parser<'t> { - pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> { - Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) } + pub(super) fn new(inp: &'t Input) -> Parser<'t> { + Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0) } } pub(crate) fn finish(self) -> Vec { @@ -55,7 +55,7 @@ impl<'t> Parser<'t> { assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck"); self.steps.set(steps + 1); - self.tokens.kind(self.pos + n) + self.inp.kind(self.pos + n) } /// Checks if the current token is `kind`. @@ -91,7 +91,7 @@ impl<'t> Parser<'t> { T![<<=] => self.at_composite3(n, T![<], T![<], T![=]), T![>>=] => self.at_composite3(n, T![>], T![>], T![=]), - _ => self.tokens.kind(self.pos + n) == kind, + _ => self.inp.kind(self.pos + n) == kind, } } @@ -130,17 +130,17 @@ impl<'t> Parser<'t> { } fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool { - self.tokens.kind(self.pos + n) == k1 - && self.tokens.kind(self.pos + n + 1) == k2 - && self.tokens.is_joint(self.pos + n) + self.inp.kind(self.pos + n) == k1 + && self.inp.kind(self.pos + n + 1) == k2 + && self.inp.is_joint(self.pos + n) } fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool { - self.tokens.kind(self.pos + n) == k1 - && self.tokens.kind(self.pos + n + 1) == k2 - && self.tokens.kind(self.pos + n + 2) == k3 - && self.tokens.is_joint(self.pos + n) - && self.tokens.is_joint(self.pos + n + 1) + self.inp.kind(self.pos + n) == k1 + && self.inp.kind(self.pos + n + 1) == k2 + && self.inp.kind(self.pos + n + 2) == k3 + && self.inp.is_joint(self.pos + n) + && self.inp.is_joint(self.pos + n + 1) } /// Checks if the current token is in `kinds`. @@ -150,7 +150,7 @@ impl<'t> Parser<'t> { /// Checks if the current token is contextual keyword with text `t`. pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool { - self.tokens.contextual_kind(self.pos) == kw + self.inp.contextual_kind(self.pos) == kw } /// Starts a new node in the syntax tree. All nodes and tokens diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs index 6721e5aa81..20c7101a03 100644 --- a/crates/syntax/src/parsing.rs +++ b/crates/syntax/src/parsing.rs @@ -12,9 +12,9 @@ pub(crate) use crate::parsing::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let lexed = parser::LexedStr::new(text); - let parser_tokens = lexed.to_tokens(); - let tree_traversal = parser::parse_source_file(&parser_tokens); - let (node, errors, _eof) = build_tree(lexed, tree_traversal, false); + let parser_input = lexed.to_input(); + let parser_output = parser::parse_source_file(&parser_input); + let (node, errors, _eof) = build_tree(lexed, parser_output, false); (node, errors) } @@ -27,9 +27,9 @@ pub(crate) fn parse_text_as( if lexed.errors().next().is_some() { return Err(()); } - let parser_tokens = lexed.to_tokens(); - let tree_traversal = parser::parse(&parser_tokens, entry_point); - let (node, errors, eof) = build_tree(lexed, tree_traversal, true); + let parser_input = lexed.to_input(); + let parser_output = parser::parse(&parser_input, entry_point); + let (node, errors, eof) = build_tree(lexed, parser_output, true); if !errors.is_empty() || !eof { return Err(()); diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index dca97a7f58..bbe7209e72 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -89,7 +89,7 @@ fn reparse_block( let text = get_text_after_edit(node.clone().into(), edit); let lexed = parser::LexedStr::new(text.as_str()); - let parser_tokens = lexed.to_tokens(); + let parser_input = lexed.to_input(); if !is_balanced(&lexed) { return None; } diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs index c435791746..a5bc3f58dc 100644 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ b/crates/syntax/src/parsing/text_tree_sink.rs @@ -2,7 +2,7 @@ use std::mem; -use parser::{LexedStr, TreeTraversal}; +use parser::LexedStr; use crate::{ ast, @@ -14,7 +14,7 @@ use crate::{ pub(crate) fn build_tree( lexed: LexedStr<'_>, - tree_traversal: TreeTraversal, + parser_output: parser::Output, synthetic_root: bool, ) -> (GreenNode, Vec, bool) { let mut builder = TextTreeSink::new(lexed); @@ -23,14 +23,14 @@ pub(crate) fn build_tree( builder.start_node(SyntaxKind::SOURCE_FILE); } - for event in tree_traversal.iter() { + for event in parser_output.iter() { match event { - parser::TraversalStep::Token { kind, n_raw_tokens } => { + parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => { builder.token(kind, n_raw_tokens) } - parser::TraversalStep::EnterNode { kind } => builder.start_node(kind), - parser::TraversalStep::LeaveNode => builder.finish_node(), - parser::TraversalStep::Error { msg } => { + parser::Step::Enter { kind } => builder.start_node(kind), + parser::Step::Exit => builder.finish_node(), + parser::Step::Error { msg } => { let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap(); builder.inner.error(msg.to_string(), text_pos); } @@ -45,7 +45,7 @@ pub(crate) fn build_tree( /// Bridges the parser with our specific syntax tree representation. /// /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. -pub(crate) struct TextTreeSink<'a> { +struct TextTreeSink<'a> { lexed: LexedStr<'a>, pos: usize, state: State, From f692fafee8bf90d603c36668f43b151e5e111ec3 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 25 Dec 2021 22:02:12 +0300 Subject: [PATCH 3/3] rename --- crates/syntax/src/parsing/reparsing.rs | 2 +- crates/syntax/src/parsing/text_tree_sink.rs | 81 ++++++++++----------- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index bbe7209e72..a6abe3cccf 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -94,7 +94,7 @@ fn reparse_block( return None; } - let tree_traversal = reparser.parse(&parser_tokens); + let tree_traversal = reparser.parse(&parser_input); let (green, new_parser_errors, _eof) = build_tree(lexed, tree_traversal, false); diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs index a5bc3f58dc..f40c549e3d 100644 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ b/crates/syntax/src/parsing/text_tree_sink.rs @@ -17,10 +17,10 @@ pub(crate) fn build_tree( parser_output: parser::Output, synthetic_root: bool, ) -> (GreenNode, Vec, bool) { - let mut builder = TextTreeSink::new(lexed); + let mut builder = Builder::new(lexed); if synthetic_root { - builder.start_node(SyntaxKind::SOURCE_FILE); + builder.enter(SyntaxKind::SOURCE_FILE); } for event in parser_output.iter() { @@ -28,8 +28,8 @@ pub(crate) fn build_tree( parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => { builder.token(kind, n_raw_tokens) } - parser::Step::Enter { kind } => builder.start_node(kind), - parser::Step::Exit => builder.finish_node(), + parser::Step::Enter { kind } => builder.enter(kind), + parser::Step::Exit => builder.exit(), parser::Step::Error { msg } => { let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap(); builder.inner.error(msg.to_string(), text_pos); @@ -37,15 +37,12 @@ pub(crate) fn build_tree( } } if synthetic_root { - builder.finish_node() + builder.exit() } - builder.finish_eof() + builder.build() } -/// Bridges the parser with our specific syntax tree representation. -/// -/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. -struct TextTreeSink<'a> { +struct Builder<'a> { lexed: LexedStr<'a>, pos: usize, state: State, @@ -58,7 +55,35 @@ enum State { PendingFinish, } -impl<'a> TextTreeSink<'a> { +impl<'a> Builder<'a> { + fn new(lexed: parser::LexedStr<'a>) -> Self { + Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() } + } + + fn build(mut self) -> (GreenNode, Vec, bool) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingFinish => { + self.eat_trivias(); + self.inner.finish_node(); + } + State::PendingStart | State::Normal => unreachable!(), + } + + let (node, mut errors) = self.inner.finish_raw(); + for (i, err) in self.lexed.errors() { + let text_range = self.lexed.text_range(i); + let text_range = TextRange::new( + text_range.start.try_into().unwrap(), + text_range.end.try_into().unwrap(), + ); + errors.push(SyntaxError::new(err, text_range)) + } + + let is_eof = self.pos == self.lexed.len(); + + (node, errors, is_eof) + } + fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => unreachable!(), @@ -69,7 +94,7 @@ impl<'a> TextTreeSink<'a> { self.do_token(kind, n_tokens as usize); } - fn start_node(&mut self, kind: SyntaxKind) { + fn enter(&mut self, kind: SyntaxKind) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => { self.inner.start_node(kind); @@ -93,43 +118,13 @@ impl<'a> TextTreeSink<'a> { self.eat_n_trivias(n_attached_trivias); } - fn finish_node(&mut self) { + fn exit(&mut self) { match mem::replace(&mut self.state, State::PendingFinish) { State::PendingStart => unreachable!(), State::PendingFinish => self.inner.finish_node(), State::Normal => (), } } -} - -impl<'a> TextTreeSink<'a> { - pub(super) fn new(lexed: parser::LexedStr<'a>) -> Self { - Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() } - } - - pub(super) fn finish_eof(mut self) -> (GreenNode, Vec, bool) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingFinish => { - self.eat_trivias(); - self.inner.finish_node(); - } - State::PendingStart | State::Normal => unreachable!(), - } - - let (node, mut errors) = self.inner.finish_raw(); - for (i, err) in self.lexed.errors() { - let text_range = self.lexed.text_range(i); - let text_range = TextRange::new( - text_range.start.try_into().unwrap(), - text_range.end.try_into().unwrap(), - ); - errors.push(SyntaxError::new(err, text_range)) - } - - let is_eof = self.pos == self.lexed.len(); - - (node, errors, is_eof) - } fn eat_trivias(&mut self) { while self.pos < self.lexed.len() {