From d94124e3b4a4380ad23c1a317fd3943664b7182a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 4 Feb 2018 13:53:47 +0300 Subject: [PATCH] Somewhat more san directory structure --- src/parser/event.rs | 140 ++++++++++++++++++ src/parser/event_parser/mod.rs | 74 --------- .../{event_parser => }/grammar/attributes.rs | 0 .../{event_parser => }/grammar/expressions.rs | 0 .../grammar/items/consts.rs | 0 .../{event_parser => }/grammar/items/mod.rs | 0 .../grammar/items/structs.rs | 0 .../grammar/items/traits.rs | 0 .../grammar/items/use_item.rs | 0 src/parser/{event_parser => }/grammar/mod.rs | 0 .../{event_parser => }/grammar/paths.rs | 0 .../{event_parser => }/grammar/type_params.rs | 0 .../{event_parser => }/grammar/types.rs | 0 src/parser/mod.rs | 89 ++--------- src/parser/{event_parser => }/parser.rs | 2 +- 15 files changed, 153 insertions(+), 152 deletions(-) create mode 100644 src/parser/event.rs delete mode 100644 src/parser/event_parser/mod.rs rename src/parser/{event_parser => }/grammar/attributes.rs (100%) rename src/parser/{event_parser => }/grammar/expressions.rs (100%) rename src/parser/{event_parser => }/grammar/items/consts.rs (100%) rename src/parser/{event_parser => }/grammar/items/mod.rs (100%) rename src/parser/{event_parser => }/grammar/items/structs.rs (100%) rename src/parser/{event_parser => }/grammar/items/traits.rs (100%) rename src/parser/{event_parser => }/grammar/items/use_item.rs (100%) rename src/parser/{event_parser => }/grammar/mod.rs (100%) rename src/parser/{event_parser => }/grammar/paths.rs (100%) rename src/parser/{event_parser => }/grammar/type_params.rs (100%) rename src/parser/{event_parser => }/grammar/types.rs (100%) rename src/parser/{event_parser => }/parser.rs (99%) diff --git a/src/parser/event.rs b/src/parser/event.rs new file mode 100644 index 0000000000..fd6bdc0863 --- /dev/null +++ b/src/parser/event.rs @@ -0,0 +1,140 @@ +use {File, FileBuilder, Sink, SyntaxKind, Token}; +use syntax_kinds::TOMBSTONE; +use super::is_insignificant; + +/// `Parser` produces a flat list of `Event`s. +/// They are converted to a tree-structure in +/// a separate pass, via `TreeBuilder`. +#[derive(Debug)] +pub(crate) enum Event { + /// This event signifies the start of the node. + /// It should be either abandoned (in which case the + /// `kind` is `TOMBSTONE`, and the event is ignored), + /// or completed via a `Finish` event. + /// + /// All tokens between a `Start` and a `Finish` would + /// become the children of the respective node. + /// + /// For left-recursive syntactic constructs, the parser produces + /// a child node before it sees a parent. `forward_parent` + /// exists to allow to tweak parent-child relationships. + /// + /// Consider this path + /// + /// foo::bar + /// + /// The events for it would look like this: + /// + /// + /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH + /// | /\ + /// | | + /// +------forward-parent------+ + /// + /// And the tree would look like this + /// + /// +--PATH---------+ + /// | | | + /// | | | + /// | '::' 'bar' + /// | + /// PATH + /// | + /// 'foo' + /// + /// See also `CompleteMarker::precede`. + Start { + kind: SyntaxKind, + forward_parent: Option, + }, + + /// Complete the previous `Start` event + Finish, + + /// Produce a single leaf-element. + /// `n_raw_tokens` is used to glue complex contextual tokens. + /// For example, lexer tokenizes `>>` as `>`, `>`, and + /// `n_raw_tokens = 2` is used to produced a single `>>`. + Token { + kind: SyntaxKind, + n_raw_tokens: u8, + }, + + Error { + message: String, + }, +} + +pub(super) fn to_file(text: String, tokens: &[Token], events: Vec) -> File { + let mut builder = FileBuilder::new(text); + let mut idx = 0; + + let mut holes = Vec::new(); + let mut forward_parents = Vec::new(); + + for (i, event) in events.iter().enumerate() { + if holes.last() == Some(&i) { + holes.pop(); + continue; + } + + match event { + &Event::Start { + kind: TOMBSTONE, .. + } => (), + + &Event::Start { .. } => { + forward_parents.clear(); + let mut idx = i; + loop { + let (kind, fwd) = match events[idx] { + Event::Start { + kind, + forward_parent, + } => (kind, forward_parent), + _ => unreachable!(), + }; + forward_parents.push((idx, kind)); + if let Some(fwd) = fwd { + idx += fwd as usize; + } else { + break; + } + } + for &(idx, kind) in forward_parents.iter().into_iter().rev() { + builder.start_internal(kind); + holes.push(idx); + } + holes.pop(); + } + &Event::Finish => { + while idx < tokens.len() { + let token = tokens[idx]; + if is_insignificant(token.kind) { + idx += 1; + builder.leaf(token.kind, token.len); + } else { + break; + } + } + builder.finish_internal() + } + &Event::Token { + kind: _, + mut n_raw_tokens, + } => loop { + let token = tokens[idx]; + if !is_insignificant(token.kind) { + n_raw_tokens -= 1; + } + idx += 1; + builder.leaf(token.kind, token.len); + if n_raw_tokens == 0 { + break; + } + }, + &Event::Error { ref message } => builder.error().message(message.clone()).emit(), + } + } + builder.finish() +} diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs deleted file mode 100644 index 7823c476c2..0000000000 --- a/src/parser/event_parser/mod.rs +++ /dev/null @@ -1,74 +0,0 @@ -use {SyntaxKind, Token}; - -#[macro_use] -mod parser; -mod grammar; - -/// `Parser` produces a flat list of `Event`s. -/// They are converted to a tree-structure in -/// a separate pass, via `TreeBuilder`. -#[derive(Debug)] -pub(crate) enum Event { - /// This event signifies the start of the node. - /// It should be either abandoned (in which case the - /// `kind` is `TOMBSTONE`, and the event is ignored), - /// or completed via a `Finish` event. - /// - /// All tokens between a `Start` and a `Finish` would - /// become the children of the respective node. - /// - /// For left-recursive syntactic constructs, the parser produces - /// a child node before it sees a parent. `forward_parent` - /// exists to allow to tweak parent-child relationships. - /// - /// Consider this path - /// - /// foo::bar - /// - /// The events for it would look like this: - /// - /// - /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH - /// | /\ - /// | | - /// +------forward-parent------+ - /// - /// And the tree would look like this - /// - /// +--PATH---------+ - /// | | | - /// | | | - /// | '::' 'bar' - /// | - /// PATH - /// | - /// 'foo' - /// - /// See also `CompleteMarker::precede`. - Start { - kind: SyntaxKind, - forward_parent: Option, - }, - - /// Complete the previous `Start` event - Finish, - - /// Produce a single leaf-element. - /// `n_raw_tokens` is used to glue complex contextual tokens. - /// For example, lexer tokenizes `>>` as `>`, `>`, and - /// `n_raw_tokens = 2` is used to produced a single `>>`. - Token { - kind: SyntaxKind, - n_raw_tokens: u8, - }, - - Error { - message: String, - }, -} - -pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec { - let mut parser = parser::Parser::new(text, raw_tokens); - grammar::file(&mut parser); - parser.into_events() -} diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/grammar/attributes.rs similarity index 100% rename from src/parser/event_parser/grammar/attributes.rs rename to src/parser/grammar/attributes.rs diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/grammar/expressions.rs similarity index 100% rename from src/parser/event_parser/grammar/expressions.rs rename to src/parser/grammar/expressions.rs diff --git a/src/parser/event_parser/grammar/items/consts.rs b/src/parser/grammar/items/consts.rs similarity index 100% rename from src/parser/event_parser/grammar/items/consts.rs rename to src/parser/grammar/items/consts.rs diff --git a/src/parser/event_parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs similarity index 100% rename from src/parser/event_parser/grammar/items/mod.rs rename to src/parser/grammar/items/mod.rs diff --git a/src/parser/event_parser/grammar/items/structs.rs b/src/parser/grammar/items/structs.rs similarity index 100% rename from src/parser/event_parser/grammar/items/structs.rs rename to src/parser/grammar/items/structs.rs diff --git a/src/parser/event_parser/grammar/items/traits.rs b/src/parser/grammar/items/traits.rs similarity index 100% rename from src/parser/event_parser/grammar/items/traits.rs rename to src/parser/grammar/items/traits.rs diff --git a/src/parser/event_parser/grammar/items/use_item.rs b/src/parser/grammar/items/use_item.rs similarity index 100% rename from src/parser/event_parser/grammar/items/use_item.rs rename to src/parser/grammar/items/use_item.rs diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/grammar/mod.rs similarity index 100% rename from src/parser/event_parser/grammar/mod.rs rename to src/parser/grammar/mod.rs diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/grammar/paths.rs similarity index 100% rename from src/parser/event_parser/grammar/paths.rs rename to src/parser/grammar/paths.rs diff --git a/src/parser/event_parser/grammar/type_params.rs b/src/parser/grammar/type_params.rs similarity index 100% rename from src/parser/event_parser/grammar/type_params.rs rename to src/parser/grammar/type_params.rs diff --git a/src/parser/event_parser/grammar/types.rs b/src/parser/grammar/types.rs similarity index 100% rename from src/parser/event_parser/grammar/types.rs rename to src/parser/grammar/types.rs diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c5525ff9cd..22ccb49216 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,88 +1,23 @@ -use {File, FileBuilder, Sink, SyntaxKind, Token}; +use {File, SyntaxKind, Token}; use syntax_kinds::*; -mod event_parser; -use self::event_parser::Event; +#[macro_use] +mod parser; +mod event; +mod grammar; +use self::event::Event; /// Parse a sequence of tokens into the representative node tree pub fn parse(text: String, tokens: &[Token]) -> File { - let events = event_parser::parse(&text, tokens); - from_events_to_file(text, tokens, events) + let events = parse_into_events(&text, tokens); + event::to_file(text, tokens, events) } -fn from_events_to_file(text: String, tokens: &[Token], events: Vec) -> File { - let mut builder = FileBuilder::new(text); - let mut idx = 0; - - let mut holes = Vec::new(); - let mut forward_parents = Vec::new(); - - for (i, event) in events.iter().enumerate() { - if holes.last() == Some(&i) { - holes.pop(); - continue; - } - - match event { - &Event::Start { - kind: TOMBSTONE, .. - } => (), - - &Event::Start { .. } => { - forward_parents.clear(); - let mut idx = i; - loop { - let (kind, fwd) = match events[idx] { - Event::Start { - kind, - forward_parent, - } => (kind, forward_parent), - _ => unreachable!(), - }; - forward_parents.push((idx, kind)); - if let Some(fwd) = fwd { - idx += fwd as usize; - } else { - break; - } - } - for &(idx, kind) in forward_parents.iter().into_iter().rev() { - builder.start_internal(kind); - holes.push(idx); - } - holes.pop(); - } - &Event::Finish => { - while idx < tokens.len() { - let token = tokens[idx]; - if is_insignificant(token.kind) { - idx += 1; - builder.leaf(token.kind, token.len); - } else { - break; - } - } - builder.finish_internal() - } - &Event::Token { - kind: _, - mut n_raw_tokens, - } => loop { - let token = tokens[idx]; - if !is_insignificant(token.kind) { - n_raw_tokens -= 1; - } - idx += 1; - builder.leaf(token.kind, token.len); - if n_raw_tokens == 0 { - break; - } - }, - &Event::Error { ref message } => builder.error().message(message.clone()).emit(), - } - } - builder.finish() +pub(crate) fn parse_into_events<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec { + let mut parser = parser::Parser::new(text, raw_tokens); + grammar::file(&mut parser); + parser.into_events() } fn is_insignificant(kind: SyntaxKind) -> bool { diff --git a/src/parser/event_parser/parser.rs b/src/parser/parser.rs similarity index 99% rename from src/parser/event_parser/parser.rs rename to src/parser/parser.rs index 6cf6ac9b9d..3f4c8a07d0 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/parser.rs @@ -1,6 +1,6 @@ use {SyntaxKind, TextUnit, Token}; use super::Event; -use super::super::is_insignificant; +use super::is_insignificant; use SyntaxKind::{EOF, TOMBSTONE}; pub(crate) struct Marker {