From 2acb21e8f72896c7a2855ca6042d0ee1870d8643 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 22:44:06 +0300 Subject: [PATCH] merge parse_impl and parser_api --- crates/ra_syntax/src/parsing/parser_api.rs | 108 +++++++++++-- crates/ra_syntax/src/parsing/parser_impl.rs | 163 ++------------------ 2 files changed, 101 insertions(+), 170 deletions(-) diff --git a/crates/ra_syntax/src/parsing/parser_api.rs b/crates/ra_syntax/src/parsing/parser_api.rs index aed23a6a49..92d7895d3e 100644 --- a/crates/ra_syntax/src/parsing/parser_api.rs +++ b/crates/ra_syntax/src/parsing/parser_api.rs @@ -1,10 +1,14 @@ +use std::cell::Cell; + use drop_bomb::DropBomb; use crate::{ - SyntaxKind::{self, ERROR}, + syntax_error::ParseError, + SyntaxKind::{self, ERROR, EOF, TOMBSTONE}, parsing::{ + TokenSource, TokenPos, token_set::TokenSet, - parser_impl::ParserImpl, + parser_impl::event::Event, }, }; @@ -17,9 +21,22 @@ use crate::{ /// tree, but rather a flat stream of events of the form /// "start expression, consume number literal, /// finish expression". See `Event` docs for more. -pub(crate) struct Parser<'t>(pub(super) ParserImpl<'t>); +pub(crate) struct Parser<'t> { + token_source: &'t dyn TokenSource, + pos: TokenPos, + events: Vec, + steps: Cell, +} impl<'t> Parser<'t> { + pub(super) fn new(token_source: &'t dyn TokenSource) -> Parser<'t> { + Parser { token_source, pos: TokenPos::default(), events: Vec::new(), steps: Cell::new(0) } + } + + pub(crate) fn finish(self) -> Vec { + self.events + } + /// Returns the kind of the current token. /// If parser has already reached the end of input, /// the special `EOF` kind is returned. @@ -32,7 +49,13 @@ impl<'t> Parser<'t> { /// /// Useful for parsing things like `>>`. pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> { - self.0.current2() + let c1 = self.token_source.token_kind(self.pos); + let c2 = self.token_source.token_kind(self.pos + 1); + if self.token_source.is_token_joint_to_next(self.pos) { + Some((c1, c2)) + } else { + None + } } /// Returns the kinds of the current three tokens, if they are not separated @@ -40,13 +63,25 @@ impl<'t> Parser<'t> { /// /// Useful for parsing things like `=>>`. pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> { - self.0.current3() + let c1 = self.token_source.token_kind(self.pos); + let c2 = self.token_source.token_kind(self.pos + 1); + let c3 = self.token_source.token_kind(self.pos + 2); + if self.token_source.is_token_joint_to_next(self.pos) + && self.token_source.is_token_joint_to_next(self.pos + 1) + { + Some((c1, c2, c3)) + } else { + None + } } /// Lookahead operation: returns the kind of the next nth /// token. pub(crate) fn nth(&self, n: u32) -> SyntaxKind { - self.0.nth(n) + let steps = self.steps.get(); + assert!(steps <= 10_000_000, "the parser seems stuck"); + self.steps.set(steps + 1); + self.token_source.token_kind(self.pos + n) } /// Checks if the current token is `kind`. @@ -60,20 +95,26 @@ impl<'t> Parser<'t> { } /// Checks if the current token is contextual keyword with text `t`. - pub(crate) fn at_contextual_kw(&self, t: &str) -> bool { - self.0.at_kw(t) + pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool { + self.token_source.is_keyword(self.pos, kw) } /// Starts a new node in the syntax tree. All nodes and tokens /// consumed between the `start` and the corresponding `Marker::complete` /// belong to the same node. pub(crate) fn start(&mut self) -> Marker { - Marker::new(self.0.start()) + let pos = self.events.len() as u32; + self.push_event(Event::tombstone()); + Marker::new(pos) } /// Advances the parser by one token unconditionally. pub(crate) fn bump(&mut self) { - self.0.bump(); + let kind = self.nth(0); + if kind == EOF { + return; + } + self.do_bump(kind, 1); } /// Advances the parser by one token, remapping its kind. @@ -83,14 +124,18 @@ impl<'t> Parser<'t> { /// `union` keyword, and keyword is what ends up in the /// final tree. pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) { - self.0.bump_remap(kind); + if self.nth(0) == EOF { + // TODO: panic!? + return; + } + self.do_bump(kind, 1); } /// Advances the parser by `n` tokens, remapping its kind. /// This is useful to create compound tokens from parts. For /// example, an `<<` token is two consecutive remapped `<` tokens pub(crate) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) { - self.0.bump_compound(kind, n); + self.do_bump(kind, n); } /// Emit error with the `message` @@ -98,7 +143,8 @@ impl<'t> Parser<'t> { /// structured errors with spans and notes, like rustc /// does. pub(crate) fn error>(&mut self, message: T) { - self.0.error(message.into()) + let msg = ParseError(message.into()); + self.push_event(Event::Error { msg }) } /// Consume the next token if `kind` matches. @@ -136,6 +182,15 @@ impl<'t> Parser<'t> { m.complete(self, ERROR); }; } + + fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { + self.pos += u32::from(n_raw_tokens); + self.push_event(Event::Token { kind, n_raw_tokens }); + } + + fn push_event(&mut self, event: Event) { + self.events.push(event) + } } /// See `Parser::start`. @@ -154,7 +209,14 @@ impl Marker { /// operation like `.precede()` to deal with forward_parent. pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker { self.bomb.defuse(); - p.0.complete(self.pos, kind); + let idx = self.pos as usize; + match p.events[idx] { + Event::Start { kind: ref mut slot, .. } => { + *slot = kind; + } + _ => unreachable!(), + } + p.push_event(Event::Finish); CompletedMarker::new(self.pos, kind) } @@ -162,7 +224,13 @@ impl Marker { /// are attached to its parent instead. pub(crate) fn abandon(mut self, p: &mut Parser) { self.bomb.defuse(); - p.0.abandon(self.pos); + let idx = self.pos as usize; + if idx == p.events.len() - 1 { + match p.events.pop() { + Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (), + _ => unreachable!(), + } + } } } @@ -186,7 +254,15 @@ impl CompletedMarker { /// then mark `NEWSTART` as `START`'s parent with saving its relative /// distance to `NEWSTART` into forward_parent(=2 in this case); pub(crate) fn precede(self, p: &mut Parser) -> Marker { - Marker::new(p.0.precede(self.0)) + let new_pos = p.start(); + let idx = self.0 as usize; + match p.events[idx] { + Event::Start { ref mut forward_parent, .. } => { + *forward_parent = Some(new_pos.pos - self.0); + } + _ => unreachable!(), + } + new_pos } pub(crate) fn kind(&self) -> SyntaxKind { diff --git a/crates/ra_syntax/src/parsing/parser_impl.rs b/crates/ra_syntax/src/parsing/parser_impl.rs index 89439e0746..6eed0e656a 100644 --- a/crates/ra_syntax/src/parsing/parser_impl.rs +++ b/crates/ra_syntax/src/parsing/parser_impl.rs @@ -1,20 +1,13 @@ -mod event; -pub(crate) mod input; +pub(super) mod event; +pub(super) mod input; -use std::cell::Cell; - -use crate::{ - syntax_error::ParseError, - parsing::{ - TreeSink, TokenSource, TokenPos, - lexer::Token, - parser_api::Parser, - parser_impl::event::{Event, EventProcessor}, - }, +use crate::parsing::{ + TreeSink, TokenSource, + lexer::Token, + parser_api::Parser, + parser_impl::event::EventProcessor, }; -use crate::SyntaxKind::{self, EOF, TOMBSTONE}; - /// Parse a sequence of tokens into the representative node tree pub(super) fn parse_with( sink: S, @@ -24,147 +17,9 @@ pub(super) fn parse_with( ) -> S::Tree { let mut events = { let input = input::ParserInput::new(text, tokens); - let parser_impl = ParserImpl::new(&input); - let mut parser_api = Parser(parser_impl); + let mut parser_api = Parser::new(&input); parser(&mut parser_api); - parser_api.0.into_events() + parser_api.finish() }; EventProcessor::new(sink, text, tokens, &mut events).process().finish() } - -/// Implementation details of `Parser`, extracted -/// to a separate struct in order not to pollute -/// the public API of the `Parser`. -pub(super) struct ParserImpl<'a> { - token_source: &'a dyn TokenSource, - pos: TokenPos, - events: Vec, - steps: Cell, -} - -impl<'a> ParserImpl<'a> { - fn new(token_source: &'a dyn TokenSource) -> ParserImpl<'a> { - ParserImpl { - token_source, - pos: TokenPos::default(), - events: Vec::new(), - steps: Cell::new(0), - } - } - - fn into_events(self) -> Vec { - assert_eq!(self.nth(0), EOF); - self.events - } - - pub(super) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> { - let c1 = self.token_source.token_kind(self.pos); - let c2 = self.token_source.token_kind(self.pos + 1); - if self.token_source.is_token_joint_to_next(self.pos) { - Some((c1, c2)) - } else { - None - } - } - - pub(super) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> { - let c1 = self.token_source.token_kind(self.pos); - let c2 = self.token_source.token_kind(self.pos + 1); - let c3 = self.token_source.token_kind(self.pos + 2); - if self.token_source.is_token_joint_to_next(self.pos) - && self.token_source.is_token_joint_to_next(self.pos + 1) - { - Some((c1, c2, c3)) - } else { - None - } - } - - /// Get the syntax kind of the nth token. - pub(super) fn nth(&self, n: u32) -> SyntaxKind { - let steps = self.steps.get(); - assert!(steps <= 10_000_000, "the parser seems stuck"); - self.steps.set(steps + 1); - self.token_source.token_kind(self.pos + n) - } - - pub(super) fn at_kw(&self, kw: &str) -> bool { - self.token_source.is_keyword(self.pos, kw) - } - - /// Start parsing right behind the last event. - pub(super) fn start(&mut self) -> u32 { - let pos = self.events.len() as u32; - self.push_event(Event::tombstone()); - pos - } - - /// Advances the parser by one token unconditionally. - pub(super) fn bump(&mut self) { - let kind = self.nth(0); - if kind == EOF { - return; - } - self.do_bump(kind, 1); - } - - pub(super) fn bump_remap(&mut self, kind: SyntaxKind) { - if self.nth(0) == EOF { - // TODO: panic!? - return; - } - self.do_bump(kind, 1); - } - - pub(super) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) { - self.do_bump(kind, n); - } - - fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { - self.pos += u32::from(n_raw_tokens); - self.push_event(Event::Token { kind, n_raw_tokens }); - } - - /// Append one Error event to the back of events. - pub(super) fn error(&mut self, msg: String) { - self.push_event(Event::Error { msg: ParseError(msg) }) - } - - /// Complete an event with appending a `Finish` event. - pub(super) fn complete(&mut self, pos: u32, kind: SyntaxKind) { - match self.events[pos as usize] { - Event::Start { kind: ref mut slot, .. } => { - *slot = kind; - } - _ => unreachable!(), - } - self.push_event(Event::Finish); - } - - /// Ignore the dummy `Start` event. - pub(super) fn abandon(&mut self, pos: u32) { - let idx = pos as usize; - if idx == self.events.len() - 1 { - match self.events.pop() { - Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (), - _ => unreachable!(), - } - } - } - - /// Save the relative distance of a completed event to its forward_parent. - pub(super) fn precede(&mut self, pos: u32) -> u32 { - let new_pos = self.start(); - match self.events[pos as usize] { - Event::Start { ref mut forward_parent, .. } => { - *forward_parent = Some(new_pos - pos); - } - _ => unreachable!(), - } - new_pos - } - - fn push_event(&mut self, event: Event) { - self.events.push(event) - } -}