From fcb1eef3232c3fc673bf5f98595708e108c3950c Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 25 May 2019 20:31:53 +0800 Subject: [PATCH 1/3] Change TokenSource to iteration based --- crates/ra_mbe/src/subtree_parser.rs | 4 +- crates/ra_mbe/src/subtree_source.rs | 61 ++++++++++++------- crates/ra_mbe/src/syntax_bridge.rs | 42 +++++++------ crates/ra_parser/src/lib.rs | 56 +++++++++++------ crates/ra_parser/src/parser.rs | 57 +++++++++-------- crates/ra_syntax/src/parsing.rs | 4 +- crates/ra_syntax/src/parsing/reparsing.rs | 4 +- .../src/parsing/text_token_source.rs | 49 ++++++++++----- 8 files changed, 174 insertions(+), 103 deletions(-) diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs index 709b87a384..9cc989b23a 100644 --- a/crates/ra_mbe/src/subtree_parser.rs +++ b/crates/ra_mbe/src/subtree_parser.rs @@ -68,13 +68,13 @@ impl<'a> Parser<'a> { fn parse(self, f: F) -> Option where - F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), + F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink), { let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]); let mut src = SubtreeTokenSource::new(&buffer); let mut sink = OffsetTokenSink { token_pos: 0, error: false }; - f(&src, &mut sink); + f(&mut src, &mut sink); let r = self.finish(sink.token_pos, &mut src); if sink.error { diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 972af4a7c5..c4f79f38a3 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -1,6 +1,7 @@ -use ra_parser::{TokenSource}; +use ra_parser::{TokenSource, Token}; use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; use std::cell::{RefCell, Cell}; +use std::sync::Arc; use tt::buffer::{TokenBuffer, Cursor}; pub(crate) trait Querier { @@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> { return cached[pos].clone(); } - fn collect_token_trees(&mut self, n: usize) -> Vec { + fn collect_token_trees(&self, n: usize) -> Vec { let mut res = vec![]; let mut pos = 0; @@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> { } pub(crate) struct SubtreeTokenSource<'a> { - walker: SubtreeWalk<'a>, + walker: Arc>, + curr: (Token, usize), } impl<'a> SubtreeTokenSource<'a> { pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { - SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) } + let mut res = SubtreeTokenSource { + walker: Arc::new(SubtreeWalk::new(buffer.begin())), + curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), + }; + res.curr = (res.mk_token(0), 0); + res } - pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a> - where - 'a: 'b, - { - &self.walker + pub fn querier(&self) -> Arc> { + self.walker.clone() } pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec { let res = self.walker.collect_token_trees(parsed_tokens); res } + + fn mk_token(&self, pos: usize) -> Token { + match self.walker.get(pos) { + Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next }, + None => Token { kind: EOF, is_jointed_to_next: false }, + } + } } impl<'a> TokenSource for SubtreeTokenSource<'a> { - fn token_kind(&self, pos: usize) -> SyntaxKind { - if let Some(tok) = self.walker.get(pos) { - tok.kind - } else { - SyntaxKind::EOF - } + fn current(&self) -> Token { + self.curr.0 } - fn is_token_joint_to_next(&self, pos: usize) -> bool { - match self.walker.get(pos) { - Some(t) => t.is_joint_to_next, - _ => false, - } + + /// Lookahead n token + fn lookahead_nth(&self, n: usize) -> Token { + self.mk_token(self.curr.1 + n) } - fn is_keyword(&self, pos: usize, kw: &str) -> bool { - match self.walker.get(pos) { + + /// bump cursor to next token + fn bump(&mut self) { + if self.current().kind == EOF { + return; + } + + self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1) + } + + /// Is the current token a specified keyword? + fn is_keyword(&self, kw: &str) -> bool { + match self.walker.get(self.curr.1) { Some(t) => t.text == *kw, _ => false, } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0a75305b47..0aab5ea8b2 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -48,9 +48,10 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke /// Parses the token tree (result of macro expansion) to an expression pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_expr(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_expr(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -64,9 +65,10 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, Expand /// Parses the token tree (result of macro expansion) to a Pattern pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_pat(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_pat(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -78,9 +80,10 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandEr /// Parses the token tree (result of macro expansion) to a Type pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_ty(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_ty(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -93,9 +96,10 @@ pub fn token_tree_to_macro_stmts( tt: &tt::Subtree, ) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -108,9 +112,10 @@ pub fn token_tree_to_macro_items( tt: &tt::Subtree, ) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_macro_items(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_macro_items(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -121,9 +126,10 @@ pub fn token_tree_to_macro_items( /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse(&mut token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); ast::SourceFile::cast(&syntax).unwrap().to_owned() } diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs index 697d1b7949..3d88be6425 100644 --- a/crates/ra_parser/src/lib.rs +++ b/crates/ra_parser/src/lib.rs @@ -31,12 +31,26 @@ pub struct ParseError(pub String); /// /// Hopefully this will allow us to treat text and token trees in the same way! pub trait TokenSource { - /// What is the current token? - fn token_kind(&self, pos: usize) -> SyntaxKind; - /// Is the current token joined to the next one (`> >` vs `>>`). - fn is_token_joint_to_next(&self, pos: usize) -> bool; + fn current(&self) -> Token; + + /// Lookahead n token + fn lookahead_nth(&self, n: usize) -> Token; + + /// bump cursor to next token + fn bump(&mut self); + /// Is the current token a specified keyword? - fn is_keyword(&self, pos: usize, kw: &str) -> bool; + fn is_keyword(&self, kw: &str) -> bool; +} + +/// `TokenCursor` abstracts the cursor of `TokenSource` operates one. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub struct Token { + /// What is the current token? + pub kind: SyntaxKind, + + /// Is the current token joined to the next one (`> >` vs `>>`). + pub is_jointed_to_next: bool, } /// `TreeSink` abstracts details of a particular syntax tree implementation. @@ -54,7 +68,7 @@ pub trait TreeSink { fn error(&mut self, error: ParseError); } -fn parse_from_tokens(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) +fn parse_from_tokens(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) where F: FnOnce(&mut parser::Parser), { @@ -65,61 +79,65 @@ where } /// Parse given tokens into the given sink as a rust file. -pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::root); } /// Parse given tokens into the given sink as a path -pub fn parse_path(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_path(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::path); } /// Parse given tokens into the given sink as a expression -pub fn parse_expr(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_expr(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::expr); } /// Parse given tokens into the given sink as a ty -pub fn parse_ty(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_ty(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::type_); } /// Parse given tokens into the given sink as a pattern -pub fn parse_pat(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_pat(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::pattern); } /// Parse given tokens into the given sink as a statement -pub fn parse_stmt(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, with_semi: bool) { +pub fn parse_stmt( + token_source: &mut dyn TokenSource, + tree_sink: &mut dyn TreeSink, + with_semi: bool, +) { parse_from_tokens(token_source, tree_sink, |p| grammar::stmt(p, with_semi)); } /// Parse given tokens into the given sink as a block -pub fn parse_block(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_block(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::block); } -pub fn parse_meta(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_meta(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::meta_item); } /// Parse given tokens into the given sink as an item -pub fn parse_item(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_item(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::item); } /// Parse given tokens into the given sink as an visibility qualifier -pub fn parse_vis(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_vis(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, |p| { grammar::opt_visibility(p); }); } -pub fn parse_macro_items(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_macro_items(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::macro_items); } -pub fn parse_macro_stmts(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +pub fn parse_macro_stmts(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { parse_from_tokens(token_source, tree_sink, grammar::macro_stmts); } @@ -140,7 +158,7 @@ impl Reparser { /// /// Tokens must start with `{`, end with `}` and form a valid brace /// sequence. - pub fn parse(self, token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { let Reparser(r) = self; let mut p = parser::Parser::new(token_source); r(&mut p); diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs index 4434dfb096..8f654f04c0 100644 --- a/crates/ra_parser/src/parser.rs +++ b/crates/ra_parser/src/parser.rs @@ -19,15 +19,14 @@ use crate::{ /// "start expression, consume number literal, /// finish expression". See `Event` docs for more. pub(crate) struct Parser<'t> { - token_source: &'t dyn TokenSource, - token_pos: usize, + token_source: &'t mut dyn TokenSource, events: Vec, steps: Cell, } impl<'t> Parser<'t> { - pub(super) fn new(token_source: &'t dyn TokenSource) -> Parser<'t> { - Parser { token_source, token_pos: 0, events: Vec::new(), steps: Cell::new(0) } + pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> { + Parser { token_source, events: Vec::new(), steps: Cell::new(0) } } pub(crate) fn finish(self) -> Vec { @@ -49,7 +48,7 @@ impl<'t> Parser<'t> { let c1 = self.nth(0); let c2 = self.nth(1); - if self.token_source.is_token_joint_to_next(self.token_pos) { + if self.token_source.current().is_jointed_to_next { Some((c1, c2)) } else { None @@ -64,8 +63,8 @@ impl<'t> Parser<'t> { let c1 = self.nth(0); let c2 = self.nth(1); let c3 = self.nth(2); - if self.token_source.is_token_joint_to_next(self.token_pos) - && self.token_source.is_token_joint_to_next(self.token_pos + 1) + if self.token_source.current().is_jointed_to_next + && self.token_source.lookahead_nth(1).is_jointed_to_next { Some((c1, c2, c3)) } else { @@ -76,6 +75,8 @@ impl<'t> Parser<'t> { /// Lookahead operation: returns the kind of the next nth /// token. pub(crate) fn nth(&self, n: usize) -> SyntaxKind { + assert!(n <= 3); + let steps = self.steps.get(); assert!(steps <= 10_000_000, "the parser seems stuck"); self.steps.set(steps + 1); @@ -86,7 +87,7 @@ impl<'t> Parser<'t> { let mut i = 0; loop { - let mut kind = self.token_source.token_kind(self.token_pos + i); + let mut kind = self.token_source.lookahead_nth(i).kind; if let Some((composited, step)) = self.is_composite(kind, i) { kind = composited; i += step; @@ -115,7 +116,7 @@ impl<'t> Parser<'t> { /// Checks if the current token is contextual keyword with text `t`. pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool { - self.token_source.is_keyword(self.token_pos, kw) + self.token_source.is_keyword(kw) } /// Starts a new node in the syntax tree. All nodes and tokens @@ -130,12 +131,12 @@ impl<'t> Parser<'t> { /// Advances the parser by one token unconditionally /// Mainly use in `token_tree` parsing pub(crate) fn bump_raw(&mut self) { - let mut kind = self.token_source.token_kind(self.token_pos); + let mut kind = self.token_source.current().kind; // Skip dollars, do_bump will eat these later let mut i = 0; while kind == SyntaxKind::L_DOLLAR || kind == SyntaxKind::R_DOLLAR { - kind = self.token_source.token_kind(self.token_pos + i); + kind = self.token_source.lookahead_nth(i).kind; i += 1; } @@ -236,7 +237,11 @@ impl<'t> Parser<'t> { fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { self.eat_dollars(); - self.token_pos += usize::from(n_raw_tokens); + + for _ in 0..n_raw_tokens { + self.token_source.bump(); + } + self.push_event(Event::Token { kind, n_raw_tokens }); } @@ -249,10 +254,14 @@ impl<'t> Parser<'t> { // We assume the dollars will not occuried between // mult-byte tokens - let jn1 = self.token_source.is_token_joint_to_next(self.token_pos + n); - let la2 = self.token_source.token_kind(self.token_pos + n + 1); - let jn2 = self.token_source.is_token_joint_to_next(self.token_pos + n + 1); - let la3 = self.token_source.token_kind(self.token_pos + n + 2); + let first = self.token_source.lookahead_nth(n); + let second = self.token_source.lookahead_nth(n + 1); + let third = self.token_source.lookahead_nth(n + 2); + + let jn1 = first.is_jointed_to_next; + let la2 = second.kind; + let jn2 = second.is_jointed_to_next; + let la3 = third.kind; match kind { T![.] if jn1 && la2 == T![.] && jn2 && la3 == T![.] => Some((T![...], 3)), @@ -271,9 +280,9 @@ impl<'t> Parser<'t> { fn eat_dollars(&mut self) { loop { - match self.token_source.token_kind(self.token_pos) { + match self.token_source.current().kind { k @ SyntaxKind::L_DOLLAR | k @ SyntaxKind::R_DOLLAR => { - self.token_pos += 1; + self.token_source.bump(); self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); } _ => { @@ -286,9 +295,9 @@ impl<'t> Parser<'t> { pub(crate) fn eat_l_dollars(&mut self) -> usize { let mut ate_count = 0; loop { - match self.token_source.token_kind(self.token_pos) { + match self.token_source.current().kind { k @ SyntaxKind::L_DOLLAR => { - self.token_pos += 1; + self.token_source.bump(); self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); ate_count += 1; } @@ -302,9 +311,9 @@ impl<'t> Parser<'t> { pub(crate) fn eat_r_dollars(&mut self, max_count: usize) -> usize { let mut ate_count = 0; loop { - match self.token_source.token_kind(self.token_pos) { + match self.token_source.current().kind { k @ SyntaxKind::R_DOLLAR => { - self.token_pos += 1; + self.token_source.bump(); self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); ate_count += 1; @@ -320,12 +329,12 @@ impl<'t> Parser<'t> { } pub(crate) fn at_l_dollar(&self) -> bool { - let kind = self.token_source.token_kind(self.token_pos); + let kind = self.token_source.current().kind; (kind == SyntaxKind::L_DOLLAR) } pub(crate) fn at_r_dollar(&self) -> bool { - let kind = self.token_source.token_kind(self.token_pos); + let kind = self.token_source.current().kind; (kind == SyntaxKind::R_DOLLAR) } } diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index 15d69c5ab0..4c1fa6c4f3 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -17,8 +17,8 @@ pub(crate) use self::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let tokens = tokenize(&text); - let token_source = text_token_source::TextTokenSource::new(text, &tokens); + let mut token_source = text_token_source::TextTokenSource::new(text, &tokens); let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens); - ra_parser::parse(&token_source, &mut tree_sink); + ra_parser::parse(&mut token_source, &mut tree_sink); tree_sink.finish() } diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 6de02a15a7..3b6687f61b 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -85,9 +85,9 @@ fn reparse_block<'node>( if !is_balanced(&tokens) { return None; } - let token_source = TextTokenSource::new(&text, &tokens); + let mut token_source = TextTokenSource::new(&text, &tokens); let mut tree_sink = TextTreeSink::new(&text, &tokens); - reparser.parse(&token_source, &mut tree_sink); + reparser.parse(&mut token_source, &mut tree_sink); let (green, new_errors) = tree_sink.finish(); Some((node.replace_with(green), new_errors, node.range())) } diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs index a6277f66fc..71d2947f77 100644 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ b/crates/ra_syntax/src/parsing/text_token_source.rs @@ -1,7 +1,8 @@ use ra_parser::TokenSource; +use ra_parser::Token as PToken; use crate::{ - SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, + SyntaxKind::EOF, TextRange, TextUnit, parsing::lexer::Token, }; @@ -23,31 +24,50 @@ pub(crate) struct TextTokenSource<'t> { /// ``` /// tokens: `[struct, Foo, {, }]` tokens: Vec, + + /// Current token and position + curr: (PToken, usize), } impl<'t> TokenSource for TextTokenSource<'t> { - fn token_kind(&self, pos: usize) -> SyntaxKind { - if !(pos < self.tokens.len()) { - return EOF; - } - self.tokens[pos].kind + fn current(&self) -> PToken { + return self.curr.0; } - fn is_token_joint_to_next(&self, pos: usize) -> bool { - if !(pos + 1 < self.tokens.len()) { - return true; - } - self.start_offsets[pos] + self.tokens[pos].len == self.start_offsets[pos + 1] + + fn lookahead_nth(&self, n: usize) -> PToken { + mk_token(self.curr.1 + n, &self.start_offsets, &self.tokens) } - fn is_keyword(&self, pos: usize, kw: &str) -> bool { + + fn bump(&mut self) { + if self.curr.0.kind == EOF { + return; + } + + let pos = self.curr.1 + 1; + self.curr = (mk_token(pos, &self.start_offsets, &self.tokens), pos); + } + + fn is_keyword(&self, kw: &str) -> bool { + let pos = self.curr.1; if !(pos < self.tokens.len()) { return false; } let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); - self.text[range] == *kw } } +fn mk_token(pos: usize, start_offsets: &[TextUnit], tokens: &[Token]) -> PToken { + let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF); + let is_jointed_to_next = if pos + 1 < start_offsets.len() { + start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1] + } else { + false + }; + + PToken { kind, is_jointed_to_next } +} + impl<'t> TextTokenSource<'t> { /// Generate input from tokens(expect comment and whitespace). pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { @@ -62,6 +82,7 @@ impl<'t> TextTokenSource<'t> { len += token.len; } - TextTokenSource { text, start_offsets, tokens } + let first = mk_token(0, &start_offsets, &tokens); + TextTokenSource { text, start_offsets, tokens, curr: (first, 0) } } } From 816147c4b508117fad4ca4d3cb6d77a3c4a683a9 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 25 May 2019 21:55:46 +0800 Subject: [PATCH 2/3] Simplify token_tree_to_xxx --- crates/ra_mbe/src/syntax_bridge.rs | 67 +++++++++--------------------- 1 file changed, 20 insertions(+), 47 deletions(-) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0aab5ea8b2..6265fe462d 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -45,18 +45,29 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // // -/// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { +fn token_tree_to_syntax_node(tt: &tt::Subtree, f: F) -> Result, ExpandError> +where + F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink), +{ let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); let mut token_source = SubtreeTokenSource::new(&buffer); let querier = token_source.querier(); let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_expr(&mut token_source, &mut tree_sink); + f(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } - let syntax = tree_sink.inner.finish(); + if tree_sink.roots.len() != 1 { + return Err(ExpandError::ConversionError); + } + + Ok(tree_sink.inner.finish()) +} + +/// Parses the token tree (result of macro expansion) to an expression +pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_expr)?; ast::Expr::cast(&syntax) .map(|m| m.to_owned()) .ok_or_else(|| crate::ExpandError::ConversionError) @@ -64,30 +75,13 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, Expand /// Parses the token tree (result of macro expansion) to a Pattern pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_pat(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_pat)?; ast::Pat::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) to a Type pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_ty(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_ty)?; ast::TypeRef::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } @@ -95,15 +89,7 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, Expan pub fn token_tree_to_macro_stmts( tt: &tt::Subtree, ) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_stmts)?; ast::MacroStmts::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } @@ -111,26 +97,13 @@ pub fn token_tree_to_macro_stmts( pub fn token_tree_to_macro_items( tt: &tt::Subtree, ) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_macro_items(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_items)?; ast::MacroItems::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse(&mut token_source, &mut tree_sink); - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse).unwrap(); ast::SourceFile::cast(&syntax).unwrap().to_owned() } From 90764fc54b2be1e0fc5d6ac9c9e960d7bb059b14 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 26 May 2019 02:41:00 +0800 Subject: [PATCH 3/3] Remove duplicated code --- crates/ra_mbe/src/syntax_bridge.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 6265fe462d..0a7e50c4e2 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -58,10 +58,6 @@ where return Err(ExpandError::ConversionError); } - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - Ok(tree_sink.inner.finish()) }