Merge #1328

1328: Change TokenSource to iteration based r=matklad a=edwin0cheng

This PR change the `TokenSource` trait from random access to be an iteration based trait:
```rust

/// `TokenSource` abstracts the source of the tokens parser operates one.
///
/// Hopefully this will allow us to treat text and token trees in the same way!
pub trait TokenSource {
    fn current(&self) -> Token;

    /// Lookahead n token
    fn lookahead_nth(&self, n: usize) -> Token;

    /// bump cursor to next token
    fn bump(&mut self);

    /// Is the current token a specified keyword?
    fn is_keyword(&self, kw: &str) -> bool;
}

/// `TokenCursor` abstracts the cursor of `TokenSource` operates one.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub struct Token {
    /// What is the current token?
    pub kind: SyntaxKind,

    /// Is the current token joined to the next one (`> >` vs `>>`).
    pub is_jointed_to_next: bool,
}
```

Note that the refactoring based on this new trait will be separated to incoming PRs

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

crates/ra_mbe/src/subtree_parser.rs

@@ -68,13 +68,13 @@ impl<'a> Parser<'a> {
 
     fn parse<F>(self, f: F) -> Option<tt::TokenTree>
     where
-        F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
+        F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
     {
         let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
         let mut src = SubtreeTokenSource::new(&buffer);
         let mut sink = OffsetTokenSink { token_pos: 0, error: false };
 
-        f(&src, &mut sink);
+        f(&mut src, &mut sink);
 
         let r = self.finish(sink.token_pos, &mut src);
         if sink.error {

crates/ra_mbe/src/subtree_source.rs

@@ -1,6 +1,7 @@
-use ra_parser::{TokenSource};
+use ra_parser::{TokenSource, Token};
 use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use std::cell::{RefCell, Cell};
+use std::sync::Arc;
 use tt::buffer::{TokenBuffer, Cursor};
 
 pub(crate) trait Querier {
@@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> {
         return cached[pos].clone();
     }
 
-    fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> {
+    fn collect_token_trees(&self, n: usize) -> Vec<tt::TokenTree> {
         let mut res = vec![];
 
         let mut pos = 0;
@@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> {
 }
 
 pub(crate) struct SubtreeTokenSource<'a> {
-    walker: SubtreeWalk<'a>,
+    walker: Arc<SubtreeWalk<'a>>,
+    curr: (Token, usize),
 }
 
 impl<'a> SubtreeTokenSource<'a> {
     pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
-        SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) }
+        let mut res = SubtreeTokenSource {
+            walker: Arc::new(SubtreeWalk::new(buffer.begin())),
+            curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
+        };
+        res.curr = (res.mk_token(0), 0);
+        res
     }
 
-    pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a>
+    pub fn querier(&self) -> Arc<SubtreeWalk<'a>> {
-    where
+        self.walker.clone()
-        'a: 'b,
-    {
-        &self.walker
     }
 
     pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
         let res = self.walker.collect_token_trees(parsed_tokens);
         res
     }
+
+    fn mk_token(&self, pos: usize) -> Token {
+        match self.walker.get(pos) {
+            Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
+            None => Token { kind: EOF, is_jointed_to_next: false },
+        }
+    }
 }
 
 impl<'a> TokenSource for SubtreeTokenSource<'a> {
-    fn token_kind(&self, pos: usize) -> SyntaxKind {
+    fn current(&self) -> Token {
-        if let Some(tok) = self.walker.get(pos) {
+        self.curr.0
-            tok.kind
-        } else {
-            SyntaxKind::EOF
-        }
     }
-    fn is_token_joint_to_next(&self, pos: usize) -> bool {
+
-        match self.walker.get(pos) {
+    /// Lookahead n token
-            Some(t) => t.is_joint_to_next,
+    fn lookahead_nth(&self, n: usize) -> Token {
-            _ => false,
+        self.mk_token(self.curr.1 + n)
-        }
     }
-    fn is_keyword(&self, pos: usize, kw: &str) -> bool {
+
-        match self.walker.get(pos) {
+    /// bump cursor to next token
+    fn bump(&mut self) {
+        if self.current().kind == EOF {
+            return;
+        }
+
+        self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1)
+    }
+
+    /// Is the current token a specified keyword?
+    fn is_keyword(&self, kw: &str) -> bool {
+        match self.walker.get(self.curr.1) {
             Some(t) => t.text == *kw,
             _ => false,
         }

crates/ra_mbe/src/syntax_bridge.rs

@@ -45,17 +45,25 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
 //
 //
 
-/// Parses the token tree (result of macro expansion) to an expression
+fn token_tree_to_syntax_node<F>(tt: &tt::Subtree, f: F) -> Result<TreeArc<SyntaxNode>, ExpandError>
-pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
+where
+    F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink),
+{
     let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
+    let mut token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
+    let querier = token_source.querier();
-    ra_parser::parse_expr(&token_source, &mut tree_sink);
+    let mut tree_sink = TtTreeSink::new(querier.as_ref());
+    f(&mut token_source, &mut tree_sink);
     if tree_sink.roots.len() != 1 {
         return Err(ExpandError::ConversionError);
     }
 
-    let syntax = tree_sink.inner.finish();
+    Ok(tree_sink.inner.finish())
+}
+
+/// Parses the token tree (result of macro expansion) to an expression
+pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_expr)?;
     ast::Expr::cast(&syntax)
         .map(|m| m.to_owned())
         .ok_or_else(|| crate::ExpandError::ConversionError)
@@ -63,28 +71,13 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand
 
 /// Parses the token tree (result of macro expansion) to a Pattern
 pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_pat)?;
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_pat(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-
-    let syntax = tree_sink.inner.finish();
     ast::Pat::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
 /// Parses the token tree (result of macro expansion) to a Type
 pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_ty)?;
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_ty(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
     ast::TypeRef::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
@@ -92,14 +85,7 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, Expan
 pub fn token_tree_to_macro_stmts(
     tt: &tt::Subtree,
 ) -> Result<TreeArc<ast::MacroStmts>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_stmts)?;
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_macro_stmts(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
     ast::MacroStmts::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
@@ -107,24 +93,13 @@ pub fn token_tree_to_macro_stmts(
 pub fn token_tree_to_macro_items(
     tt: &tt::Subtree,
 ) -> Result<TreeArc<ast::MacroItems>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_items)?;
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_macro_items(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
     ast::MacroItems::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
 /// Parses the token tree (result of macro expansion) as a sequence of items
 pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse).unwrap();
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse(&token_source, &mut tree_sink);
-    let syntax = tree_sink.inner.finish();
     ast::SourceFile::cast(&syntax).unwrap().to_owned()
 }
 

crates/ra_parser/src/lib.rs

@@ -31,12 +31,26 @@ pub struct ParseError(pub String);
 ///
 /// Hopefully this will allow us to treat text and token trees in the same way!
 pub trait TokenSource {
-    /// What is the current token?
+    fn current(&self) -> Token;
-    fn token_kind(&self, pos: usize) -> SyntaxKind;
+
-    /// Is the current token joined to the next one (`> >` vs `>>`).
+    /// Lookahead n token
-    fn is_token_joint_to_next(&self, pos: usize) -> bool;
+    fn lookahead_nth(&self, n: usize) -> Token;
+
+    /// bump cursor to next token
+    fn bump(&mut self);
+
     /// Is the current token a specified keyword?
-    fn is_keyword(&self, pos: usize, kw: &str) -> bool;
+    fn is_keyword(&self, kw: &str) -> bool;
+}
+
+/// `TokenCursor` abstracts the cursor of `TokenSource` operates one.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub struct Token {
+    /// What is the current token?
+    pub kind: SyntaxKind,
+
+    /// Is the current token joined to the next one (`> >` vs `>>`).
+    pub is_jointed_to_next: bool,
 }
 
 /// `TreeSink` abstracts details of a particular syntax tree implementation.
@@ -54,7 +68,7 @@ pub trait TreeSink {
     fn error(&mut self, error: ParseError);
 }
 
-fn parse_from_tokens<F>(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F)
+fn parse_from_tokens<F>(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F)
 where
     F: FnOnce(&mut parser::Parser),
 {
@@ -65,61 +79,65 @@ where
 }
 
 /// Parse given tokens into the given sink as a rust file.
-pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::root);
 }
 
 /// Parse given tokens into the given sink as a path
-pub fn parse_path(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_path(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::path);
 }
 
 /// Parse given tokens into the given sink as a expression
-pub fn parse_expr(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_expr(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::expr);
 }
 
 /// Parse given tokens into the given sink as a ty
-pub fn parse_ty(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_ty(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::type_);
 }
 
 /// Parse given tokens into the given sink as a pattern
-pub fn parse_pat(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_pat(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::pattern);
 }
 
 /// Parse given tokens into the given sink as a statement
-pub fn parse_stmt(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, with_semi: bool) {
+pub fn parse_stmt(
+    token_source: &mut dyn TokenSource,
+    tree_sink: &mut dyn TreeSink,
+    with_semi: bool,
+) {
     parse_from_tokens(token_source, tree_sink, |p| grammar::stmt(p, with_semi));
 }
 
 /// Parse given tokens into the given sink as a block
-pub fn parse_block(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_block(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::block);
 }
 
-pub fn parse_meta(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_meta(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::meta_item);
 }
 
 /// Parse given tokens into the given sink as an item
-pub fn parse_item(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_item(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::item);
 }
 
 /// Parse given tokens into the given sink as an visibility qualifier
-pub fn parse_vis(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_vis(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, |p| {
         grammar::opt_visibility(p);
     });
 }
 
-pub fn parse_macro_items(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_macro_items(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::macro_items);
 }
 
-pub fn parse_macro_stmts(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+pub fn parse_macro_stmts(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::macro_stmts);
 }
 
@@ -140,7 +158,7 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
         let Reparser(r) = self;
         let mut p = parser::Parser::new(token_source);
         r(&mut p);

crates/ra_parser/src/parser.rs

@@ -19,15 +19,14 @@ use crate::{
 /// "start expression, consume number literal,
 /// finish expression". See `Event` docs for more.
 pub(crate) struct Parser<'t> {
-    token_source: &'t dyn TokenSource,
+    token_source: &'t mut dyn TokenSource,
-    token_pos: usize,
     events: Vec<Event>,
     steps: Cell<u32>,
 }
 
 impl<'t> Parser<'t> {
-    pub(super) fn new(token_source: &'t dyn TokenSource) -> Parser<'t> {
+    pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> {
-        Parser { token_source, token_pos: 0, events: Vec::new(), steps: Cell::new(0) }
+        Parser { token_source, events: Vec::new(), steps: Cell::new(0) }
     }
 
     pub(crate) fn finish(self) -> Vec<Event> {
@@ -49,7 +48,7 @@ impl<'t> Parser<'t> {
         let c1 = self.nth(0);
         let c2 = self.nth(1);
 
-        if self.token_source.is_token_joint_to_next(self.token_pos) {
+        if self.token_source.current().is_jointed_to_next {
             Some((c1, c2))
         } else {
             None
@@ -64,8 +63,8 @@ impl<'t> Parser<'t> {
         let c1 = self.nth(0);
         let c2 = self.nth(1);
         let c3 = self.nth(2);
-        if self.token_source.is_token_joint_to_next(self.token_pos)
+        if self.token_source.current().is_jointed_to_next
-            && self.token_source.is_token_joint_to_next(self.token_pos + 1)
+            && self.token_source.lookahead_nth(1).is_jointed_to_next
         {
             Some((c1, c2, c3))
         } else {
@@ -76,6 +75,8 @@ impl<'t> Parser<'t> {
     /// Lookahead operation: returns the kind of the next nth
     /// token.
     pub(crate) fn nth(&self, n: usize) -> SyntaxKind {
+        assert!(n <= 3);
+
         let steps = self.steps.get();
         assert!(steps <= 10_000_000, "the parser seems stuck");
         self.steps.set(steps + 1);
@@ -86,7 +87,7 @@ impl<'t> Parser<'t> {
         let mut i = 0;
 
         loop {
-            let mut kind = self.token_source.token_kind(self.token_pos + i);
+            let mut kind = self.token_source.lookahead_nth(i).kind;
             if let Some((composited, step)) = self.is_composite(kind, i) {
                 kind = composited;
                 i += step;
@@ -115,7 +116,7 @@ impl<'t> Parser<'t> {
 
     /// Checks if the current token is contextual keyword with text `t`.
     pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
-        self.token_source.is_keyword(self.token_pos, kw)
+        self.token_source.is_keyword(kw)
     }
 
     /// Starts a new node in the syntax tree. All nodes and tokens
@@ -130,12 +131,12 @@ impl<'t> Parser<'t> {
     /// Advances the parser by one token unconditionally
     /// Mainly use in `token_tree` parsing
     pub(crate) fn bump_raw(&mut self) {
-        let mut kind = self.token_source.token_kind(self.token_pos);
+        let mut kind = self.token_source.current().kind;
 
         // Skip dollars, do_bump will eat these later
         let mut i = 0;
         while kind == SyntaxKind::L_DOLLAR || kind == SyntaxKind::R_DOLLAR {
-            kind = self.token_source.token_kind(self.token_pos + i);
+            kind = self.token_source.lookahead_nth(i).kind;
             i += 1;
         }
 
@@ -236,7 +237,11 @@ impl<'t> Parser<'t> {
 
     fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
         self.eat_dollars();
-        self.token_pos += usize::from(n_raw_tokens);
+
+        for _ in 0..n_raw_tokens {
+            self.token_source.bump();
+        }
+
         self.push_event(Event::Token { kind, n_raw_tokens });
     }
 
@@ -249,10 +254,14 @@ impl<'t> Parser<'t> {
         // We assume the dollars will not occuried between
         // mult-byte tokens
 
-        let jn1 = self.token_source.is_token_joint_to_next(self.token_pos + n);
+        let first = self.token_source.lookahead_nth(n);
-        let la2 = self.token_source.token_kind(self.token_pos + n + 1);
+        let second = self.token_source.lookahead_nth(n + 1);
-        let jn2 = self.token_source.is_token_joint_to_next(self.token_pos + n + 1);
+        let third = self.token_source.lookahead_nth(n + 2);
-        let la3 = self.token_source.token_kind(self.token_pos + n + 2);
+
+        let jn1 = first.is_jointed_to_next;
+        let la2 = second.kind;
+        let jn2 = second.is_jointed_to_next;
+        let la3 = third.kind;
 
         match kind {
             T![.] if jn1 && la2 == T![.] && jn2 && la3 == T![.] => Some((T![...], 3)),
@@ -271,9 +280,9 @@ impl<'t> Parser<'t> {
 
     fn eat_dollars(&mut self) {
         loop {
-            match self.token_source.token_kind(self.token_pos) {
+            match self.token_source.current().kind {
                 k @ SyntaxKind::L_DOLLAR | k @ SyntaxKind::R_DOLLAR => {
-                    self.token_pos += 1;
+                    self.token_source.bump();
                     self.push_event(Event::Token { kind: k, n_raw_tokens: 1 });
                 }
                 _ => {
@@ -286,9 +295,9 @@ impl<'t> Parser<'t> {
     pub(crate) fn eat_l_dollars(&mut self) -> usize {
         let mut ate_count = 0;
         loop {
-            match self.token_source.token_kind(self.token_pos) {
+            match self.token_source.current().kind {
                 k @ SyntaxKind::L_DOLLAR => {
-                    self.token_pos += 1;
+                    self.token_source.bump();
                     self.push_event(Event::Token { kind: k, n_raw_tokens: 1 });
                     ate_count += 1;
                 }
@@ -302,9 +311,9 @@ impl<'t> Parser<'t> {
     pub(crate) fn eat_r_dollars(&mut self, max_count: usize) -> usize {
         let mut ate_count = 0;
         loop {
-            match self.token_source.token_kind(self.token_pos) {
+            match self.token_source.current().kind {
                 k @ SyntaxKind::R_DOLLAR => {
-                    self.token_pos += 1;
+                    self.token_source.bump();
                     self.push_event(Event::Token { kind: k, n_raw_tokens: 1 });
                     ate_count += 1;
 
@@ -320,12 +329,12 @@ impl<'t> Parser<'t> {
     }
 
     pub(crate) fn at_l_dollar(&self) -> bool {
-        let kind = self.token_source.token_kind(self.token_pos);
+        let kind = self.token_source.current().kind;
         (kind == SyntaxKind::L_DOLLAR)
     }
 
     pub(crate) fn at_r_dollar(&self) -> bool {
-        let kind = self.token_source.token_kind(self.token_pos);
+        let kind = self.token_source.current().kind;
         (kind == SyntaxKind::R_DOLLAR)
     }
 }

crates/ra_syntax/src/parsing.rs

@@ -17,8 +17,8 @@ pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     let tokens = tokenize(&text);
-    let token_source = text_token_source::TextTokenSource::new(text, &tokens);
+    let mut token_source = text_token_source::TextTokenSource::new(text, &tokens);
     let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens);
-    ra_parser::parse(&token_source, &mut tree_sink);
+    ra_parser::parse(&mut token_source, &mut tree_sink);
     tree_sink.finish()
 }

crates/ra_syntax/src/parsing/reparsing.rs

@@ -85,9 +85,9 @@ fn reparse_block<'node>(
     if !is_balanced(&tokens) {
         return None;
     }
-    let token_source = TextTokenSource::new(&text, &tokens);
+    let mut token_source = TextTokenSource::new(&text, &tokens);
     let mut tree_sink = TextTreeSink::new(&text, &tokens);
-    reparser.parse(&token_source, &mut tree_sink);
+    reparser.parse(&mut token_source, &mut tree_sink);
     let (green, new_errors) = tree_sink.finish();
     Some((node.replace_with(green), new_errors, node.range()))
 }

crates/ra_syntax/src/parsing/text_token_source.rs

@@ -1,7 +1,8 @@
 use ra_parser::TokenSource;
+use ra_parser::Token as PToken;
 
 use crate::{
-    SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit,
+    SyntaxKind::EOF, TextRange, TextUnit,
     parsing::lexer::Token,
 };
 
@@ -23,31 +24,50 @@ pub(crate) struct TextTokenSource<'t> {
     /// ```
     /// tokens: `[struct, Foo, {, }]`
     tokens: Vec<Token>,
+
+    /// Current token and position
+    curr: (PToken, usize),
 }
 
 impl<'t> TokenSource for TextTokenSource<'t> {
-    fn token_kind(&self, pos: usize) -> SyntaxKind {
+    fn current(&self) -> PToken {
-        if !(pos < self.tokens.len()) {
+        return self.curr.0;
-            return EOF;
-        }
-        self.tokens[pos].kind
     }
-    fn is_token_joint_to_next(&self, pos: usize) -> bool {
+
-        if !(pos + 1 < self.tokens.len()) {
+    fn lookahead_nth(&self, n: usize) -> PToken {
-            return true;
+        mk_token(self.curr.1 + n, &self.start_offsets, &self.tokens)
-        }
-        self.start_offsets[pos] + self.tokens[pos].len == self.start_offsets[pos + 1]
     }
-    fn is_keyword(&self, pos: usize, kw: &str) -> bool {
+
+    fn bump(&mut self) {
+        if self.curr.0.kind == EOF {
+            return;
+        }
+
+        let pos = self.curr.1 + 1;
+        self.curr = (mk_token(pos, &self.start_offsets, &self.tokens), pos);
+    }
+
+    fn is_keyword(&self, kw: &str) -> bool {
+        let pos = self.curr.1;
         if !(pos < self.tokens.len()) {
             return false;
         }
         let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len);
-
         self.text[range] == *kw
     }
 }
 
+fn mk_token(pos: usize, start_offsets: &[TextUnit], tokens: &[Token]) -> PToken {
+    let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF);
+    let is_jointed_to_next = if pos + 1 < start_offsets.len() {
+        start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1]
+    } else {
+        false
+    };
+
+    PToken { kind, is_jointed_to_next }
+}
+
 impl<'t> TextTokenSource<'t> {
     /// Generate input from tokens(expect comment and whitespace).
     pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
@@ -62,6 +82,7 @@ impl<'t> TextTokenSource<'t> {
             len += token.len;
         }
 
-        TextTokenSource { text, start_offsets, tokens }
+        let first = mk_token(0, &start_offsets, &tokens);
+        TextTokenSource { text, start_offsets, tokens, curr: (first, 0) }
     }
 }