Add TokenConvertor trait

crates/ra_mbe/src/syntax_bridge.rs

@@ -3,12 +3,11 @@
 use ra_parser::{FragmentKind, ParseError, TreeSink};
 use ra_syntax::{
     ast::{self, make::tokens::doc_comment},
-    tokenize, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind,
+    tokenize, AstToken, Parse, SmolStr, SyntaxKind,
     SyntaxKind::*,
-    SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, Token, T,
+    SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextUnit, Token as RawToken, T,
 };
 use rustc_hash::FxHashMap;
-use std::iter::successors;
 use tt::buffer::{Cursor, TokenBuffer};
 
 use crate::subtree_source::SubtreeTokenSource;
@@ -50,10 +49,8 @@ pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenM
 /// will consume).
 pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
     let global_offset = node.text_range().start();
-    let mut c = Convertor {
+    let mut c = Convertor::new(node, global_offset);
-        id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
+    let subtree = c.go()?;
-    };
-    let subtree = c.go(node)?;
     Some((subtree, c.id_alloc.map))
 }
 
@@ -237,16 +234,6 @@ impl TokenIdAlloc {
         token_id
     }
 
-    fn delim(&mut self, open_abs_range: TextRange, close_abs_range: TextRange) -> tt::TokenId {
-        let open_relative_range = open_abs_range - self.global_offset;
-        let close_relative_range = close_abs_range - self.global_offset;
-        let token_id = tt::TokenId(self.next_id);
-        self.next_id += 1;
-
-        self.map.insert_delim(token_id, open_relative_range, close_relative_range);
-        token_id
-    }
-
     fn open_delim(&mut self, open_abs_range: TextRange) -> tt::TokenId {
         let token_id = tt::TokenId(self.next_id);
         self.next_id += 1;
@@ -264,11 +251,15 @@ struct RawConvertor<'a> {
     text: &'a str,
     offset: TextUnit,
     id_alloc: TokenIdAlloc,
-    inner: std::slice::Iter<'a, Token>,
+    inner: std::slice::Iter<'a, RawToken>,
 }
 
 trait SrcToken {
-    fn kind() -> SyntaxKind;
+    fn kind(&self) -> SyntaxKind;
+
+    fn to_char(&self) -> Option<char>;
+
+    fn to_text(&self) -> SmolStr;
 }
 
 trait TokenConvertor {
@@ -291,10 +282,6 @@ trait TokenConvertor  {
         Some(subtree)
     }
 
-    fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
-
-    fn peek(&self) -> Option<Self::Token>;
-
     fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
         let (token, range) = match self.bump() {
             None => return,
@@ -303,8 +290,7 @@ trait TokenConvertor  {
 
         let k: SyntaxKind = token.kind();
         if k == COMMENT {
-            let node = doc_comment(&self.text[range]);
+            if let Some(tokens) = self.convert_doc_comment(&token) {
-            if let Some(tokens) = convert_doc_comment(&node) {
                 result.extend(tokens);
             }
             return;
@@ -320,40 +306,39 @@ trait TokenConvertor  {
 
             if let Some((kind, closed)) = delim {
                 let mut subtree = tt::Subtree::default();
-                let id = self.id_alloc.open_delim(range);
+                let id = self.id_alloc().open_delim(range);
                 subtree.delimiter = Some(tt::Delimiter { kind, id });
 
-                while self.peek().map(|it| it.kind != closed).unwrap_or(false) {
+                while self.peek().map(|it| it.kind() != closed).unwrap_or(false) {
                     self.collect_leaf(&mut subtree.token_trees);
                 }
                 let last_range = match self.bump() {
                     None => return,
                     Some(it) => it.1,
                 };
-                self.id_alloc.close_delim(id, last_range);
+                self.id_alloc().close_delim(id, last_range);
                 subtree.into()
             } else {
                 let spacing = match self.peek() {
                     Some(next)
-                        if next.kind.is_trivia()
+                        if next.kind().is_trivia()
-                            || next.kind == T!['[']
+                            || next.kind() == T!['[']
-                            || next.kind == T!['{']
+                            || next.kind() == T!['{']
-                            || next.kind == T!['('] =>
+                            || next.kind() == T!['('] =>
                     {
                         tt::Spacing::Alone
                     }
-                    Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
+                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
                     _ => tt::Spacing::Alone,
                 };
-                let char =
+                let char = token.to_char().expect("Token from lexer must be single char");
-                    self.text[range].chars().next().expect("Token from lexer must be single char");
 
-                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into()
+                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
             }
         } else {
             macro_rules! make_leaf {
                 ($i:ident) => {
-                    tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
+                    tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into()
                 };
             }
             let leaf: tt::Leaf = match k {
@@ -367,237 +352,168 @@ trait TokenConvertor  {
             leaf.into()
         });
     }
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
+
+    fn peek(&self) -> Option<Self::Token>;
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc;
 }
 
-impl RawConvertor<'_> {
+impl<'a> SrcToken for (RawToken, &'a str) {
-    fn go(&mut self) -> Option<tt::Subtree> {
+    fn kind(&self) -> SyntaxKind {
-        let mut subtree = tt::Subtree::default();
+        self.0.kind
-        subtree.delimiter = None;
-        while self.peek().is_some() {
-            self.collect_leaf(&mut subtree.token_trees);
-        }
-        if subtree.token_trees.is_empty() {
-            return None;
-        }
-        if subtree.token_trees.len() == 1 {
-            if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
-                return Some(first.clone());
-            }
-        }
-        Some(subtree)
     }
 
-    fn bump(&mut self) -> Option<(Token, TextRange)> {
+    fn to_char(&self) -> Option<char> {
+        self.1.chars().next()
+    }
+
+    fn to_text(&self) -> SmolStr {
+        self.1.into()
+    }
+}
+
+impl RawConvertor<'_> {}
+
+impl<'a> TokenConvertor for RawConvertor<'a> {
+    type Token = (RawToken, &'a str);
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(&doc_comment(token.1))
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
         let token = self.inner.next()?;
         let range = TextRange::offset_len(self.offset, token.len);
         self.offset += token.len;
-        Some((*token, range))
+
+        Some(((*token, &self.text[range]), range))
     }
 
-    fn peek(&self) -> Option<Token> {
+    fn peek(&self) -> Option<Self::Token> {
-        self.inner.as_slice().get(0).cloned()
+        let token = self.inner.as_slice().get(0).cloned();
+
+        token.map(|it| {
+            let range = TextRange::offset_len(self.offset, it.len);
+            (it, &self.text[range])
+        })
     }
 
-
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
-    fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
+        &mut self.id_alloc
-        let (token, range) = match self.bump() {
-            None => return,
-            Some(it) => it,
-        };
-
-        let k: SyntaxKind = token.kind;
-        if k == COMMENT {
-            let node = doc_comment(&self.text[range]);
-            if let Some(tokens) = convert_doc_comment(&node) {
-                result.extend(tokens);
-            }
-            return;
-        }
-
-        result.push(if k.is_punct() {
-            let delim = match k {
-                T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
-                T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
-                T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
-                _ => None,
-            };
-
-            if let Some((kind, closed)) = delim {
-                let mut subtree = tt::Subtree::default();
-                let id = self.id_alloc.open_delim(range);
-                subtree.delimiter = Some(tt::Delimiter { kind, id });
-
-                while self.peek().map(|it| it.kind != closed).unwrap_or(false) {
-                    self.collect_leaf(&mut subtree.token_trees);
-                }
-                let last_range = match self.bump() {
-                    None => return,
-                    Some(it) => it.1,
-                };
-                self.id_alloc.close_delim(id, last_range);
-                subtree.into()
-            } else {
-                let spacing = match self.peek() {
-                    Some(next)
-                        if next.kind.is_trivia()
-                            || next.kind == T!['[']
-                            || next.kind == T!['{']
-                            || next.kind == T!['('] =>
-                    {
-                        tt::Spacing::Alone
-                    }
-                    Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
-                    _ => tt::Spacing::Alone,
-                };
-                let char =
-                    self.text[range].chars().next().expect("Token from lexer must be single char");
-
-                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into()
-            }
-        } else {
-            macro_rules! make_leaf {
-                ($i:ident) => {
-                    tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
-                };
-            }
-            let leaf: tt::Leaf = match k {
-                T![true] | T![false] => make_leaf!(Literal),
-                IDENT | LIFETIME => make_leaf!(Ident),
-                k if k.is_keyword() => make_leaf!(Ident),
-                k if k.is_literal() => make_leaf!(Literal),
-                _ => return,
-            };
-
-            leaf.into()
-        });
     }
 }
 
-// FIXME: There are some duplicate logic between RawConvertor and Convertor
-// It would be nice to refactor to converting SyntaxNode to ra_parser::Token and thus
-// use RawConvertor directly. But performance-wise it may not be a good idea ?
 struct Convertor {
     id_alloc: TokenIdAlloc,
+    current: Option<SyntaxToken>,
+    range: TextRange,
+    punct_offset: Option<(SyntaxToken, TextUnit)>,
 }
 
 impl Convertor {
-    fn go(&mut self, tt: &SyntaxNode) -> Option<tt::Subtree> {
+    fn new(node: &SyntaxNode, global_offset: TextUnit) -> Convertor {
-        // This tree is empty
+        Convertor {
-        if tt.first_child_or_token().is_none() {
+            id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
-            return Some(tt::Subtree { token_trees: vec![], delimiter: None });
+            current: node.first_token(),
+            range: node.text_range(),
+            punct_offset: None,
+        }
+    }
 }
 
-        let first_child = tt.first_child_or_token()?;
+enum SynToken {
-        let last_child = tt.last_child_or_token()?;
+    Ordiniary(SyntaxToken),
+    Punch(SyntaxToken, TextUnit),
+}
 
-        // ignore trivial first_child and last_child
+impl SynToken {
-        let first_child = successors(Some(first_child), |it| {
+    fn token(&self) -> &SyntaxToken {
-            if it.kind().is_trivia() {
+        match self {
-                it.next_sibling_or_token()
+            SynToken::Ordiniary(it) => it,
+            SynToken::Punch(it, _) => it,
+        }
+    }
+}
+
+impl SrcToken for SynToken {
+    fn kind(&self) -> SyntaxKind {
+        self.token().kind()
+    }
+    fn to_char(&self) -> Option<char> {
+        match self {
+            SynToken::Ordiniary(_) => None,
+            SynToken::Punch(it, i) => it.text().chars().nth(i.to_usize()),
+        }
+    }
+    fn to_text(&self) -> SmolStr {
+        self.token().text().clone()
+    }
+}
+
+impl TokenConvertor for Convertor {
+    type Token = SynToken;
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(token.token())
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
+        let curr = self.current.clone()?;
+        if !curr.text_range().is_subrange(&self.range) {
+            return None;
+        }
+
+        if let Some((punct, offset)) = self.punct_offset.clone() {
+            if offset.to_usize() + 1 < punct.text().len() {
+                let offset = offset + TextUnit::from_usize(1);
+                let range = punct.text_range();
+                self.punct_offset = Some((punct, offset));
+                let range = TextRange::offset_len(range.start() + offset, TextUnit::from_usize(1));
+                return Some((SynToken::Punch(curr, offset), range));
+            }
+        }
+
+        self.current = curr.next_token();
+
+        let token = if curr.kind().is_punct() {
+            let range = curr.text_range();
+            self.punct_offset = Some((curr.clone(), TextUnit::from_usize(0)));
+            (SynToken::Punch(curr, TextUnit::from_usize(0)), range)
         } else {
-                None
+            self.punct_offset = None;
-            }
+            let range = curr.text_range();
-        })
+            (SynToken::Ordiniary(curr), range)
-        .last()
+        };
-        .unwrap();
+
-        if first_child.kind().is_trivia() {
+        Some(token)
-            return Some(tt::Subtree { token_trees: vec![], delimiter: None });
     }
 
-        let last_child = successors(Some(last_child), |it| {
+    fn peek(&self) -> Option<Self::Token> {
-            if it.kind().is_trivia() {
+        let curr = self.current.clone()?;
-                it.prev_sibling_or_token()
+        if !curr.text_range().is_subrange(&self.range) {
+            return None;
+        }
+
+        if let Some((punct, mut offset)) = self.punct_offset.clone() {
+            offset = offset + TextUnit::from_usize(1);
+            if offset.to_usize() < punct.text().len() {
+                return Some(SynToken::Punch(punct, offset));
+            }
+        }
+
+        let token = if curr.kind().is_punct() {
+            SynToken::Punch(curr, TextUnit::from_usize(0))
         } else {
-                None
+            SynToken::Ordiniary(curr)
-            }
-        })
-        .last()
-        .unwrap();
-
-        let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) {
-            (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true),
-            (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true),
-            (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true),
-            _ => (None, false),
         };
-        let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
+        Some(token)
-            kind,
-            id: self.id_alloc.delim(first_child.text_range(), last_child.text_range()),
-        });
-
-        let mut token_trees = Vec::new();
-        let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
-
-        while let Some(child) = child_iter.next() {
-            if skip_first && (child == first_child || child == last_child) {
-                continue;
     }
 
-            match child {
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
-                NodeOrToken::Token(token) => {
+        &mut self.id_alloc
-                    if let Some(doc_tokens) = convert_doc_comment(&token) {
-                        token_trees.extend(doc_tokens);
-                    } else if token.kind().is_trivia() {
-                        continue;
-                    } else if token.kind().is_punct() {
-                        // we need to pull apart joined punctuation tokens
-                        let last_spacing = match child_iter.peek() {
-                            Some(NodeOrToken::Token(token)) => {
-                                if token.kind().is_punct() {
-                                    tt::Spacing::Joint
-                                } else {
-                                    tt::Spacing::Alone
-                                }
-                            }
-                            _ => tt::Spacing::Alone,
-                        };
-                        let spacing_iter = std::iter::repeat(tt::Spacing::Joint)
-                            .take(token.text().len() - 1)
-                            .chain(std::iter::once(last_spacing));
-                        for (char, spacing) in token.text().chars().zip(spacing_iter) {
-                            token_trees.push(
-                                tt::Leaf::from(tt::Punct {
-                                    char,
-                                    spacing,
-                                    id: self.id_alloc.alloc(token.text_range()),
-                                })
-                                .into(),
-                            );
-                        }
-                    } else {
-                        macro_rules! make_leaf {
-                            ($i:ident) => {
-                                tt::$i {
-                                    id: self.id_alloc.alloc(token.text_range()),
-                                    text: token.text().clone(),
-                                }
-                                .into()
-                            };
-                        }
-
-                        let child: tt::Leaf = match token.kind() {
-                            T![true] | T![false] => make_leaf!(Literal),
-                            IDENT | LIFETIME => make_leaf!(Ident),
-                            k if k.is_keyword() => make_leaf!(Ident),
-                            k if k.is_literal() => make_leaf!(Literal),
-                            _ => return None,
-                        };
-                        token_trees.push(child.into());
-                    }
-                }
-                NodeOrToken::Node(node) => {
-                    let child_subtree = self.go(&node)?;
-                    if child_subtree.delimiter.is_none() && node.kind() != SyntaxKind::TOKEN_TREE {
-                        token_trees.extend(child_subtree.token_trees);
-                    } else {
-                        token_trees.push(child_subtree.into());
-                    }
-                }
-            };
-        }
-
-        let res = tt::Subtree { delimiter, token_trees };
-        Some(res)
     }
 }
 

crates/ra_mbe/src/tests.rs

@@ -1449,8 +1449,8 @@ impl MacroFixture {
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
-        let (invocation_tt, _) =
+        let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
-            ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
+            .ok_or_else(|| ExpandError::ConversionError)?;
 
         self.rules.expand(&invocation_tt).result()
     }
@@ -1694,5 +1694,5 @@ fn test_expand_bad_literal() {
         macro_rules! foo { ($i:literal) => {}; }
     "#,
     )
-    .assert_expand_err(r#"foo!(&k");"#, &ExpandError::BindingError("".to_string()));
+    .assert_expand_err(r#"foo!(&k");"#, &ExpandError::ConversionError);
 }