Add test for delim bug

crates/ra_mbe/src/syntax_bridge.rs

@@ -267,6 +267,108 @@ struct RawConvertor<'a> {
     inner: std::slice::Iter<'a, Token>,
 }
 
+trait SrcToken {
+    fn kind() -> SyntaxKind;
+}
+
+trait TokenConvertor  {
+    type Token : SrcToken;
+
+    fn go(&mut self) -> Option<tt::Subtree> {
+        let mut subtree = tt::Subtree::default();
+        subtree.delimiter = None;
+        while self.peek().is_some() {
+            self.collect_leaf(&mut subtree.token_trees);
+        }
+        if subtree.token_trees.is_empty() {
+            return None;
+        }
+        if subtree.token_trees.len() == 1 {
+            if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
+                return Some(first.clone());
+            }
+        }
+        Some(subtree)
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
+
+    fn peek(&self) -> Option<Self::Token>;
+
+    fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
+        let (token, range) = match self.bump() {
+            None => return,
+            Some(it) => it,
+        };
+
+        let k: SyntaxKind = token.kind();
+        if k == COMMENT {
+            let node = doc_comment(&self.text[range]);
+            if let Some(tokens) = convert_doc_comment(&node) {
+                result.extend(tokens);
+            }
+            return;
+        }
+
+        result.push(if k.is_punct() {
+            let delim = match k {
+                T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
+                T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
+                T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
+                _ => None,
+            };
+
+            if let Some((kind, closed)) = delim {
+                let mut subtree = tt::Subtree::default();
+                let id = self.id_alloc.open_delim(range);
+                subtree.delimiter = Some(tt::Delimiter { kind, id });
+
+                while self.peek().map(|it| it.kind != closed).unwrap_or(false) {
+                    self.collect_leaf(&mut subtree.token_trees);
+                }
+                let last_range = match self.bump() {
+                    None => return,
+                    Some(it) => it.1,
+                };
+                self.id_alloc.close_delim(id, last_range);
+                subtree.into()
+            } else {
+                let spacing = match self.peek() {
+                    Some(next)
+                        if next.kind.is_trivia()
+                            || next.kind == T!['[']
+                            || next.kind == T!['{']
+                            || next.kind == T!['('] =>
+                    {
+                        tt::Spacing::Alone
+                    }
+                    Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
+                    _ => tt::Spacing::Alone,
+                };
+                let char =
+                    self.text[range].chars().next().expect("Token from lexer must be single char");
+
+                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into()
+            }
+        } else {
+            macro_rules! make_leaf {
+                ($i:ident) => {
+                    tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
+                };
+            }
+            let leaf: tt::Leaf = match k {
+                T![true] | T![false] => make_leaf!(Literal),
+                IDENT | LIFETIME => make_leaf!(Ident),
+                k if k.is_keyword() => make_leaf!(Ident),
+                k if k.is_literal() => make_leaf!(Literal),
+                _ => return,
+            };
+
+            leaf.into()
+        });
+    }
+}
+
 impl RawConvertor<'_> {
     fn go(&mut self) -> Option<tt::Subtree> {
         let mut subtree = tt::Subtree::default();
@@ -296,6 +398,7 @@ impl RawConvertor<'_> {
         self.inner.as_slice().get(0).cloned()
     }
     
+
     fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
         let (token, range) = match self.bump() {
             None => return,

crates/ra_mbe/src/tests.rs

@@ -427,22 +427,28 @@ MACRO_ITEMS@[0; 40)
     );
 }
 
+fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
+    if let tt::TokenTree::Subtree(subtree) = tt {
+        return &subtree;
+    }
+    unreachable!("It is not a subtree");
+}
+fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
+    if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
+        return lit;
+    }
+    unreachable!("It is not a literal");
+}
+
+fn to_punct(tt: &tt::TokenTree) -> &tt::Punct {
+    if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt {
+        return lit;
+    }
+    unreachable!("It is not a Punct");
+}
+
 #[test]
 fn test_expand_literals_to_token_tree() {
-    fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
-        if let tt::TokenTree::Subtree(subtree) = tt {
-            return &subtree;
-        }
-        unreachable!("It is not a subtree");
-    }
-
-    fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
-        if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
-            return lit;
-        }
-        unreachable!("It is not a literal");
-    }
-
     let expansion = parse_macro(
         r#"
             macro_rules! literals {
@@ -470,6 +476,22 @@ fn test_expand_literals_to_token_tree() {
     assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
 }
 
+#[test]
+fn test_attr_to_token_tree() {
+    let expansion = parse_to_token_tree_by_syntax(
+        r#"
+            #[derive(Copy)]
+            struct Foo;
+            "#,
+    );
+
+    assert_eq!(to_punct(&expansion.token_trees[0]).char, '#');
+    assert_eq!(
+        to_subtree(&expansion.token_trees[1]).delimiter_kind(),
+        Some(tt::DelimiterKind::Bracket)
+    );
+}
+
 #[test]
 fn test_two_idents() {
     parse_macro(
@@ -1517,6 +1539,16 @@ pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
     MacroFixture { rules }
 }
 
+pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
+    let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0;
+
+    let parsed = parse_to_token_tree(ra_fixture).unwrap().0;
+    assert_eq!(tt, parsed);
+
+    parsed
+}
+
 fn debug_dump_ignore_spaces(node: &ra_syntax::SyntaxNode) -> String {
     let mut level = 0;
     let mut buf = String::new();