Merge #7433

7433: Support Macro v2 in mbe r=jonas-schievink a=edwin0cheng

Added `mbe::MacroDef` for Macro v2.

cc @jonas-schievink 

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

crates/mbe/Cargo.toml

@@ -17,6 +17,5 @@ log = "0.4.8"
 syntax = { path = "../syntax", version = "0.0.0" }
 parser = { path = "../parser", version = "0.0.0" }
 tt = { path = "../tt", version = "0.0.0" }
-
-[dev-dependencies]
 test_utils = { path = "../test_utils" }
+

crates/mbe/src/lib.rs

@@ -14,6 +14,7 @@ mod tests;
 
 use std::fmt;
 
+use test_utils::mark;
 pub use tt::{Delimiter, DelimiterKind, Punct};
 
 use crate::{
@@ -76,6 +77,14 @@ pub struct MacroRules {
     shift: Shift,
 }
 
+/// For Macro 2.0
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MacroDef {
+    rules: Vec<Rule>,
+    /// Highest id of the token we have in TokenMap
+    shift: Shift,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 struct Rule {
     lhs: MetaTemplate,
@@ -179,7 +188,7 @@ impl MacroRules {
         let mut src = TtIter::new(tt);
         let mut rules = Vec::new();
         while src.len() > 0 {
-            let rule = Rule::parse(&mut src)?;
+            let rule = Rule::parse(&mut src, true)?;
             rules.push(rule);
             if let Err(()) = src.expect_char(';') {
                 if src.len() > 0 {
@@ -200,7 +209,58 @@ impl MacroRules {
         // apply shift
         let mut tt = tt.clone();
         self.shift.shift_all(&mut tt);
-        mbe_expander::expand(self, &tt)
+        mbe_expander::expand_rules(&self.rules, &tt)
+    }
+
+    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
+        self.shift.shift(id)
+    }
+
+    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
+        match self.shift.unshift(id) {
+            Some(id) => (id, Origin::Call),
+            None => (id, Origin::Def),
+        }
+    }
+}
+
+impl MacroDef {
+    pub fn parse(tt: &tt::Subtree) -> Result<MacroDef, ParseError> {
+        let mut src = TtIter::new(tt);
+        let mut rules = Vec::new();
+
+        if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() {
+            mark::hit!(parse_macro_def_rules);
+            while src.len() > 0 {
+                let rule = Rule::parse(&mut src, true)?;
+                rules.push(rule);
+                if let Err(()) = src.expect_char(';') {
+                    if src.len() > 0 {
+                        return Err(ParseError::Expected("expected `;`".to_string()));
+                    }
+                    break;
+                }
+            }
+        } else {
+            mark::hit!(parse_macro_def_simple);
+            let rule = Rule::parse(&mut src, false)?;
+            if src.len() != 0 {
+                return Err(ParseError::Expected("remain tokens in macro def".to_string()));
+            }
+            rules.push(rule);
+        }
+        for rule in rules.iter() {
+            validate(&rule.lhs)?;
+        }
+
+        Ok(MacroDef { rules, shift: Shift::new(tt) })
+    }
+
+    pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+        // apply shift
+        let mut tt = tt.clone();
+        self.shift.shift_all(&mut tt);
+        mbe_expander::expand_rules(&self.rules, &tt)
     }
 
     pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
@@ -216,12 +276,14 @@ impl MacroRules {
 }
 
 impl Rule {
-    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
+    fn parse(src: &mut TtIter, expect_arrow: bool) -> Result<Rule, ParseError> {
         let lhs = src
             .expect_subtree()
             .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;
-        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
-        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        if expect_arrow {
+            src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
+            src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        }
         let rhs = src
             .expect_subtree()
             .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;

crates/mbe/src/mbe_expander.rs

@@ -10,11 +10,10 @@ use syntax::SmolStr;
 
 use crate::{ExpandError, ExpandResult};
 
-pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
-    expand_rules(&rules.rules, input)
-}
-
-fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+pub(crate) fn expand_rules(
+    rules: &[crate::Rule],
+    input: &tt::Subtree,
+) -> ExpandResult<tt::Subtree> {
     let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
     for rule in rules {
         let new_match = match matcher::match_(&rule.lhs, input) {

crates/mbe/src/tests.rs

@@ -6,7 +6,7 @@ use syntax::{
     SyntaxKind::{ERROR, IDENT},
     SyntaxNode, WalkEvent, T,
 };
-use test_utils::assert_eq_text;
+use test_utils::{assert_eq_text, mark};
 
 use super::*;
 
@@ -675,6 +675,36 @@ fn test_match_literal() {
     .assert_expand_items("foo! ['('];", "fn foo () {}");
 }
 
+#[test]
+fn test_parse_macro_def_simple() {
+    mark::check!(parse_macro_def_simple);
+
+    parse_macro2(
+        r#"
+macro foo($id:ident) {
+    fn $id() {}
+}
+"#,
+    )
+    .assert_expand_items("foo!(bar);", "fn bar () {}");
+}
+
+#[test]
+fn test_parse_macro_def_rules() {
+    mark::check!(parse_macro_def_rules);
+
+    parse_macro2(
+        r#"
+macro foo {
+    ($id:ident) => {
+        fn $id() {}
+    }
+}
+"#,
+    )
+    .assert_expand_items("foo!(bar);", "fn bar () {}");
+}
+
 // The following tests are port from intellij-rust directly
 // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
 
@@ -1699,95 +1729,122 @@ pub(crate) struct MacroFixture {
     rules: MacroRules,
 }
 
-impl MacroFixture {
-    pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
-        self.try_expand_tt(invocation).unwrap()
-    }
-
-    fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
-        let source_file = ast::SourceFile::parse(invocation).tree();
-        let macro_invocation =
-            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
-
-        let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
-            .ok_or_else(|| ExpandError::ConversionError)?;
-
-        self.rules.expand(&invocation_tt).result()
-    }
-
-    fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
-        assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
-    }
-
-    fn expand_items(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node()
-    }
-
-    fn expand_statements(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node()
-    }
-
-    fn expand_expr(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node()
-    }
-
-    fn assert_expand_tt(&self, invocation: &str, expected: &str) {
-        let expansion = self.expand_tt(invocation);
-        assert_eq!(expansion.to_string(), expected);
-    }
-
-    fn assert_expand(&self, invocation: &str, expected: &str) {
-        let expansion = self.expand_tt(invocation);
-        let actual = format!("{:?}", expansion);
-        test_utils::assert_eq_text!(&expected.trim(), &actual.trim());
-    }
-
-    fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture {
-        self.assert_expansion(FragmentKind::Items, invocation, expected);
-        self
-    }
-
-    fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture {
-        self.assert_expansion(FragmentKind::Statements, invocation, expected);
-        self
-    }
-
-    fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) {
-        let expanded = self.expand_tt(invocation);
-        assert_eq!(expanded.to_string(), expected);
-
-        let expected = expected.replace("$crate", "C_C__C");
-
-        // wrap the given text to a macro call
-        let expected = {
-            let wrapped = format!("wrap_macro!( {} )", expected);
-            let wrapped = ast::SourceFile::parse(&wrapped);
-            let wrapped =
-                wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-            let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0;
-            wrapped.delimiter = None;
-            wrapped
-        };
-
-        let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node();
-        let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string();
-
-        let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node();
-        let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string();
-
-        let expected_tree = expected_tree.replace("C_C__C", "$crate");
-        assert_eq!(
-            expanded_tree, expected_tree,
-            "\nleft:\n{}\nright:\n{}",
-            expanded_tree, expected_tree,
-        );
-    }
+pub(crate) struct MacroFixture2 {
+    rules: MacroDef,
 }
 
-fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree {
+macro_rules! impl_fixture {
+    ($name:ident) => {
+        impl $name {
+            pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
+                self.try_expand_tt(invocation).unwrap()
+            }
+
+            fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
+                let source_file = ast::SourceFile::parse(invocation).tree();
+                let macro_invocation =
+                    source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+                let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
+                    .ok_or_else(|| ExpandError::ConversionError)?;
+
+                self.rules.expand(&invocation_tt).result()
+            }
+
+            #[allow(unused)]
+            fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
+                assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
+            }
+
+            #[allow(unused)]
+            fn expand_items(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node()
+            }
+
+            #[allow(unused)]
+            fn expand_statements(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Statements)
+                    .unwrap()
+                    .0
+                    .syntax_node()
+            }
+
+            #[allow(unused)]
+            fn expand_expr(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node()
+            }
+
+            #[allow(unused)]
+            fn assert_expand_tt(&self, invocation: &str, expected: &str) {
+                let expansion = self.expand_tt(invocation);
+                assert_eq!(expansion.to_string(), expected);
+            }
+
+            #[allow(unused)]
+            fn assert_expand(&self, invocation: &str, expected: &str) {
+                let expansion = self.expand_tt(invocation);
+                let actual = format!("{:?}", expansion);
+                test_utils::assert_eq_text!(&expected.trim(), &actual.trim());
+            }
+
+            fn assert_expand_items(&self, invocation: &str, expected: &str) -> &$name {
+                self.assert_expansion(FragmentKind::Items, invocation, expected);
+                self
+            }
+
+            #[allow(unused)]
+            fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &$name {
+                self.assert_expansion(FragmentKind::Statements, invocation, expected);
+                self
+            }
+
+            fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) {
+                let expanded = self.expand_tt(invocation);
+                assert_eq!(expanded.to_string(), expected);
+
+                let expected = expected.replace("$crate", "C_C__C");
+
+                // wrap the given text to a macro call
+                let expected = {
+                    let wrapped = format!("wrap_macro!( {} )", expected);
+                    let wrapped = ast::SourceFile::parse(&wrapped);
+                    let wrapped = wrapped
+                        .tree()
+                        .syntax()
+                        .descendants()
+                        .find_map(ast::TokenTree::cast)
+                        .unwrap();
+                    let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0;
+                    wrapped.delimiter = None;
+                    wrapped
+                };
+
+                let expanded_tree =
+                    token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node();
+                let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string();
+
+                let expected_tree =
+                    token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node();
+                let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string();
+
+                let expected_tree = expected_tree.replace("C_C__C", "$crate");
+                assert_eq!(
+                    expanded_tree, expected_tree,
+                    "\nleft:\n{}\nright:\n{}",
+                    expanded_tree, expected_tree,
+                );
+            }
+        }
+    };
+}
+
+impl_fixture!(MacroFixture);
+impl_fixture!(MacroFixture2);
+
+fn parse_macro_rules_to_tt(ra_fixture: &str) -> tt::Subtree {
     let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
     let macro_definition =
         source_file.syntax().descendants().find_map(ast::MacroRules::cast).unwrap();
@@ -1804,14 +1861,36 @@ fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree {
     definition_tt
 }
 
+fn parse_macro_def_to_tt(ra_fixture: &str) -> tt::Subtree {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
+    let macro_definition =
+        source_file.syntax().descendants().find_map(ast::MacroDef::cast).unwrap();
+
+    let (definition_tt, _) = ast_to_token_tree(&macro_definition.body().unwrap()).unwrap();
+
+    let parsed =
+        parse_to_token_tree(&ra_fixture[macro_definition.body().unwrap().syntax().text_range()])
+            .unwrap()
+            .0;
+    assert_eq!(definition_tt, parsed);
+
+    definition_tt
+}
+
 pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
-    let definition_tt = parse_macro_to_tt(ra_fixture);
+    let definition_tt = parse_macro_rules_to_tt(ra_fixture);
     let rules = MacroRules::parse(&definition_tt).unwrap();
     MacroFixture { rules }
 }
 
+pub(crate) fn parse_macro2(ra_fixture: &str) -> MacroFixture2 {
+    let definition_tt = parse_macro_def_to_tt(ra_fixture);
+    let rules = MacroDef::parse(&definition_tt).unwrap();
+    MacroFixture2 { rules }
+}
+
 pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError {
-    let definition_tt = parse_macro_to_tt(ra_fixture);
+    let definition_tt = parse_macro_rules_to_tt(ra_fixture);
 
     match MacroRules::parse(&definition_tt) {
         Ok(_) => panic!("Expect error"),