Add TokenId Shif in macro_rules

crates/ra_mbe/src/lib.rs

@@ -42,6 +42,8 @@ pub use crate::syntax_bridge::{
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct MacroRules {
     pub(crate) rules: Vec<Rule>,
+    /// Highest id of the token we have in TokenMap
+    pub(crate) shift: Option<u32>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -50,6 +52,42 @@ pub(crate) struct Rule {
     pub(crate) rhs: tt::Subtree,
 }
 
+/// Find the "shift" (the highest id of the TokenId) inside a subtree
+fn find_subtree_shift(tt: &tt::Subtree, mut cur: Option<u32>) -> Option<u32> {
+    use std::cmp::max;
+
+    for t in &tt.token_trees {
+        cur = match t {
+            tt::TokenTree::Leaf(leaf) => match leaf {
+                tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => {
+                    Some(max(cur.unwrap_or(0), ident.id.0))
+                }
+                _ => cur,
+            },
+            tt::TokenTree::Subtree(tt) => find_subtree_shift(tt, cur),
+        }
+    }
+
+    cur
+}
+
+/// Shift given TokenTree token id
+fn shift_token_tree(tt: &mut tt::Subtree, shift: u32) {
+    for t in tt.token_trees.iter_mut() {
+        match t {
+            tt::TokenTree::Leaf(leaf) => match leaf {
+                tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => {
+                    // Note that TokenId is started from zero,
+                    // We have to add 1 to prevent duplication.
+                    ident.id.0 += shift + 1;
+                }
+                _ => (),
+            },
+            tt::TokenTree::Subtree(tt) => shift_token_tree(tt, shift),
+        }
+    }
+}
+
 impl MacroRules {
     pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
         // Note: this parsing can be implemented using mbe machinery itself, by
@@ -72,10 +110,17 @@ impl MacroRules {
             validate(&rule.lhs)?;
         }
 
-        Ok(MacroRules { rules })
+        Ok(MacroRules { rules, shift: find_subtree_shift(tt, None) })
     }
+
     pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
-        mbe_expander::expand(self, tt)
+        // apply shift
+        let mut tt = tt.clone();
+        if let Some(shift) = self.shift {
+            shift_token_tree(&mut tt, shift)
+        }
+
+        mbe_expander::expand(self, &tt)
     }
 }
 

crates/ra_mbe/src/tests.rs

@@ -58,6 +58,33 @@ mod rule_parsing {
 // * Port the test to rust and add it to this module
 // * Make it pass :-)
 
+#[test]
+fn test_token_id_shift() {
+    let macro_definition = r#"
+macro_rules! foobar {
+    ($e:ident) => { foo bar $e }
+}
+"#;
+    let rules = create_rules(macro_definition);
+    let expansion = expand(&rules, "foobar!(baz);");
+
+    fn get_id(t: &tt::TokenTree) -> Option<u32> {
+        if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t {
+            return Some(ident.id.0);
+        }
+        None
+    }
+
+    assert_eq!(expansion.token_trees.len(), 3);
+    // ($e:ident) => { foo bar $e }
+    //   0 1            2   3   4
+    assert_eq!(get_id(&expansion.token_trees[0]), Some(2));
+    assert_eq!(get_id(&expansion.token_trees[1]), Some(3));
+
+    // So baz should be 5
+    assert_eq!(get_id(&expansion.token_trees[2]), Some(5));
+}
+
 #[test]
 fn test_convert_tt() {
     let macro_definition = r#"