use usual token tree for macro expansion

crates/ra_mbe/src/lib.rs

@@ -3,30 +3,19 @@
 /// interface, although it contains some code to bridge `SyntaxNode`s and
 /// `TokenTree`s as well!
 
-macro_rules! impl_froms {
+mod parser;
-    ($e:ident: $($v:ident), *) => {
-        $(
-            impl From<$v> for $e {
-                fn from(it: $v) -> $e {
-                    $e::$v(it)
-                }
-            }
-        )*
-    }
-}
-
-mod mbe_parser;
 mod mbe_expander;
 mod syntax_bridge;
-mod tt_cursor;
+mod tt_iter;
 mod subtree_source;
-mod subtree_parser;
-
-use ra_syntax::SmolStr;
-use smallvec::SmallVec;
 
 pub use tt::{Delimiter, Punct};
 
+use crate::{
+    parser::{parse_pattern, Op},
+    tt_iter::TtIter,
+};
+
 #[derive(Debug, PartialEq, Eq)]
 pub enum ParseError {
     Expected(String),
@@ -38,6 +27,7 @@ pub enum ExpandError {
     UnexpectedToken,
     BindingError(String),
     ConversionError,
+    InvalidRepeat,
 }
 
 pub use crate::syntax_bridge::{
@@ -54,97 +44,72 @@ pub struct MacroRules {
     pub(crate) rules: Vec<Rule>,
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct Rule {
+    pub(crate) lhs: tt::Subtree,
+    pub(crate) rhs: tt::Subtree,
+}
+
 impl MacroRules {
     pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
-        mbe_parser::parse(tt)
+        let mut src = TtIter::new(tt);
+        let mut rules = Vec::new();
+        while src.len() > 0 {
+            let rule = Rule::parse(&mut src)?;
+            rules.push(rule);
+            if let Err(()) = src.expect_char(';') {
+                if src.len() > 0 {
+                    return Err(ParseError::Expected("expected `:`".to_string()));
+                }
+                break;
+            }
+        }
+        Ok(MacroRules { rules })
     }
     pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
         mbe_expander::expand(self, tt)
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+impl Rule {
-pub(crate) struct Rule {
+    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
-    pub(crate) lhs: Subtree,
+        let mut lhs = src
-    pub(crate) rhs: Subtree,
+            .expect_subtree()
-}
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
-
+            .clone();
-#[derive(Clone, Debug, PartialEq, Eq)]
+        validate(&lhs)?;
-pub(crate) enum TokenTree {
+        lhs.delimiter = tt::Delimiter::None;
-    Leaf(Leaf),
+        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
-    Subtree(Subtree),
+        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
-    Repeat(Repeat),
+        let mut rhs = src
-}
+            .expect_subtree()
-impl_froms!(TokenTree: Leaf, Subtree, Repeat);
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
-
+            .clone();
-#[derive(Clone, Debug, PartialEq, Eq)]
+        rhs.delimiter = tt::Delimiter::None;
-pub(crate) enum Leaf {
+        Ok(crate::Rule { lhs, rhs })
-    Literal(Literal),
-    Punct(Punct),
-    Ident(Ident),
-    Var(Var),
-}
-impl_froms!(Leaf: Literal, Punct, Ident, Var);
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct Subtree {
-    pub(crate) delimiter: Delimiter,
-    pub(crate) token_trees: Vec<TokenTree>,
-}
-
-#[derive(Clone, Debug, Eq)]
-pub(crate) enum Separator {
-    Literal(tt::Literal),
-    Ident(tt::Ident),
-    Puncts(SmallVec<[tt::Punct; 3]>),
-}
-
-// Note that when we compare a Separator, we just care about its textual value.
-impl PartialEq for crate::Separator {
-    fn eq(&self, other: &crate::Separator) -> bool {
-        use crate::Separator::*;
-
-        match (self, other) {
-            (Ident(ref a), Ident(ref b)) => a.text == b.text,
-            (Literal(ref a), Literal(ref b)) => a.text == b.text,
-            (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
-                let a_iter = a.iter().map(|a| a.char);
-                let b_iter = b.iter().map(|b| b.char);
-                a_iter.eq(b_iter)
-            }
-            _ => false,
-        }
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
-pub(crate) struct Repeat {
+    for op in parse_pattern(pattern) {
-    pub(crate) subtree: Subtree,
+        let op = match op {
-    pub(crate) kind: RepeatKind,
+            Ok(it) => it,
-    pub(crate) separator: Option<Separator>,
+            Err(e) => {
-}
+                let msg = match e {
-
+                    ExpandError::InvalidRepeat => "invalid repeat".to_string(),
-#[derive(Clone, Debug, PartialEq, Eq)]
+                    _ => "invalid macro definition".to_string(),
-pub(crate) enum RepeatKind {
+                };
-    ZeroOrMore,
+                return Err(ParseError::Expected(msg));
-    OneOrMore,
+            }
-    ZeroOrOne,
+        };
-}
+        match op {
-
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
-#[derive(Clone, Debug, PartialEq, Eq)]
+                validate(subtree)?
-pub(crate) struct Literal {
+            }
-    pub(crate) text: SmolStr,
+            _ => (),
-}
+        }
-
+    }
-#[derive(Clone, Debug, PartialEq, Eq)]
+    Ok(())
-pub(crate) struct Ident {
-    pub(crate) text: SmolStr,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct Var {
-    pub(crate) text: SmolStr,
-    pub(crate) kind: Option<SmolStr>,
 }
 
 #[cfg(test)]

crates/ra_mbe/src/mbe_expander.rs

@@ -8,7 +8,6 @@ mod transcriber;
 use ra_syntax::SmolStr;
 use rustc_hash::FxHashMap;
 
-use crate::tt_cursor::TtCursor;
 use crate::ExpandError;
 
 pub(crate) fn expand(
@@ -19,12 +18,8 @@ pub(crate) fn expand(
 }
 
 fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
-    let mut input = TtCursor::new(input);
+    let bindings = matcher::match_(&rule.lhs, input)?;
-    let bindings = matcher::match_lhs(&rule.lhs, &mut input)?;
+    let res = transcriber::transcribe(&rule.rhs, &bindings)?;
-    if !input.is_eof() {
-        return Err(ExpandError::UnexpectedToken);
-    }
-    let res = transcriber::transcribe(&bindings, &rule.rhs)?;
     Ok(res)
 }
 
@@ -103,13 +98,6 @@ mod tests {
 
     #[test]
     fn test_expand_rule() {
-        // FIXME: The missing $var check should be in parsing phase
-        // assert_err(
-        //     "($i:ident) => ($j)",
-        //     "foo!{a}",
-        //     ExpandError::BindingError(String::from("could not find binding `j`")),
-        // );
-
         assert_err(
             "($($i:ident);*) => ($i)",
             "foo!{a}",
@@ -118,9 +106,6 @@ mod tests {
             )),
         );
 
-        assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
-        assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
-
         // FIXME:
         // Add an err test case for ($($i:ident)) => ($())
     }

crates/ra_mbe/src/mbe_expander/matcher.rs

@@ -1,11 +1,14 @@
 use crate::{
     mbe_expander::{Binding, Bindings, Fragment},
-    tt_cursor::TtCursor,
+    parser::{parse_pattern, Op, RepeatKind, Separator},
+    subtree_source::SubtreeTokenSource,
+    tt_iter::TtIter,
     ExpandError,
 };
 
-use ra_parser::FragmentKind::*;
+use ra_parser::{FragmentKind::*, TreeSink};
-use ra_syntax::SmolStr;
+use ra_syntax::{SmolStr, SyntaxKind};
+use tt::buffer::{Cursor, TokenBuffer};
 
 impl Bindings {
     fn push_optional(&mut self, name: &SmolStr) {
@@ -42,121 +45,247 @@ impl Bindings {
         }
         Ok(())
     }
-
-    fn merge(&mut self, nested: Bindings) {
-        self.inner.extend(nested.inner);
-    }
 }
 
-pub(super) fn match_lhs(
+macro_rules! err {
-    pattern: &crate::Subtree,
+    () => {
-    input: &mut TtCursor,
+        ExpandError::BindingError(format!(""))
-) -> Result<Bindings, ExpandError> {
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
+}
+
+macro_rules! bail {
+    ($($tt:tt)*) => {
+        return Err(err!($($tt)*))
+    };
+}
+
+pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> {
+    assert!(pattern.delimiter == tt::Delimiter::None);
+
     let mut res = Bindings::default();
-    for pat in pattern.token_trees.iter() {
+    let mut src = TtIter::new(src);
-        match pat {
-            crate::TokenTree::Leaf(leaf) => match leaf {
-                crate::Leaf::Var(crate::Var { text, kind }) => {
-                    let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
-                    match match_meta_var(kind.as_str(), input)? {
-                        Some(fragment) => {
-                            res.inner.insert(text.clone(), Binding::Fragment(fragment));
-                        }
-                        None => res.push_optional(text),
-                    }
-                }
-                crate::Leaf::Punct(punct) => {
-                    if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-                }
-                crate::Leaf::Ident(ident) => {
-                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-                }
-                crate::Leaf::Literal(literal) => {
-                    if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-                }
-            },
-            crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
-                // Dirty hack to make macro-expansion terminate.
-                // This should be replaced by a propper macro-by-example implementation
-                let mut limit = 65536;
-                let mut counter = 0;
 
-                let mut memento = input.save();
+    match_subtree(&mut res, pattern, &mut src)?;
 
-                loop {
+    if src.len() > 0 {
-                    match match_lhs(subtree, input) {
+        bail!("leftover tokens");
-                        Ok(nested) => {
-                            limit -= 1;
-                            if limit == 0 {
-                                log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
-                                break;
-                            }
-
-                            memento = input.save();
-                            res.push_nested(counter, nested)?;
-                            counter += 1;
-                            if counter == 1 {
-                                if let crate::RepeatKind::ZeroOrOne = kind {
-                                    break;
-                                }
-                            }
-
-                            if let Some(separator) = separator {
-                                if !input
-                                    .eat_seperator()
-                                    .map(|sep| sep == *separator)
-                                    .unwrap_or(false)
-                                {
-                                    input.rollback(memento);
-                                    break;
-                                }
-                            }
-                        }
-                        Err(_) => {
-                            input.rollback(memento);
-                            break;
-                        }
-                    }
-                }
-
-                match kind {
-                    crate::RepeatKind::OneOrMore if counter == 0 => {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-                    _ if counter == 0 => {
-                        // Collect all empty variables in subtrees
-                        collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
-                    }
-                    _ => {}
-                }
-            }
-            crate::TokenTree::Subtree(subtree) => {
-                let input_subtree =
-                    input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
-                if subtree.delimiter != input_subtree.delimiter {
-                    return Err(ExpandError::UnexpectedToken);
-                }
-
-                let mut input = TtCursor::new(input_subtree);
-                let bindings = match_lhs(&subtree, &mut input)?;
-                if !input.is_eof() {
-                    return Err(ExpandError::UnexpectedToken);
-                }
-
-                res.merge(bindings);
-            }
-        }
     }
+
     Ok(res)
 }
 
-fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
+fn match_subtree(
+    bindings: &mut Bindings,
+    pattern: &tt::Subtree,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
+                let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?;
+                match (lhs, rhs) {
+                    (
+                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    _ => Err(ExpandError::UnexpectedToken)?,
+                }
+            }
+            Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
+                let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
+                if lhs.delimiter != rhs.delimiter {
+                    bail!("mismatched delimiter")
+                }
+                let mut src = TtIter::new(rhs);
+                match_subtree(bindings, lhs, &mut src)?;
+                if src.len() > 0 {
+                    bail!("leftover tokens");
+                }
+            }
+            Op::Var { name, kind } => {
+                let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
+                match match_meta_var(kind.as_str(), src)? {
+                    Some(fragment) => {
+                        bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
+                    }
+                    None => bindings.push_optional(name),
+                }
+                ()
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                match_repeat(bindings, subtree, kind, separator, src)?
+            }
+        }
+    }
+    Ok(())
+}
+
+impl<'a> TtIter<'a> {
+    fn eat_separator(&mut self, separator: &Separator) -> bool {
+        let mut fork = self.clone();
+        let ok = match separator {
+            Separator::Ident(lhs) => match fork.expect_ident() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Literal(lhs) => match fork.expect_literal() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
+                Ok(rhs) => rhs.char == lhs.char,
+                _ => false,
+            }),
+        };
+        if ok {
+            *self = fork;
+        }
+        ok
+    }
+
+    pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
+        let ident = self.expect_ident()?;
+        // check if it start from "`"
+        if ident.text.chars().next() != Some('\'') {
+            return Err(());
+        }
+        Ok(ident)
+    }
+
+    pub(crate) fn expect_fragment(
+        &mut self,
+        fragment_kind: ra_parser::FragmentKind,
+    ) -> Result<tt::TokenTree, ()> {
+        pub(crate) struct OffsetTokenSink<'a> {
+            pub(crate) cursor: Cursor<'a>,
+            pub(crate) error: bool,
+        }
+
+        impl<'a> TreeSink for OffsetTokenSink<'a> {
+            fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
+                for _ in 0..n_tokens {
+                    self.cursor = self.cursor.bump_subtree();
+                }
+            }
+            fn start_node(&mut self, _kind: SyntaxKind) {}
+            fn finish_node(&mut self) {}
+            fn error(&mut self, _error: ra_parser::ParseError) {
+                self.error = true;
+            }
+        }
+
+        let buffer = TokenBuffer::new(self.inner.as_slice());
+        let mut src = SubtreeTokenSource::new(&buffer);
+        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
+
+        ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind);
+
+        if !sink.cursor.is_root() || sink.error {
+            return Err(());
+        }
+
+        let mut curr = buffer.begin();
+        let mut res = vec![];
+
+        while curr != sink.cursor {
+            if let Some(token) = curr.token_tree() {
+                res.push(token);
+            }
+            curr = curr.bump();
+        }
+        self.inner = self.inner.as_slice()[res.len()..].iter();
+        match res.len() {
+            0 => Err(()),
+            1 => Ok(res[0].clone()),
+            _ => Ok(tt::TokenTree::Subtree(tt::Subtree {
+                delimiter: tt::Delimiter::None,
+                token_trees: res.into_iter().cloned().collect(),
+            })),
+        }
+    }
+
+    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_fragment(Visibility) {
+            Ok(tt) => {
+                *self = fork;
+                Some(tt)
+            }
+            Err(()) => None,
+        }
+    }
+}
+
+pub(super) fn match_repeat(
+    bindings: &mut Bindings,
+    pattern: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a propper macro-by-example implementation
+    let mut limit = 65536;
+    let mut counter = 0;
+
+    for i in 0.. {
+        let mut fork = src.clone();
+
+        if let Some(separator) = &separator {
+            if i != 0 && !fork.eat_separator(separator) {
+                break;
+            }
+        }
+
+        let mut nested = Bindings::default();
+        match match_subtree(&mut nested, pattern, &mut fork) {
+            Ok(()) => {
+                limit -= 1;
+                if limit == 0 {
+                    log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator);
+                    break;
+                }
+                *src = fork;
+
+                bindings.push_nested(counter, nested)?;
+                counter += 1;
+                if counter == 1 {
+                    if let RepeatKind::ZeroOrOne = kind {
+                        break;
+                    }
+                }
+            }
+            Err(_) => break,
+        }
+    }
+
+    match (kind, counter) {
+        (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken),
+        (_, 0) => {
+            // Collect all empty variables in subtrees
+            let mut vars = Vec::new();
+            collect_vars(&mut vars, pattern)?;
+            for var in vars {
+                bindings.push_empty(&var)
+            }
+        }
+        _ => (),
+    }
+    Ok(())
+}
+
+fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, ExpandError> {
     let fragment = match kind {
         "path" => Path,
         "expr" => Expr,
@@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
         _ => {
             let tt = match kind {
                 "ident" => {
-                    let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
+                    let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone();
                     tt::Leaf::from(ident).into()
                 }
-                "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
+                "tt" => input.next().ok_or_else(|| err!())?.clone(),
-                "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
+                "lifetime" => {
+                    let ident = input.expect_lifetime().map_err(|()| err!())?;
+                    tt::Leaf::Ident(ident.clone()).into()
+                }
                 "literal" => {
-                    let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
+                    let literal = input.expect_literal().map_err(|()| err!())?.clone();
                     tt::Leaf::from(literal).into()
                 }
                 // `vis` is optional
-                "vis" => match input.try_eat_vis() {
+                "vis" => match input.eat_vis() {
                     Some(vis) => vis,
                     None => return Ok(None),
                 },
@@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
             return Ok(Some(Fragment::Tokens(tt)));
         }
     };
-    let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
+    let tt = input.expect_fragment(fragment).map_err(|()| err!())?;
     let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
     Ok(Some(fragment))
 }
 
-fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
+fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
-    let mut res = Vec::new();
+    for op in parse_pattern(pattern) {
-
+        match op? {
-    for tkn in subtree.token_trees.iter() {
+            Op::Var { name, .. } => buf.push(name.clone()),
-        match tkn {
+            Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
-            crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
-                res.push(text.clone());
+            Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
-            }
-            crate::TokenTree::Subtree(subtree) => {
-                res.extend(collect_vars(subtree));
-            }
-            crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
-                res.extend(collect_vars(subtree));
-            }
-            _ => {}
         }
     }
-
+    Ok(())
-    res
 }

crates/ra_mbe/src/mbe_expander/transcriber.rs

@@ -1,16 +1,20 @@
+//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like
+//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
+
 use ra_syntax::SmolStr;
 
 use crate::{
     mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_template, Op, RepeatKind, Separator},
     ExpandError,
 };
 
 impl Bindings {
-    fn contains(&self, name: &SmolStr) -> bool {
+    fn contains(&self, name: &str) -> bool {
         self.inner.contains_key(name)
     }
 
-    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
+    fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
         let mut b = self.inner.get(name).ok_or_else(|| {
             ExpandError::BindingError(format!("could not find binding `{}`", name))
         })?;
@@ -43,11 +47,12 @@ impl Bindings {
 }
 
 pub(super) fn transcribe(
+    template: &tt::Subtree,
     bindings: &Bindings,
-    template: &crate::Subtree,
 ) -> Result<tt::Subtree, ExpandError> {
+    assert!(template.delimiter == tt::Delimiter::None);
     let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
-    expand_subtree(template, &mut ctx)
+    expand_subtree(&mut ctx, template)
 }
 
 #[derive(Debug)]
@@ -57,159 +62,158 @@ struct ExpandCtx<'a> {
     var_expanded: bool,
 }
 
-fn expand_subtree(
+fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
-    template: &crate::Subtree,
-    ctx: &mut ExpandCtx,
-) -> Result<tt::Subtree, ExpandError> {
     let mut buf: Vec<tt::TokenTree> = Vec::new();
-    for tt in template.token_trees.iter() {
+    for op in parse_template(template) {
-        let tt = expand_tt(tt, ctx)?;
+        match op? {
-        push_fragment(&mut buf, tt);
+            Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()),
+            Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
+                let tt = expand_subtree(ctx, tt)?;
+                buf.push(tt.into());
+            }
+            Op::Var { name, kind: _ } => {
+                let fragment = expand_var(ctx, name)?;
+                push_fragment(&mut buf, fragment);
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                let fragment = expand_repeat(ctx, subtree, kind, separator)?;
+                push_fragment(&mut buf, fragment)
+            }
+        }
     }
-
     Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
 }
 
-fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
+fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> {
-    let res: tt::TokenTree = match template {
+    let res = if v == "crate" {
-        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
+        // FIXME: Properly handle $crate token
-        crate::TokenTree::Repeat(repeat) => {
+        let tt =
-            let mut buf: Vec<tt::TokenTree> = Vec::new();
+            tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
-            ctx.nesting.push(0);
+                .into();
-            // Dirty hack to make macro-expansion terminate.
+        Fragment::Tokens(tt)
-            // This should be replaced by a propper macro-by-example implementation
+    } else if !ctx.bindings.contains(v) {
-            let mut limit = 65536;
+        // Note that it is possible to have a `$var` inside a macro which is not bound.
-            let mut has_seps = 0;
+        // For example:
-            let mut counter = 0;
+        // ```
-
+        // macro_rules! foo {
-            // We store the old var expanded value, and restore it later
+        //     ($a:ident, $b:ident, $c:tt) => {
-            // It is because before this `$repeat`,
+        //         macro_rules! bar {
-            // it is possible some variables already expanad in the same subtree
+        //             ($bi:ident) => {
-            //
+        //                 fn $bi() -> u8 {$c}
-            // `some_var_expanded` keep check if the deeper subtree has expanded variables
+        //             }
-            let mut some_var_expanded = false;
+        //         }
-            let old_var_expanded = ctx.var_expanded;
+        //     }
-            ctx.var_expanded = false;
+        // ```
-
+        // We just treat it a normal tokens
-            while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
+        let tt = tt::Subtree {
-                // if no var expanded in the child, we count it as a fail
+            delimiter: tt::Delimiter::None,
-                if !ctx.var_expanded {
+            token_trees: vec![
-                    break;
+                tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(),
-                }
+                tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
-
+                    .into(),
-                // Reset `ctx.var_expandeded` to see if there is other expanded variable
+            ],
-                // in the next matching
-                some_var_expanded = true;
-                ctx.var_expanded = false;
-
-                counter += 1;
-                limit -= 1;
-                if limit == 0 {
-                    log::warn!(
-                        "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
-                        template,
-                        ctx
-                    );
-                    break;
-                }
-
-                let idx = ctx.nesting.pop().unwrap();
-                ctx.nesting.push(idx + 1);
-                push_subtree(&mut buf, t);
-
-                if let Some(ref sep) = repeat.separator {
-                    match sep {
-                        crate::Separator::Ident(ident) => {
-                            has_seps = 1;
-                            buf.push(tt::Leaf::from(ident.clone()).into());
-                        }
-                        crate::Separator::Literal(lit) => {
-                            has_seps = 1;
-                            buf.push(tt::Leaf::from(lit.clone()).into());
-                        }
-
-                        crate::Separator::Puncts(puncts) => {
-                            has_seps = puncts.len();
-                            for punct in puncts {
-                                buf.push(tt::Leaf::from(*punct).into());
-                            }
-                        }
-                    }
-                }
-
-                if let crate::RepeatKind::ZeroOrOne = repeat.kind {
-                    break;
-                }
-            }
-
-            // Restore the `var_expanded` by combining old one and the new one
-            ctx.var_expanded = some_var_expanded || old_var_expanded;
-
-            ctx.nesting.pop().unwrap();
-            for _ in 0..has_seps {
-                buf.pop();
-            }
-
-            if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
-                return Err(ExpandError::UnexpectedToken);
-            }
-
-            // Check if it is a single token subtree without any delimiter
-            // e.g {Delimiter:None> ['>'] /Delimiter:None>}
-            tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
         }
-        crate::TokenTree::Leaf(leaf) => match leaf {
+        .into();
-            crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident {
+        Fragment::Tokens(tt)
-                text: ident.text.clone(),
+    } else {
-                id: tt::TokenId::unspecified(),
+        let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone();
-            })
+        ctx.var_expanded = true;
-            .into(),
+        fragment
-            crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
+    };
-            crate::Leaf::Var(v) => {
+    Ok(res)
-                if v.text == "crate" {
+}
-                    // FIXME: Properly handle $crate token
+
-                    tt::Leaf::from(tt::Ident {
+fn expand_repeat(
-                        text: "$crate".into(),
+    ctx: &mut ExpandCtx,
-                        id: tt::TokenId::unspecified(),
+    template: &tt::Subtree,
-                    })
+    kind: RepeatKind,
-                    .into()
+    separator: Option<Separator>,
-                } else if !ctx.bindings.contains(&v.text) {
+) -> Result<Fragment, ExpandError> {
-                    // Note that it is possible to have a `$var` inside a macro which is not bound.
+    let mut buf: Vec<tt::TokenTree> = Vec::new();
-                    // For example:
+    ctx.nesting.push(0);
-                    // ```
+    // Dirty hack to make macro-expansion terminate.
-                    // macro_rules! foo {
+    // This should be replaced by a propper macro-by-example implementation
-                    //     ($a:ident, $b:ident, $c:tt) => {
+    let mut limit = 65536;
-                    //         macro_rules! bar {
+    let mut has_seps = 0;
-                    //             ($bi:ident) => {
+    let mut counter = 0;
-                    //                 fn $bi() -> u8 {$c}
+
-                    //             }
+    // We store the old var expanded value, and restore it later
-                    //         }
+    // It is because before this `$repeat`,
-                    //     }
+    // it is possible some variables already expanad in the same subtree
-                    // ```
+    //
-                    // We just treat it a normal tokens
+    // `some_var_expanded` keep check if the deeper subtree has expanded variables
-                    tt::Subtree {
+    let mut some_var_expanded = false;
-                        delimiter: tt::Delimiter::None,
+    let old_var_expanded = ctx.var_expanded;
-                        token_trees: vec![
+    ctx.var_expanded = false;
-                            tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
+
-                                .into(),
+    while let Ok(mut t) = expand_subtree(ctx, template) {
-                            tt::Leaf::from(tt::Ident {
+        t.delimiter = tt::Delimiter::None;
-                                text: v.text.clone(),
+        // if no var expanded in the child, we count it as a fail
-                                id: tt::TokenId::unspecified(),
+        if !ctx.var_expanded {
-                            })
+            break;
-                            .into(),
+        }
-                        ],
+
+        // Reset `ctx.var_expandeded` to see if there is other expanded variable
+        // in the next matching
+        some_var_expanded = true;
+        ctx.var_expanded = false;
+
+        counter += 1;
+        limit -= 1;
+        if limit == 0 {
+            log::warn!(
+                "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
+                template,
+                ctx
+            );
+            break;
+        }
+
+        let idx = ctx.nesting.pop().unwrap();
+        ctx.nesting.push(idx + 1);
+        push_subtree(&mut buf, t);
+
+        if let Some(ref sep) = separator {
+            match sep {
+                Separator::Ident(ident) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(ident.clone()).into());
+                }
+                Separator::Literal(lit) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(lit.clone()).into());
+                }
+
+                Separator::Puncts(puncts) => {
+                    has_seps = puncts.len();
+                    for punct in puncts {
+                        buf.push(tt::Leaf::from(*punct).into());
                     }
-                    .into()
-                } else {
-                    let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
-                    ctx.var_expanded = true;
-                    return Ok(fragment);
                 }
             }
-            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
+        }
-        },
+
-    };
+        if RepeatKind::ZeroOrOne == kind {
-    Ok(Fragment::Tokens(res))
+            break;
+        }
+    }
+
+    // Restore the `var_expanded` by combining old one and the new one
+    ctx.var_expanded = some_var_expanded || old_var_expanded;
+
+    ctx.nesting.pop().unwrap();
+    for _ in 0..has_seps {
+        buf.pop();
+    }
+
+    if RepeatKind::OneOrMore == kind && counter == 0 {
+        return Err(ExpandError::UnexpectedToken);
+    }
+
+    // Check if it is a single token subtree without any delimiter
+    // e.g {Delimiter:None> ['>'] /Delimiter:None>}
+    let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into();
+    Ok(Fragment::Tokens(tt))
 }
 
 fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {

crates/ra_mbe/src/mbe_parser.rs

@@ -1,187 +0,0 @@
-use crate::tt_cursor::TtCursor;
-/// This module parses a raw `tt::TokenStream` into macro-by-example token
-/// stream. This is a *mostly* identify function, expect for handling of
-/// `$var:tt_kind` and `$(repeat),*` constructs.
-use crate::ParseError;
-
-pub(crate) fn parse(tt: &tt::Subtree) -> Result<crate::MacroRules, ParseError> {
-    let mut parser = TtCursor::new(tt);
-    let mut rules = Vec::new();
-    while !parser.is_eof() {
-        rules.push(parse_rule(&mut parser)?);
-        if let Err(e) = parser.expect_char(';') {
-            if !parser.is_eof() {
-                return Err(e);
-            }
-            break;
-        }
-    }
-    Ok(crate::MacroRules { rules })
-}
-
-fn parse_rule(p: &mut TtCursor) -> Result<crate::Rule, ParseError> {
-    let lhs = parse_subtree(p.eat_subtree()?, false)?;
-    p.expect_char('=')?;
-    p.expect_char('>')?;
-    let mut rhs = parse_subtree(p.eat_subtree()?, true)?;
-    rhs.delimiter = crate::Delimiter::None;
-    Ok(crate::Rule { lhs, rhs })
-}
-
-fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool {
-    if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit {
-        if lit.text == "true" || lit.text == "false" {
-            return true;
-        }
-    }
-
-    false
-}
-
-fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> {
-    let mut token_trees = Vec::new();
-    let mut p = TtCursor::new(tt);
-    while let Some(tt) = p.eat() {
-        let child: crate::TokenTree = match tt {
-            tt::TokenTree::Leaf(leaf) => match leaf {
-                tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => {
-                    // mbe var can be an ident or keyword, including `true` and `false`
-                    if p.at_ident().is_some() || is_boolean_literal(p.current()) {
-                        crate::Leaf::from(parse_var(&mut p, transcriber)?).into()
-                    } else if let Some(tt::TokenTree::Subtree(_)) = p.current() {
-                        parse_repeat(&mut p, transcriber)?.into()
-                    } else {
-                        // Treat it as normal punct
-                        crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into()
-                    }
-                }
-                tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(),
-                tt::Leaf::Ident(tt::Ident { text, .. }) => {
-                    crate::Leaf::from(crate::Ident { text: text.clone() }).into()
-                }
-                tt::Leaf::Literal(tt::Literal { text }) => {
-                    crate::Leaf::from(crate::Literal { text: text.clone() }).into()
-                }
-            },
-            tt::TokenTree::Subtree(subtree) => parse_subtree(&subtree, transcriber)?.into(),
-        };
-        token_trees.push(child);
-    }
-    Ok(crate::Subtree { token_trees, delimiter: tt.delimiter })
-}
-
-fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> {
-    let text = {
-        if is_boolean_literal(p.current()) {
-            let lit = p.eat_literal().unwrap();
-            lit.text.clone()
-        } else {
-            let ident = p.eat_ident().unwrap();
-            ident.text.clone()
-        }
-    };
-
-    let kind = if !transcriber && p.at_char(':') {
-        p.bump();
-        if let Some(ident) = p.eat_ident() {
-            Some(ident.text.clone())
-        } else {
-            p.rev_bump();
-            None
-        }
-    } else {
-        None
-    };
-
-    Ok(crate::Var { text, kind })
-}
-
-fn mk_repeat(
-    rep: char,
-    subtree: crate::Subtree,
-    separator: Option<crate::Separator>,
-) -> Result<crate::Repeat, ParseError> {
-    let kind = match rep {
-        '*' => crate::RepeatKind::ZeroOrMore,
-        '+' => crate::RepeatKind::OneOrMore,
-        '?' => crate::RepeatKind::ZeroOrOne,
-        _ => return Err(ParseError::Expected(String::from("repeat"))),
-    };
-    Ok(crate::Repeat { subtree, kind, separator })
-}
-
-fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> {
-    let subtree = p.eat_subtree()?;
-    let mut subtree = parse_subtree(subtree, transcriber)?;
-    subtree.delimiter = crate::Delimiter::None;
-
-    if let Some(rep) = p.at_punct() {
-        match rep.char {
-            '*' | '+' | '?' => {
-                p.bump();
-                return mk_repeat(rep.char, subtree, None);
-            }
-            _ => {}
-        }
-    }
-
-    let sep = p.eat_seperator().ok_or_else(|| ParseError::Expected(String::from("separator")))?;
-    let rep = p.eat_punct().ok_or_else(|| ParseError::Expected(String::from("repeat")))?;
-
-    mk_repeat(rep.char, subtree, Some(sep))
-}
-
-#[cfg(test)]
-mod tests {
-    use ra_syntax::{ast, AstNode};
-
-    use super::*;
-    use crate::ast_to_token_tree;
-
-    #[test]
-    fn test_invalid_parse() {
-        expect_err("invalid", "subtree");
-
-        is_valid("($i:ident) => ()");
-        is_valid("($($i:ident)*) => ($_)");
-        is_valid("($($true:ident)*) => ($true)");
-        is_valid("($($false:ident)*) => ($false)");
-
-        expect_err("$i:ident => ()", "subtree");
-        expect_err("($i:ident) ()", "`=`");
-        expect_err("($($i:ident)_) => ()", "repeat");
-    }
-
-    fn expect_err(macro_body: &str, expected: &str) {
-        assert_eq!(
-            create_rules(&format_macro(macro_body)),
-            Err(ParseError::Expected(String::from(expected)))
-        );
-    }
-
-    fn is_valid(macro_body: &str) {
-        assert!(create_rules(&format_macro(macro_body)).is_ok());
-    }
-
-    fn format_macro(macro_body: &str) -> String {
-        format!(
-            "
-        macro_rules! foo {{
-            {}
-        }}
-",
-            macro_body
-        )
-    }
-
-    fn create_rules(macro_definition: &str) -> Result<crate::MacroRules, ParseError> {
-        let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
-        let macro_definition =
-            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
-
-        let (definition_tt, _) =
-            ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
-        parse(&definition_tt)
-    }
-
-}

crates/ra_mbe/src/parser.rs

@@ -0,0 +1,187 @@
+//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
+//! trees.
+
+use ra_syntax::SmolStr;
+use smallvec::SmallVec;
+
+use crate::{tt_iter::TtIter, ExpandError};
+
+#[derive(Debug)]
+pub(crate) enum Op<'a> {
+    Var { name: &'a SmolStr, kind: Option<&'a SmolStr> },
+    Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> },
+    TokenTree(&'a tt::TokenTree),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) enum RepeatKind {
+    ZeroOrMore,
+    OneOrMore,
+    ZeroOrOne,
+}
+
+#[derive(Clone, Debug, Eq)]
+pub(crate) enum Separator {
+    Literal(tt::Literal),
+    Ident(tt::Ident),
+    Puncts(SmallVec<[tt::Punct; 3]>),
+}
+
+// Note that when we compare a Separator, we just care about its textual value.
+impl PartialEq for Separator {
+    fn eq(&self, other: &Separator) -> bool {
+        use Separator::*;
+
+        match (self, other) {
+            (Ident(ref a), Ident(ref b)) => a.text == b.text,
+            (Literal(ref a), Literal(ref b)) => a.text == b.text,
+            (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
+                let a_iter = a.iter().map(|a| a.char);
+                let b_iter = b.iter().map(|b| b.char);
+                a_iter.eq(b_iter)
+            }
+            _ => false,
+        }
+    }
+}
+
+pub(crate) fn parse_template<'a>(
+    template: &'a tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
+    parse_inner(template, Mode::Template)
+}
+
+pub(crate) fn parse_pattern<'a>(
+    pattern: &'a tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
+    parse_inner(pattern, Mode::Pattern)
+}
+
+#[derive(Clone, Copy)]
+enum Mode {
+    Pattern,
+    Template,
+}
+
+fn parse_inner<'a>(
+    src: &'a tt::Subtree,
+    mode: Mode,
+) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
+    let mut src = TtIter::new(src);
+    std::iter::from_fn(move || {
+        let first = src.next()?;
+        Some(next_op(first, &mut src, mode))
+    })
+}
+
+macro_rules! err {
+    ($($tt:tt)*) => {
+        ExpandError::UnexpectedToken
+    };
+}
+
+macro_rules! bail {
+    ($($tt:tt)*) => {
+        return Err(err!($($tt)*))
+    };
+}
+
+fn next_op<'a>(
+    first: &'a tt::TokenTree,
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Op<'a>, ExpandError> {
+    let res = match first {
+        tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
+            let second = src.next().ok_or_else(|| err!("bad var 1"))?;
+            match second {
+                tt::TokenTree::Subtree(subtree) => {
+                    let (separator, kind) = parse_repeat(src)?;
+                    Op::Repeat { subtree, separator, kind }
+                }
+                tt::TokenTree::Leaf(leaf) => match leaf {
+                    tt::Leaf::Punct(..) => Err(ExpandError::UnexpectedToken)?,
+                    tt::Leaf::Ident(ident) => {
+                        let name = &ident.text;
+                        let kind = eat_fragment_kind(src, mode)?;
+                        Op::Var { name, kind }
+                    }
+                    tt::Leaf::Literal(lit) => {
+                        if is_boolean_literal(lit) {
+                            let name = &lit.text;
+                            let kind = eat_fragment_kind(src, mode)?;
+                            Op::Var { name, kind }
+                        } else {
+                            bail!("bad var 2");
+                        }
+                    }
+                },
+            }
+        }
+        tt => Op::TokenTree(tt),
+    };
+    Ok(res)
+}
+
+fn eat_fragment_kind<'a>(
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Option<&'a SmolStr>, ExpandError> {
+    if let Mode::Pattern = mode {
+        src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
+        let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
+        return Ok(Some(&ident.text));
+    };
+    Ok(None)
+}
+
+fn is_boolean_literal(lit: &tt::Literal) -> bool {
+    match lit.text.as_str() {
+        "true" | "false" => true,
+        _ => false,
+    }
+}
+
+///TOOD: impl for slice iter
+fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
+    let mut separator = Separator::Puncts(SmallVec::new());
+    for tt in src {
+        let tt = match tt {
+            tt::TokenTree::Leaf(leaf) => leaf,
+            tt::TokenTree::Subtree(_) => Err(ExpandError::InvalidRepeat)?,
+        };
+        let has_sep = match &separator {
+            Separator::Puncts(puncts) => puncts.len() != 0,
+            _ => true,
+        };
+        match tt {
+            tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
+                Err(ExpandError::InvalidRepeat)?
+            }
+            tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
+            tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
+            tt::Leaf::Punct(punct) => {
+                let repeat_kind = match punct.char {
+                    '*' => RepeatKind::ZeroOrMore,
+                    '+' => RepeatKind::OneOrMore,
+                    '?' => RepeatKind::ZeroOrOne,
+                    _ => {
+                        match &mut separator {
+                            Separator::Puncts(puncts) => {
+                                if puncts.len() == 3 {
+                                    Err(ExpandError::InvalidRepeat)?
+                                }
+                                puncts.push(punct.clone())
+                            }
+                            _ => Err(ExpandError::InvalidRepeat)?,
+                        }
+                        continue;
+                    }
+                };
+                let separator = if has_sep { Some(separator) } else { None };
+                return Ok((separator, repeat_kind));
+            }
+        }
+    }
+    Err(ExpandError::InvalidRepeat)
+}

crates/ra_mbe/src/subtree_parser.rs

@@ -1,91 +0,0 @@
-use crate::subtree_source::SubtreeTokenSource;
-
-use ra_parser::{FragmentKind, TokenSource, TreeSink};
-use ra_syntax::SyntaxKind;
-use tt::buffer::{Cursor, TokenBuffer};
-
-struct OffsetTokenSink<'a> {
-    cursor: Cursor<'a>,
-    error: bool,
-}
-
-impl<'a> OffsetTokenSink<'a> {
-    pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> {
-        if !self.cursor.is_root() {
-            return vec![];
-        }
-
-        let mut curr = begin;
-        let mut res = vec![];
-
-        while self.cursor != curr {
-            if let Some(token) = curr.token_tree() {
-                res.push(token);
-            }
-            curr = curr.bump();
-        }
-
-        res
-    }
-}
-
-impl<'a> TreeSink for OffsetTokenSink<'a> {
-    fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
-        for _ in 0..n_tokens {
-            self.cursor = self.cursor.bump_subtree();
-        }
-    }
-    fn start_node(&mut self, _kind: SyntaxKind) {}
-    fn finish_node(&mut self) {}
-    fn error(&mut self, _error: ra_parser::ParseError) {
-        self.error = true;
-    }
-}
-
-pub(crate) struct Parser<'a> {
-    subtree: &'a tt::Subtree,
-    cur_pos: &'a mut usize,
-}
-
-impl<'a> Parser<'a> {
-    pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> {
-        Parser { cur_pos, subtree }
-    }
-
-    pub fn parse_fragment(self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
-        self.parse(|token_source, tree_skink| {
-            ra_parser::parse_fragment(token_source, tree_skink, fragment_kind)
-        })
-    }
-
-    fn parse<F>(self, f: F) -> Option<tt::TokenTree>
-    where
-        F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
-    {
-        let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
-        let mut src = SubtreeTokenSource::new(&buffer);
-        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
-
-        f(&mut src, &mut sink);
-
-        let r = self.finish(buffer.begin(), &mut sink);
-        if sink.error {
-            return None;
-        }
-        r
-    }
-
-    fn finish(self, begin: Cursor, sink: &mut OffsetTokenSink) -> Option<tt::TokenTree> {
-        let res = sink.collect(begin);
-        *self.cur_pos += res.len();
-
-        match res.len() {
-            0 => None,
-            1 => Some(res[0].clone()),
-            _ => Some(tt::TokenTree::Subtree(tt::Subtree {
-                delimiter: tt::Delimiter::None,
-                token_trees: res.into_iter().cloned().collect(),
-            })),
-        }
-    }
-}

crates/ra_mbe/src/tests.rs

@@ -3,6 +3,54 @@ use test_utils::assert_eq_text;
 
 use super::*;
 
+mod rule_parsing {
+    use ra_syntax::{ast, AstNode};
+
+    use super::*;
+    use crate::ast_to_token_tree;
+
+    #[test]
+    fn test_valid_arms() {
+        fn check(macro_body: &str) {
+            let m = parse_macro_arm(macro_body);
+            m.unwrap();
+        }
+
+        check("($i:ident) => ()");
+        check("($($i:ident)*) => ($_)");
+        check("($($true:ident)*) => ($true)");
+        check("($($false:ident)*) => ($false)");
+    }
+
+    #[test]
+    fn test_invalid_arms() {
+        fn check(macro_body: &str, err: &str) {
+            let m = parse_macro_arm(macro_body);
+            assert_eq!(m, Err(ParseError::Expected(String::from(err))));
+        }
+
+        check("invalid", "expected subtree");
+
+        check("$i:ident => ()", "expected subtree");
+        check("($i:ident) ()", "expected `=`");
+        check("($($i:ident)_) => ()", "invalid repeat");
+
+        check("($i) => ($i)", "invalid macro definition");
+        check("($i:) => ($i)", "invalid macro definition");
+    }
+
+    fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {
+        let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition);
+        let source_file = ast::SourceFile::parse(&macro_definition).ok().unwrap();
+        let macro_definition =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (definition_tt, _) =
+            ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+        crate::MacroRules::parse(&definition_tt)
+    }
+}
+
 // Good first issue (although a slightly challenging one):
 //
 // * Pick a random test from here

crates/ra_mbe/src/tt_cursor.rs

@@ -1,281 +0,0 @@
-use crate::{subtree_parser::Parser, ParseError};
-
-use ra_parser::FragmentKind;
-use smallvec::{smallvec, SmallVec};
-
-#[derive(Debug, Clone)]
-pub(crate) struct TtCursor<'a> {
-    subtree: &'a tt::Subtree,
-    pos: usize,
-}
-
-pub(crate) struct TtCursorMemento {
-    pos: usize,
-}
-
-impl<'a> TtCursor<'a> {
-    pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> {
-        TtCursor { subtree, pos: 0 }
-    }
-
-    pub(crate) fn is_eof(&self) -> bool {
-        self.pos == self.subtree.token_trees.len()
-    }
-
-    pub(crate) fn current(&self) -> Option<&'a tt::TokenTree> {
-        self.subtree.token_trees.get(self.pos)
-    }
-
-    pub(crate) fn at_punct(&self) -> Option<&'a tt::Punct> {
-        match self.current() {
-            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(it))) => Some(it),
-            _ => None,
-        }
-    }
-
-    pub(crate) fn at_char(&self, char: char) -> bool {
-        match self.at_punct() {
-            Some(tt::Punct { char: c, .. }) if *c == char => true,
-            _ => false,
-        }
-    }
-
-    pub(crate) fn at_ident(&mut self) -> Option<&'a tt::Ident> {
-        match self.current() {
-            Some(tt::TokenTree::Leaf(tt::Leaf::Ident(i))) => Some(i),
-            _ => None,
-        }
-    }
-
-    pub(crate) fn at_literal(&mut self) -> Option<&'a tt::Literal> {
-        match self.current() {
-            Some(tt::TokenTree::Leaf(tt::Leaf::Literal(i))) => Some(i),
-            _ => None,
-        }
-    }
-
-    pub(crate) fn bump(&mut self) {
-        self.pos += 1;
-    }
-    pub(crate) fn rev_bump(&mut self) {
-        self.pos -= 1;
-    }
-
-    pub(crate) fn eat(&mut self) -> Option<&'a tt::TokenTree> {
-        self.current().map(|it| {
-            self.bump();
-            it
-        })
-    }
-
-    pub(crate) fn eat_subtree(&mut self) -> Result<&'a tt::Subtree, ParseError> {
-        match self.current() {
-            Some(tt::TokenTree::Subtree(sub)) => {
-                self.bump();
-                Ok(sub)
-            }
-            _ => Err(ParseError::Expected(String::from("subtree"))),
-        }
-    }
-
-    pub(crate) fn eat_punct(&mut self) -> Option<&'a tt::Punct> {
-        self.at_punct().map(|it| {
-            self.bump();
-            it
-        })
-    }
-
-    pub(crate) fn eat_ident(&mut self) -> Option<&'a tt::Ident> {
-        self.at_ident().map(|i| {
-            self.bump();
-            i
-        })
-    }
-
-    pub(crate) fn eat_literal(&mut self) -> Option<&'a tt::Literal> {
-        self.at_literal().map(|i| {
-            self.bump();
-            i
-        })
-    }
-
-    pub(crate) fn eat_fragment(&mut self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
-        let parser = Parser::new(&mut self.pos, self.subtree);
-        parser.parse_fragment(fragment_kind)
-    }
-
-    pub(crate) fn eat_lifetime(&mut self) -> Option<tt::TokenTree> {
-        // check if it start from "`"
-        if let Some(ident) = self.at_ident() {
-            if ident.text.chars().next()? != '\'' {
-                return None;
-            }
-        }
-
-        self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into())
-    }
-
-    pub(crate) fn try_eat_vis(&mut self) -> Option<tt::TokenTree> {
-        // `vis` matcher is optional
-        let old_pos = self.pos;
-        let parser = Parser::new(&mut self.pos, self.subtree);
-
-        let res = parser.parse_fragment(FragmentKind::Visibility);
-        if res.is_none() {
-            self.pos = old_pos;
-        }
-        res
-    }
-
-    pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {
-        if self.at_char(char) {
-            self.bump();
-            Ok(())
-        } else {
-            Err(ParseError::Expected(format!("`{}`", char)))
-        }
-    }
-
-    fn eat_punct3(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
-        let sec = *self.eat_punct()?;
-        let third = *self.eat_punct()?;
-        Some(smallvec![p, sec, third])
-    }
-
-    fn eat_punct2(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
-        let sec = *self.eat_punct()?;
-        Some(smallvec![p, sec])
-    }
-
-    fn eat_multi_char_punct<'b, I>(
-        &mut self,
-        p: tt::Punct,
-        iter: &mut TokenPeek<'b, I>,
-    ) -> Option<SmallVec<[tt::Punct; 3]>>
-    where
-        I: Iterator<Item = &'b tt::TokenTree>,
-    {
-        if let Some((m, _)) = iter.current_punct3(p) {
-            if let r @ Some(_) = match m {
-                ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => {
-                    self.eat_punct3(p)
-                }
-                _ => None,
-            } {
-                return r;
-            }
-        }
-
-        if let Some((m, _)) = iter.current_punct2(p) {
-            if let r @ Some(_) = match m {
-                ('<', '=')
-                | ('>', '=')
-                | ('+', '=')
-                | ('-', '=')
-                | ('|', '=')
-                | ('&', '=')
-                | ('^', '=')
-                | ('/', '=')
-                | ('*', '=')
-                | ('%', '=')
-                | ('&', '&')
-                | ('|', '|')
-                | ('<', '<')
-                | ('>', '>')
-                | ('-', '>')
-                | ('!', '=')
-                | ('=', '>')
-                | ('=', '=')
-                | ('.', '.')
-                | (':', ':') => self.eat_punct2(p),
-
-                _ => None,
-            } {
-                return r;
-            }
-        }
-
-        None
-    }
-
-    pub(crate) fn eat_seperator(&mut self) -> Option<crate::Separator> {
-        match self.eat()? {
-            tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
-                Some(crate::Separator::Literal(lit.clone()))
-            }
-            tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
-                Some(crate::Separator::Ident(ident.clone()))
-            }
-            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
-                match punct.char {
-                    '*' | '+' | '?' => return None,
-                    _ => {}
-                };
-
-                // FIXME: The parser is only handle some compositeable punct,
-                // But at this phase, some punct still is jointed.
-                // So we by pass that check here.
-                let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter());
-                let puncts = self.eat_multi_char_punct(*punct, &mut peekable);
-                let puncts = puncts.unwrap_or_else(|| smallvec![*punct]);
-
-                Some(crate::Separator::Puncts(puncts))
-            }
-            _ => None,
-        }
-    }
-
-    #[must_use]
-    pub(crate) fn save(&self) -> TtCursorMemento {
-        TtCursorMemento { pos: self.pos }
-    }
-
-    pub(crate) fn rollback(&mut self, memento: TtCursorMemento) {
-        self.pos = memento.pos;
-    }
-}
-
-pub(crate) struct TokenPeek<'a, I>
-where
-    I: Iterator<Item = &'a tt::TokenTree>,
-{
-    iter: itertools::MultiPeek<I>,
-}
-
-// helper function
-fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
-    if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
-        return Some(pp);
-    }
-    None
-}
-
-impl<'a, I> TokenPeek<'a, I>
-where
-    I: Iterator<Item = &'a tt::TokenTree>,
-{
-    pub fn new(iter: I) -> Self {
-        TokenPeek { iter: itertools::multipeek(iter) }
-    }
-
-    pub fn current_punct2(&mut self, p: tt::Punct) -> Option<((char, char), bool)> {
-        if p.spacing != tt::Spacing::Joint {
-            return None;
-        }
-
-        self.iter.reset_peek();
-        let p1 = to_punct(self.iter.peek()?)?;
-        Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
-    }
-
-    pub fn current_punct3(&mut self, p: tt::Punct) -> Option<((char, char, char), bool)> {
-        self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
-            if !last_joint {
-                None
-            } else {
-                let p2 = to_punct(*self.iter.peek()?)?;
-                Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
-            }
-        })
-    }
-}

crates/ra_mbe/src/tt_iter.rs

@@ -0,0 +1,67 @@
+#[derive(Debug, Clone)]
+pub(crate) struct TtIter<'a> {
+    pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
+}
+
+impl<'a> TtIter<'a> {
+    pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
+        TtIter { inner: subtree.token_trees.iter() }
+    }
+
+    pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => {
+                Ok(())
+            }
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Subtree(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Ident(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Literal(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Punct(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+}
+
+impl<'a> Iterator for TtIter<'a> {
+    type Item = &'a tt::TokenTree;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {}