Infect mbe crate with generic span type parameter

crates/mbe/src/benchmark.rs

@@ -6,10 +6,11 @@ use syntax::{
     AstNode, SmolStr,
 };
 use test_utils::{bench, bench_fixture, skip_slow_tests};
+use tt::{Span, TokenId};
 
 use crate::{
     parser::{MetaVarKind, Op, RepeatKind, Separator},
-    syntax_node_to_token_tree, tt, DeclarativeMacro,
+    syntax_node_to_token_tree, DeclarativeMacro,
 };
 
 #[test]
@@ -54,7 +55,7 @@ fn macro_rules_fixtures() -> FxHashMap<String, DeclarativeMacro> {
         .collect()
 }
 
-fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
+fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree<TokenId>> {
     let fixture = bench_fixture::numerous_macro_rules();
     let source_file = ast::SourceFile::parse(&fixture).ok().unwrap();
 
@@ -71,7 +72,9 @@ fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
 }
 
 /// Generate random invocation fixtures from rules
-fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(String, tt::Subtree)> {
+fn invocation_fixtures(
+    rules: &FxHashMap<String, DeclarativeMacro>,
+) -> Vec<(String, tt::Subtree<TokenId>)> {
     let mut seed = 123456789;
     let mut res = Vec::new();
 
@@ -93,8 +96,8 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
                 loop {
                     let mut subtree = tt::Subtree {
                         delimiter: tt::Delimiter {
-                            open: tt::TokenId::UNSPECIFIED,
+                            open: tt::TokenId::DUMMY,
-                            close: tt::TokenId::UNSPECIFIED,
+                            close: tt::TokenId::DUMMY,
                             kind: tt::DelimiterKind::Invisible,
                         },
                         token_trees: vec![],
@@ -116,7 +119,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
     }
     return res;
 
-    fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) {
+    fn collect_from_op(op: &Op<TokenId>, parent: &mut tt::Subtree<TokenId>, seed: &mut usize) {
         return match op {
             Op::Var { kind, .. } => match kind.as_ref() {
                 Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")),
@@ -202,36 +205,30 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
             *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c);
             *seed
         }
-        fn make_ident(ident: &str) -> tt::TokenTree {
+        fn make_ident(ident: &str) -> tt::TokenTree<TokenId> {
-            tt::Leaf::Ident(tt::Ident {
+            tt::Leaf::Ident(tt::Ident { span: tt::TokenId::DUMMY, text: SmolStr::new(ident) })
-                span: tt::TokenId::unspecified(),
+                .into()
-                text: SmolStr::new(ident),
-            })
-            .into()
         }
-        fn make_punct(char: char) -> tt::TokenTree {
+        fn make_punct(char: char) -> tt::TokenTree<TokenId> {
             tt::Leaf::Punct(tt::Punct {
-                span: tt::TokenId::unspecified(),
+                span: tt::TokenId::DUMMY,
                 char,
                 spacing: tt::Spacing::Alone,
             })
             .into()
         }
-        fn make_literal(lit: &str) -> tt::TokenTree {
+        fn make_literal(lit: &str) -> tt::TokenTree<TokenId> {
-            tt::Leaf::Literal(tt::Literal {
+            tt::Leaf::Literal(tt::Literal { span: tt::TokenId::DUMMY, text: SmolStr::new(lit) })
-                span: tt::TokenId::unspecified(),
+                .into()
-                text: SmolStr::new(lit),
-            })
-            .into()
         }
         fn make_subtree(
             kind: tt::DelimiterKind,
-            token_trees: Option<Vec<tt::TokenTree>>,
+            token_trees: Option<Vec<tt::TokenTree<TokenId>>>,
-        ) -> tt::TokenTree {
+        ) -> tt::TokenTree<TokenId> {
             tt::Subtree {
                 delimiter: tt::Delimiter {
-                    open: tt::TokenId::unspecified(),
+                    open: tt::TokenId::DUMMY,
-                    close: tt::TokenId::unspecified(),
+                    close: tt::TokenId::DUMMY,
                     kind,
                 },
                 token_trees: token_trees.unwrap_or_default(),

crates/mbe/src/expander.rs

@@ -7,15 +7,16 @@ mod transcriber;
 
 use rustc_hash::FxHashMap;
 use syntax::SmolStr;
+use tt::Span;
 
-use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult};
+use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
 
-pub(crate) fn expand_rules(
+pub(crate) fn expand_rules<S: Span>(
-    rules: &[crate::Rule],
+    rules: &[crate::Rule<S>],
-    input: &tt::Subtree,
+    input: &tt::Subtree<S>,
     is_2021: bool,
-) -> ExpandResult<tt::Subtree> {
+) -> ExpandResult<tt::Subtree<S>> {
-    let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
+    let mut match_: Option<(matcher::Match<S>, &crate::Rule<S>)> = None;
     for rule in rules {
         let new_match = matcher::match_(&rule.lhs, input, is_2021);
 
@@ -47,7 +48,7 @@ pub(crate) fn expand_rules(
         ExpandResult { value, err: match_.err.or(transcribe_err) }
     } else {
         ExpandResult::new(
-            tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
+            tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] },
             ExpandError::NoMatchingRule,
         )
     }
@@ -98,23 +99,29 @@ pub(crate) fn expand_rules(
 /// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
 /// `tt::TokenTree`, where the index to select a particular `TokenTree` among
 /// many is not a plain `usize`, but a `&[usize]`.
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
-struct Bindings {
+struct Bindings<S> {
-    inner: FxHashMap<SmolStr, Binding>,
+    inner: FxHashMap<SmolStr, Binding<S>>,
+}
+
+impl<S> Default for Bindings<S> {
+    fn default() -> Self {
+        Self { inner: Default::default() }
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-enum Binding {
+enum Binding<S> {
-    Fragment(Fragment),
+    Fragment(Fragment<S>),
-    Nested(Vec<Binding>),
+    Nested(Vec<Binding<S>>),
     Empty,
     Missing(MetaVarKind),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-enum Fragment {
+enum Fragment<S> {
     /// token fragments are just copy-pasted into the output
-    Tokens(tt::TokenTree),
+    Tokens(tt::TokenTree<S>),
     /// Expr ast fragments are surrounded with `()` on insertion to preserve
     /// precedence. Note that this impl is different from the one currently in
     /// `rustc` -- `rustc` doesn't translate fragments into token trees at all.
@@ -122,7 +129,7 @@ enum Fragment {
     /// At one point in time, we tried to use "fake" delimiters here à la
     /// proc-macro delimiter=none. As we later discovered, "none" delimiters are
     /// tricky to handle in the parser, and rustc doesn't handle those either.
-    Expr(tt::TokenTree),
+    Expr(tt::TokenTree<S>),
     /// There are roughly two types of paths: paths in expression context, where a
     /// separator `::` between an identifier and its following generic argument list
     /// is mandatory, and paths in type context, where `::` can be omitted.
@@ -132,5 +139,5 @@ enum Fragment {
     /// and is trasncribed as an expression-context path, verbatim transcription
     /// would cause a syntax error. We need to fix it up just before transcribing;
     /// see `transcriber::fix_up_and_push_path_tt()`.
-    Path(tt::TokenTree),
+    Path(tt::TokenTree<S>),
 }

crates/mbe/src/expander/matcher.rs

@@ -63,21 +63,20 @@ use std::rc::Rc;
 
 use smallvec::{smallvec, SmallVec};
 use syntax::SmolStr;
+use tt::Span;
 
 use crate::{
     expander::{Binding, Bindings, ExpandResult, Fragment},
     parser::{MetaVarKind, Op, RepeatKind, Separator},
-    tt,
     tt_iter::TtIter,
     ExpandError, MetaTemplate, ValueResult,
 };
 
-impl Bindings {
+impl<S: Span> Bindings<S> {
     fn push_optional(&mut self, name: &SmolStr) {
         // FIXME: Do we have a better way to represent an empty token ?
         // Insert an empty subtree for empty token
-        let tt =
+        let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![] }.into();
-            tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into();
         self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
     }
 
@@ -85,14 +84,14 @@ impl Bindings {
         self.inner.insert(name.clone(), Binding::Empty);
     }
 
-    fn bindings(&self) -> impl Iterator<Item = &Binding> {
+    fn bindings(&self) -> impl Iterator<Item = &Binding<S>> {
         self.inner.values()
     }
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
-pub(super) struct Match {
+pub(super) struct Match<S> {
-    pub(super) bindings: Bindings,
+    pub(super) bindings: Bindings<S>,
     /// We currently just keep the first error and count the rest to compare matches.
     pub(super) err: Option<ExpandError>,
     pub(super) err_count: usize,
@@ -102,7 +101,19 @@ pub(super) struct Match {
     pub(super) bound_count: usize,
 }
 
-impl Match {
+impl<S> Default for Match<S> {
+    fn default() -> Self {
+        Self {
+            bindings: Default::default(),
+            err: Default::default(),
+            err_count: Default::default(),
+            unmatched_tts: Default::default(),
+            bound_count: Default::default(),
+        }
+    }
+}
+
+impl<S> Match<S> {
     fn add_err(&mut self, err: ExpandError) {
         let prev_err = self.err.take();
         self.err = prev_err.or(Some(err));
@@ -111,12 +122,16 @@ impl Match {
 }
 
 /// Matching errors are added to the `Match`.
-pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree, is_2021: bool) -> Match {
+pub(super) fn match_<S: Span>(
+    pattern: &MetaTemplate<S>,
+    input: &tt::Subtree<S>,
+    is_2021: bool,
+) -> Match<S> {
     let mut res = match_loop(pattern, input, is_2021);
     res.bound_count = count(res.bindings.bindings());
     return res;
 
-    fn count<'a>(bindings: impl Iterator<Item = &'a Binding>) -> usize {
+    fn count<'a, S: 'a>(bindings: impl Iterator<Item = &'a Binding<S>>) -> usize {
         bindings
             .map(|it| match it {
                 Binding::Fragment(_) => 1,
@@ -129,10 +144,10 @@ pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree, is_2021: bool)
 }
 
 #[derive(Debug, Clone)]
-enum BindingKind {
+enum BindingKind<S> {
     Empty(SmolStr),
     Optional(SmolStr),
-    Fragment(SmolStr, Fragment),
+    Fragment(SmolStr, Fragment<S>),
     Missing(SmolStr, MetaVarKind),
     Nested(usize, usize),
 }
@@ -146,13 +161,18 @@ enum LinkNode<T> {
     Parent { idx: usize, len: usize },
 }
 
-#[derive(Default)]
+struct BindingsBuilder<S> {
-struct BindingsBuilder {
+    nodes: Vec<Vec<LinkNode<Rc<BindingKind<S>>>>>,
-    nodes: Vec<Vec<LinkNode<Rc<BindingKind>>>>,
     nested: Vec<Vec<LinkNode<usize>>>,
 }
 
-impl BindingsBuilder {
+impl<S> Default for BindingsBuilder<S> {
+    fn default() -> Self {
+        Self { nodes: Default::default(), nested: Default::default() }
+    }
+}
+
+impl<S: Span> BindingsBuilder<S> {
     fn alloc(&mut self) -> BindingsIdx {
         let idx = self.nodes.len();
         self.nodes.push(Vec::new());
@@ -189,7 +209,7 @@ impl BindingsBuilder {
         self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone()))));
     }
 
-    fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) {
+    fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment<S>) {
         self.nodes[idx.0]
             .push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment))));
     }
@@ -210,11 +230,11 @@ impl BindingsBuilder {
         idx.0 = new_idx;
     }
 
-    fn build(self, idx: &BindingsIdx) -> Bindings {
+    fn build(self, idx: &BindingsIdx) -> Bindings<S> {
         self.build_inner(&self.nodes[idx.0])
     }
 
-    fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind>>]) -> Bindings {
+    fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind<S>>>]) -> Bindings<S> {
         let mut bindings = Bindings::default();
         let mut nodes = Vec::new();
         self.collect_nodes(link_nodes, &mut nodes);
@@ -264,7 +284,7 @@ impl BindingsBuilder {
         &'a self,
         id: usize,
         len: usize,
-        nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind>>]>,
+        nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind<S>>>]>,
     ) {
         self.nested[id].iter().take(len).for_each(|it| match it {
             LinkNode::Node(id) => nested_refs.push(&self.nodes[*id]),
@@ -272,7 +292,7 @@ impl BindingsBuilder {
         });
     }
 
-    fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings>) {
+    fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings<S>>) {
         let last = &self.nodes[idx];
         let mut nested_refs: Vec<&[_]> = Vec::new();
         self.nested[nested_idx].iter().for_each(|it| match *it {
@@ -283,7 +303,7 @@ impl BindingsBuilder {
         nested.extend(nested_refs.into_iter().map(|iter| self.build_inner(iter)));
     }
 
-    fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind>) {
+    fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind<S>>) {
         self.nodes[id].iter().take(len).for_each(|it| match it {
             LinkNode::Node(it) => nodes.push(it),
             LinkNode::Parent { idx, len } => self.collect_nodes_ref(*idx, *len, nodes),
@@ -292,8 +312,8 @@ impl BindingsBuilder {
 
     fn collect_nodes<'a>(
         &'a self,
-        link_nodes: &'a [LinkNode<Rc<BindingKind>>],
+        link_nodes: &'a [LinkNode<Rc<BindingKind<S>>>],
-        nodes: &mut Vec<&'a BindingKind>,
+        nodes: &mut Vec<&'a BindingKind<S>>,
     ) {
         link_nodes.iter().for_each(|it| match it {
             LinkNode::Node(it) => nodes.push(it),
@@ -303,22 +323,22 @@ impl BindingsBuilder {
 }
 
 #[derive(Debug, Clone)]
-struct MatchState<'t> {
+struct MatchState<'t, S> {
     /// The position of the "dot" in this matcher
-    dot: OpDelimitedIter<'t>,
+    dot: OpDelimitedIter<'t, S>,
 
     /// Token subtree stack
     /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
     /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
     /// that where the bottom of the stack is the outermost matcher.
-    stack: SmallVec<[OpDelimitedIter<'t>; 4]>,
+    stack: SmallVec<[OpDelimitedIter<'t, S>; 4]>,
 
     /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
     /// before we enter the repetition.
-    up: Option<Box<MatchState<'t>>>,
+    up: Option<Box<MatchState<'t, S>>>,
 
     /// The separator if we are in a repetition.
-    sep: Option<Separator>,
+    sep: Option<Separator<S>>,
 
     /// The KleeneOp of this sequence if we are in a repetition.
     sep_kind: Option<RepeatKind>,
@@ -330,7 +350,7 @@ struct MatchState<'t> {
     bindings: BindingsIdx,
 
     /// Cached result of meta variable parsing
-    meta_result: Option<(TtIter<'t>, ExpandResult<Option<Fragment>>)>,
+    meta_result: Option<(TtIter<'t, S>, ExpandResult<Option<Fragment<S>>>)>,
 
     /// Is error occurred in this state, will `poised` to "parent"
     is_error: bool,
@@ -355,16 +375,16 @@ struct MatchState<'t> {
 /// - `bb_items`: the set of items that are waiting for the black-box parser.
 /// - `error_items`: the set of items in errors, used for error-resilient parsing
 #[inline]
-fn match_loop_inner<'t>(
+fn match_loop_inner<'t, S: Span>(
-    src: TtIter<'t>,
+    src: TtIter<'t, S>,
-    stack: &[TtIter<'t>],
+    stack: &[TtIter<'t, S>],
-    res: &mut Match,
+    res: &mut Match<S>,
-    bindings_builder: &mut BindingsBuilder,
+    bindings_builder: &mut BindingsBuilder<S>,
-    cur_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    cur_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
-    bb_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    bb_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
-    next_items: &mut Vec<MatchState<'t>>,
+    next_items: &mut Vec<MatchState<'t, S>>,
-    eof_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    eof_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
-    error_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    error_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
     is_2021: bool,
 ) {
     macro_rules! try_push {
@@ -468,7 +488,7 @@ fn match_loop_inner<'t>(
                 if let Ok(subtree) = src.clone().expect_subtree() {
                     if subtree.delimiter.kind == delimiter.kind {
                         item.stack.push(item.dot);
-                        item.dot = tokens.iter_delimited(Some(delimiter));
+                        item.dot = tokens.iter_delimited(Some(*delimiter));
                         cur_items.push(item);
                     }
                 }
@@ -587,9 +607,9 @@ fn match_loop_inner<'t>(
     }
 }
 
-fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match {
+fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021: bool) -> Match<S> {
     let mut src = TtIter::new(src);
-    let mut stack: SmallVec<[TtIter<'_>; 1]> = SmallVec::new();
+    let mut stack: SmallVec<[TtIter<'_, S>; 1]> = SmallVec::new();
     let mut res = Match::default();
     let mut error_recover_item = None;
 
@@ -736,11 +756,11 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree, is_2021: bool) -> Match
     }
 }
 
-fn match_meta_var(
+fn match_meta_var<S: Span>(
     kind: MetaVarKind,
-    input: &mut TtIter<'_>,
+    input: &mut TtIter<'_, S>,
     is_2021: bool,
-) -> ExpandResult<Option<Fragment>> {
+) -> ExpandResult<Option<Fragment<S>>> {
     let fragment = match kind {
         MetaVarKind::Path => {
             return input
@@ -811,7 +831,7 @@ fn match_meta_var(
     input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens))
 }
 
-fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) {
+fn collect_vars<S: Span>(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate<S>) {
     for op in pattern.iter() {
         match op {
             Op::Var { name, .. } => collector_fun(name.clone()),
@@ -824,38 +844,38 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
         }
     }
 }
-impl MetaTemplate {
+impl<S: Span> MetaTemplate<S> {
-    fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> {
+    fn iter_delimited(&self, delimited: Option<tt::Delimiter<S>>) -> OpDelimitedIter<'_, S> {
         OpDelimitedIter {
             inner: &self.0,
             idx: 0,
-            delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED),
+            delimited: delimited.unwrap_or(tt::Delimiter::UNSPECIFIED),
         }
     }
 }
 
 #[derive(Debug, Clone, Copy)]
-enum OpDelimited<'a> {
+enum OpDelimited<'a, S> {
-    Op(&'a Op),
+    Op(&'a Op<S>),
     Open,
     Close,
 }
 
 #[derive(Debug, Clone, Copy)]
-struct OpDelimitedIter<'a> {
+struct OpDelimitedIter<'a, S> {
-    inner: &'a [Op],
+    inner: &'a [Op<S>],
-    delimited: &'a tt::Delimiter,
+    delimited: tt::Delimiter<S>,
     idx: usize,
 }
 
-impl<'a> OpDelimitedIter<'a> {
+impl<'a, S: Span> OpDelimitedIter<'a, S> {
     fn is_eof(&self) -> bool {
         let len = self.inner.len()
             + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
         self.idx >= len
     }
 
-    fn peek(&self) -> Option<OpDelimited<'a>> {
+    fn peek(&self) -> Option<OpDelimited<'a, S>> {
         match self.delimited.kind {
             tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
             _ => match self.idx {
@@ -871,8 +891,8 @@ impl<'a> OpDelimitedIter<'a> {
     }
 }
 
-impl<'a> Iterator for OpDelimitedIter<'a> {
+impl<'a, S: Span> Iterator for OpDelimitedIter<'a, S> {
-    type Item = OpDelimited<'a>;
+    type Item = OpDelimited<'a, S>;
 
     fn next(&mut self) -> Option<Self::Item> {
         let res = self.peek();
@@ -888,8 +908,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> {
     }
 }
 
-impl TtIter<'_> {
+impl<S: Span> TtIter<'_, S> {
-    fn expect_separator(&mut self, separator: &Separator) -> bool {
+    fn expect_separator(&mut self, separator: &Separator<S>) -> bool {
         let mut fork = self.clone();
         let ok = match separator {
             Separator::Ident(lhs) => match fork.expect_ident_or_underscore() {
@@ -919,7 +939,7 @@ impl TtIter<'_> {
         ok
     }
 
-    fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+    fn expect_tt(&mut self) -> Result<tt::TokenTree<S>, ()> {
         if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) {
             if punct.char == '\'' {
                 self.expect_lifetime()
@@ -936,7 +956,7 @@ impl TtIter<'_> {
         }
     }
 
-    fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+    fn expect_lifetime(&mut self) -> Result<tt::TokenTree<S>, ()> {
         let punct = self.expect_single_punct()?;
         if punct.char != '\'' {
             return Err(());
@@ -953,7 +973,7 @@ impl TtIter<'_> {
         .into())
     }
 
-    fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
+    fn eat_char(&mut self, c: char) -> Option<tt::TokenTree<S>> {
         let mut fork = self.clone();
         match fork.expect_char(c) {
             Ok(_) => {

crates/mbe/src/expander/transcriber.rs

@@ -2,20 +2,20 @@
 //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
 
 use syntax::SmolStr;
+use tt::{Delimiter, Span};
 
 use crate::{
     expander::{Binding, Bindings, Fragment},
     parser::{MetaVarKind, Op, RepeatKind, Separator},
-    tt::{self, Delimiter},
     CountError, ExpandError, ExpandResult, MetaTemplate,
 };
 
-impl Bindings {
+impl<S: Span> Bindings<S> {
     fn contains(&self, name: &str) -> bool {
         self.inner.contains_key(name)
     }
 
-    fn get(&self, name: &str) -> Result<&Binding, ExpandError> {
+    fn get(&self, name: &str) -> Result<&Binding<S>, ExpandError> {
         match self.inner.get(name) {
             Some(binding) => Ok(binding),
             None => Err(ExpandError::binding_error(format!("could not find binding `{name}`"))),
@@ -26,7 +26,7 @@ impl Bindings {
         &self,
         name: &str,
         nesting: &mut [NestingState],
-    ) -> Result<Fragment, ExpandError> {
+    ) -> Result<Fragment<S>, ExpandError> {
         macro_rules! binding_err {
             ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) };
         }
@@ -54,15 +54,15 @@ impl Bindings {
             Binding::Missing(it) => Ok(match it {
                 MetaVarKind::Stmt => {
                     Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
-                        span: tt::TokenId::unspecified(),
+                        span: S::DUMMY,
                         char: ';',
                         spacing: tt::Spacing::Alone,
                     })))
                 }
                 MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
                     delimiter: tt::Delimiter {
-                        open: tt::TokenId::unspecified(),
+                        open: S::DUMMY,
-                        close: tt::TokenId::unspecified(),
+                        close: S::DUMMY,
                         kind: tt::DelimiterKind::Brace,
                     },
                     token_trees: vec![],
@@ -82,19 +82,19 @@ impl Bindings {
                 | MetaVarKind::Ident => {
                     Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                         text: SmolStr::new_inline("missing"),
-                        span: tt::TokenId::unspecified(),
+                        span: S::DUMMY,
                     })))
                 }
                 MetaVarKind::Lifetime => {
                     Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                         text: SmolStr::new_inline("'missing"),
-                        span: tt::TokenId::unspecified(),
+                        span: S::DUMMY,
                     })))
                 }
                 MetaVarKind::Literal => {
                     Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                         text: SmolStr::new_inline("\"missing\""),
-                        span: tt::TokenId::unspecified(),
+                        span: S::DUMMY,
                     })))
                 }
             }),
@@ -108,12 +108,12 @@ impl Bindings {
     }
 }
 
-pub(super) fn transcribe(
+pub(super) fn transcribe<S: Span>(
-    template: &MetaTemplate,
+    template: &MetaTemplate<S>,
-    bindings: &Bindings,
+    bindings: &Bindings<S>,
-) -> ExpandResult<tt::Subtree> {
+) -> ExpandResult<tt::Subtree<S>> {
     let mut ctx = ExpandCtx { bindings, nesting: Vec::new() };
-    let mut arena: Vec<tt::TokenTree> = Vec::new();
+    let mut arena: Vec<tt::TokenTree<S>> = Vec::new();
     expand_subtree(&mut ctx, template, None, &mut arena)
 }
 
@@ -129,17 +129,17 @@ struct NestingState {
 }
 
 #[derive(Debug)]
-struct ExpandCtx<'a> {
+struct ExpandCtx<'a, S> {
-    bindings: &'a Bindings,
+    bindings: &'a Bindings<S>,
     nesting: Vec<NestingState>,
 }
 
-fn expand_subtree(
+fn expand_subtree<S: Span>(
-    ctx: &mut ExpandCtx<'_>,
+    ctx: &mut ExpandCtx<'_, S>,
-    template: &MetaTemplate,
+    template: &MetaTemplate<S>,
-    delimiter: Option<Delimiter>,
+    delimiter: Option<Delimiter<S>>,
-    arena: &mut Vec<tt::TokenTree>,
+    arena: &mut Vec<tt::TokenTree<S>>,
-) -> ExpandResult<tt::Subtree> {
+) -> ExpandResult<tt::Subtree<S>> {
     // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
     let start_elements = arena.len();
     let mut err = None;
@@ -180,7 +180,7 @@ fn expand_subtree(
                 arena.push(
                     tt::Leaf::Literal(tt::Literal {
                         text: index.to_string().into(),
-                        span: tt::TokenId::unspecified(),
+                        span: S::DUMMY,
                     })
                     .into(),
                 );
@@ -237,11 +237,8 @@ fn expand_subtree(
                     }
                 };
                 arena.push(
-                    tt::Leaf::Literal(tt::Literal {
+                    tt::Leaf::Literal(tt::Literal { text: c.to_string().into(), span: S::DUMMY })
-                        text: c.to_string().into(),
+                        .into(),
-                        span: tt::TokenId::unspecified(),
-                    })
-                    .into(),
                 );
             }
         }
@@ -257,7 +254,11 @@ fn expand_subtree(
     }
 }
 
-fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
+fn expand_var<S: Span>(
+    ctx: &mut ExpandCtx<'_, S>,
+    v: &SmolStr,
+    id: S,
+) -> ExpandResult<Fragment<S>> {
     // We already handle $crate case in mbe parser
     debug_assert!(v != "crate");
 
@@ -296,14 +297,14 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
     }
 }
 
-fn expand_repeat(
+fn expand_repeat<S: Span>(
-    ctx: &mut ExpandCtx<'_>,
+    ctx: &mut ExpandCtx<'_, S>,
-    template: &MetaTemplate,
+    template: &MetaTemplate<S>,
     kind: RepeatKind,
-    separator: &Option<Separator>,
+    separator: &Option<Separator<S>>,
-    arena: &mut Vec<tt::TokenTree>,
+    arena: &mut Vec<tt::TokenTree<S>>,
-) -> ExpandResult<Fragment> {
+) -> ExpandResult<Fragment<S>> {
-    let mut buf: Vec<tt::TokenTree> = Vec::new();
+    let mut buf: Vec<tt::TokenTree<S>> = Vec::new();
     ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
     // Dirty hack to make macro-expansion terminate.
     // This should be replaced by a proper macro-by-example implementation
@@ -342,7 +343,7 @@ fn expand_repeat(
             continue;
         }
 
-        t.delimiter = tt::Delimiter::unspecified();
+        t.delimiter = tt::Delimiter::UNSPECIFIED;
         push_subtree(&mut buf, t);
 
         if let Some(sep) = separator {
@@ -376,7 +377,7 @@ fn expand_repeat(
 
     // Check if it is a single token subtree without any delimiter
     // e.g {Delimiter:None> ['>'] /Delimiter:None>}
-    let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into();
+    let tt = tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: buf }.into();
 
     if RepeatKind::OneOrMore == kind && counter == 0 {
         return ExpandResult {
@@ -387,14 +388,14 @@ fn expand_repeat(
     ExpandResult { value: Fragment::Tokens(tt), err }
 }
 
-fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
+fn push_fragment<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, fragment: Fragment<S>) {
     match fragment {
         Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
         Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => {
             if tt.delimiter.kind == tt::DelimiterKind::Invisible {
                 tt.delimiter = tt::Delimiter {
-                    open: tt::TokenId::UNSPECIFIED,
+                    open: S::DUMMY,
-                    close: tt::TokenId::UNSPECIFIED,
+                    close: S::DUMMY,
                     kind: tt::DelimiterKind::Parenthesis,
                 };
             }
@@ -405,7 +406,7 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
     }
 }
 
-fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
+fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) {
     match tt.delimiter.kind {
         tt::DelimiterKind::Invisible => buf.extend(tt.token_trees),
         _ => buf.push(tt.into()),
@@ -415,7 +416,7 @@ fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
 /// Inserts the path separator `::` between an identifier and its following generic
 /// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why
 /// we need this fixup.
-fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
+fn fix_up_and_push_path_tt<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, subtree: tt::Subtree<S>) {
     stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible));
     let mut prev_was_ident = false;
     // Note that we only need to fix up the top-level `TokenTree`s because the
@@ -432,7 +433,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
                     tt::Leaf::Punct(tt::Punct {
                         char: ':',
                         spacing: tt::Spacing::Joint,
-                        span: tt::Span::unspecified(),
+                        span: S::DUMMY,
                     })
                     .into(),
                 );
@@ -440,7 +441,7 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
                     tt::Leaf::Punct(tt::Punct {
                         char: ':',
                         spacing: tt::Spacing::Alone,
-                        span: tt::Span::unspecified(),
+                        span: S::DUMMY,
                     })
                     .into(),
                 );
@@ -453,9 +454,9 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) {
 
 /// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth
 /// defined by the metavar expression.
-fn count(
+fn count<S>(
-    ctx: &ExpandCtx<'_>,
+    ctx: &ExpandCtx<'_, S>,
-    binding: &Binding,
+    binding: &Binding<S>,
     our_depth: usize,
     count_depth: Option<usize>,
 ) -> Result<usize, CountError> {

crates/mbe/src/lib.rs

@@ -18,8 +18,8 @@ mod to_parser_input;
 mod benchmark;
 mod token_map;
 
-use ::tt::token_id as tt;
 use stdx::impl_from;
+use tt::{Span, TokenId};
 
 use std::fmt;
 
@@ -28,8 +28,9 @@ use crate::{
     tt_iter::TtIter,
 };
 
-pub use self::tt::{Delimiter, DelimiterKind, Punct};
+// FIXME: we probably should re-think  `token_tree_to_syntax_node` interfaces
 pub use ::parser::TopEntryPoint;
+pub use tt::{Delimiter, DelimiterKind, Punct};
 
 pub use crate::{
     syntax_bridge::{
@@ -125,7 +126,7 @@ impl fmt::Display for CountError {
 /// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct DeclarativeMacro {
-    rules: Box<[Rule]>,
+    rules: Box<[Rule<TokenId>]>,
     /// Highest id of the token we have in TokenMap
     shift: Shift,
     // This is used for correctly determining the behavior of the pat fragment
@@ -135,23 +136,23 @@ pub struct DeclarativeMacro {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
-struct Rule {
+struct Rule<S> {
-    lhs: MetaTemplate,
+    lhs: MetaTemplate<S>,
-    rhs: MetaTemplate,
+    rhs: MetaTemplate<S>,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct Shift(u32);
 
 impl Shift {
-    pub fn new(tt: &tt::Subtree) -> Shift {
+    pub fn new(tt: &tt::Subtree<TokenId>) -> Shift {
         // Note that TokenId is started from zero,
         // We have to add 1 to prevent duplication.
         let value = max_id(tt).map_or(0, |it| it + 1);
         return Shift(value);
 
         // Find the max token id inside a subtree
-        fn max_id(subtree: &tt::Subtree) -> Option<u32> {
+        fn max_id(subtree: &tt::Subtree<TokenId>) -> Option<u32> {
             let filter =
                 |tt: &_| match tt {
                     tt::TokenTree::Subtree(subtree) => {
@@ -177,7 +178,7 @@ impl Shift {
     }
 
     /// Shift given TokenTree token id
-    pub fn shift_all(self, tt: &mut tt::Subtree) {
+    pub fn shift_all(self, tt: &mut tt::Subtree<TokenId>) {
         for t in &mut tt.token_trees {
             match t {
                 tt::TokenTree::Leaf(
@@ -224,7 +225,7 @@ impl DeclarativeMacro {
     }
 
     /// The old, `macro_rules! m {}` flavor.
-    pub fn parse_macro_rules(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
+    pub fn parse_macro_rules(tt: &tt::Subtree<TokenId>, is_2021: bool) -> DeclarativeMacro {
         // Note: this parsing can be implemented using mbe machinery itself, by
         // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
         // manually seems easier.
@@ -260,7 +261,7 @@ impl DeclarativeMacro {
     }
 
     /// The new, unstable `macro m {}` flavor.
-    pub fn parse_macro2(tt: &tt::Subtree, is_2021: bool) -> DeclarativeMacro {
+    pub fn parse_macro2(tt: &tt::Subtree<TokenId>, is_2021: bool) -> DeclarativeMacro {
         let mut src = TtIter::new(tt);
         let mut rules = Vec::new();
         let mut err = None;
@@ -310,7 +311,7 @@ impl DeclarativeMacro {
         DeclarativeMacro { rules: rules.into_boxed_slice(), shift: Shift::new(tt), is_2021, err }
     }
 
-    pub fn expand(&self, mut tt: tt::Subtree) -> ExpandResult<tt::Subtree> {
+    pub fn expand(&self, mut tt: tt::Subtree<TokenId>) -> ExpandResult<tt::Subtree<TokenId>> {
         self.shift.shift_all(&mut tt);
         expander::expand_rules(&self.rules, &tt, self.is_2021)
     }
@@ -335,8 +336,8 @@ impl DeclarativeMacro {
     }
 }
 
-impl Rule {
+impl<S: Span> Rule<S> {
-    fn parse(src: &mut TtIter<'_>, expect_arrow: bool) -> Result<Self, ParseError> {
+    fn parse(src: &mut TtIter<'_, S>, expect_arrow: bool) -> Result<Self, ParseError> {
         let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
         if expect_arrow {
             src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
@@ -351,7 +352,7 @@ impl Rule {
     }
 }
 
-fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
+fn validate<S: Span>(pattern: &MetaTemplate<S>) -> Result<(), ParseError> {
     for op in pattern.iter() {
         match op {
             Op::Subtree { tokens, .. } => validate(tokens)?,

crates/mbe/src/parser.rs

@@ -3,8 +3,9 @@
 
 use smallvec::{smallvec, SmallVec};
 use syntax::SmolStr;
+use tt::Span;
 
-use crate::{tt, tt_iter::TtIter, ParseError};
+use crate::{tt_iter::TtIter, ParseError};
 
 /// Consider
 ///
@@ -20,22 +21,22 @@ use crate::{tt, tt_iter::TtIter, ParseError};
 /// Stuff to the right is a [`MetaTemplate`] template which is used to produce
 /// output.
 #[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>);
+pub(crate) struct MetaTemplate<S>(pub(crate) Box<[Op<S>]>);
 
-impl MetaTemplate {
+impl<S: Span> MetaTemplate<S> {
-    pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result<MetaTemplate, ParseError> {
+    pub(crate) fn parse_pattern(pattern: &tt::Subtree<S>) -> Result<Self, ParseError> {
         MetaTemplate::parse(pattern, Mode::Pattern)
     }
 
-    pub(crate) fn parse_template(template: &tt::Subtree) -> Result<MetaTemplate, ParseError> {
+    pub(crate) fn parse_template(template: &tt::Subtree<S>) -> Result<Self, ParseError> {
         MetaTemplate::parse(template, Mode::Template)
     }
 
-    pub(crate) fn iter(&self) -> impl Iterator<Item = &Op> {
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &Op<S>> {
         self.0.iter()
     }
 
-    fn parse(tt: &tt::Subtree, mode: Mode) -> Result<MetaTemplate, ParseError> {
+    fn parse(tt: &tt::Subtree<S>, mode: Mode) -> Result<Self, ParseError> {
         let mut src = TtIter::new(tt);
 
         let mut res = Vec::new();
@@ -49,16 +50,16 @@ impl MetaTemplate {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) enum Op {
+pub(crate) enum Op<S> {
-    Var { name: SmolStr, kind: Option<MetaVarKind>, id: tt::TokenId },
+    Var { name: SmolStr, kind: Option<MetaVarKind>, id: S },
-    Ignore { name: SmolStr, id: tt::TokenId },
+    Ignore { name: SmolStr, id: S },
     Index { depth: usize },
     Count { name: SmolStr, depth: Option<usize> },
-    Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
+    Repeat { tokens: MetaTemplate<S>, kind: RepeatKind, separator: Option<Separator<S>> },
-    Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter },
+    Subtree { tokens: MetaTemplate<S>, delimiter: tt::Delimiter<S> },
-    Literal(tt::Literal),
+    Literal(tt::Literal<S>),
-    Punct(SmallVec<[tt::Punct; 3]>),
+    Punct(SmallVec<[tt::Punct<S>; 3]>),
-    Ident(tt::Ident),
+    Ident(tt::Ident<S>),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -87,15 +88,15 @@ pub(crate) enum MetaVarKind {
 }
 
 #[derive(Clone, Debug, Eq)]
-pub(crate) enum Separator {
+pub(crate) enum Separator<S> {
-    Literal(tt::Literal),
+    Literal(tt::Literal<S>),
-    Ident(tt::Ident),
+    Ident(tt::Ident<S>),
-    Puncts(SmallVec<[tt::Punct; 3]>),
+    Puncts(SmallVec<[tt::Punct<S>; 3]>),
 }
 
 // Note that when we compare a Separator, we just care about its textual value.
-impl PartialEq for Separator {
+impl<S> PartialEq for Separator<S> {
-    fn eq(&self, other: &Separator) -> bool {
+    fn eq(&self, other: &Separator<S>) -> bool {
         use Separator::*;
 
         match (self, other) {
@@ -117,11 +118,11 @@ enum Mode {
     Template,
 }
 
-fn next_op(
+fn next_op<S: Span>(
-    first_peeked: &tt::TokenTree,
+    first_peeked: &tt::TokenTree<S>,
-    src: &mut TtIter<'_>,
+    src: &mut TtIter<'_, S>,
     mode: Mode,
-) -> Result<Op, ParseError> {
+) -> Result<Op<S>, ParseError> {
     let res = match first_peeked {
         tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
             src.next().expect("first token already peeked");
@@ -212,7 +213,10 @@ fn next_op(
     Ok(res)
 }
 
-fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<MetaVarKind>, ParseError> {
+fn eat_fragment_kind<S: Span>(
+    src: &mut TtIter<'_, S>,
+    mode: Mode,
+) -> Result<Option<MetaVarKind>, ParseError> {
     if let Mode::Pattern = mode {
         src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
         let ident = src
@@ -240,11 +244,13 @@ fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<MetaVarK
     Ok(None)
 }
 
-fn is_boolean_literal(lit: &tt::Literal) -> bool {
+fn is_boolean_literal<S>(lit: &tt::Literal<S>) -> bool {
     matches!(lit.text.as_str(), "true" | "false")
 }
 
-fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind), ParseError> {
+fn parse_repeat<S: Span>(
+    src: &mut TtIter<'_, S>,
+) -> Result<(Option<Separator<S>>, RepeatKind), ParseError> {
     let mut separator = Separator::Puncts(SmallVec::new());
     for tt in src {
         let tt = match tt {
@@ -281,7 +287,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind),
     Err(ParseError::InvalidRepeat)
 }
 
-fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
+fn parse_metavar_expr<S: Span>(src: &mut TtIter<'_, S>) -> Result<Op<S>, ()> {
     let func = src.expect_ident()?;
     let args = src.expect_subtree()?;
 
@@ -314,7 +320,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
     Ok(op)
 }
 
-fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {
+fn parse_depth<S: Span>(src: &mut TtIter<'_, S>) -> Result<usize, ()> {
     if src.len() == 0 {
         Ok(0)
     } else if let tt::Leaf::Literal(lit) = src.expect_literal()? {
@@ -325,7 +331,7 @@ fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {
     }
 }
 
-fn try_eat_comma(src: &mut TtIter<'_>) -> bool {
+fn try_eat_comma<S: Span>(src: &mut TtIter<'_, S>) -> bool {
     if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek_n(0) {
         let _ = src.next();
         return true;

crates/mbe/src/syntax_bridge.rs

@@ -8,23 +8,19 @@ use syntax::{
     SyntaxKind::*,
     SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
 };
-
+use tt::{
-use crate::{
+    buffer::{Cursor, TokenBuffer},
-    to_parser_input::to_parser_input,
+    TokenId,
-    tt::{
-        self,
-        buffer::{Cursor, TokenBuffer},
-    },
-    tt_iter::TtIter,
-    TokenMap,
 };
 
+use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
+
 #[cfg(test)]
 mod tests;
 
 /// Convert the syntax node to a `TokenTree` (what macro
 /// will consume).
-pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) {
+pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree<TokenId>, TokenMap) {
     let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications(
         node,
         Default::default(),
@@ -43,7 +39,7 @@ pub fn syntax_node_to_token_tree_with_modifications(
     next_id: u32,
     replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
     append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
-) -> (tt::Subtree, TokenMap, u32) {
+) -> (tt::Subtree<TokenId>, TokenMap, u32) {
     let global_offset = node.text_range().start();
     let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append);
     let subtree = convert_tokens(&mut c);
@@ -108,7 +104,7 @@ pub struct SyntheticToken {
 // * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
 
 pub fn token_tree_to_syntax_node(
-    tt: &tt::Subtree,
+    tt: &tt::Subtree<TokenId>,
     entry_point: parser::TopEntryPoint,
 ) -> (Parse<SyntaxNode>, TokenMap) {
     let buffer = match tt {
@@ -138,7 +134,7 @@ pub fn token_tree_to_syntax_node(
 }
 
 /// Convert a string to a `TokenTree`
-pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
+pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree<TokenId>, TokenMap)> {
     let lexed = parser::LexedStr::new(text);
     if lexed.errors().next().is_some() {
         return None;
@@ -159,7 +155,7 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
 }
 
 /// Split token tree with separate expr: $($e:expr)SEP*
-pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
+pub fn parse_exprs_with_sep(tt: &tt::Subtree<TokenId>, sep: char) -> Vec<tt::Subtree<TokenId>> {
     if tt.token_trees.is_empty() {
         return Vec::new();
     }
@@ -195,9 +191,9 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
     res
 }
 
-fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
+fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree<TokenId> {
     struct StackEntry {
-        subtree: tt::Subtree,
+        subtree: tt::Subtree<TokenId>,
         idx: usize,
         open_range: TextRange,
     }
@@ -296,7 +292,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
                     .into()
                 };
             }
-            let leaf: tt::Leaf = match kind {
+            let leaf: tt::Leaf<TokenId> = match kind {
                 T![true] | T![false] => make_leaf!(Ident),
                 IDENT => make_leaf!(Ident),
                 UNDERSCORE => make_leaf!(Ident),
@@ -335,7 +331,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
         let parent = stack.last_mut();
 
         conv.id_alloc().close_delim(entry.idx, None);
-        let leaf: tt::Leaf = tt::Punct {
+        let leaf: tt::Leaf<TokenId> = tt::Punct {
             span: conv.id_alloc().alloc(entry.open_range, None),
             char: match entry.subtree.delimiter.kind {
                 tt::DelimiterKind::Parenthesis => '(',
@@ -514,7 +510,7 @@ fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
 fn convert_doc_comment(
     token: &syntax::SyntaxToken,
     span: tt::TokenId,
-) -> Option<Vec<tt::TokenTree>> {
+) -> Option<Vec<tt::TokenTree<TokenId>>> {
     cov_mark::hit!(test_meta_doc_comments);
     let comment = ast::Comment::cast(token.clone())?;
     let doc = comment.kind().doc?;
@@ -537,11 +533,11 @@ fn convert_doc_comment(
     return Some(token_trees);
 
     // Helper functions
-    fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree {
+    fn mk_ident(s: &str, span: tt::TokenId) -> tt::TokenTree<TokenId> {
         tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), span }))
     }
 
-    fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree {
+    fn mk_punct(c: char, span: tt::TokenId) -> tt::TokenTree<TokenId> {
         tt::TokenTree::from(tt::Leaf::from(tt::Punct {
             char: c,
             spacing: tt::Spacing::Alone,
@@ -549,7 +545,7 @@ fn convert_doc_comment(
         }))
     }
 
-    fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree {
+    fn mk_doc_literal(comment: &ast::Comment, span: tt::TokenId) -> tt::TokenTree<TokenId> {
         let lit = tt::Literal { text: doc_comment_text(comment), span };
 
         tt::TokenTree::from(tt::Leaf::from(lit))
@@ -636,7 +632,7 @@ trait TokenConverter: Sized {
         &self,
         token: &Self::Token,
         span: tt::TokenId,
-    ) -> Option<Vec<tt::TokenTree>>;
+    ) -> Option<Vec<tt::TokenTree<TokenId>>>;
 
     fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
 
@@ -666,7 +662,11 @@ impl SrcToken<RawConverter<'_>> for usize {
 impl TokenConverter for RawConverter<'_> {
     type Token = usize;
 
-    fn convert_doc_comment(&self, &token: &usize, span: tt::TokenId) -> Option<Vec<tt::TokenTree>> {
+    fn convert_doc_comment(
+        &self,
+        &token: &usize,
+        span: tt::TokenId,
+    ) -> Option<Vec<tt::TokenTree<TokenId>>> {
         let text = self.lexed.text(token);
         convert_doc_comment(&doc_comment(text), span)
     }
@@ -819,7 +819,7 @@ impl TokenConverter for Converter {
         &self,
         token: &Self::Token,
         span: tt::TokenId,
-    ) -> Option<Vec<tt::TokenTree>> {
+    ) -> Option<Vec<tt::TokenTree<TokenId>>> {
         convert_doc_comment(token.token()?, span)
     }
 
@@ -899,7 +899,7 @@ impl TokenConverter for Converter {
 
 struct TtTreeSink<'a> {
     buf: String,
-    cursor: Cursor<'a>,
+    cursor: Cursor<'a, TokenId>,
     open_delims: FxHashMap<tt::TokenId, TextSize>,
     text_pos: TextSize,
     inner: SyntaxTreeBuilder,
@@ -907,7 +907,7 @@ struct TtTreeSink<'a> {
 }
 
 impl<'a> TtTreeSink<'a> {
-    fn new(cursor: Cursor<'a>) -> Self {
+    fn new(cursor: Cursor<'a, TokenId>) -> Self {
         TtTreeSink {
             buf: String::new(),
             cursor,

crates/mbe/src/to_parser_input.rs

@@ -3,9 +3,9 @@
 
 use syntax::{SyntaxKind, SyntaxKind::*, T};
 
-use crate::tt::buffer::TokenBuffer;
+use tt::{buffer::TokenBuffer, Span};
 
-pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
+pub(crate) fn to_parser_input<S: Span>(buffer: &TokenBuffer<'_, S>) -> parser::Input {
     let mut res = parser::Input::default();
 
     let mut current = buffer.begin();

crates/mbe/src/tt_iter.rs

@@ -3,16 +3,17 @@
 
 use smallvec::{smallvec, SmallVec};
 use syntax::SyntaxKind;
+use tt::Span;
 
-use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult};
+use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult};
 
 #[derive(Debug, Clone)]
-pub(crate) struct TtIter<'a> {
+pub(crate) struct TtIter<'a, S> {
-    pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
+    pub(crate) inner: std::slice::Iter<'a, tt::TokenTree<S>>,
 }
 
-impl<'a> TtIter<'a> {
+impl<'a, S: Span> TtIter<'a, S> {
-    pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
+    pub(crate) fn new(subtree: &'a tt::Subtree<S>) -> TtIter<'a, S> {
         TtIter { inner: subtree.token_trees.iter() }
     }
 
@@ -36,35 +37,35 @@ impl<'a> TtIter<'a> {
         }
     }
 
-    pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
+    pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree<S>, ()> {
         match self.next() {
             Some(tt::TokenTree::Subtree(it)) => Ok(it),
             _ => Err(()),
         }
     }
 
-    pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
+    pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf<S>, ()> {
         match self.next() {
             Some(tt::TokenTree::Leaf(it)) => Ok(it),
             _ => Err(()),
         }
     }
 
-    pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
+    pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident<S>, ()> {
         match self.expect_leaf()? {
             tt::Leaf::Ident(it) if it.text != "_" => Ok(it),
             _ => Err(()),
         }
     }
 
-    pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> {
+    pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident<S>, ()> {
         match self.expect_leaf()? {
             tt::Leaf::Ident(it) => Ok(it),
             _ => Err(()),
         }
     }
 
-    pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> {
+    pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf<S>, ()> {
         let it = self.expect_leaf()?;
         match it {
             tt::Leaf::Literal(_) => Ok(it),
@@ -73,7 +74,7 @@ impl<'a> TtIter<'a> {
         }
     }
 
-    pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct, ()> {
+    pub(crate) fn expect_single_punct(&mut self) -> Result<&'a tt::Punct<S>, ()> {
         match self.expect_leaf()? {
             tt::Leaf::Punct(it) => Ok(it),
             _ => Err(()),
@@ -84,7 +85,7 @@ impl<'a> TtIter<'a> {
     ///
     /// This method currently may return a single quotation, which is part of lifetime ident and
     /// conceptually not a punct in the context of mbe. Callers should handle this.
-    pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct; 3]>, ()> {
+    pub(crate) fn expect_glued_punct(&mut self) -> Result<SmallVec<[tt::Punct<S>; 3]>, ()> {
         let tt::TokenTree::Leaf(tt::Leaf::Punct(first)) = self.next().ok_or(())?.clone() else {
             return Err(());
         };
@@ -126,7 +127,7 @@ impl<'a> TtIter<'a> {
     pub(crate) fn expect_fragment(
         &mut self,
         entry_point: parser::PrefixEntryPoint,
-    ) -> ExpandResult<Option<tt::TokenTree>> {
+    ) -> ExpandResult<Option<tt::TokenTree<S>>> {
         let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
         let parser_input = to_parser_input(&buffer);
         let tree_traversal = entry_point.parse(&parser_input);
@@ -181,13 +182,13 @@ impl<'a> TtIter<'a> {
         ExpandResult { value: res, err }
     }
 
-    pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree> {
+    pub(crate) fn peek_n(&self, n: usize) -> Option<&'a tt::TokenTree<S>> {
         self.inner.as_slice().get(n)
     }
 }
 
-impl<'a> Iterator for TtIter<'a> {
+impl<'a, S> Iterator for TtIter<'a, S> {
-    type Item = &'a tt::TokenTree;
+    type Item = &'a tt::TokenTree<S>;
     fn next(&mut self) -> Option<Self::Item> {
         self.inner.next()
     }
@@ -197,4 +198,4 @@ impl<'a> Iterator for TtIter<'a> {
     }
 }
 
-impl std::iter::ExactSizeIterator for TtIter<'_> {}
+impl<S> std::iter::ExactSizeIterator for TtIter<'_, S> {}

crates/tt/src/lib.rs

@@ -47,70 +47,44 @@ pub mod token_id {
         pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>;
         pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>;
     }
+}
 
-    impl Delimiter {
+pub trait Span: std::fmt::Debug + Copy + Sized {
-        pub const UNSPECIFIED: Self = Self {
+    const DUMMY: Self;
-            open: TokenId::UNSPECIFIED,
+    fn is_dummy(&self) -> bool;
-            close: TokenId::UNSPECIFIED,
+}
-            kind: DelimiterKind::Invisible,
+impl Span for TokenId {
-        };
+    const DUMMY: Self = TokenId(!0);
-        pub const fn unspecified() -> Self {
-            Self::UNSPECIFIED
-        }
-    }
-    impl Subtree {
-        pub const fn empty() -> Self {
-            Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
-        }
-    }
-    impl TokenTree {
-        pub const fn empty() -> Self {
-            Self::Subtree(Subtree::empty())
-        }
-    }
 
-    impl Subtree {
+    fn is_dummy(&self) -> bool {
-        pub fn visit_ids(&mut self, f: &mut impl FnMut(TokenId) -> TokenId) {
+        *self == Self::DUMMY
-            self.delimiter.open = f(self.delimiter.open);
-            self.delimiter.close = f(self.delimiter.close);
-            self.token_trees.iter_mut().for_each(|tt| match tt {
-                crate::TokenTree::Leaf(leaf) => match leaf {
-                    crate::Leaf::Literal(it) => it.span = f(it.span),
-                    crate::Leaf::Punct(it) => it.span = f(it.span),
-                    crate::Leaf::Ident(it) => it.span = f(it.span),
-                },
-                crate::TokenTree::Subtree(s) => s.visit_ids(f),
-            })
-        }
     }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct SyntaxContext(pub u32);
 
-// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-// pub struct Span {
-//     pub id: TokenId,
-//     pub ctx: SyntaxContext,
-// }
-// pub type Span = (TokenId, SyntaxContext);
-
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TokenTree<Span> {
+pub enum TokenTree<S> {
-    Leaf(Leaf<Span>),
+    Leaf(Leaf<S>),
-    Subtree(Subtree<Span>),
+    Subtree(Subtree<S>),
 }
-impl_from!(Leaf<Span>, Subtree<Span> for TokenTree);
+impl_from!(Leaf<S>, Subtree<S> for TokenTree);
-
+impl<S: Span> TokenTree<S> {
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+    pub const fn empty() -> Self {
-pub enum Leaf<Span> {
+        Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] })
-    Literal(Literal<Span>),
+    }
-    Punct(Punct<Span>),
-    Ident(Ident<Span>),
 }
 
-impl<Span> Leaf<Span> {
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-    pub fn span(&self) -> &Span {
+pub enum Leaf<S> {
+    Literal(Literal<S>),
+    Punct(Punct<S>),
+    Ident(Ident<S>),
+}
+
+impl<S> Leaf<S> {
+    pub fn span(&self) -> &S {
         match self {
             Leaf::Literal(it) => &it.span,
             Leaf::Punct(it) => &it.span,
@@ -118,21 +92,49 @@ impl<Span> Leaf<Span> {
         }
     }
 }
-impl_from!(Literal<Span>, Punct<Span>, Ident<Span> for Leaf);
+impl_from!(Literal<S>, Punct<S>, Ident<S> for Leaf);
 
 #[derive(Clone, PartialEq, Eq, Hash)]
-pub struct Subtree<Span> {
+pub struct Subtree<S> {
-    pub delimiter: Delimiter<Span>,
+    pub delimiter: Delimiter<S>,
-    pub token_trees: Vec<TokenTree<Span>>,
+    pub token_trees: Vec<TokenTree<S>>,
+}
+
+impl<S: Span> Subtree<S> {
+    pub const fn empty() -> Self {
+        Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
+    }
+
+    pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) {
+        self.delimiter.open = f(self.delimiter.open);
+        self.delimiter.close = f(self.delimiter.close);
+        self.token_trees.iter_mut().for_each(|tt| match tt {
+            crate::TokenTree::Leaf(leaf) => match leaf {
+                crate::Leaf::Literal(it) => it.span = f(it.span),
+                crate::Leaf::Punct(it) => it.span = f(it.span),
+                crate::Leaf::Ident(it) => it.span = f(it.span),
+            },
+            crate::TokenTree::Subtree(s) => s.visit_ids(f),
+        })
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub struct Delimiter<Span> {
+pub struct Delimiter<S> {
-    pub open: Span,
+    pub open: S,
-    pub close: Span,
+    pub close: S,
     pub kind: DelimiterKind,
 }
 
+impl<S: Span> Delimiter<S> {
+    pub const UNSPECIFIED: Self =
+        Self { open: S::DUMMY, close: S::DUMMY, kind: DelimiterKind::Invisible };
+    pub const fn unspecified() -> Self {
+        Self::UNSPECIFIED
+    }
+}
+
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum DelimiterKind {
     Parenthesis,
@@ -142,16 +144,16 @@ pub enum DelimiterKind {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Literal<Span> {
+pub struct Literal<S> {
     pub text: SmolStr,
-    pub span: Span,
+    pub span: S,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Punct<Span> {
+pub struct Punct<S> {
     pub char: char,
     pub spacing: Spacing,
-    pub span: Span,
+    pub span: S,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -162,9 +164,9 @@ pub enum Spacing {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
-pub struct Ident<Span> {
+pub struct Ident<S> {
     pub text: SmolStr,
-    pub span: Span,
+    pub span: S,
 }
 
 impl<S> Ident<S> {
@@ -173,9 +175,9 @@ impl<S> Ident<S> {
     }
 }
 
-fn print_debug_subtree<Span: fmt::Debug>(
+fn print_debug_subtree<S: fmt::Debug>(
     f: &mut fmt::Formatter<'_>,
-    subtree: &Subtree<Span>,
+    subtree: &Subtree<S>,
     level: usize,
 ) -> fmt::Result {
     let align = "  ".repeat(level);
@@ -203,9 +205,9 @@ fn print_debug_subtree<Span: fmt::Debug>(
     Ok(())
 }
 
-fn print_debug_token<Span: fmt::Debug>(
+fn print_debug_token<S: fmt::Debug>(
     f: &mut fmt::Formatter<'_>,
-    tkn: &TokenTree<Span>,
+    tkn: &TokenTree<S>,
     level: usize,
 ) -> fmt::Result {
     let align = "  ".repeat(level);
@@ -231,13 +233,13 @@ fn print_debug_token<Span: fmt::Debug>(
     Ok(())
 }
 
-impl<Span: fmt::Debug> fmt::Debug for Subtree<Span> {
+impl<S: fmt::Debug> fmt::Debug for Subtree<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         print_debug_subtree(f, self, 0)
     }
 }
 
-impl<Span> fmt::Display for TokenTree<Span> {
+impl<S> fmt::Display for TokenTree<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
@@ -246,7 +248,7 @@ impl<Span> fmt::Display for TokenTree<Span> {
     }
 }
 
-impl<Span> fmt::Display for Subtree<Span> {
+impl<S> fmt::Display for Subtree<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let (l, r) = match self.delimiter.kind {
             DelimiterKind::Parenthesis => ("(", ")"),
@@ -274,7 +276,7 @@ impl<Span> fmt::Display for Subtree<Span> {
     }
 }
 
-impl<Span> fmt::Display for Leaf<Span> {
+impl<S> fmt::Display for Leaf<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Leaf::Ident(it) => fmt::Display::fmt(it, f),
@@ -284,25 +286,25 @@ impl<Span> fmt::Display for Leaf<Span> {
     }
 }
 
-impl<Span> fmt::Display for Ident<Span> {
+impl<S> fmt::Display for Ident<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt::Display::fmt(&self.text, f)
     }
 }
 
-impl<Span> fmt::Display for Literal<Span> {
+impl<S> fmt::Display for Literal<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt::Display::fmt(&self.text, f)
     }
 }
 
-impl<Span> fmt::Display for Punct<Span> {
+impl<S> fmt::Display for Punct<S> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt::Display::fmt(&self.char, f)
     }
 }
 
-impl<Span> Subtree<Span> {
+impl<S> Subtree<S> {
     /// Count the number of tokens recursively
     pub fn count(&self) -> usize {
         let children_count = self
@@ -318,7 +320,7 @@ impl<Span> Subtree<Span> {
     }
 }
 
-impl<Span> Subtree<Span> {
+impl<S> Subtree<S> {
     /// A simple line string used for debugging
     pub fn as_debug_string(&self) -> String {
         let delim = match self.delimiter.kind {
@@ -366,8 +368,8 @@ impl<Span> Subtree<Span> {
 
 pub mod buffer;
 
-pub fn pretty<Span>(tkns: &[TokenTree<Span>]) -> String {
+pub fn pretty<S>(tkns: &[TokenTree<S>]) -> String {
-    fn tokentree_to_text<Span>(tkn: &TokenTree<Span>) -> String {
+    fn tokentree_to_text<S>(tkn: &TokenTree<S>) -> String {
         match tkn {
             TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(),
             TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(),