Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-28 05:53:45 +00:00
Code Issues Projects Releases Packages Wiki Activity

Swap Subtree::token_trees from Vec to boxed slice

Browse source
This commit is contained in:
GnomedDev 2024-02-03 22:47:13 +00:00
parent 8f6a72871e
commit 8011b56827
No known key found for this signature in database
GPG key ID: 9BF10F8372B254D1
22 changed files with 194 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/hir-def/src/nameres/collector.rs
View file

@ -2125,7 +2125,7 @@ impl ModCollector<'_, '_> {
let is_export = export_attr.exists(); let is_export = export_attr.exists();
let local_inner = if is_export { let local_inner = if is_export {
export_attr.tt_values().flat_map(|it| &it.token_trees).any(|it| match it { export_attr.tt_values().flat_map(|it| it.token_trees.iter()).any(|it| match it {
tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
ident.text.contains("local_inner_macros") ident.text.contains("local_inner_macros")
} }

2
crates/hir-expand/src/attrs.rs
View file

@ -123,7 +123,7 @@ impl RawAttrs {
.filter_map(|(idx, attr)| Attr::from_tt(db, attr, index.with_cfg_attr(idx))); .filter_map(|(idx, attr)| Attr::from_tt(db, attr, index.with_cfg_attr(idx)));
let cfg_options = &crate_graph[krate].cfg_options; let cfg_options = &crate_graph[krate].cfg_options;
let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() }; let cfg = Subtree { delimiter: subtree.delimiter, token_trees: Box::from(cfg) };
let cfg = CfgExpr::parse(&cfg); let cfg = CfgExpr::parse(&cfg);
if cfg_options.check(&cfg) == Some(false) { if cfg_options.check(&cfg) == Some(false) {
smallvec![] smallvec![]

5
crates/hir-expand/src/builtin_attr_macro.rs
View file

@ -137,5 +137,8 @@ pub fn pseudo_derive_attr_expansion(
token_trees.extend(tt.iter().cloned()); token_trees.extend(tt.iter().cloned());
token_trees.push(mk_leaf(']')); token_trees.push(mk_leaf(']'));
} }
ExpandResult::ok(tt::Subtree { delimiter: tt.delimiter, token_trees }) ExpandResult::ok(tt::Subtree {
delimiter: tt.delimiter,
token_trees: token_trees.into_boxed_slice(),
})
} }

26
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -155,10 +155,10 @@ fn line_expand(
// not incremental // not incremental
ExpandResult::ok(tt::Subtree { ExpandResult::ok(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(span), delimiter: tt::Delimiter::invisible_spanned(span),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: "0u32".into(), text: "0u32".into(),
span, span,
}))], }))]),
}) })
} }
@ -208,11 +208,11 @@ fn assert_expand(
[cond, panic_args @ ..] => { [cond, panic_args @ ..] => {
let comma = tt::Subtree { let comma = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(call_site_span), delimiter: tt::Delimiter::invisible_spanned(call_site_span),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char: ',', char: ',',
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
span: call_site_span, span: call_site_span,
}))], }))]),
}; };
let cond = cond.clone(); let cond = cond.clone();
let panic_args = itertools::Itertools::intersperse(panic_args.iter().cloned(), comma); let panic_args = itertools::Itertools::intersperse(panic_args.iter().cloned(), comma);
@ -359,7 +359,12 @@ fn panic_expand(
close: call_site_span, close: call_site_span,
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}; };
call.token_trees.push(tt::TokenTree::Subtree(subtree));
// FIXME(slow): quote! have a way to expand to builder to make this a vec!
let mut mutable_trees = std::mem::take(&mut call.token_trees).into_vec();
mutable_trees.push(tt::TokenTree::Subtree(subtree));
call.token_trees = mutable_trees.into_boxed_slice();
ExpandResult::ok(call) ExpandResult::ok(call)
} }
@ -388,7 +393,12 @@ fn unreachable_expand(
close: call_site_span, close: call_site_span,
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}; };
call.token_trees.push(tt::TokenTree::Subtree(subtree));
// FIXME(slow): quote! have a way to expand to builder to make this a vec!
let mut mutable_trees = std::mem::take(&mut call.token_trees).into_vec();
mutable_trees.push(tt::TokenTree::Subtree(subtree));
call.token_trees = mutable_trees.into_boxed_slice();
ExpandResult::ok(call) ExpandResult::ok(call)
} }
@ -675,10 +685,10 @@ fn include_bytes_expand(
// FIXME: actually read the file here if the user asked for macro expansion // FIXME: actually read the file here if the user asked for macro expansion
let res = tt::Subtree { let res = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(span), delimiter: tt::Delimiter::invisible_spanned(span),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: r#"b"""#.into(), text: r#"b"""#.into(),
span, span,
}))], }))]),
}; };
ExpandResult::ok(res) ExpandResult::ok(res)
} }

8
crates/hir-expand/src/db.rs
View file

@ -524,7 +524,7 @@ fn macro_expand(
return ExpandResult { return ExpandResult {
value: CowArc::Owned(tt::Subtree { value: CowArc::Owned(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(loc.call_site), delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
token_trees: Vec::new(), token_trees: Box::new([]),
}), }),
// FIXME: We should make sure to enforce an invariant that invalid macro // FIXME: We should make sure to enforce an invariant that invalid macro
// calls do not reach this call path! // calls do not reach this call path!
@ -586,7 +586,7 @@ fn macro_expand(
return value.map(|()| { return value.map(|()| {
CowArc::Owned(tt::Subtree { CowArc::Owned(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(loc.call_site), delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
token_trees: vec![], token_trees: Box::new([]),
}) })
}); });
} }
@ -601,7 +601,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
return ExpandResult { return ExpandResult {
value: Arc::new(tt::Subtree { value: Arc::new(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(loc.call_site), delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
token_trees: Vec::new(), token_trees: Box::new([]),
}), }),
// FIXME: We should make sure to enforce an invariant that invalid macro // FIXME: We should make sure to enforce an invariant that invalid macro
// calls do not reach this call path! // calls do not reach this call path!
@ -635,7 +635,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
return value.map(|()| { return value.map(|()| {
Arc::new(tt::Subtree { Arc::new(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(loc.call_site), delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
token_trees: vec![], token_trees: Box::new([]),
}) })
}); });
} }

6
crates/hir-expand/src/fixup.rs
View file

@ -312,7 +312,7 @@ pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo)
} }
fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) { fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
let tts = std::mem::take(&mut tt.token_trees); let tts = std::mem::take(&mut tt.token_trees).into_vec();
tt.token_trees = tts tt.token_trees = tts
.into_iter() .into_iter()
// delete all fake nodes // delete all fake nodes
@ -343,7 +343,7 @@ fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
// we have a fake node here, we need to replace it again with the original // we have a fake node here, we need to replace it again with the original
let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone(); let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone();
if original.delimiter.kind == tt::DelimiterKind::Invisible { if original.delimiter.kind == tt::DelimiterKind::Invisible {
original.token_trees.into() SmallVec::from(original.token_trees.into_vec())
} else { } else {
SmallVec::from_const([original.into()]) SmallVec::from_const([original.into()])
} }
@ -383,7 +383,7 @@ mod tests {
fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool { fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
a.delimiter.kind == b.delimiter.kind a.delimiter.kind == b.delimiter.kind
&& a.token_trees.len() == b.token_trees.len() && a.token_trees.len() == b.token_trees.len()
&& a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b)) && a.token_trees.iter().zip(b.token_trees.iter()).all(|(a, b)| check_tt_eq(a, b))
} }
fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool { fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool {

3
crates/hir-expand/src/lib.rs
View file

@ -66,6 +66,7 @@ pub mod tt {
pub type Delimiter = ::tt::Delimiter<Span>; pub type Delimiter = ::tt::Delimiter<Span>;
pub type DelimSpan = ::tt::DelimSpan<Span>; pub type DelimSpan = ::tt::DelimSpan<Span>;
pub type Subtree = ::tt::Subtree<Span>; pub type Subtree = ::tt::Subtree<Span>;
pub type SubtreeBuilder = ::tt::SubtreeBuilder<Span>;
pub type Leaf = ::tt::Leaf<Span>; pub type Leaf = ::tt::Leaf<Span>;
pub type Literal = ::tt::Literal<Span>; pub type Literal = ::tt::Literal<Span>;
pub type Punct = ::tt::Punct<Span>; pub type Punct = ::tt::Punct<Span>;
@ -760,7 +761,7 @@ impl ExpansionInfo {
( (
Arc::new(tt::Subtree { Arc::new(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(loc.call_site), delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
token_trees: Vec::new(), token_trees: Box::new([]),
}), }),
SyntaxFixupUndoInfo::NONE, SyntaxFixupUndoInfo::NONE,
) )

9
crates/hir-expand/src/quote.rs
View file

@ -31,7 +31,7 @@ macro_rules! __quote {
open: $span, open: $span,
close: $span, close: $span,
}, },
token_trees: $crate::quote::IntoTt::to_tokens(children), token_trees: $crate::quote::IntoTt::to_tokens(children).into_boxed_slice(),
} }
} }
}; };
@ -146,7 +146,7 @@ impl IntoTt for Vec<crate::tt::TokenTree> {
fn to_subtree(self, span: Span) -> crate::tt::Subtree { fn to_subtree(self, span: Span) -> crate::tt::Subtree {
crate::tt::Subtree { crate::tt::Subtree {
delimiter: crate::tt::Delimiter::invisible_spanned(span), delimiter: crate::tt::Delimiter::invisible_spanned(span),
token_trees: self, token_trees: self.into_boxed_slice(),
} }
} }
@ -296,8 +296,9 @@ mod tests {
// } // }
let struct_name = mk_ident("Foo"); let struct_name = mk_ident("Foo");
let fields = [mk_ident("name"), mk_ident("id")]; let fields = [mk_ident("name"), mk_ident("id")];
let fields = let fields = fields
fields.iter().flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees); .iter()
.flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees.into_vec());
let list = crate::tt::Subtree { let list = crate::tt::Subtree {
delimiter: crate::tt::Delimiter { delimiter: crate::tt::Delimiter {

72
crates/mbe/src/benchmark.rs
View file

@ -100,17 +100,19 @@ fn invocation_fixtures(
// So we just skip any error cases and try again // So we just skip any error cases and try again
let mut try_cnt = 0; let mut try_cnt = 0;
loop { loop {
let mut subtree = tt::Subtree { let mut token_trees = Vec::new();
for op in rule.lhs.iter() {
collect_from_op(op, &mut token_trees, &mut seed);
}
let subtree = tt::Subtree {
delimiter: tt::Delimiter { delimiter: tt::Delimiter {
open: DUMMY, open: DUMMY,
close: DUMMY, close: DUMMY,
kind: tt::DelimiterKind::Invisible, kind: tt::DelimiterKind::Invisible,
}, },
token_trees: vec![], token_trees: token_trees.into_boxed_slice(),
}; };
for op in rule.lhs.iter() {
collect_from_op(op, &mut subtree, &mut seed);
}
if it.expand(&subtree, |_| (), true, DUMMY).err.is_none() { if it.expand(&subtree, |_| (), true, DUMMY).err.is_none() {
res.push((name.clone(), subtree)); res.push((name.clone(), subtree));
break; break;
@ -127,45 +129,45 @@ fn invocation_fixtures(
fn collect_from_op( fn collect_from_op(
op: &Op<DummyTestSpanData>, op: &Op<DummyTestSpanData>,
parent: &mut tt::Subtree<DummyTestSpanData>, token_trees: &mut Vec<tt::TokenTree<DummyTestSpanData>>,
seed: &mut usize, seed: &mut usize,
) { ) {
return match op { return match op {
Op::Var { kind, .. } => match kind.as_ref() { Op::Var { kind, .. } => match kind.as_ref() {
Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")), Some(MetaVarKind::Ident) => token_trees.push(make_ident("foo")),
Some(MetaVarKind::Ty) => parent.token_trees.push(make_ident("Foo")), Some(MetaVarKind::Ty) => token_trees.push(make_ident("Foo")),
Some(MetaVarKind::Tt) => parent.token_trees.push(make_ident("foo")), Some(MetaVarKind::Tt) => token_trees.push(make_ident("foo")),
Some(MetaVarKind::Vis) => parent.token_trees.push(make_ident("pub")), Some(MetaVarKind::Vis) => token_trees.push(make_ident("pub")),
Some(MetaVarKind::Pat) => parent.token_trees.push(make_ident("foo")), Some(MetaVarKind::Pat) => token_trees.push(make_ident("foo")),
Some(MetaVarKind::Path) => parent.token_trees.push(make_ident("foo")), Some(MetaVarKind::Path) => token_trees.push(make_ident("foo")),
Some(MetaVarKind::Literal) => parent.token_trees.push(make_literal("1")), Some(MetaVarKind::Literal) => token_trees.push(make_literal("1")),
Some(MetaVarKind::Expr) => parent.token_trees.push(make_ident("foo")), Some(MetaVarKind::Expr) => token_trees.push(make_ident("foo")),
Some(MetaVarKind::Lifetime) => { Some(MetaVarKind::Lifetime) => {
parent.token_trees.push(make_punct('\'')); token_trees.push(make_punct('\''));
parent.token_trees.push(make_ident("a")); token_trees.push(make_ident("a"));
} }
Some(MetaVarKind::Block) => { Some(MetaVarKind::Block) => {
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None)) token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
} }
Some(MetaVarKind::Item) => { Some(MetaVarKind::Item) => {
parent.token_trees.push(make_ident("fn")); token_trees.push(make_ident("fn"));
parent.token_trees.push(make_ident("foo")); token_trees.push(make_ident("foo"));
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None)); token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None)); token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
} }
Some(MetaVarKind::Meta) => { Some(MetaVarKind::Meta) => {
parent.token_trees.push(make_ident("foo")); token_trees.push(make_ident("foo"));
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None)); token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
} }
None => (), None => (),
Some(kind) => panic!("Unhandled kind {kind:?}"), Some(kind) => panic!("Unhandled kind {kind:?}"),
}, },
Op::Literal(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()), Op::Literal(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()), Op::Ident(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
Op::Punct(puncts) => { Op::Punct(puncts) => {
for punct in puncts { for punct in puncts {
parent.token_trees.push(tt::Leaf::from(*punct).into()); token_trees.push(tt::Leaf::from(*punct).into());
} }
} }
Op::Repeat { tokens, kind, separator } => { Op::Repeat { tokens, kind, separator } => {
@ -177,20 +179,20 @@ fn invocation_fixtures(
}; };
for i in 0..cnt { for i in 0..cnt {
for it in tokens.iter() { for it in tokens.iter() {
collect_from_op(it, parent, seed); collect_from_op(it, token_trees, seed);
} }
if i + 1 != cnt { if i + 1 != cnt {
if let Some(sep) = separator { if let Some(sep) = separator {
match sep { match sep {
Separator::Literal(it) => { Separator::Literal(it) => {
parent.token_trees.push(tt::Leaf::Literal(it.clone()).into()) token_trees.push(tt::Leaf::Literal(it.clone()).into())
} }
Separator::Ident(it) => { Separator::Ident(it) => {
parent.token_trees.push(tt::Leaf::Ident(it.clone()).into()) token_trees.push(tt::Leaf::Ident(it.clone()).into())
} }
Separator::Puncts(puncts) => { Separator::Puncts(puncts) => {
for it in puncts { for it in puncts {
parent.token_trees.push(tt::Leaf::Punct(*it).into()) token_trees.push(tt::Leaf::Punct(*it).into())
} }
} }
}; };
@ -199,11 +201,15 @@ fn invocation_fixtures(
} }
} }
Op::Subtree { tokens, delimiter } => { Op::Subtree { tokens, delimiter } => {
let mut subtree = tt::Subtree { delimiter: *delimiter, token_trees: Vec::new() }; let mut subtree = Vec::new();
tokens.iter().for_each(|it| { tokens.iter().for_each(|it| {
collect_from_op(it, &mut subtree, seed); collect_from_op(it, &mut subtree, seed);
}); });
parent.token_trees.push(subtree.into());
let subtree =
tt::Subtree { delimiter: *delimiter, token_trees: subtree.into_boxed_slice() };
token_trees.push(subtree.into());
} }
Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Length { .. } => {} Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Length { .. } => {}
}; };
@ -230,7 +236,7 @@ fn invocation_fixtures(
) -> tt::TokenTree<DummyTestSpanData> { ) -> tt::TokenTree<DummyTestSpanData> {
tt::Subtree { tt::Subtree {
delimiter: tt::Delimiter { open: DUMMY, close: DUMMY, kind }, delimiter: tt::Delimiter { open: DUMMY, close: DUMMY, kind },
token_trees: token_trees.unwrap_or_default(), token_trees: token_trees.map(Vec::into_boxed_slice).unwrap_or_default(),
} }
.into() .into()
} }

2
crates/mbe/src/expander.rs
View file

@ -58,7 +58,7 @@ pub(crate) fn expand_rules<S: Span>(
ExpandResult::new( ExpandResult::new(
tt::Subtree { tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(call_site), delimiter: tt::Delimiter::invisible_spanned(call_site),
token_trees: vec![], token_trees: Box::new([]),
}, },
ExpandError::NoMatchingRule, ExpandError::NoMatchingRule,
) )

8
crates/mbe/src/expander/matcher.rs
View file

@ -798,7 +798,7 @@ fn match_meta_var<S: Span>(
tt.map(|tt| match tt { tt.map(|tt| match tt {
tt::TokenTree::Leaf(leaf) => tt::Subtree { tt::TokenTree::Leaf(leaf) => tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(*leaf.span()), delimiter: tt::Delimiter::invisible_spanned(*leaf.span()),
token_trees: vec![leaf.into()], token_trees: Box::new([leaf.into()]),
}, },
tt::TokenTree::Subtree(mut s) => { tt::TokenTree::Subtree(mut s) => {
if s.delimiter.kind == tt::DelimiterKind::Invisible { if s.delimiter.kind == tt::DelimiterKind::Invisible {
@ -832,7 +832,7 @@ fn match_meta_var<S: Span>(
None => lit.into(), None => lit.into(),
Some(neg) => tt::TokenTree::Subtree(tt::Subtree { Some(neg) => tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(*literal.span()), delimiter: tt::Delimiter::invisible_spanned(*literal.span()),
token_trees: vec![neg, lit.into()], token_trees: Box::new([neg, lit.into()]),
}), }),
} }
}) })
@ -989,10 +989,10 @@ impl<S: Span> TtIter<'_, S> {
close: ident.span, close: ident.span,
kind: tt::DelimiterKind::Invisible, kind: tt::DelimiterKind::Invisible,
}, },
token_trees: vec![ token_trees: Box::new([
tt::Leaf::Punct(*punct).into(), tt::Leaf::Punct(*punct).into(),
tt::Leaf::Ident(ident.clone()).into(), tt::Leaf::Ident(ident.clone()).into(),
], ]),
} }
.into()) .into())
} }

14
crates/mbe/src/expander/transcriber.rs
View file

@ -83,7 +83,7 @@ impl<S: Span> Bindings<S> {
close: span, close: span,
kind: tt::DelimiterKind::Brace, kind: tt::DelimiterKind::Brace,
}, },
token_trees: vec![], token_trees: Box::new([]),
})), })),
// FIXME: Meta and Item should get proper defaults // FIXME: Meta and Item should get proper defaults
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
@ -349,11 +349,11 @@ fn expand_var<S: Span>(
// We just treat it a normal tokens // We just treat it a normal tokens
let tt = tt::Subtree { let tt = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(id), delimiter: tt::Delimiter::invisible_spanned(id),
token_trees: vec![ token_trees: Box::new([
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id })
.into(), .into(),
tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(),
], ]),
} }
.into(); .into();
ExpandResult::ok(Fragment::Tokens(tt)) ExpandResult::ok(Fragment::Tokens(tt))
@ -406,7 +406,7 @@ fn expand_repeat<S: Span>(
value: Fragment::Tokens( value: Fragment::Tokens(
tt::Subtree { tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(ctx.call_site), delimiter: tt::Delimiter::invisible_spanned(ctx.call_site),
token_trees: vec![], token_trees: Box::new([]),
} }
.into(), .into(),
), ),
@ -455,7 +455,7 @@ fn expand_repeat<S: Span>(
// e.g {Delimiter:None> ['>'] /Delimiter:None>} // e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { let tt = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(ctx.call_site), delimiter: tt::Delimiter::invisible_spanned(ctx.call_site),
token_trees: buf, token_trees: buf.into_boxed_slice(),
} }
.into(); .into();
@ -486,7 +486,7 @@ fn push_fragment<S: Span>(
fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) { fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) {
match tt.delimiter.kind { match tt.delimiter.kind {
tt::DelimiterKind::Invisible => buf.extend(tt.token_trees), tt::DelimiterKind::Invisible => buf.extend(Vec::from(tt.token_trees)),
_ => buf.push(tt.into()), _ => buf.push(tt.into()),
} }
} }
@ -504,7 +504,7 @@ fn fix_up_and_push_path_tt<S: Span>(
// Note that we only need to fix up the top-level `TokenTree`s because the // Note that we only need to fix up the top-level `TokenTree`s because the
// context of the paths in the descendant `Subtree`s won't be changed by the // context of the paths in the descendant `Subtree`s won't be changed by the
// mbe transcription. // mbe transcription.
for tt in subtree.token_trees { for tt in Vec::from(subtree.token_trees) {
if prev_was_ident { if prev_was_ident {
// Pedantically, `(T) -> U` in `FnOnce(T) -> U` is treated as a generic // Pedantically, `(T) -> U` in `FnOnce(T) -> U` is treated as a generic
// argument list and thus needs `::` between it and `FnOnce`. However in // argument list and thus needs `::` between it and `FnOnce`. However in

14
crates/mbe/src/syntax_bridge.rs
View file

@ -130,7 +130,7 @@ where
tt::Subtree { tt::Subtree {
delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. }, delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. },
token_trees, token_trees,
} => TokenBuffer::from_tokens(token_trees.as_slice()), } => TokenBuffer::from_tokens(token_trees),
_ => TokenBuffer::from_subtree(tt), _ => TokenBuffer::from_subtree(tt),
}; };
let parser_input = to_parser_input(&buffer); let parser_input = to_parser_input(&buffer);
@ -227,14 +227,14 @@ where
C: TokenConverter<S>, C: TokenConverter<S>,
S: Span, S: Span,
{ {
let entry = tt::Subtree { let entry = tt::SubtreeBuilder {
delimiter: tt::Delimiter::invisible_spanned(conv.call_site()), delimiter: tt::Delimiter::invisible_spanned(conv.call_site()),
token_trees: vec![], token_trees: vec![],
}; };
let mut stack = NonEmptyVec::new(entry); let mut stack = NonEmptyVec::new(entry);
while let Some((token, abs_range)) = conv.bump() { while let Some((token, abs_range)) = conv.bump() {
let tt::Subtree { delimiter, token_trees } = stack.last_mut(); let tt::SubtreeBuilder { delimiter, token_trees } = stack.last_mut();
let tt = match token.as_leaf() { let tt = match token.as_leaf() {
Some(leaf) => tt::TokenTree::Leaf(leaf.clone()), Some(leaf) => tt::TokenTree::Leaf(leaf.clone()),
@ -260,7 +260,7 @@ where
if matches!(expected, Some(expected) if expected == kind) { if matches!(expected, Some(expected) if expected == kind) {
if let Some(mut subtree) = stack.pop() { if let Some(mut subtree) = stack.pop() {
subtree.delimiter.close = conv.span_for(abs_range); subtree.delimiter.close = conv.span_for(abs_range);
stack.last_mut().token_trees.push(subtree.into()); stack.last_mut().token_trees.push(subtree.build().into());
} }
continue; continue;
} }
@ -275,7 +275,7 @@ where
// Start a new subtree // Start a new subtree
if let Some(kind) = delim { if let Some(kind) = delim {
let open = conv.span_for(abs_range); let open = conv.span_for(abs_range);
stack.push(tt::Subtree { stack.push(tt::SubtreeBuilder {
delimiter: tt::Delimiter { delimiter: tt::Delimiter {
open, open,
// will be overwritten on subtree close above // will be overwritten on subtree close above
@ -361,7 +361,7 @@ where
parent.token_trees.extend(entry.token_trees); parent.token_trees.extend(entry.token_trees);
} }
let subtree = stack.into_last(); let subtree = stack.into_last().build();
if let [tt::TokenTree::Subtree(first)] = &*subtree.token_trees { if let [tt::TokenTree::Subtree(first)] = &*subtree.token_trees {
first.clone() first.clone()
} else { } else {
@ -454,7 +454,7 @@ fn convert_doc_comment<S: Copy>(
}; };
// Make `doc="\" Comments\"" // Make `doc="\" Comments\""
let meta_tkns = vec![mk_ident("doc"), mk_punct('='), mk_doc_literal(&comment)]; let meta_tkns = Box::new([mk_ident("doc"), mk_punct('='), mk_doc_literal(&comment)]);
// Make `#![]` // Make `#![]`
let mut token_trees = Vec::with_capacity(3); let mut token_trees = Vec::with_capacity(3);

2
crates/mbe/src/tt_iter.rs
View file

@ -180,7 +180,7 @@ impl<'a, S: Span> TtIter<'a, S> {
[] | [_] => res.pop(), [] | [_] => res.pop(),
[first, ..] => Some(tt::TokenTree::Subtree(tt::Subtree { [first, ..] => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(first.first_span()), delimiter: tt::Delimiter::invisible_spanned(first.first_span()),
token_trees: res, token_trees: res.into_boxed_slice(),
})), })),
}; };
ExpandResult { value: res, err } ExpandResult { value: res, err }

59
crates/proc-macro-api/src/msg.rs
View file

@ -187,23 +187,9 @@ mod tests {
file_id: FileId::from_raw(0), file_id: FileId::from_raw(0),
ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)), ast_id: ErasedFileAstId::from_raw(RawIdx::from(0)),
}; };
let mut subtree = Subtree {
delimiter: Delimiter { let token_trees = Box::new([
open: Span { TokenTree::Leaf(
range: TextRange::empty(TextSize::new(0)),
anchor,
ctx: SyntaxContextId::ROOT,
},
close: Span {
range: TextRange::empty(TextSize::new(13)),
anchor,
ctx: SyntaxContextId::ROOT,
},
kind: DelimiterKind::Invisible,
},
token_trees: Vec::new(),
};
subtree.token_trees.push(TokenTree::Leaf(
Ident { Ident {
text: "struct".into(), text: "struct".into(),
span: Span { span: Span {
@ -213,8 +199,8 @@ mod tests {
}, },
} }
.into(), .into(),
)); ),
subtree.token_trees.push(TokenTree::Leaf( TokenTree::Leaf(
Ident { Ident {
text: "Foo".into(), text: "Foo".into(),
span: Span { span: Span {
@ -224,8 +210,8 @@ mod tests {
}, },
} }
.into(), .into(),
)); ),
subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal { TokenTree::Leaf(Leaf::Literal(Literal {
text: "Foo".into(), text: "Foo".into(),
span: Span { span: Span {
@ -233,8 +219,8 @@ mod tests {
anchor, anchor,
ctx: SyntaxContextId::ROOT, ctx: SyntaxContextId::ROOT,
}, },
}))); })),
subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct { TokenTree::Leaf(Leaf::Punct(Punct {
char: '@', char: '@',
span: Span { span: Span {
range: TextRange::at(TextSize::new(11), TextSize::of('@')), range: TextRange::at(TextSize::new(11), TextSize::of('@')),
@ -242,8 +228,8 @@ mod tests {
ctx: SyntaxContextId::ROOT, ctx: SyntaxContextId::ROOT,
}, },
spacing: Spacing::Joint, spacing: Spacing::Joint,
}))); })),
subtree.token_trees.push(TokenTree::Subtree(Subtree { TokenTree::Subtree(Subtree {
delimiter: Delimiter { delimiter: Delimiter {
open: Span { open: Span {
range: TextRange::at(TextSize::new(12), TextSize::of('{')), range: TextRange::at(TextSize::new(12), TextSize::of('{')),
@ -257,9 +243,26 @@ mod tests {
}, },
kind: DelimiterKind::Brace, kind: DelimiterKind::Brace,
}, },
token_trees: vec![], token_trees: Box::new([]),
})); }),
subtree ]);
Subtree {
delimiter: Delimiter {
open: Span {
range: TextRange::empty(TextSize::new(0)),
anchor,
ctx: SyntaxContextId::ROOT,
},
close: Span {
range: TextRange::empty(TextSize::new(13)),
anchor,
ctx: SyntaxContextId::ROOT,
},
kind: DelimiterKind::Invisible,
},
token_trees,
}
} }
#[test] #[test]

2
crates/proc-macro-api/src/msg/flat.rs
View file

@ -370,7 +370,7 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> {
self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32];
for child in &subtree.token_trees { for child in subtree.token_trees.iter() {
let idx_tag = match child { let idx_tag = match child {
tt::TokenTree::Subtree(it) => { tt::TokenTree::Subtree(it) => {
let idx = self.enqueue(it); let idx = self.enqueue(it);

4
crates/proc-macro-srv/src/server/rust_analyzer_span.rs
View file

@ -104,7 +104,7 @@ impl server::TokenStream for RaSpanServer {
delimiter: delim_to_internal(group.delimiter, group.span), delimiter: delim_to_internal(group.delimiter, group.span),
token_trees: match group.stream { token_trees: match group.stream {
Some(stream) => stream.into_iter().collect(), Some(stream) => stream.into_iter().collect(),
None => Vec::new(), None => Box::new([]),
}, },
}; };
let tree = tt::TokenTree::from(group); let tree = tt::TokenTree::from(group);
@ -221,7 +221,7 @@ impl server::TokenStream for RaSpanServer {
stream: if subtree.token_trees.is_empty() { stream: if subtree.token_trees.is_empty() {
None None
} else { } else {
Some(subtree.token_trees.into_iter().collect()) Some(subtree.token_trees.into_vec().into_iter().collect())
}, },
span: bridge::DelimSpan::from_single(subtree.delimiter.open), span: bridge::DelimSpan::from_single(subtree.delimiter.open),
}), }),

10
crates/proc-macro-srv/src/server/token_id.rs
View file

@ -94,7 +94,7 @@ impl server::TokenStream for TokenIdServer {
delimiter: delim_to_internal(group.delimiter, group.span), delimiter: delim_to_internal(group.delimiter, group.span),
token_trees: match group.stream { token_trees: match group.stream {
Some(stream) => stream.into_iter().collect(), Some(stream) => stream.into_iter().collect(),
None => Vec::new(), None => Box::new([]),
}, },
}; };
let tree = TokenTree::from(group); let tree = TokenTree::from(group);
@ -206,7 +206,7 @@ impl server::TokenStream for TokenIdServer {
stream: if subtree.token_trees.is_empty() { stream: if subtree.token_trees.is_empty() {
None None
} else { } else {
Some(TokenStream { token_trees: subtree.token_trees }) Some(TokenStream { token_trees: subtree.token_trees.into_vec() })
}, },
span: bridge::DelimSpan::from_single(subtree.delimiter.open), span: bridge::DelimSpan::from_single(subtree.delimiter.open),
}), }),
@ -338,7 +338,7 @@ mod tests {
close: tt::TokenId(0), close: tt::TokenId(0),
kind: tt::DelimiterKind::Brace, kind: tt::DelimiterKind::Brace,
}, },
token_trees: vec![], token_trees: Box::new([]),
}), }),
], ],
}; };
@ -354,10 +354,10 @@ mod tests {
close: tt::TokenId(0), close: tt::TokenId(0),
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}, },
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(), text: "a".into(),
span: tt::TokenId(0), span: tt::TokenId(0),
}))], }))]),
}); });
let t1 = TokenStream::from_str("(a)", tt::TokenId(0)).unwrap(); let t1 = TokenStream::from_str("(a)", tt::TokenId(0)).unwrap();

8
crates/proc-macro-srv/src/server/token_stream.rs
View file

@ -15,14 +15,14 @@ impl<S> Default for TokenStream<S> {
impl<S> TokenStream<S> { impl<S> TokenStream<S> {
pub(crate) fn new() -> Self { pub(crate) fn new() -> Self {
TokenStream { token_trees: vec![] } TokenStream::default()
} }
pub(crate) fn with_subtree(subtree: tt::Subtree<S>) -> Self { pub(crate) fn with_subtree(subtree: tt::Subtree<S>) -> Self {
if subtree.delimiter.kind != tt::DelimiterKind::Invisible { if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else { } else {
TokenStream { token_trees: subtree.token_trees } TokenStream { token_trees: subtree.token_trees.into_vec() }
} }
} }
@ -36,7 +36,7 @@ impl<S> TokenStream<S> {
close: call_site, close: call_site,
kind: tt::DelimiterKind::Invisible, kind: tt::DelimiterKind::Invisible,
}, },
token_trees: self.token_trees, token_trees: self.token_trees.into_boxed_slice(),
} }
} }
@ -83,7 +83,7 @@ impl<S> Extend<TokenStream<S>> for TokenStream<S> {
tt::TokenTree::Subtree(subtree) tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind == tt::DelimiterKind::Invisible => if subtree.delimiter.kind == tt::DelimiterKind::Invisible =>
{ {
self.token_trees.extend(subtree.token_trees); self.token_trees.extend(subtree.token_trees.into_vec().into_iter());
} }
_ => { _ => {
self.token_trees.push(tkn); self.token_trees.push(tkn);

2
crates/test-fixture/src/lib.rs
View file

@ -598,7 +598,7 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
}; };
token_trees.push(tt); token_trees.push(tt);
} }
Subtree { delimiter: input.delimiter, token_trees } Subtree { delimiter: input.delimiter, token_trees: token_trees.into_boxed_slice() }
} }
Ok(traverse(input)) Ok(traverse(input))
} }

2
crates/tt/src/buffer.rs
View file

@ -106,7 +106,7 @@ impl<'t, Span> TokenBuffer<'t, Span> {
for (child_idx, (subtree, tt)) in children { for (child_idx, (subtree, tt)) in children {
let idx = TokenBuffer::new_inner( let idx = TokenBuffer::new_inner(
subtree.token_trees.as_slice(), &*subtree.token_trees,
buffers, buffers,
Some(EntryPtr(EntryId(res), child_idx + 1)), Some(EntryPtr(EntryId(res), child_idx + 1)),
); );

28
crates/tt/src/lib.rs
View file

@ -23,10 +23,10 @@ pub enum TokenTree<S> {
} }
impl_from!(Leaf<S>, Subtree<S> for TokenTree); impl_from!(Leaf<S>, Subtree<S> for TokenTree);
impl<S: Span> TokenTree<S> { impl<S: Span> TokenTree<S> {
pub const fn empty(span: S) -> Self { pub fn empty(span: S) -> Self {
Self::Subtree(Subtree { Self::Subtree(Subtree {
delimiter: Delimiter::invisible_spanned(span), delimiter: Delimiter::invisible_spanned(span),
token_trees: vec![], token_trees: Box::new([]),
}) })
} }
@ -34,7 +34,7 @@ impl<S: Span> TokenTree<S> {
match self { match self {
TokenTree::Leaf(_) => Subtree { TokenTree::Leaf(_) => Subtree {
delimiter: Delimiter::invisible_delim_spanned(span), delimiter: Delimiter::invisible_delim_spanned(span),
token_trees: vec![self], token_trees: Box::new([self]),
}, },
TokenTree::Subtree(s) => s, TokenTree::Subtree(s) => s,
} }
@ -69,12 +69,12 @@ impl_from!(Literal<S>, Punct<S>, Ident<S> for Leaf);
#[derive(Clone, PartialEq, Eq, Hash)] #[derive(Clone, PartialEq, Eq, Hash)]
pub struct Subtree<S> { pub struct Subtree<S> {
pub delimiter: Delimiter<S>, pub delimiter: Delimiter<S>,
pub token_trees: Vec<TokenTree<S>>, pub token_trees: Box<[TokenTree<S>]>,
} }
impl<S: Span> Subtree<S> { impl<S: Span> Subtree<S> {
pub const fn empty(span: DelimSpan<S>) -> Self { pub fn empty(span: DelimSpan<S>) -> Self {
Subtree { delimiter: Delimiter::invisible_delim_spanned(span), token_trees: vec![] } Subtree { delimiter: Delimiter::invisible_delim_spanned(span), token_trees: Box::new([]) }
} }
pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) { pub fn visit_ids(&mut self, f: &mut impl FnMut(S) -> S) {
@ -91,6 +91,18 @@ impl<S: Span> Subtree<S> {
} }
} }
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct SubtreeBuilder<S> {
pub delimiter: Delimiter<S>,
pub token_trees: Vec<TokenTree<S>>,
}
impl<S> SubtreeBuilder<S> {
pub fn build(self) -> Subtree<S> {
Subtree { delimiter: self.delimiter, token_trees: self.token_trees.into_boxed_slice() }
}
}
#[derive(Debug, Copy, Clone, PartialEq)] #[derive(Debug, Copy, Clone, PartialEq)]
pub struct DelimSpan<S> { pub struct DelimSpan<S> {
pub open: S, pub open: S,
@ -241,7 +253,7 @@ impl<S> fmt::Display for Subtree<S> {
}; };
f.write_str(l)?; f.write_str(l)?;
let mut needs_space = false; let mut needs_space = false;
for tt in &self.token_trees { for tt in self.token_trees.iter() {
if needs_space { if needs_space {
f.write_str(" ")?; f.write_str(" ")?;
} }
@ -316,7 +328,7 @@ impl<S> Subtree<S> {
let mut res = String::new(); let mut res = String::new();
res.push_str(delim.0); res.push_str(delim.0);
let mut last = None; let mut last = None;
for child in &self.token_trees { for child in self.token_trees.iter() {
let s = match child { let s = match child {
TokenTree::Leaf(it) => { TokenTree::Leaf(it) => {
let s = match it { let s = match it {