Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-26 21:13:37 +00:00
Code Issues Projects Releases Packages Wiki Activity

Make tt generic over the span data

Browse source
This commit is contained in:
Lukas Wirth 2023-01-31 11:49:49 +01:00
parent d805c74c51
commit 41a46a78f2
48 changed files with 806 additions and 569 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
crates/base-db/src/fixture.rs
View file

@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
use test_utils::{ use test_utils::{
extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER, extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
}; };
use tt::Subtree; use tt::token_id::Subtree;
use vfs::{file_set::FileSet, VfsPath}; use vfs::{file_set::FileSet, VfsPath};
use crate::{ use crate::{
@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
_: &Env, _: &Env,
) -> Result<Subtree, ProcMacroExpansionError> { ) -> Result<Subtree, ProcMacroExpansionError> {
fn traverse(input: &Subtree) -> Subtree { fn traverse(input: &Subtree) -> Subtree {
let mut res = Subtree::default(); let mut token_trees = vec![];
res.delimiter = input.delimiter;
for tt in input.token_trees.iter().rev() { for tt in input.token_trees.iter().rev() {
let tt = match tt { let tt = match tt {
tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()), tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)), tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
}; };
res.token_trees.push(tt); token_trees.push(tt);
} }
res Subtree { delimiter: input.delimiter, token_trees }
} }
Ok(traverse(input)) Ok(traverse(input))
} }

2
crates/base-db/src/input.rs
View file

@ -12,7 +12,7 @@ use cfg::CfgOptions;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use stdx::hash::{NoHashHashMap, NoHashHashSet}; use stdx::hash::{NoHashHashMap, NoHashHashSet};
use syntax::SmolStr; use syntax::SmolStr;
use tt::Subtree; use tt::token_id::Subtree;
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath}; use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
/// Files are grouped into source roots. A source root is a directory on the /// Files are grouped into source roots. A source root is a directory on the

4
crates/cfg/src/cfg_expr.rs
View file

@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
} }
impl CfgExpr { impl CfgExpr {
pub fn parse(tt: &tt::Subtree) -> CfgExpr { pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid) next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
} }
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates. /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@ -85,7 +85,7 @@ impl CfgExpr {
} }
} }
fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> { fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
let name = match it.next() { let name = match it.next() {
None => return None, None => return None,
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

4
crates/hir-def/src/adt.rs
View file

@ -2,6 +2,7 @@
use std::sync::Arc; use std::sync::Arc;
use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use base_db::CrateId; use base_db::CrateId;
use either::Either; use either::Either;
use hir_expand::{ use hir_expand::{
@ -12,7 +13,6 @@ use intern::Interned;
use la_arena::{Arena, ArenaMap}; use la_arena::{Arena, ArenaMap};
use rustc_abi::{Integer, IntegerType}; use rustc_abi::{Integer, IntegerType};
use syntax::ast::{self, HasName, HasVisibility}; use syntax::ast::{self, HasName, HasVisibility};
use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use crate::{ use crate::{
body::{CfgExpander, LowerCtx}, body::{CfgExpander, LowerCtx},
@ -82,7 +82,7 @@ fn repr_from_value(
fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> { fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
match tt.delimiter { match tt.delimiter {
Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {} Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
_ => return None, _ => return None,
} }

5
crates/hir-def/src/attr.rs
View file

@ -16,7 +16,6 @@ use syntax::{
ast::{self, HasAttrs, IsString}, ast::{self, HasAttrs, IsString},
AstPtr, AstToken, SmolStr, TextRange, TextSize, AstPtr, AstToken, SmolStr, TextRange, TextSize,
}; };
use tt::Subtree;
use crate::{ use crate::{
db::DefDatabase, db::DefDatabase,
@ -234,7 +233,7 @@ impl Attrs {
pub fn has_doc_hidden(&self) -> bool { pub fn has_doc_hidden(&self) -> bool {
self.by_key("doc").tt_values().any(|tt| { self.by_key("doc").tt_values().any(|tt| {
tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) && tt.delimiter.kind == DelimiterKind::Parenthesis &&
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden") matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
}) })
} }
@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
} }
impl<'attr> AttrQuery<'attr> { impl<'attr> AttrQuery<'attr> {
pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> { pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
self.attrs().filter_map(|attr| attr.token_tree_value()) self.attrs().filter_map(|attr| attr.token_tree_value())
} }

2
crates/hir-def/src/data.rs
View file

@ -142,7 +142,7 @@ impl FunctionData {
} }
} }
fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> { fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
let mut indices = Vec::new(); let mut indices = Vec::new();
for args in tt.token_trees.chunks(2) { for args in tt.token_trees.chunks(2) {
match &args[0] { match &args[0] {

23
crates/hir-def/src/lib.rs
View file

@ -79,6 +79,8 @@ use nameres::DefMap;
use stdx::impl_from; use stdx::impl_from;
use syntax::ast; use syntax::ast;
use ::tt::token_id as tt;
use crate::{ use crate::{
adt::VariantData, adt::VariantData,
builtin_type::BuiltinType, builtin_type::BuiltinType,
@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
def: MacroDefId, def: MacroDefId,
is_derive: bool, is_derive: bool,
) -> MacroCallId { ) -> MacroCallId {
let mut arg = match macro_attr.input.as_deref() { let arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()), Some(AttrInput::TokenTree(tt, map)) => (
_ => Default::default(), {
let mut tt = tt.clone();
tt.delimiter = tt::Delimiter::UNSPECIFIED;
tt
},
map.clone(),
),
_ => (tt::Subtree::empty(), Default::default()),
}; };
// The parentheses are always disposed here. def.as_lazy_macro(
arg.0.delimiter = None;
let res = def.as_lazy_macro(
db.upcast(), db.upcast(),
krate, krate,
MacroCallKind::Attr { MacroCallKind::Attr {
@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
invoc_attr_index: macro_attr.id, invoc_attr_index: macro_attr.id,
is_derive, is_derive,
}, },
); )
res
} }
intern::impl_internable!( intern::impl_internable!(
crate::type_ref::TypeRef, crate::type_ref::TypeRef,

8
crates/hir-def/src/macro_expansion_tests.rs
View file

@ -30,7 +30,7 @@ use syntax::{
SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT}, SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
SyntaxNode, TextRange, T, SyntaxNode, TextRange, T,
}; };
use tt::{Subtree, TokenId}; use tt::token_id::{Subtree, TokenId};
use crate::{ use crate::{
db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver, db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
tree.token_trees.iter().for_each(|tree| match tree { tree.token_trees.iter().for_each(|tree| match tree {
tt::TokenTree::Leaf(leaf) => { tt::TokenTree::Leaf(leaf) => {
let id = match leaf { let id = match leaf {
tt::Leaf::Literal(it) => it.id, tt::Leaf::Literal(it) => it.span,
tt::Leaf::Punct(it) => it.id, tt::Leaf::Punct(it) => it.span,
tt::Leaf::Ident(it) => it.id, tt::Leaf::Ident(it) => it.span,
}; };
ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id))); ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
} }

12
crates/hir-def/src/nameres/collector.rs
View file

@ -46,6 +46,7 @@ use crate::{
}, },
path::{ImportAlias, ModPath, PathKind}, path::{ImportAlias, ModPath, PathKind},
per_ns::PerNs, per_ns::PerNs,
tt,
visibility::{RawVisibility, Visibility}, visibility::{RawVisibility, Visibility},
AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId, AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc, FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
.enumerate() .enumerate()
.map(|(idx, it)| { .map(|(idx, it)| {
// FIXME: a hacky way to create a Name from string. // FIXME: a hacky way to create a Name from string.
let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() }; let name =
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
( (
name.as_name(), name.as_name(),
ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)), ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@ -451,7 +453,10 @@ impl DefCollector<'_> {
directive.module_id, directive.module_id,
MacroCallKind::Attr { MacroCallKind::Attr {
ast_id: ast_id.ast_id, ast_id: ast_id.ast_id,
attr_args: Default::default(), attr_args: std::sync::Arc::new((
tt::Subtree::empty(),
Default::default(),
)),
invoc_attr_index: attr.id, invoc_attr_index: attr.id,
is_derive: false, is_derive: false,
}, },
@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
let name = match attrs.by_key("rustc_builtin_macro").string_value() { let name = match attrs.by_key("rustc_builtin_macro").string_value() {
Some(it) => { Some(it) => {
// FIXME: a hacky way to create a Name from string. // FIXME: a hacky way to create a Name from string.
name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name(); name =
tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
&name &name
} }
None => { None => {

2
crates/hir-def/src/nameres/proc_macro.rs
View file

@ -1,9 +1,9 @@
//! Nameres-specific procedural macro data and helpers. //! Nameres-specific procedural macro data and helpers.
use hir_expand::name::{AsName, Name}; use hir_expand::name::{AsName, Name};
use tt::{Leaf, TokenTree};
use crate::attr::Attrs; use crate::attr::Attrs;
use crate::tt::{Leaf, TokenTree};
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub struct ProcMacroDef { pub struct ProcMacroDef {

9
crates/hir-expand/src/attrs.rs
View file

@ -8,13 +8,13 @@ use intern::Interned;
use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct}; use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode}; use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
use tt::Subtree;
use crate::{ use crate::{
db::AstDatabase, db::AstDatabase,
hygiene::Hygiene, hygiene::Hygiene,
mod_path::{ModPath, PathKind}, mod_path::{ModPath, PathKind},
name::AsName, name::AsName,
tt::{self, Subtree},
InFile, InFile,
}; };
@ -117,7 +117,10 @@ impl RawAttrs {
let index = attr.id; let index = attr.id;
let attrs = let attrs =
parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| { parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
let tree = Subtree { delimiter: None, token_trees: attr.to_vec() }; let tree = Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: attr.to_vec(),
};
// FIXME hygiene // FIXME hygiene
let hygiene = Hygiene::new_unhygienic(); let hygiene = Hygiene::new_unhygienic();
Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx)) Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
@ -266,7 +269,7 @@ impl Attr {
pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> { pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
let args = self.token_tree_value()?; let args = self.token_tree_value()?;
if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) { if args.delimiter.kind != DelimiterKind::Parenthesis {
return None; return None;
} }
let paths = args let paths = args

6
crates/hir-expand/src/builtin_attr_macro.rs
View file

@ -1,6 +1,6 @@
//! Builtin attributes. //! Builtin attributes.
use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind}; use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
macro_rules! register_builtin { macro_rules! register_builtin {
( $(($name:ident, $variant:ident) => $expand:ident),* ) => { ( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@ -97,7 +97,7 @@ fn derive_attr_expand(
let loc = db.lookup_intern_macro_call(id); let loc = db.lookup_intern_macro_call(id);
let derives = match &loc.kind { let derives = match &loc.kind {
MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0, MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
_ => return ExpandResult::ok(Default::default()), _ => return ExpandResult::ok(tt::Subtree::empty()),
}; };
pseudo_derive_attr_expansion(tt, derives) pseudo_derive_attr_expansion(tt, derives)
} }
@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char, char,
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})) }))
}; };

10
crates/hir-expand/src/builtin_derive_macro.rs
View file

@ -3,11 +3,11 @@
use base_db::{CrateOrigin, LangCrateOrigin}; use base_db::{CrateOrigin, LangCrateOrigin};
use tracing::debug; use tracing::debug;
use crate::tt::{self, TokenId};
use syntax::{ use syntax::{
ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName}, ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
match_ast, match_ast,
}; };
use tt::TokenId;
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
})?; })?;
let name_token_id = let name_token_id =
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified); token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
let name_token = tt::Ident { id: name_token_id, text: name.text().into() }; let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
let param_types = params let param_types = params
.into_iter() .into_iter()
.flat_map(|param_list| param_list.type_or_const_params()) .flat_map(|param_list| param_list.type_or_const_params())
@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
let ty = param let ty = param
.ty() .ty()
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0) .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
.unwrap_or_default(); .unwrap_or_else(tt::Subtree::empty);
Some(ty) Some(ty)
} else { } else {
None None
@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> { fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
let info = match parse_adt(tt) { let info = match parse_adt(tt) {
Ok(info) => info, Ok(info) => info,
Err(e) => return ExpandResult::only_err(e), Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
}; };
let (params, args): (Vec<_>, Vec<_>) = info let (params, args): (Vec<_>, Vec<_>) = info
.param_types .param_types
@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu
.enumerate() .enumerate()
.map(|(idx, param_ty)| { .map(|(idx, param_ty)| {
let ident = tt::Leaf::Ident(tt::Ident { let ident = tt::Leaf::Ident(tt::Ident {
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
text: format!("T{idx}").into(), text: format!("T{idx}").into(),
}); });
let ident_ = ident.clone(); let ident_ = ident.clone();

66
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -9,7 +9,9 @@ use syntax::{
SmolStr, SmolStr,
}; };
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; use crate::{
db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
};
macro_rules! register_builtin { macro_rules! register_builtin {
( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => { ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => {
@ -61,7 +63,7 @@ macro_rules! register_builtin {
}; };
} }
#[derive(Debug, Default)] #[derive(Debug)]
pub struct ExpandedEager { pub struct ExpandedEager {
pub(crate) subtree: tt::Subtree, pub(crate) subtree: tt::Subtree,
/// The included file ID of the include macro. /// The included file ID of the include macro.
@ -116,7 +118,7 @@ register_builtin! {
} }
const DOLLAR_CRATE: tt::Ident = const DOLLAR_CRATE: tt::Ident =
tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() }; tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() };
fn module_path_expand( fn module_path_expand(
_db: &dyn AstDatabase, _db: &dyn AstDatabase,
@ -162,7 +164,7 @@ fn stringify_expand(
_id: MacroCallId, _id: MacroCallId,
tt: &tt::Subtree, tt: &tt::Subtree,
) -> ExpandResult<tt::Subtree> { ) -> ExpandResult<tt::Subtree> {
let pretty = tt::pretty(&tt.token_trees); let pretty = ::tt::pretty(&tt.token_trees);
let expanded = quote! { let expanded = quote! {
#pretty #pretty
@ -194,11 +196,11 @@ fn assert_expand(
let expanded = match &*args { let expanded = match &*args {
[cond, panic_args @ ..] => { [cond, panic_args @ ..] => {
let comma = tt::Subtree { let comma = tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char: ',', char: ',',
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))], }))],
}; };
let cond = cond.clone(); let cond = cond.clone();
@ -247,7 +249,10 @@ fn format_args_expand(
let mut args = parse_exprs_with_sep(tt, ','); let mut args = parse_exprs_with_sep(tt, ',');
if args.is_empty() { if args.is_empty() {
return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into()); return ExpandResult::with_err(
tt::Subtree::empty(),
mbe::ExpandError::NoMatchingRule.into(),
);
} }
for arg in &mut args { for arg in &mut args {
// Remove `key =`. // Remove `key =`.
@ -282,7 +287,7 @@ fn asm_expand(
for tt in tt.token_trees.chunks(2) { for tt in tt.token_trees.chunks(2) {
match tt { match tt {
[tt::TokenTree::Leaf(tt::Leaf::Literal(lit))] [tt::TokenTree::Leaf(tt::Leaf::Literal(lit))]
| [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] => | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] =>
{ {
let krate = DOLLAR_CRATE.clone(); let krate = DOLLAR_CRATE.clone();
literals.push(quote!(#krate::format_args!(#lit);)); literals.push(quote!(#krate::format_args!(#lit);));
@ -400,7 +405,7 @@ fn concat_expand(
// FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses // FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses
// to ensure the right parsing order, so skip the parentheses here. Ideally we'd // to ensure the right parsing order, so skip the parentheses here. Ideally we'd
// implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623 // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623
if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t { if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t {
if let [tt] = &**token_trees { if let [tt] = &**token_trees {
if delim.kind == tt::DelimiterKind::Parenthesis { if delim.kind == tt::DelimiterKind::Parenthesis {
t = tt; t = tt;
@ -459,9 +464,7 @@ fn concat_bytes_expand(
} }
} }
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (), tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (),
tt::TokenTree::Subtree(tree) tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => {
if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) =>
{
if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) { if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) {
err.get_or_insert(e); err.get_or_insert(e);
break; break;
@ -473,7 +476,7 @@ fn concat_bytes_expand(
} }
} }
} }
let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() }; let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!([#ident])), err } ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
} }
@ -521,7 +524,7 @@ fn concat_idents_expand(
} }
} }
} }
let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() }; let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!(#ident)), err } ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
} }
@ -572,7 +575,10 @@ fn include_expand(
Ok((subtree, file_id)) => { Ok((subtree, file_id)) => {
ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) }) ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) })
} }
Err(e) => ExpandResult::only_err(e), Err(e) => ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
),
} }
} }
@ -582,15 +588,18 @@ fn include_bytes_expand(
tt: &tt::Subtree, tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> { ) -> ExpandResult<ExpandedEager> {
if let Err(e) = parse_string(tt) { if let Err(e) = parse_string(tt) {
return ExpandResult::only_err(e); return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
);
} }
// FIXME: actually read the file here if the user asked for macro expansion // FIXME: actually read the file here if the user asked for macro expansion
let res = tt::Subtree { let res = tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: r#"b"""#.into(), text: r#"b"""#.into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))], }))],
}; };
ExpandResult::ok(ExpandedEager::new(res)) ExpandResult::ok(ExpandedEager::new(res))
@ -603,7 +612,12 @@ fn include_str_expand(
) -> ExpandResult<ExpandedEager> { ) -> ExpandResult<ExpandedEager> {
let path = match parse_string(tt) { let path = match parse_string(tt) {
Ok(it) => it, Ok(it) => it,
Err(e) => return ExpandResult::only_err(e), Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
}; };
// FIXME: we're not able to read excluded files (which is most of them because // FIXME: we're not able to read excluded files (which is most of them because
@ -635,7 +649,12 @@ fn env_expand(
) -> ExpandResult<ExpandedEager> { ) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) { let key = match parse_string(tt) {
Ok(it) => it, Ok(it) => it,
Err(e) => return ExpandResult::only_err(e), Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
}; };
let mut err = None; let mut err = None;
@ -666,7 +685,12 @@ fn option_env_expand(
) -> ExpandResult<ExpandedEager> { ) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) { let key = match parse_string(tt) {
Ok(it) => it, Ok(it) => it,
Err(e) => return ExpandResult::only_err(e), Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
}; };
let expanded = match get_env_inner(db, arg_id, &key) { let expanded = match get_env_inner(db, arg_id, &key) {

14
crates/hir-expand/src/db.rs
View file

@ -14,7 +14,7 @@ use syntax::{
use crate::{ use crate::{
ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup, ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup,
hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
}; };
@ -175,7 +175,7 @@ pub fn expand_speculative(
match attr.token_tree() { match attr.token_tree() {
Some(token_tree) => { Some(token_tree) => {
let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax()); let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax());
tree.delimiter = None; tree.delimiter = tt::Delimiter::unspecified();
let shift = mbe::Shift::new(&tt); let shift = mbe::Shift::new(&tt);
shift.shift_all(&mut tree); shift.shift_all(&mut tree);
@ -210,7 +210,7 @@ pub fn expand_speculative(
// Otherwise the expand query will fetch the non speculative attribute args and pass those instead. // Otherwise the expand query will fetch the non speculative attribute args and pass those instead.
let mut speculative_expansion = match loc.def.kind { let mut speculative_expansion = match loc.def.kind {
MacroDefKind::ProcMacro(expander, ..) => { MacroDefKind::ProcMacro(expander, ..) => {
tt.delimiter = None; tt.delimiter = tt::Delimiter::unspecified();
expander.expand(db, loc.krate, &tt, attr_arg.as_ref()) expander.expand(db, loc.krate, &tt, attr_arg.as_ref())
} }
MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => {
@ -316,9 +316,8 @@ fn macro_arg(
if loc.def.is_proc_macro() { if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included // proc macros expect their inputs without parentheses, MBEs expect it with them included
tt.delimiter = None; tt.delimiter = tt::Delimiter::unspecified();
} }
Some(Arc::new((tt, tmap, fixups.undo_info))) Some(Arc::new((tt, tmap, fixups.undo_info)))
} }
@ -479,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult<tt::
let macro_arg = match db.macro_arg(id) { let macro_arg = match db.macro_arg(id) {
Some(it) => it, Some(it) => it,
None => { None => {
return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into())) return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("No arguments for proc-macro".into()),
)
} }
}; };

6
crates/hir-expand/src/eager.rs
View file

@ -108,7 +108,7 @@ pub fn expand_eager_macro(
.value .value
.token_tree() .token_tree()
.map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0) .map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
.unwrap_or_default(); .unwrap_or_else(tt::Subtree::empty);
let ast_map = db.ast_id_map(macro_call.file_id); let ast_map = db.ast_id_map(macro_call.file_id);
let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value)); let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@ -165,9 +165,9 @@ pub fn expand_eager_macro(
} }
} }
fn to_subtree(node: &SyntaxNode) -> tt::Subtree { fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree {
let mut subtree = mbe::syntax_node_to_token_tree(node).0; let mut subtree = mbe::syntax_node_to_token_tree(node).0;
subtree.delimiter = None; subtree.delimiter = crate::tt::Delimiter::unspecified();
subtree subtree
} }

18
crates/hir-expand/src/fixup.rs
View file

@ -9,7 +9,7 @@ use syntax::{
ast::{self, AstNode, HasLoopBody}, ast::{self, AstNode, HasLoopBody},
match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
}; };
use tt::Subtree; use tt::token_id::Subtree;
/// The result of calculating fixes for a syntax node -- a bunch of changes /// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to /// (appending to and replacing nodes), the information that is needed to
@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups(
tt.token_trees = tts tt.token_trees = tts
.into_iter() .into_iter()
.filter(|tt| match tt { .filter(|tt| match tt {
tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID), tt::TokenTree::Leaf(leaf) => {
token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID)
}
tt::TokenTree::Subtree(st) => { tt::TokenTree::Subtree(st) => {
st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID)) token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID)
} }
}) })
.flat_map(|tt| match tt { .flat_map(|tt| match tt {
@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups(
SmallVec::from_const([tt.into()]) SmallVec::from_const([tt.into()])
} }
tt::TokenTree::Leaf(leaf) => { tt::TokenTree::Leaf(leaf) => {
if let Some(id) = token_map.synthetic_token_id(leaf.id()) { if let Some(id) = token_map.synthetic_token_id(*leaf.span()) {
let original = undo_info.original[id.0 as usize].clone(); let original = undo_info.original[id.0 as usize].clone();
if original.delimiter.is_none() { if original.delimiter.kind == tt::DelimiterKind::Invisible {
original.token_trees.into() original.token_trees.into()
} else { } else {
SmallVec::from_const([original.into()]) SmallVec::from_const([original.into()])
@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups(
mod tests { mod tests {
use expect_test::{expect, Expect}; use expect_test::{expect, Expect};
use crate::tt;
use super::reverse_fixups; use super::reverse_fixups;
// The following three functions are only meant to check partial structural equivalence of // The following three functions are only meant to check partial structural equivalence of
@ -341,7 +345,7 @@ mod tests {
} }
fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool { fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind) a.delimiter.kind == b.delimiter.kind
&& a.token_trees.len() == b.token_trees.len() && a.token_trees.len() == b.token_trees.len()
&& a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b)) && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
} }
@ -386,7 +390,7 @@ mod tests {
let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node()); let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
assert!( assert!(
check_subtree_eq(&tt, &original_as_tt), check_subtree_eq(&tt, &original_as_tt),
"different token tree: {tt:?}, {original_as_tt:?}" "different token tree: {tt:?},\n{original_as_tt:?}"
); );
} }

2
crates/hir-expand/src/hygiene.rs
View file

@ -128,7 +128,7 @@ struct HygieneInfo {
attr_input_or_mac_def_start: Option<InFile<TextSize>>, attr_input_or_mac_def_start: Option<InFile<TextSize>>,
macro_def: Arc<TokenExpander>, macro_def: Arc<TokenExpander>,
macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
macro_arg_shift: mbe::Shift, macro_arg_shift: mbe::Shift,
exp_map: Arc<mbe::TokenMap>, exp_map: Arc<mbe::TokenMap>,
} }

2
crates/hir-expand/src/lib.rs
View file

@ -22,6 +22,8 @@ mod fixup;
pub use mbe::{Origin, ValueResult}; pub use mbe::{Origin, ValueResult};
use ::tt::token_id as tt;
use std::{fmt, hash::Hash, iter, sync::Arc}; use std::{fmt, hash::Hash, iter, sync::Arc};
use base_db::{ use base_db::{

2
crates/hir-expand/src/name.rs
View file

@ -191,7 +191,7 @@ impl AsName for ast::NameOrNameRef {
} }
} }
impl AsName for tt::Ident { impl<Span> AsName for tt::Ident<Span> {
fn as_name(&self) -> Name { fn as_name(&self) -> Name {
Name::resolve(&self.text) Name::resolve(&self.text)
} }

24
crates/hir-expand/src/proc_macro.rs
View file

@ -3,7 +3,7 @@
use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind}; use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
use stdx::never; use stdx::never;
use crate::{db::AstDatabase, ExpandError, ExpandResult}; use crate::{db::AstDatabase, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct ProcMacroExpander { pub struct ProcMacroExpander {
@ -39,7 +39,10 @@ impl ProcMacroExpander {
Ok(proc_macros) => proc_macros, Ok(proc_macros) => proc_macros,
Err(_) => { Err(_) => {
never!("Non-dummy expander even though there are no proc macros"); never!("Non-dummy expander even though there are no proc macros");
return ExpandResult::only_err(ExpandError::Other("Internal error".into())); return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
} }
}; };
let proc_macro = match proc_macros.get(id.0 as usize) { let proc_macro = match proc_macros.get(id.0 as usize) {
@ -50,7 +53,10 @@ impl ProcMacroExpander {
proc_macros.len(), proc_macros.len(),
id.0 id.0
); );
return ExpandResult::only_err(ExpandError::Other("Internal error".into())); return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
} }
}; };
@ -69,13 +75,17 @@ impl ProcMacroExpander {
} }
} }
ProcMacroExpansionError::System(text) ProcMacroExpansionError::System(text)
| ProcMacroExpansionError::Panic(text) => { | ProcMacroExpansionError::Panic(text) => ExpandResult::with_err(
ExpandResult::only_err(ExpandError::Other(text.into())) tt::Subtree::empty(),
} ExpandError::Other(text.into()),
),
}, },
} }
} }
None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)), None => ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::UnresolvedProcMacro(self.krate),
),
} }
} }
} }

112
crates/hir-expand/src/quote.rs
View file

@ -9,17 +9,18 @@
#[macro_export] #[macro_export]
macro_rules! __quote { macro_rules! __quote {
() => { () => {
Vec::<tt::TokenTree>::new() Vec::<crate::tt::TokenTree>::new()
}; };
( @SUBTREE $delim:ident $($tt:tt)* ) => { ( @SUBTREE $delim:ident $($tt:tt)* ) => {
{ {
let children = $crate::__quote!($($tt)*); let children = $crate::__quote!($($tt)*);
tt::Subtree { crate::tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: crate::tt::Delimiter {
kind: tt::DelimiterKind::$delim, kind: crate::tt::DelimiterKind::$delim,
id: tt::TokenId::unspecified(), open: crate::tt::TokenId::unspecified(),
}), close: crate::tt::TokenId::unspecified(),
},
token_trees: $crate::quote::IntoTt::to_tokens(children), token_trees: $crate::quote::IntoTt::to_tokens(children),
} }
} }
@ -28,10 +29,10 @@ macro_rules! __quote {
( @PUNCT $first:literal ) => { ( @PUNCT $first:literal ) => {
{ {
vec![ vec![
tt::Leaf::Punct(tt::Punct { crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first, char: $first,
spacing: tt::Spacing::Alone, spacing: crate::tt::Spacing::Alone,
id: tt::TokenId::unspecified(), span: crate::tt::TokenId::unspecified(),
}).into() }).into()
] ]
} }
@ -40,15 +41,15 @@ macro_rules! __quote {
( @PUNCT $first:literal, $sec:literal ) => { ( @PUNCT $first:literal, $sec:literal ) => {
{ {
vec![ vec![
tt::Leaf::Punct(tt::Punct { crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first, char: $first,
spacing: tt::Spacing::Joint, spacing: crate::tt::Spacing::Joint,
id: tt::TokenId::unspecified(), span: crate::tt::TokenId::unspecified(),
}).into(), }).into(),
tt::Leaf::Punct(tt::Punct { crate::tt::Leaf::Punct(crate::tt::Punct {
char: $sec, char: $sec,
spacing: tt::Spacing::Alone, spacing: crate::tt::Spacing::Alone,
id: tt::TokenId::unspecified(), span: crate::tt::TokenId::unspecified(),
}).into() }).into()
] ]
} }
@ -67,7 +68,7 @@ macro_rules! __quote {
( ## $first:ident $($tail:tt)* ) => { ( ## $first:ident $($tail:tt)* ) => {
{ {
let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<tt::TokenTree>>(); let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<crate::tt::TokenTree>>();
let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
tokens.append(&mut tail_tokens); tokens.append(&mut tail_tokens);
tokens tokens
@ -86,9 +87,9 @@ macro_rules! __quote {
// Ident // Ident
( $tt:ident ) => { ( $tt:ident ) => {
vec![ { vec![ {
tt::Leaf::Ident(tt::Ident { crate::tt::Leaf::Ident(crate::tt::Ident {
text: stringify!($tt).into(), text: stringify!($tt).into(),
id: tt::TokenId::unspecified(), span: crate::tt::TokenId::unspecified(),
}).into() }).into()
}] }]
}; };
@ -127,42 +128,42 @@ macro_rules! quote {
} }
pub(crate) trait IntoTt { pub(crate) trait IntoTt {
fn to_subtree(self) -> tt::Subtree; fn to_subtree(self) -> crate::tt::Subtree;
fn to_tokens(self) -> Vec<tt::TokenTree>; fn to_tokens(self) -> Vec<crate::tt::TokenTree>;
} }
impl IntoTt for Vec<tt::TokenTree> { impl IntoTt for Vec<crate::tt::TokenTree> {
fn to_subtree(self) -> tt::Subtree { fn to_subtree(self) -> crate::tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self } crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self }
} }
fn to_tokens(self) -> Vec<tt::TokenTree> { fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
self self
} }
} }
impl IntoTt for tt::Subtree { impl IntoTt for crate::tt::Subtree {
fn to_subtree(self) -> tt::Subtree { fn to_subtree(self) -> crate::tt::Subtree {
self self
} }
fn to_tokens(self) -> Vec<tt::TokenTree> { fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
vec![tt::TokenTree::Subtree(self)] vec![crate::tt::TokenTree::Subtree(self)]
} }
} }
pub(crate) trait ToTokenTree { pub(crate) trait ToTokenTree {
fn to_token(self) -> tt::TokenTree; fn to_token(self) -> crate::tt::TokenTree;
} }
impl ToTokenTree for tt::TokenTree { impl ToTokenTree for crate::tt::TokenTree {
fn to_token(self) -> tt::TokenTree { fn to_token(self) -> crate::tt::TokenTree {
self self
} }
} }
impl ToTokenTree for tt::Subtree { impl ToTokenTree for crate::tt::Subtree {
fn to_token(self) -> tt::TokenTree { fn to_token(self) -> crate::tt::TokenTree {
self.into() self.into()
} }
} }
@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees {
($($ty:ty => $this:ident $im:block);*) => { ($($ty:ty => $this:ident $im:block);*) => {
$( $(
impl ToTokenTree for $ty { impl ToTokenTree for $ty {
fn to_token($this) -> tt::TokenTree { fn to_token($this) -> crate::tt::TokenTree {
let leaf: tt::Leaf = $im.into(); let leaf: crate::tt::Leaf = $im.into();
leaf.into() leaf.into()
} }
} }
impl ToTokenTree for &$ty { impl ToTokenTree for &$ty {
fn to_token($this) -> tt::TokenTree { fn to_token($this) -> crate::tt::TokenTree {
let leaf: tt::Leaf = $im.clone().into(); let leaf: crate::tt::Leaf = $im.clone().into();
leaf.into() leaf.into()
} }
} }
@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees {
} }
impl_to_to_tokentrees! { impl_to_to_tokentrees! {
u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
tt::Leaf => self { self }; crate::tt::Leaf => self { self };
tt::Literal => self { self }; crate::tt::Literal => self { self };
tt::Ident => self { self }; crate::tt::Ident => self { self };
tt::Punct => self { self }; crate::tt::Punct => self { self };
&str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}; &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}};
String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}} String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}
} }
#[cfg(test)] #[cfg(test)]
@ -223,8 +224,8 @@ mod tests {
assert_eq!(quote!(#s).to_string(), "\"hello\""); assert_eq!(quote!(#s).to_string(), "\"hello\"");
} }
fn mk_ident(name: &str) -> tt::Ident { fn mk_ident(name: &str) -> crate::tt::Ident {
tt::Ident { text: name.into(), id: tt::TokenId::unspecified() } crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() }
} }
#[test] #[test]
@ -234,7 +235,7 @@ mod tests {
let quoted = quote!(#a); let quoted = quote!(#a);
assert_eq!(quoted.to_string(), "hello"); assert_eq!(quoted.to_string(), "hello");
let t = format!("{quoted:?}"); let t = format!("{quoted:?}");
assert_eq!(t, "SUBTREE $\n IDENT hello 4294967295"); assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295");
} }
#[test] #[test]
@ -263,11 +264,12 @@ mod tests {
let fields = [mk_ident("name"), mk_ident("id")]; let fields = [mk_ident("name"), mk_ident("id")];
let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees); let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees);
let list = tt::Subtree { let list = crate::tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: crate::tt::Delimiter {
kind: tt::DelimiterKind::Brace, kind: crate::tt::DelimiterKind::Brace,
id: tt::TokenId::unspecified(), open: crate::tt::TokenId::unspecified(),
}), close: crate::tt::TokenId::unspecified(),
},
token_trees: fields.collect(), token_trees: fields.collect(),
}; };

28
crates/mbe/src/benchmark.rs
View file

@ -9,7 +9,7 @@ use test_utils::{bench, bench_fixture, skip_slow_tests};
use crate::{ use crate::{
parser::{MetaVarKind, Op, RepeatKind, Separator}, parser::{MetaVarKind, Op, RepeatKind, Separator},
syntax_node_to_token_tree, DeclarativeMacro, syntax_node_to_token_tree, tt, DeclarativeMacro,
}; };
#[test] #[test]
@ -91,7 +91,14 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
// So we just skip any error cases and try again // So we just skip any error cases and try again
let mut try_cnt = 0; let mut try_cnt = 0;
loop { loop {
let mut subtree = tt::Subtree::default(); let mut subtree = tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Invisible,
},
token_trees: vec![],
};
for op in rule.lhs.iter() { for op in rule.lhs.iter() {
collect_from_op(op, &mut subtree, &mut seed); collect_from_op(op, &mut subtree, &mut seed);
} }
@ -196,12 +203,15 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
*seed *seed
} }
fn make_ident(ident: &str) -> tt::TokenTree { fn make_ident(ident: &str) -> tt::TokenTree {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) }) tt::Leaf::Ident(tt::Ident {
.into() span: tt::TokenId::unspecified(),
text: SmolStr::new(ident),
})
.into()
} }
fn make_punct(char: char) -> tt::TokenTree { fn make_punct(char: char) -> tt::TokenTree {
tt::Leaf::Punct(tt::Punct { tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
char, char,
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
}) })
@ -209,7 +219,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
} }
fn make_literal(lit: &str) -> tt::TokenTree { fn make_literal(lit: &str) -> tt::TokenTree {
tt::Leaf::Literal(tt::Literal { tt::Leaf::Literal(tt::Literal {
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
text: SmolStr::new(lit), text: SmolStr::new(lit),
}) })
.into() .into()
@ -219,7 +229,11 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
token_trees: Option<Vec<tt::TokenTree>>, token_trees: Option<Vec<tt::TokenTree>>,
) -> tt::TokenTree { ) -> tt::TokenTree {
tt::Subtree { tt::Subtree {
delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }), delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind,
},
token_trees: token_trees.unwrap_or_default(), token_trees: token_trees.unwrap_or_default(),
} }
.into() .into()

7
crates/mbe/src/expander.rs
View file

@ -8,7 +8,7 @@ mod transcriber;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use syntax::SmolStr; use syntax::SmolStr;
use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult};
pub(crate) fn expand_rules( pub(crate) fn expand_rules(
rules: &[crate::Rule], rules: &[crate::Rule],
@ -45,7 +45,10 @@ pub(crate) fn expand_rules(
transcriber::transcribe(&rule.rhs, &match_.bindings); transcriber::transcribe(&rule.rhs, &match_.bindings);
ExpandResult { value, err: match_.err.or(transcribe_err) } ExpandResult { value, err: match_.err.or(transcribe_err) }
} else { } else {
ExpandResult::only_err(ExpandError::NoMatchingRule) ExpandResult::with_err(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
ExpandError::NoMatchingRule,
)
} }
} }

41
crates/mbe/src/expander/matcher.rs
View file

@ -67,6 +67,7 @@ use syntax::SmolStr;
use crate::{ use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment}, expander::{Binding, Bindings, ExpandResult, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator}, parser::{MetaVarKind, Op, RepeatKind, Separator},
tt,
tt_iter::TtIter, tt_iter::TtIter,
ExpandError, MetaTemplate, ValueResult, ExpandError, MetaTemplate, ValueResult,
}; };
@ -75,7 +76,8 @@ impl Bindings {
fn push_optional(&mut self, name: &SmolStr) { fn push_optional(&mut self, name: &SmolStr) {
// FIXME: Do we have a better way to represent an empty token ? // FIXME: Do we have a better way to represent an empty token ?
// Insert an empty subtree for empty token // Insert an empty subtree for empty token
let tt = tt::Subtree::default().into(); let tt =
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into();
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
} }
@ -462,9 +464,9 @@ fn match_loop_inner<'t>(
} }
OpDelimited::Op(Op::Subtree { tokens, delimiter }) => { OpDelimited::Op(Op::Subtree { tokens, delimiter }) => {
if let Ok(subtree) = src.clone().expect_subtree() { if let Ok(subtree) = src.clone().expect_subtree() {
if subtree.delimiter_kind() == delimiter.map(|it| it.kind) { if subtree.delimiter.kind == delimiter.kind {
item.stack.push(item.dot); item.stack.push(item.dot);
item.dot = tokens.iter_delimited(delimiter.as_ref()); item.dot = tokens.iter_delimited(Some(delimiter));
cur_items.push(item); cur_items.push(item);
} }
} }
@ -663,8 +665,8 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
} }
res.add_err(ExpandError::LeftoverTokens); res.add_err(ExpandError::LeftoverTokens);
if let Some(error_reover_item) = error_recover_item { if let Some(error_recover_item) = error_recover_item {
res.bindings = bindings_builder.build(&error_reover_item); res.bindings = bindings_builder.build(&error_recover_item);
} }
return res; return res;
} }
@ -782,7 +784,7 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Opt
match neg { match neg {
None => lit.into(), None => lit.into(),
Some(neg) => tt::TokenTree::Subtree(tt::Subtree { Some(neg) => tt::TokenTree::Subtree(tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::unspecified(),
token_trees: vec![neg, lit.into()], token_trees: vec![neg, lit.into()],
}), }),
} }
@ -810,7 +812,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
} }
impl MetaTemplate { impl MetaTemplate {
fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> { fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> {
OpDelimitedIter { inner: &self.0, idx: 0, delimited } OpDelimitedIter {
inner: &self.0,
idx: 0,
delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED),
}
} }
} }
@ -824,20 +830,21 @@ enum OpDelimited<'a> {
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
struct OpDelimitedIter<'a> { struct OpDelimitedIter<'a> {
inner: &'a [Op], inner: &'a [Op],
delimited: Option<&'a tt::Delimiter>, delimited: &'a tt::Delimiter,
idx: usize, idx: usize,
} }
impl<'a> OpDelimitedIter<'a> { impl<'a> OpDelimitedIter<'a> {
fn is_eof(&self) -> bool { fn is_eof(&self) -> bool {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
self.idx >= len self.idx >= len
} }
fn peek(&self) -> Option<OpDelimited<'a>> { fn peek(&self) -> Option<OpDelimited<'a>> {
match self.delimited { match self.delimited.kind {
None => self.inner.get(self.idx).map(OpDelimited::Op), tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
Some(_) => match self.idx { _ => match self.idx {
0 => Some(OpDelimited::Open), 0 => Some(OpDelimited::Open),
i if i == self.inner.len() + 1 => Some(OpDelimited::Close), i if i == self.inner.len() + 1 => Some(OpDelimited::Close),
i => self.inner.get(i - 1).map(OpDelimited::Op), i => self.inner.get(i - 1).map(OpDelimited::Op),
@ -860,7 +867,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> {
} }
fn size_hint(&self) -> (usize, Option<usize>) { fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
let remain = len.saturating_sub(self.idx); let remain = len.saturating_sub(self.idx);
(remain, Some(remain)) (remain, Some(remain))
} }
@ -904,7 +912,10 @@ impl<'a> TtIter<'a> {
} else { } else {
let puncts = self.expect_glued_punct()?; let puncts = self.expect_glued_punct()?;
let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect(); let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect();
Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees })) Ok(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees,
}))
} }
} else { } else {
self.next().ok_or(()).cloned() self.next().ok_or(()).cloned()
@ -919,7 +930,7 @@ impl<'a> TtIter<'a> {
let ident = self.expect_ident_or_underscore()?; let ident = self.expect_ident_or_underscore()?;
Ok(tt::Subtree { Ok(tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::unspecified(),
token_trees: vec![ token_trees: vec![
tt::Leaf::Punct(*punct).into(), tt::Leaf::Punct(*punct).into(),
tt::Leaf::Ident(ident.clone()).into(), tt::Leaf::Ident(ident.clone()).into(),

70
crates/mbe/src/expander/transcriber.rs
View file

@ -2,11 +2,11 @@
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use syntax::SmolStr; use syntax::SmolStr;
use tt::{Delimiter, Subtree};
use crate::{ use crate::{
expander::{Binding, Bindings, Fragment}, expander::{Binding, Bindings, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator}, parser::{MetaVarKind, Op, RepeatKind, Separator},
tt::{self, Delimiter},
ExpandError, ExpandResult, MetaTemplate, ExpandError, ExpandResult, MetaTemplate,
}; };
@ -44,22 +44,23 @@ impl Bindings {
Binding::Missing(it) => Ok(match it { Binding::Missing(it) => Ok(match it {
MetaVarKind::Stmt => { MetaVarKind::Stmt => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
char: ';', char: ';',
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
}))) })))
} }
MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace, kind: tt::DelimiterKind::Brace,
}), },
token_trees: vec![], token_trees: vec![],
})), })),
// FIXME: Meta and Item should get proper defaults // FIXME: Meta and Item should get proper defaults
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![], token_trees: vec![],
})) }))
} }
@ -71,19 +72,19 @@ impl Bindings {
| MetaVarKind::Ident => { | MetaVarKind::Ident => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("missing"), text: SmolStr::new_inline("missing"),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))) })))
} }
MetaVarKind::Lifetime => { MetaVarKind::Lifetime => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("'missing"), text: SmolStr::new_inline("'missing"),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))) })))
} }
MetaVarKind::Literal => { MetaVarKind::Literal => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("\"missing\""), text: SmolStr::new_inline("\"missing\""),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))) })))
} }
}), }),
@ -143,7 +144,7 @@ fn expand_subtree(
} }
Op::Subtree { tokens, delimiter } => { Op::Subtree { tokens, delimiter } => {
let ExpandResult { value: tt, err: e } = let ExpandResult { value: tt, err: e } =
expand_subtree(ctx, tokens, *delimiter, arena); expand_subtree(ctx, tokens, Some(*delimiter), arena);
err = err.or(e); err = err.or(e);
arena.push(tt.into()); arena.push(tt.into());
} }
@ -170,7 +171,7 @@ fn expand_subtree(
arena.push( arena.push(
tt::Leaf::Literal(tt::Literal { tt::Leaf::Literal(tt::Literal {
text: index.to_string().into(), text: index.to_string().into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}) })
.into(), .into(),
); );
@ -179,7 +180,13 @@ fn expand_subtree(
} }
// drain the elements added in this instance of expand_subtree // drain the elements added in this instance of expand_subtree
let tts = arena.drain(start_elements..).collect(); let tts = arena.drain(start_elements..).collect();
ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err } ExpandResult {
value: tt::Subtree {
delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified),
token_trees: tts,
},
err,
}
} }
fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> { fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@ -201,17 +208,24 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
// ``` // ```
// We just treat it a normal tokens // We just treat it a normal tokens
let tt = tt::Subtree { let tt = tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![ token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(), tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id })
tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(), .into(),
tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(),
], ],
} }
.into(); .into();
ExpandResult::ok(Fragment::Tokens(tt)) ExpandResult::ok(Fragment::Tokens(tt))
} else { } else {
ctx.bindings.get(v, &mut ctx.nesting).map_or_else( ctx.bindings.get(v, &mut ctx.nesting).map_or_else(
|e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, |e| ExpandResult {
value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![],
})),
err: Some(e),
},
ExpandResult::ok, ExpandResult::ok,
) )
} }
@ -249,7 +263,10 @@ fn expand_repeat(
ctx ctx
); );
return ExpandResult { return ExpandResult {
value: Fragment::Tokens(Subtree::default().into()), value: Fragment::Tokens(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }
.into(),
),
err: Some(ExpandError::LimitExceeded), err: Some(ExpandError::LimitExceeded),
}; };
} }
@ -258,7 +275,7 @@ fn expand_repeat(
continue; continue;
} }
t.delimiter = None; t.delimiter = tt::Delimiter::unspecified();
push_subtree(&mut buf, t); push_subtree(&mut buf, t);
if let Some(sep) = separator { if let Some(sep) = separator {
@ -292,7 +309,7 @@ fn expand_repeat(
// Check if it is a single token subtree without any delimiter // Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>} // e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into();
if RepeatKind::OneOrMore == kind && counter == 0 { if RepeatKind::OneOrMore == kind && counter == 0 {
return ExpandResult { return ExpandResult {
@ -307,11 +324,12 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
match fragment { match fragment {
Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => {
if tt.delimiter.is_none() { if tt.delimiter.kind == tt::DelimiterKind::Invisible {
tt.delimiter = Some(tt::Delimiter { tt.delimiter = tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}) };
} }
buf.push(tt.into()) buf.push(tt.into())
} }
@ -320,8 +338,8 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
} }
fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
match tt.delimiter { match tt.delimiter.kind {
None => buf.extend(tt.token_trees), tt::DelimiterKind::Invisible => buf.extend(tt.token_trees),
Some(_) => buf.push(tt.into()), _ => buf.push(tt.into()),
} }
} }

55
crates/mbe/src/lib.rs
View file

@ -18,6 +18,8 @@ mod to_parser_input;
mod benchmark; mod benchmark;
mod token_map; mod token_map;
use ::tt::token_id as tt;
use std::fmt; use std::fmt;
use crate::{ use crate::{
@ -26,8 +28,8 @@ use crate::{
}; };
// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces // FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces
pub use self::tt::{Delimiter, DelimiterKind, Punct};
pub use ::parser::TopEntryPoint; pub use ::parser::TopEntryPoint;
pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::{ pub use crate::{
syntax_bridge::{ syntax_bridge::{
@ -125,24 +127,26 @@ impl Shift {
// Find the max token id inside a subtree // Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> { fn max_id(subtree: &tt::Subtree) -> Option<u32> {
let filter = |tt: &_| match tt { let filter =
tt::TokenTree::Subtree(subtree) => { |tt: &_| match tt {
let tree_id = max_id(subtree); tt::TokenTree::Subtree(subtree) => {
match subtree.delimiter { let tree_id = max_id(subtree);
Some(it) if it.id != tt::TokenId::unspecified() => { if subtree.delimiter.open != tt::TokenId::unspecified() {
Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) Some(tree_id.map_or(subtree.delimiter.open.0, |t| {
t.max(subtree.delimiter.open.0)
}))
} else {
tree_id
} }
_ => tree_id,
} }
} tt::TokenTree::Leaf(leaf) => {
tt::TokenTree::Leaf(leaf) => { let &(tt::Leaf::Ident(tt::Ident { span, .. })
let &(tt::Leaf::Ident(tt::Ident { id, .. }) | tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Punct(tt::Punct { id, .. }) | tt::Leaf::Literal(tt::Literal { span, .. })) = leaf;
| tt::Leaf::Literal(tt::Literal { id, .. })) = leaf;
(id != tt::TokenId::unspecified()).then_some(id.0) (span != tt::TokenId::unspecified()).then_some(span.0)
} }
}; };
subtree.token_trees.iter().filter_map(filter).max() subtree.token_trees.iter().filter_map(filter).max()
} }
} }
@ -152,14 +156,13 @@ impl Shift {
for t in &mut tt.token_trees { for t in &mut tt.token_trees {
match t { match t {
tt::TokenTree::Leaf( tt::TokenTree::Leaf(
tt::Leaf::Ident(tt::Ident { id, .. }) tt::Leaf::Ident(tt::Ident { span, .. })
| tt::Leaf::Punct(tt::Punct { id, .. }) | tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Literal(tt::Literal { id, .. }), | tt::Leaf::Literal(tt::Literal { span, .. }),
) => *id = self.shift(*id), ) => *span = self.shift(*span),
tt::TokenTree::Subtree(tt) => { tt::TokenTree::Subtree(tt) => {
if let Some(it) = tt.delimiter.as_mut() { tt.delimiter.open = self.shift(tt.delimiter.open);
it.id = self.shift(it.id); tt.delimiter.close = self.shift(tt.delimiter.close);
}
self.shift_all(tt) self.shift_all(tt)
} }
} }
@ -216,7 +219,7 @@ impl DeclarativeMacro {
let mut src = TtIter::new(tt); let mut src = TtIter::new(tt);
let mut rules = Vec::new(); let mut rules = Vec::new();
if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() { if tt::DelimiterKind::Brace == tt.delimiter.kind {
cov_mark::hit!(parse_macro_def_rules); cov_mark::hit!(parse_macro_def_rules);
while src.len() > 0 { while src.len() > 0 {
let rule = Rule::parse(&mut src, true)?; let rule = Rule::parse(&mut src, true)?;
@ -325,6 +328,10 @@ impl<T, E> ValueResult<T, E> {
Self { value, err: None } Self { value, err: None }
} }
pub fn with_err(value: T, err: E) -> Self {
Self { value, err: Some(err) }
}
pub fn only_err(err: E) -> Self pub fn only_err(err: E) -> Self
where where
T: Default, T: Default,

20
crates/mbe/src/parser.rs
View file

@ -4,7 +4,7 @@
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use syntax::SmolStr; use syntax::SmolStr;
use crate::{tt_iter::TtIter, ParseError}; use crate::{tt, tt_iter::TtIter, ParseError};
/// Consider /// Consider
/// ///
@ -54,7 +54,7 @@ pub(crate) enum Op {
Ignore { name: SmolStr, id: tt::TokenId }, Ignore { name: SmolStr, id: tt::TokenId },
Index { depth: u32 }, Index { depth: u32 },
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> }, Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter },
Literal(tt::Literal), Literal(tt::Literal),
Punct(SmallVec<[tt::Punct; 3]>), Punct(SmallVec<[tt::Punct; 3]>),
Ident(tt::Ident), Ident(tt::Ident),
@ -130,13 +130,13 @@ fn next_op(
Some(it) => it, Some(it) => it,
}; };
match second { match second {
tt::TokenTree::Subtree(subtree) => match subtree.delimiter_kind() { tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind {
Some(tt::DelimiterKind::Parenthesis) => { tt::DelimiterKind::Parenthesis => {
let (separator, kind) = parse_repeat(src)?; let (separator, kind) = parse_repeat(src)?;
let tokens = MetaTemplate::parse(subtree, mode)?; let tokens = MetaTemplate::parse(subtree, mode)?;
Op::Repeat { tokens, separator, kind } Op::Repeat { tokens, separator, kind }
} }
Some(tt::DelimiterKind::Brace) => match mode { tt::DelimiterKind::Brace => match mode {
Mode::Template => { Mode::Template => {
parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| { parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| {
ParseError::unexpected("invalid metavariable expression") ParseError::unexpected("invalid metavariable expression")
@ -157,18 +157,18 @@ fn next_op(
tt::TokenTree::Leaf(leaf) => match leaf { tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Ident(ident) if ident.text == "crate" => { tt::Leaf::Ident(ident) if ident.text == "crate" => {
// We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id }) Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span })
} }
tt::Leaf::Ident(ident) => { tt::Leaf::Ident(ident) => {
let kind = eat_fragment_kind(src, mode)?; let kind = eat_fragment_kind(src, mode)?;
let name = ident.text.clone(); let name = ident.text.clone();
let id = ident.id; let id = ident.span;
Op::Var { name, kind, id } Op::Var { name, kind, id }
} }
tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
let kind = eat_fragment_kind(src, mode)?; let kind = eat_fragment_kind(src, mode)?;
let name = lit.text.clone(); let name = lit.text.clone();
let id = lit.id; let id = lit.span;
Op::Var { name, kind, id } Op::Var { name, kind, id }
} }
tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode { tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode {
@ -284,7 +284,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let func = src.expect_ident()?; let func = src.expect_ident()?;
let args = src.expect_subtree()?; let args = src.expect_subtree()?;
if args.delimiter_kind() != Some(tt::DelimiterKind::Parenthesis) { if args.delimiter.kind != tt::DelimiterKind::Parenthesis {
return Err(()); return Err(());
} }
@ -293,7 +293,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let op = match &*func.text { let op = match &*func.text {
"ignore" => { "ignore" => {
let ident = args.expect_ident()?; let ident = args.expect_ident()?;
Op::Ignore { name: ident.text.clone(), id: ident.id } Op::Ignore { name: ident.text.clone(), id: ident.span }
} }
"index" => { "index" => {
let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? }; let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? };

126
crates/mbe/src/syntax_bridge.rs
View file

@ -8,9 +8,16 @@ use syntax::{
SyntaxKind::*, SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
}; };
use tt::buffer::{Cursor, TokenBuffer};
use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; use crate::{
to_parser_input::to_parser_input,
tt::{
self,
buffer::{Cursor, TokenBuffer},
},
tt_iter::TtIter,
TokenMap,
};
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;
@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node(
entry_point: parser::TopEntryPoint, entry_point: parser::TopEntryPoint,
) -> (Parse<SyntaxNode>, TokenMap) { ) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt { let buffer = match tt {
tt::Subtree { delimiter: None, token_trees } => { tt::Subtree {
TokenBuffer::from_tokens(token_trees.as_slice()) delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. },
} token_trees,
} => TokenBuffer::from_tokens(token_trees.as_slice()),
_ => TokenBuffer::from_subtree(tt), _ => TokenBuffer::from_subtree(tt),
}; };
let parser_input = to_parser_input(&buffer); let parser_input = to_parser_input(&buffer);
@ -92,8 +100,7 @@ pub fn token_tree_to_syntax_node(
parser::Step::Error { msg } => tree_sink.error(msg.to_string()), parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
} }
} }
let (parse, range_map) = tree_sink.finish(); tree_sink.finish()
(parse, range_map)
} }
/// Convert a string to a `TokenTree` /// Convert a string to a `TokenTree`
@ -132,7 +139,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
res.push(match expanded.value { res.push(match expanded.value {
None => break, None => break,
Some(tt @ tt::TokenTree::Leaf(_)) => { Some(tt @ tt::TokenTree::Leaf(_)) => {
tt::Subtree { delimiter: None, token_trees: vec![tt] } tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] }
} }
Some(tt::TokenTree::Subtree(tt)) => tt, Some(tt::TokenTree::Subtree(tt)) => tt,
}); });
@ -145,7 +152,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
} }
if iter.peek_n(0).is_some() { if iter.peek_n(0).is_some() {
res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() }); res.push(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: iter.cloned().collect(),
});
} }
res res
@ -159,7 +169,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
} }
let entry = StackEntry { let entry = StackEntry {
subtree: tt::Subtree { delimiter: None, ..Default::default() }, subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
// never used (delimiter is `None`) // never used (delimiter is `None`)
idx: !0, idx: !0,
open_range: TextRange::empty(TextSize::of('.')), open_range: TextRange::empty(TextSize::of('.')),
@ -186,7 +196,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
sub.token_trees.get_mut(2) sub.token_trees.get_mut(2)
{ {
lit.id = id lit.span = id
} }
} }
tt tt
@ -199,13 +209,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
assert_eq!(range.len(), TextSize::of('.')); assert_eq!(range.len(), TextSize::of('.'));
} }
if let Some(delim) = subtree.delimiter { let expected = match subtree.delimiter.kind {
let expected = match delim.kind { tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Parenthesis => T![')'], tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Brace => T!['}'], tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Bracket => T![']'], tt::DelimiterKind::Invisible => None,
}; };
if let Some(expected) = expected {
if kind == expected { if kind == expected {
if let Some(entry) = stack.pop() { if let Some(entry) = stack.pop() {
conv.id_alloc().close_delim(entry.idx, Some(range)); conv.id_alloc().close_delim(entry.idx, Some(range));
@ -223,9 +234,11 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
}; };
if let Some(kind) = delim { if let Some(kind) = delim {
let mut subtree = tt::Subtree::default();
let (id, idx) = conv.id_alloc().open_delim(range, synth_id); let (id, idx) = conv.id_alloc().open_delim(range, synth_id);
subtree.delimiter = Some(tt::Delimiter { id, kind }); let subtree = tt::Subtree {
delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind },
token_trees: vec![],
};
stack.push(StackEntry { subtree, idx, open_range: range }); stack.push(StackEntry { subtree, idx, open_range: range });
continue; continue;
} }
@ -240,13 +253,20 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
panic!("Token from lexer must be single char: token = {token:#?}"); panic!("Token from lexer must be single char: token = {token:#?}");
} }
}; };
tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) }) tt::Leaf::from(tt::Punct {
.into() char,
spacing,
span: conv.id_alloc().alloc(range, synth_id),
})
.into()
} else { } else {
macro_rules! make_leaf { macro_rules! make_leaf {
($i:ident) => { ($i:ident) => {
tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) } tt::$i {
.into() span: conv.id_alloc().alloc(range, synth_id),
text: token.to_text(conv),
}
.into()
}; };
} }
let leaf: tt::Leaf = match kind { let leaf: tt::Leaf = match kind {
@ -261,14 +281,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
let apostrophe = tt::Leaf::from(tt::Punct { let apostrophe = tt::Leaf::from(tt::Punct {
char: '\'', char: '\'',
spacing: tt::Spacing::Joint, spacing: tt::Spacing::Joint,
id: conv.id_alloc().alloc(r, synth_id), span: conv.id_alloc().alloc(r, synth_id),
}); });
result.push(apostrophe.into()); result.push(apostrophe.into());
let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
let ident = tt::Leaf::from(tt::Ident { let ident = tt::Leaf::from(tt::Ident {
text: SmolStr::new(&token.to_text(conv)[1..]), text: SmolStr::new(&token.to_text(conv)[1..]),
id: conv.id_alloc().alloc(r, synth_id), span: conv.id_alloc().alloc(r, synth_id),
}); });
result.push(ident.into()); result.push(ident.into());
continue; continue;
@ -289,11 +309,12 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
conv.id_alloc().close_delim(entry.idx, None); conv.id_alloc().close_delim(entry.idx, None);
let leaf: tt::Leaf = tt::Punct { let leaf: tt::Leaf = tt::Punct {
id: conv.id_alloc().alloc(entry.open_range, None), span: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.unwrap().kind { char: match entry.subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => '(', tt::DelimiterKind::Parenthesis => '(',
tt::DelimiterKind::Brace => '{', tt::DelimiterKind::Brace => '{',
tt::DelimiterKind::Bracket => '[', tt::DelimiterKind::Bracket => '[',
tt::DelimiterKind::Invisible => '$',
}, },
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
} }
@ -373,10 +394,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
token_trees.push(mk_punct('!')); token_trees.push(mk_punct('!'));
} }
token_trees.push(tt::TokenTree::from(tt::Subtree { token_trees.push(tt::TokenTree::from(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Bracket, kind: tt::DelimiterKind::Bracket,
id: tt::TokenId::unspecified(), },
}),
token_trees: meta_tkns, token_trees: meta_tkns,
})); }));
@ -386,7 +408,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
fn mk_ident(s: &str) -> tt::TokenTree { fn mk_ident(s: &str) -> tt::TokenTree {
tt::TokenTree::from(tt::Leaf::from(tt::Ident { tt::TokenTree::from(tt::Leaf::from(tt::Ident {
text: s.into(), text: s.into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})) }))
} }
@ -394,12 +416,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
tt::TokenTree::from(tt::Leaf::from(tt::Punct { tt::TokenTree::from(tt::Leaf::from(tt::Punct {
char: c, char: c,
spacing: tt::Spacing::Alone, spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})) }))
} }
fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() };
tt::TokenTree::from(tt::Leaf::from(lit)) tt::TokenTree::from(tt::Leaf::from(lit))
} }
@ -761,15 +783,16 @@ impl<'a> TtTreeSink<'a> {
} }
} }
fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
let texts = match d { let texts = match d {
tt::DelimiterKind::Parenthesis => "()", tt::DelimiterKind::Parenthesis => "()",
tt::DelimiterKind::Brace => "{}", tt::DelimiterKind::Brace => "{}",
tt::DelimiterKind::Bracket => "[]", tt::DelimiterKind::Bracket => "[]",
tt::DelimiterKind::Invisible => return None,
}; };
let idx = closing as usize; let idx = closing as usize;
&texts[idx..texts.len() - (1 - idx)] Some(&texts[idx..texts.len() - (1 - idx)])
} }
impl<'a> TtTreeSink<'a> { impl<'a> TtTreeSink<'a> {
@ -790,13 +813,16 @@ impl<'a> TtTreeSink<'a> {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
// Mark the range if needed // Mark the range if needed
let (text, id) = match leaf { let (text, id) = match leaf {
tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id), tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span),
tt::Leaf::Punct(punct) => { tt::Leaf::Punct(punct) => {
assert!(punct.char.is_ascii()); assert!(punct.char.is_ascii());
tmp = punct.char as u8; tmp = punct.char as u8;
(std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id) (
std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(),
punct.span,
)
} }
tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id), tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span),
}; };
let range = TextRange::at(self.text_pos, TextSize::of(text)); let range = TextRange::at(self.text_pos, TextSize::of(text));
self.token_map.insert(id, range); self.token_map.insert(id, range);
@ -805,10 +831,10 @@ impl<'a> TtTreeSink<'a> {
} }
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cursor = self.cursor.subtree().unwrap(); self.cursor = self.cursor.subtree().unwrap();
match subtree.delimiter { match delim_to_str(subtree.delimiter.kind, false) {
Some(d) => { Some(it) => {
self.open_delims.insert(d.id, self.text_pos); self.open_delims.insert(subtree.delimiter.open, self.text_pos);
delim_to_str(d.kind, false) it
} }
None => continue, None => continue,
} }
@ -816,15 +842,21 @@ impl<'a> TtTreeSink<'a> {
None => { None => {
let parent = self.cursor.end().unwrap(); let parent = self.cursor.end().unwrap();
self.cursor = self.cursor.bump(); self.cursor = self.cursor.bump();
match parent.delimiter { match delim_to_str(parent.delimiter.kind, true) {
Some(d) => { Some(it) => {
if let Some(open_delim) = self.open_delims.get(&d.id) { if let Some(open_delim) =
self.open_delims.get(&parent.delimiter.open)
{
let open_range = TextRange::at(*open_delim, TextSize::of('(')); let open_range = TextRange::at(*open_delim, TextSize::of('('));
let close_range = let close_range =
TextRange::at(self.text_pos, TextSize::of('(')); TextRange::at(self.text_pos, TextSize::of('('));
self.token_map.insert_delim(d.id, open_range, close_range); self.token_map.insert_delim(
parent.delimiter.open,
open_range,
close_range,
);
} }
delim_to_str(d.kind, true) it
} }
None => continue, None => continue,
} }

4
crates/mbe/src/syntax_bridge/tests.rs
View file

@ -29,8 +29,8 @@ fn check_punct_spacing(fixture: &str) {
let mut cursor = buf.begin(); let mut cursor = buf.begin();
while !cursor.eof() { while !cursor.eof() {
while let Some(token_tree) = cursor.token_tree() { while let Some(token_tree) = cursor.token_tree() {
if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree { if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree {
if let Some(expected) = annotations.remove(id) { if let Some(expected) = annotations.remove(span) {
assert_eq!(expected, *spacing); assert_eq!(expected, *spacing);
} }
} }

29
crates/mbe/src/to_parser_input.rs
View file

@ -2,7 +2,8 @@
//! format that works for our parser. //! format that works for our parser.
use syntax::{SyntaxKind, SyntaxKind::*, T}; use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
use crate::tt::buffer::TokenBuffer;
pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
let mut res = parser::Input::default(); let mut res = parser::Input::default();
@ -70,23 +71,25 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
cursor.bump() cursor.bump()
} }
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
if let Some(d) = subtree.delimiter_kind() { if let Some(kind) = match subtree.delimiter.kind {
res.push(match d { tt::DelimiterKind::Parenthesis => Some(T!['(']),
tt::DelimiterKind::Parenthesis => T!['('], tt::DelimiterKind::Brace => Some(T!['{']),
tt::DelimiterKind::Brace => T!['{'], tt::DelimiterKind::Bracket => Some(T!['[']),
tt::DelimiterKind::Bracket => T!['['], tt::DelimiterKind::Invisible => None,
}); } {
res.push(kind);
} }
cursor.subtree().unwrap() cursor.subtree().unwrap()
} }
None => match cursor.end() { None => match cursor.end() {
Some(subtree) => { Some(subtree) => {
if let Some(d) = subtree.delimiter_kind() { if let Some(kind) = match subtree.delimiter.kind {
res.push(match d { tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Parenthesis => T![')'], tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Brace => T!['}'], tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Bracket => T![']'], tt::DelimiterKind::Invisible => None,
}) } {
res.push(kind);
} }
cursor.bump() cursor.bump()
} }

7
crates/mbe/src/tt_iter.rs
View file

@ -3,9 +3,8 @@
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use syntax::SyntaxKind; use syntax::SyntaxKind;
use tt::buffer::TokenBuffer;
use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult}; use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(crate) struct TtIter<'a> { pub(crate) struct TtIter<'a> {
@ -135,7 +134,7 @@ impl<'a> TtIter<'a> {
&mut self, &mut self,
entry_point: parser::PrefixEntryPoint, entry_point: parser::PrefixEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> { ) -> ExpandResult<Option<tt::TokenTree>> {
let buffer = TokenBuffer::from_tokens(self.inner.as_slice()); let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
let parser_input = to_parser_input(&buffer); let parser_input = to_parser_input(&buffer);
let tree_traversal = entry_point.parse(&parser_input); let tree_traversal = entry_point.parse(&parser_input);
@ -178,7 +177,7 @@ impl<'a> TtIter<'a> {
1 => Some(res[0].cloned()), 1 => Some(res[0].cloned()),
0 => None, 0 => None,
_ => Some(tt::TokenTree::Subtree(tt::Subtree { _ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None, delimiter: tt::Delimiter::unspecified(),
token_trees: res.into_iter().map(|it| it.cloned()).collect(), token_trees: res.into_iter().map(|it| it.cloned()).collect(),
})), })),
}; };

9
crates/proc-macro-api/src/lib.rs
View file

@ -19,7 +19,8 @@ use std::{
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tt::Subtree;
use ::tt::token_id as tt;
use crate::{ use crate::{
msg::{ExpandMacro, FlatTree, PanicMessage}, msg::{ExpandMacro, FlatTree, PanicMessage},
@ -151,10 +152,10 @@ impl ProcMacro {
pub fn expand( pub fn expand(
&self, &self,
subtree: &Subtree, subtree: &tt::Subtree,
attr: Option<&Subtree>, attr: Option<&tt::Subtree>,
env: Vec<(String, String)>, env: Vec<(String, String)>,
) -> Result<Result<Subtree, PanicMessage>, ServerError> { ) -> Result<Result<tt::Subtree, PanicMessage>, ServerError> {
let current_dir = env let current_dir = env
.iter() .iter()
.find(|(name, _)| name == "CARGO_MANIFEST_DIR") .find(|(name, _)| name == "CARGO_MANIFEST_DIR")

18
crates/proc-macro-api/src/msg.rs
View file

@ -107,27 +107,31 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use tt::*; use crate::tt::*;
fn fixture_token_tree() -> Subtree { fn fixture_token_tree() -> Subtree {
let mut subtree = Subtree::default(); let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() };
subtree subtree
.token_trees .token_trees
.push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into())); .push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into()));
subtree subtree
.token_trees .token_trees
.push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into())); .push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into()));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal { subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal {
text: "Foo".into(), text: "Foo".into(),
id: TokenId::unspecified(), span: TokenId::unspecified(),
}))); })));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct { subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct {
char: '@', char: '@',
id: TokenId::unspecified(), span: TokenId::unspecified(),
spacing: Spacing::Joint, spacing: Spacing::Joint,
}))); })));
subtree.token_trees.push(TokenTree::Subtree(Subtree { subtree.token_trees.push(TokenTree::Subtree(Subtree {
delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }), delimiter: Delimiter {
open: TokenId(2),
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Brace,
},
token_trees: vec![], token_trees: vec![],
})); }));
subtree subtree

43
crates/proc-macro-api/src/msg/flat.rs
View file

@ -38,7 +38,8 @@
use std::collections::{HashMap, VecDeque}; use std::collections::{HashMap, VecDeque};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tt::TokenId;
use crate::tt::{self, TokenId};
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
pub struct FlatTree { pub struct FlatTree {
@ -52,7 +53,7 @@ pub struct FlatTree {
struct SubtreeRepr { struct SubtreeRepr {
id: tt::TokenId, id: tt::TokenId,
kind: Option<tt::DelimiterKind>, kind: tt::DelimiterKind,
tt: [u32; 2], tt: [u32; 2],
} }
@ -124,19 +125,19 @@ impl FlatTree {
impl SubtreeRepr { impl SubtreeRepr {
fn write(self) -> [u32; 4] { fn write(self) -> [u32; 4] {
let kind = match self.kind { let kind = match self.kind {
None => 0, tt::DelimiterKind::Invisible => 0,
Some(tt::DelimiterKind::Parenthesis) => 1, tt::DelimiterKind::Parenthesis => 1,
Some(tt::DelimiterKind::Brace) => 2, tt::DelimiterKind::Brace => 2,
Some(tt::DelimiterKind::Bracket) => 3, tt::DelimiterKind::Bracket => 3,
}; };
[self.id.0, kind, self.tt[0], self.tt[1]] [self.id.0, kind, self.tt[0], self.tt[1]]
} }
fn read([id, kind, lo, len]: [u32; 4]) -> SubtreeRepr { fn read([id, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
let kind = match kind { let kind = match kind {
0 => None, 0 => tt::DelimiterKind::Invisible,
1 => Some(tt::DelimiterKind::Parenthesis), 1 => tt::DelimiterKind::Parenthesis,
2 => Some(tt::DelimiterKind::Brace), 2 => tt::DelimiterKind::Brace,
3 => Some(tt::DelimiterKind::Bracket), 3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"), other => panic!("bad kind {other}"),
}; };
SubtreeRepr { id: TokenId(id), kind, tt: [lo, len] } SubtreeRepr { id: TokenId(id), kind, tt: [lo, len] }
@ -216,7 +217,7 @@ impl<'a> Writer<'a> {
tt::Leaf::Literal(lit) => { tt::Leaf::Literal(lit) => {
let idx = self.literal.len() as u32; let idx = self.literal.len() as u32;
let text = self.intern(&lit.text); let text = self.intern(&lit.text);
self.literal.push(LiteralRepr { id: lit.id, text }); self.literal.push(LiteralRepr { id: lit.span, text });
idx << 2 | 0b01 idx << 2 | 0b01
} }
tt::Leaf::Punct(punct) => { tt::Leaf::Punct(punct) => {
@ -224,14 +225,14 @@ impl<'a> Writer<'a> {
self.punct.push(PunctRepr { self.punct.push(PunctRepr {
char: punct.char, char: punct.char,
spacing: punct.spacing, spacing: punct.spacing,
id: punct.id, id: punct.span,
}); });
idx << 2 | 0b10 idx << 2 | 0b10
} }
tt::Leaf::Ident(ident) => { tt::Leaf::Ident(ident) => {
let idx = self.ident.len() as u32; let idx = self.ident.len() as u32;
let text = self.intern(&ident.text); let text = self.intern(&ident.text);
self.ident.push(IdentRepr { id: ident.id, text }); self.ident.push(IdentRepr { id: ident.span, text });
idx << 2 | 0b11 idx << 2 | 0b11
} }
}, },
@ -243,8 +244,8 @@ impl<'a> Writer<'a> {
fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 { fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 {
let idx = self.subtree.len(); let idx = self.subtree.len();
let delimiter_id = subtree.delimiter.map_or(TokenId::unspecified(), |it| it.id); let delimiter_id = subtree.delimiter.open;
let delimiter_kind = subtree.delimiter.map(|it| it.kind); let delimiter_kind = subtree.delimiter.kind;
self.subtree.push(SubtreeRepr { id: delimiter_id, kind: delimiter_kind, tt: [!0, !0] }); self.subtree.push(SubtreeRepr { id: delimiter_id, kind: delimiter_kind, tt: [!0, !0] });
self.work.push_back((idx, subtree)); self.work.push_back((idx, subtree));
idx as u32 idx as u32
@ -276,7 +277,11 @@ impl Reader {
let repr = &self.subtree[i]; let repr = &self.subtree[i];
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
let s = tt::Subtree { let s = tt::Subtree {
delimiter: repr.kind.map(|kind| tt::Delimiter { id: repr.id, kind }), delimiter: tt::Delimiter {
open: repr.id,
close: TokenId::UNSPECIFIED,
kind: repr.kind,
},
token_trees: token_trees token_trees: token_trees
.iter() .iter()
.copied() .copied()
@ -291,7 +296,7 @@ impl Reader {
let repr = &self.literal[idx]; let repr = &self.literal[idx];
tt::Leaf::Literal(tt::Literal { tt::Leaf::Literal(tt::Literal {
text: self.text[repr.text as usize].as_str().into(), text: self.text[repr.text as usize].as_str().into(),
id: repr.id, span: repr.id,
}) })
.into() .into()
} }
@ -300,7 +305,7 @@ impl Reader {
tt::Leaf::Punct(tt::Punct { tt::Leaf::Punct(tt::Punct {
char: repr.char, char: repr.char,
spacing: repr.spacing, spacing: repr.spacing,
id: repr.id, span: repr.id,
}) })
.into() .into()
} }
@ -308,7 +313,7 @@ impl Reader {
let repr = &self.ident[idx]; let repr = &self.ident[idx];
tt::Leaf::Ident(tt::Ident { tt::Leaf::Ident(tt::Ident {
text: self.text[repr.text as usize].as_str().into(), text: self.text[repr.text as usize].as_str().into(),
id: repr.id, span: repr.id,
}) })
.into() .into()
} }

1
crates/proc-macro-srv/src/abis/abi_1_63/mod.rs
View file

@ -11,6 +11,7 @@ mod ra_server;
use libloading::Library; use libloading::Library;
use proc_macro_api::ProcMacroKind; use proc_macro_api::ProcMacroKind;
use super::tt;
use super::PanicMessage; use super::PanicMessage;
pub use ra_server::TokenStream; pub use ra_server::TokenStream;

119
crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs
View file

@ -15,6 +15,8 @@ use std::hash::Hash;
use std::ops::Bound; use std::ops::Bound;
use std::{ascii, vec::IntoIter}; use std::{ascii, vec::IntoIter};
use crate::tt;
type Group = tt::Subtree; type Group = tt::Subtree;
type TokenTree = tt::TokenTree; type TokenTree = tt::TokenTree;
type Punct = tt::Punct; type Punct = tt::Punct;
@ -33,7 +35,7 @@ impl TokenStream {
} }
pub fn with_subtree(subtree: tt::Subtree) -> Self { pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() { if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else { } else {
TokenStream { token_trees: subtree.token_trees } TokenStream { token_trees: subtree.token_trees }
@ -41,7 +43,7 @@ impl TokenStream {
} }
pub fn into_subtree(self) -> tt::Subtree { pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees } tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
} }
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
@ -84,7 +86,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams { for item in streams {
for tkn in item { for tkn in item {
match tkn { match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind == tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees); self.token_trees.extend(subtree.token_trees);
} }
_ => { _ => {
@ -165,7 +169,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream { pub mod token_stream {
use std::str::FromStr; use std::str::FromStr;
use super::{TokenStream, TokenTree}; use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s. /// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -202,15 +206,17 @@ pub mod token_stream {
impl ToString for TokenStream { impl ToString for TokenStream {
fn to_string(&self) -> String { fn to_string(&self) -> String {
tt::pretty(&self.token_trees) ::tt::pretty(&self.token_trees)
} }
} }
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree { tt::Subtree {
delimiter: subtree delimiter: tt::Delimiter {
.delimiter open: tt::TokenId::UNSPECIFIED,
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree token_trees: subtree
.token_trees .token_trees
.into_iter() .into_iter()
@ -233,13 +239,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf { match leaf {
tt::Leaf::Literal(lit) => { tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
} }
tt::Leaf::Punct(punct) => { tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
} }
tt::Leaf::Ident(ident) => { tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
} }
} }
} }
@ -389,22 +395,22 @@ impl server::TokenStream for RustAnalyzer {
} }
} }
fn delim_to_internal(d: bridge::Delimiter) -> Option<tt::Delimiter> { fn delim_to_internal(d: bridge::Delimiter) -> tt::Delimiter {
let kind = match d { let kind = match d {
bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
bridge::Delimiter::Brace => tt::DelimiterKind::Brace, bridge::Delimiter::Brace => tt::DelimiterKind::Brace,
bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket, bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket,
bridge::Delimiter::None => return None, bridge::Delimiter::None => tt::DelimiterKind::Invisible,
}; };
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
} }
fn delim_to_external(d: Option<tt::Delimiter>) -> bridge::Delimiter { fn delim_to_external(d: tt::Delimiter) -> bridge::Delimiter {
match d.map(|it| it.kind) { match d.kind {
Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis, tt::DelimiterKind::Parenthesis => bridge::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace, tt::DelimiterKind::Brace => bridge::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket, tt::DelimiterKind::Bracket => bridge::Delimiter::Bracket,
None => bridge::Delimiter::None, tt::DelimiterKind::Invisible => bridge::Delimiter::None,
} }
} }
@ -443,23 +449,19 @@ impl server::Group for RustAnalyzer {
} }
fn span(&mut self, group: &Self::Group) -> Self::Span { fn span(&mut self, group: &Self::Group) -> Self::Span {
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified) group.delimiter.open
} }
fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
if let Some(delim) = &mut group.delimiter { group.delimiter.open = span;
delim.id = span;
}
} }
fn span_open(&mut self, group: &Self::Group) -> Self::Span { fn span_open(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters group.delimiter.open
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
} }
fn span_close(&mut self, group: &Self::Group) -> Self::Span { fn span_close(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters group.delimiter.close
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
} }
} }
@ -468,7 +470,7 @@ impl server::Punct for RustAnalyzer {
tt::Punct { tt::Punct {
char: ch, char: ch,
spacing: spacing_to_internal(spacing), spacing: spacing_to_internal(spacing),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
} }
} }
fn as_char(&mut self, punct: Self::Punct) -> char { fn as_char(&mut self, punct: Self::Punct) -> char {
@ -478,28 +480,27 @@ impl server::Punct for RustAnalyzer {
spacing_to_external(punct.spacing) spacing_to_external(punct.spacing)
} }
fn span(&mut self, punct: Self::Punct) -> Self::Span { fn span(&mut self, punct: Self::Punct) -> Self::Span {
punct.id punct.span
} }
fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
tt::Punct { id: span, ..punct } tt::Punct { span: span, ..punct }
} }
} }
impl server::Ident for RustAnalyzer { impl server::Ident for RustAnalyzer {
fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw( IdentId(self.ident_interner.intern(&IdentData(tt::Ident {
string.into(), text: if is_raw { ::tt::SmolStr::from_iter(["r#", string]) } else { string.into() },
span, span,
is_raw, })))
))))
} }
fn span(&mut self, ident: Self::Ident) -> Self::Span { fn span(&mut self, ident: Self::Ident) -> Self::Span {
self.ident_interner.get(ident.0).0.id self.ident_interner.get(ident.0).0.span
} }
fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
let data = self.ident_interner.get(ident.0); let data = self.ident_interner.get(ident.0);
let new = IdentData(tt::Ident { id: span, ..data.0.clone() }); let new = IdentData(tt::Ident { span: span, ..data.0.clone() });
IdentId(self.ident_interner.intern(&new)) IdentId(self.ident_interner.intern(&new))
} }
} }
@ -511,7 +512,7 @@ impl server::Literal for RustAnalyzer {
"".to_owned() "".to_owned()
} }
fn from_str(&mut self, s: &str) -> Result<Self::Literal, ()> { fn from_str(&mut self, s: &str) -> Result<Self::Literal, ()> {
Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() }) Ok(Literal { text: s.into(), span: tt::TokenId::unspecified() })
} }
fn symbol(&mut self, literal: &Self::Literal) -> String { fn symbol(&mut self, literal: &Self::Literal) -> String {
literal.text.to_string() literal.text.to_string()
@ -529,7 +530,7 @@ impl server::Literal for RustAnalyzer {
Ok(n) => n.to_string(), Ok(n) => n.to_string(),
Err(_) => n.parse::<u128>().unwrap().to_string(), Err(_) => n.parse::<u128>().unwrap().to_string(),
}; };
Literal { text: n.into(), id: tt::TokenId::unspecified() } Literal { text: n.into(), span: tt::TokenId::unspecified() }
} }
fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
@ -549,7 +550,7 @@ impl server::Literal for RustAnalyzer {
let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize}; let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize};
Literal { text: text.into(), id: tt::TokenId::unspecified() } Literal { text: text.into(), span: tt::TokenId::unspecified() }
} }
fn float(&mut self, n: &str) -> Self::Literal { fn float(&mut self, n: &str) -> Self::Literal {
@ -558,19 +559,19 @@ impl server::Literal for RustAnalyzer {
if !text.contains('.') { if !text.contains('.') {
text += ".0" text += ".0"
} }
Literal { text: text.into(), id: tt::TokenId::unspecified() } Literal { text: text.into(), span: tt::TokenId::unspecified() }
} }
fn f32(&mut self, n: &str) -> Self::Literal { fn f32(&mut self, n: &str) -> Self::Literal {
let n: f32 = n.parse().unwrap(); let n: f32 = n.parse().unwrap();
let text = format!("{n}f32"); let text = format!("{n}f32");
Literal { text: text.into(), id: tt::TokenId::unspecified() } Literal { text: text.into(), span: tt::TokenId::unspecified() }
} }
fn f64(&mut self, n: &str) -> Self::Literal { fn f64(&mut self, n: &str) -> Self::Literal {
let n: f64 = n.parse().unwrap(); let n: f64 = n.parse().unwrap();
let text = format!("{n}f64"); let text = format!("{n}f64");
Literal { text: text.into(), id: tt::TokenId::unspecified() } Literal { text: text.into(), span: tt::TokenId::unspecified() }
} }
fn string(&mut self, string: &str) -> Self::Literal { fn string(&mut self, string: &str) -> Self::Literal {
@ -578,11 +579,11 @@ impl server::Literal for RustAnalyzer {
for ch in string.chars() { for ch in string.chars() {
escaped.extend(ch.escape_debug()); escaped.extend(ch.escape_debug());
} }
Literal { text: format!("\"{escaped}\"").into(), id: tt::TokenId::unspecified() } Literal { text: format!("\"{escaped}\"").into(), span: tt::TokenId::unspecified() }
} }
fn character(&mut self, ch: char) -> Self::Literal { fn character(&mut self, ch: char) -> Self::Literal {
Literal { text: format!("'{ch}'").into(), id: tt::TokenId::unspecified() } Literal { text: format!("'{ch}'").into(), span: tt::TokenId::unspecified() }
} }
fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
@ -593,15 +594,15 @@ impl server::Literal for RustAnalyzer {
.map(Into::<char>::into) .map(Into::<char>::into)
.collect::<String>(); .collect::<String>();
Literal { text: format!("b\"{string}\"").into(), id: tt::TokenId::unspecified() } Literal { text: format!("b\"{string}\"").into(), span: tt::TokenId::unspecified() }
} }
fn span(&mut self, literal: &Self::Literal) -> Self::Span { fn span(&mut self, literal: &Self::Literal) -> Self::Span {
literal.id literal.span
} }
fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
literal.id = span; literal.span = span;
} }
fn subspan( fn subspan(
@ -784,17 +785,18 @@ mod tests {
token_trees: vec![ token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(), text: "struct".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})), })),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(), text: "T".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})), })),
tt::TokenTree::Subtree(tt::Subtree { tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace, kind: tt::DelimiterKind::Brace,
}), },
token_trees: vec![], token_trees: vec![],
}), }),
], ],
@ -807,13 +809,14 @@ mod tests {
fn test_ra_server_from_str() { fn test_ra_server_from_str() {
use std::str::FromStr; use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}), },
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(), text: "a".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))], }))],
}); });
@ -830,7 +833,7 @@ mod tests {
underscore.token_trees[0], underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(), text: "_".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})) }))
); );
} }

2
crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs
View file

@ -9,7 +9,7 @@ mod ra_server;
use libloading::Library; use libloading::Library;
use proc_macro_api::ProcMacroKind; use proc_macro_api::ProcMacroKind;
use super::PanicMessage; use super::{tt, PanicMessage};
pub use ra_server::TokenStream; pub use ra_server::TokenStream;

65
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs
View file

@ -22,6 +22,8 @@ pub use symbol::*;
use std::ops::Bound; use std::ops::Bound;
use crate::tt;
type Group = tt::Subtree; type Group = tt::Subtree;
type TokenTree = tt::TokenTree; type TokenTree = tt::TokenTree;
type Punct = tt::Punct; type Punct = tt::Punct;
@ -108,8 +110,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(ident) => { bridge::TokenTree::Ident(ident) => {
let text = ident.sym.text(); let text = ident.sym.text();
let text = if ident.is_raw { tt::SmolStr::from_iter(["r#", &text]) } else { text }; let text =
let ident: tt::Ident = tt::Ident { text, id: ident.span }; if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, span: ident.span };
let leaf = tt::Leaf::from(ident); let leaf = tt::Leaf::from(ident);
let tree = TokenTree::from(leaf); let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree]) Self::TokenStream::from_iter(vec![tree])
@ -118,9 +121,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Literal(literal) => { bridge::TokenTree::Literal(literal) => {
let literal = LiteralFormatter(literal); let literal = LiteralFormatter(literal);
let text = literal let text = literal
.with_stringify_parts(|parts| tt::SmolStr::from_iter(parts.iter().copied())); .with_stringify_parts(|parts| ::tt::SmolStr::from_iter(parts.iter().copied()));
let literal = tt::Literal { text, id: literal.0.span }; let literal = tt::Literal { text, span: literal.0.span };
let leaf = tt::Leaf::from(literal); let leaf = tt::Leaf::from(literal);
let tree = TokenTree::from(leaf); let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree]) Self::TokenStream::from_iter(vec![tree])
@ -130,7 +133,7 @@ impl server::TokenStream for RustAnalyzer {
let punct = tt::Punct { let punct = tt::Punct {
char: p.ch as char, char: p.ch as char,
spacing: if p.joint { Spacing::Joint } else { Spacing::Alone }, spacing: if p.joint { Spacing::Joint } else { Spacing::Alone },
id: p.span, span: p.span,
}; };
let leaf = tt::Leaf::from(punct); let leaf = tt::Leaf::from(punct);
let tree = TokenTree::from(leaf); let tree = TokenTree::from(leaf);
@ -184,7 +187,7 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(bridge::Ident { bridge::TokenTree::Ident(bridge::Ident {
sym: Symbol::intern(ident.text.trim_start_matches("r#")), sym: Symbol::intern(ident.text.trim_start_matches("r#")),
is_raw: ident.text.starts_with("r#"), is_raw: ident.text.starts_with("r#"),
span: ident.id, span: ident.span,
}) })
} }
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
@ -194,14 +197,14 @@ impl server::TokenStream for RustAnalyzer {
symbol: Symbol::intern(&lit.text), symbol: Symbol::intern(&lit.text),
// FIXME: handle suffixes // FIXME: handle suffixes
suffix: None, suffix: None,
span: lit.id, span: lit.span,
}) })
} }
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
bridge::TokenTree::Punct(bridge::Punct { bridge::TokenTree::Punct(bridge::Punct {
ch: punct.char as u8, ch: punct.char as u8,
joint: punct.spacing == Spacing::Joint, joint: punct.spacing == Spacing::Joint,
span: punct.id, span: punct.span,
}) })
} }
tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group { tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group {
@ -211,31 +214,29 @@ impl server::TokenStream for RustAnalyzer {
} else { } else {
Some(subtree.token_trees.into_iter().collect()) Some(subtree.token_trees.into_iter().collect())
}, },
span: bridge::DelimSpan::from_single( span: bridge::DelimSpan::from_single(subtree.delimiter.open),
subtree.delimiter.map_or(Span::unspecified(), |del| del.id),
),
}), }),
}) })
.collect() .collect()
} }
} }
fn delim_to_internal(d: proc_macro::Delimiter) -> Option<tt::Delimiter> { fn delim_to_internal(d: proc_macro::Delimiter) -> tt::Delimiter {
let kind = match d { let kind = match d {
proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace,
proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket,
proc_macro::Delimiter::None => return None, proc_macro::Delimiter::None => tt::DelimiterKind::Invisible,
}; };
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
} }
fn delim_to_external(d: Option<tt::Delimiter>) -> proc_macro::Delimiter { fn delim_to_external(d: tt::Delimiter) -> proc_macro::Delimiter {
match d.map(|it| it.kind) { match d.kind {
Some(tt::DelimiterKind::Parenthesis) => proc_macro::Delimiter::Parenthesis, tt::DelimiterKind::Parenthesis => proc_macro::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => proc_macro::Delimiter::Brace, tt::DelimiterKind::Brace => proc_macro::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => proc_macro::Delimiter::Bracket, tt::DelimiterKind::Bracket => proc_macro::Delimiter::Bracket,
None => proc_macro::Delimiter::None, tt::DelimiterKind::Invisible => proc_macro::Delimiter::None,
} }
} }
@ -349,7 +350,7 @@ impl server::Server for RustAnalyzer {
} }
fn intern_symbol(ident: &str) -> Self::Symbol { fn intern_symbol(ident: &str) -> Self::Symbol {
Symbol::intern(&tt::SmolStr::from(ident)) Symbol::intern(&::tt::SmolStr::from(ident))
} }
fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
@ -413,17 +414,18 @@ mod tests {
token_trees: vec![ token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(), text: "struct".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})), })),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(), text: "T".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})), })),
tt::TokenTree::Subtree(tt::Subtree { tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace, kind: tt::DelimiterKind::Brace,
}), },
token_trees: vec![], token_trees: vec![],
}), }),
], ],
@ -436,13 +438,14 @@ mod tests {
fn test_ra_server_from_str() { fn test_ra_server_from_str() {
use std::str::FromStr; use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter { delimiter: tt::Delimiter {
id: tt::TokenId::unspecified(), open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis, kind: tt::DelimiterKind::Parenthesis,
}), },
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(), text: "a".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
}))], }))],
}); });
@ -459,7 +462,7 @@ mod tests {
underscore.token_trees[0], underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(), text: "_".into(),
id: tt::TokenId::unspecified(), span: tt::TokenId::unspecified(),
})) }))
); );
} }

28
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs
View file

@ -1,6 +1,6 @@
//! TokenStream implementation used by sysroot ABI //! TokenStream implementation used by sysroot ABI
use tt::TokenTree; use crate::tt::{self, TokenTree};
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct TokenStream { pub struct TokenStream {
@ -13,7 +13,7 @@ impl TokenStream {
} }
pub fn with_subtree(subtree: tt::Subtree) -> Self { pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() { if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else { } else {
TokenStream { token_trees: subtree.token_trees } TokenStream { token_trees: subtree.token_trees }
@ -21,7 +21,7 @@ impl TokenStream {
} }
pub fn into_subtree(self) -> tt::Subtree { pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees } tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
} }
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
@ -64,7 +64,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams { for item in streams {
for tkn in item { for tkn in item {
match tkn { match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind != tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees); self.token_trees.extend(subtree.token_trees);
} }
_ => { _ => {
@ -84,7 +86,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream { pub mod token_stream {
use std::str::FromStr; use std::str::FromStr;
use super::{TokenStream, TokenTree}; use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s. /// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -121,15 +123,17 @@ pub mod token_stream {
impl ToString for TokenStream { impl ToString for TokenStream {
fn to_string(&self) -> String { fn to_string(&self) -> String {
tt::pretty(&self.token_trees) ::tt::pretty(&self.token_trees)
} }
} }
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree { tt::Subtree {
delimiter: subtree delimiter: tt::Delimiter {
.delimiter open: tt::TokenId::UNSPECIFIED,
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree token_trees: subtree
.token_trees .token_trees
.into_iter() .into_iter()
@ -152,13 +156,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf { match leaf {
tt::Leaf::Literal(lit) => { tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
} }
tt::Leaf::Punct(punct) => { tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
} }
tt::Leaf::Ident(ident) => { tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
} }
} }
} }

2
crates/proc-macro-srv/src/abis/mod.rs
View file

@ -41,6 +41,8 @@ pub(crate) use abi_sysroot::Abi as Abi_Sysroot;
use libloading::Library; use libloading::Library;
use proc_macro_api::{ProcMacroKind, RustCInfo}; use proc_macro_api::{ProcMacroKind, RustCInfo};
use crate::tt;
pub struct PanicMessage { pub struct PanicMessage {
message: Option<String>, message: Option<String>,
} }

2
crates/proc-macro-srv/src/dylib.rs
View file

@ -13,6 +13,8 @@ use object::Object;
use paths::AbsPath; use paths::AbsPath;
use proc_macro_api::{read_dylib_info, ProcMacroKind}; use proc_macro_api::{read_dylib_info, ProcMacroKind};
use crate::tt;
use super::abis::Abi; use super::abis::Abi;
const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_"; const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";

2
crates/proc-macro-srv/src/lib.rs
View file

@ -37,6 +37,8 @@ use proc_macro_api::{
ProcMacroKind, ProcMacroKind,
}; };
use ::tt::token_id as tt;
#[derive(Default)] #[derive(Default)]
pub(crate) struct ProcMacroSrv { pub(crate) struct ProcMacroSrv {
expanders: HashMap<(PathBuf, SystemTime), dylib::Expander>, expanders: HashMap<(PathBuf, SystemTime), dylib::Expander>,

26
crates/proc-macro-srv/src/tests/mod.rs
View file

@ -8,7 +8,7 @@ use expect_test::expect;
#[test] #[test]
fn test_derive_empty() { fn test_derive_empty() {
assert_expand("DeriveEmpty", r#"struct S;"#, expect![[r#"SUBTREE $"#]]); assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 4294967295 4294967295"]);
} }
#[test] #[test]
@ -17,10 +17,10 @@ fn test_derive_error() {
"DeriveError", "DeriveError",
r#"struct S;"#, r#"struct S;"#,
expect![[r##" expect![[r##"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295 IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295 PUNCH ! [alone] 4294967295
SUBTREE () 4294967295 SUBTREE () 4294967295 4294967295
LITERAL "#[derive(DeriveError)] struct S ;" 4294967295 LITERAL "#[derive(DeriveError)] struct S ;" 4294967295
PUNCH ; [alone] 4294967295"##]], PUNCH ; [alone] 4294967295"##]],
); );
@ -32,14 +32,14 @@ fn test_fn_like_macro_noop() {
"fn_like_noop", "fn_like_noop",
r#"ident, 0, 1, []"#, r#"ident, 0, 1, []"#,
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295 IDENT ident 4294967295
PUNCH , [alone] 4294967295 PUNCH , [alone] 4294967295
LITERAL 0 4294967295 LITERAL 0 4294967295
PUNCH , [alone] 4294967295 PUNCH , [alone] 4294967295
LITERAL 1 4294967295 LITERAL 1 4294967295
PUNCH , [alone] 4294967295 PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]], SUBTREE [] 4294967295 4294967295"#]],
); );
} }
@ -49,10 +49,10 @@ fn test_fn_like_macro_clone_ident_subtree() {
"fn_like_clone_tokens", "fn_like_clone_tokens",
r#"ident, []"#, r#"ident, []"#,
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295 IDENT ident 4294967295
PUNCH , [alone] 4294967295 PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]], SUBTREE [] 4294967295 4294967295"#]],
); );
} }
@ -62,7 +62,7 @@ fn test_fn_like_macro_clone_raw_ident() {
"fn_like_clone_tokens", "fn_like_clone_tokens",
"r#async", "r#async",
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT r#async 4294967295"#]], IDENT r#async 4294967295"#]],
); );
} }
@ -73,7 +73,7 @@ fn test_fn_like_mk_literals() {
"fn_like_mk_literals", "fn_like_mk_literals",
r#""#, r#""#,
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
LITERAL b"byte_string" 4294967295 LITERAL b"byte_string" 4294967295
LITERAL 'c' 4294967295 LITERAL 'c' 4294967295
LITERAL "string" 4294967295 LITERAL "string" 4294967295
@ -90,7 +90,7 @@ fn test_fn_like_mk_idents() {
"fn_like_mk_idents", "fn_like_mk_idents",
r#""#, r#""#,
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT standard 4294967295 IDENT standard 4294967295
IDENT r#raw 4294967295"#]], IDENT r#raw 4294967295"#]],
); );
@ -102,7 +102,7 @@ fn test_fn_like_macro_clone_literals() {
"fn_like_clone_tokens", "fn_like_clone_tokens",
r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#, r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#,
expect![[r#" expect![[r#"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
LITERAL 1u16 4294967295 LITERAL 1u16 4294967295
PUNCH , [alone] 4294967295 PUNCH , [alone] 4294967295
LITERAL 2_u32 4294967295 LITERAL 2_u32 4294967295
@ -126,10 +126,10 @@ fn test_attr_macro() {
r#"mod m {}"#, r#"mod m {}"#,
r#"some arguments"#, r#"some arguments"#,
expect![[r##" expect![[r##"
SUBTREE $ SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295 IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295 PUNCH ! [alone] 4294967295
SUBTREE () 4294967295 SUBTREE () 4294967295 4294967295
LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295 LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295
PUNCH ; [alone] 4294967295"##]], PUNCH ; [alone] 4294967295"##]],
); );

4
crates/rust-analyzer/src/reload.rs
View file

@ -34,6 +34,8 @@ use crate::{
op_queue::Cause, op_queue::Cause,
}; };
use ::tt::token_id as tt;
#[derive(Debug)] #[derive(Debug)]
pub(crate) enum ProjectWorkspaceProgress { pub(crate) enum ProjectWorkspaceProgress {
Begin, Begin,
@ -656,7 +658,7 @@ pub(crate) fn load_proc_macro(
_: Option<&tt::Subtree>, _: Option<&tt::Subtree>,
_: &Env, _: &Env,
) -> Result<tt::Subtree, ProcMacroExpansionError> { ) -> Result<tt::Subtree, ProcMacroExpansionError> {
Ok(tt::Subtree::default()) Ok(tt::Subtree::empty())
} }
} }
} }

9
crates/stdx/src/macros.rs
View file

@ -43,5 +43,14 @@ macro_rules! impl_from {
} }
)*)? )*)?
)* )*
};
($($variant:ident$(<$V:ident>)?),* for $enum:ident) => {
$(
impl$(<$V>)? From<$variant$(<$V>)?> for $enum$(<$V>)? {
fn from(it: $variant$(<$V>)?) -> $enum$(<$V>)? {
$enum::$variant(it)
}
}
)*
} }
} }

84
crates/tt/src/buffer.rs
View file

@ -12,10 +12,10 @@ struct EntryPtr(EntryId, usize);
/// Internal type which is used instead of `TokenTree` to represent a token tree /// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`. /// within a `TokenBuffer`.
#[derive(Debug)] #[derive(Debug)]
enum Entry<'t> { enum Entry<'t, Span> {
// Mimicking types from proc-macro. // Mimicking types from proc-macro.
Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId), Subtree(Option<&'t TokenTree<Span>>, &'t Subtree<Span>, EntryId),
Leaf(&'t TokenTree), Leaf(&'t TokenTree<Span>),
// End entries contain a pointer to the entry from the containing // End entries contain a pointer to the entry from the containing
// token tree, or None if this is the outermost level. // token tree, or None if this is the outermost level.
End(Option<EntryPtr>), End(Option<EntryPtr>),
@ -24,16 +24,21 @@ enum Entry<'t> {
/// A token tree buffer /// A token tree buffer
/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41) /// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
#[derive(Debug)] #[derive(Debug)]
pub struct TokenBuffer<'t> { pub struct TokenBuffer<'t, Span> {
buffers: Vec<Box<[Entry<'t>]>>, buffers: Vec<Box<[Entry<'t, Span>]>>,
} }
trait TokenList<'a> { trait TokenList<'a, Span> {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>); fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>);
} }
impl<'a> TokenList<'a> for &'a [TokenTree] { impl<'a, Span> TokenList<'a, Span> for &'a [TokenTree<Span>] {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) { fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End // Must contain everything in tokens and then the Entry::End
let start_capacity = self.len() + 1; let start_capacity = self.len() + 1;
let mut entries = Vec::with_capacity(start_capacity); let mut entries = Vec::with_capacity(start_capacity);
@ -53,8 +58,11 @@ impl<'a> TokenList<'a> for &'a [TokenTree] {
} }
} }
impl<'a> TokenList<'a> for &'a Subtree { impl<'a, Span> TokenList<'a, Span> for &'a Subtree<Span> {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) { fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End // Must contain everything in tokens and then the Entry::End
let mut entries = vec![]; let mut entries = vec![];
let mut children = vec![]; let mut children = vec![];
@ -64,25 +72,25 @@ impl<'a> TokenList<'a> for &'a Subtree {
} }
} }
impl<'t> TokenBuffer<'t> { impl<'t, Span> TokenBuffer<'t, Span> {
pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> { pub fn from_tokens(tokens: &'t [TokenTree<Span>]) -> TokenBuffer<'t, Span> {
Self::new(tokens) Self::new(tokens)
} }
pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> { pub fn from_subtree(subtree: &'t Subtree<Span>) -> TokenBuffer<'t, Span> {
Self::new(subtree) Self::new(subtree)
} }
fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> { fn new<T: TokenList<'t, Span>>(tokens: T) -> TokenBuffer<'t, Span> {
let mut buffers = vec![]; let mut buffers = vec![];
let idx = TokenBuffer::new_inner(tokens, &mut buffers, None); let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
assert_eq!(idx, 0); assert_eq!(idx, 0);
TokenBuffer { buffers } TokenBuffer { buffers }
} }
fn new_inner<T: TokenList<'t>>( fn new_inner<T: TokenList<'t, Span>>(
tokens: T, tokens: T,
buffers: &mut Vec<Box<[Entry<'t>]>>, buffers: &mut Vec<Box<[Entry<'t, Span>]>>,
next: Option<EntryPtr>, next: Option<EntryPtr>,
) -> usize { ) -> usize {
let (children, mut entries) = tokens.entries(); let (children, mut entries) = tokens.entries();
@ -105,25 +113,25 @@ impl<'t> TokenBuffer<'t> {
/// Creates a cursor referencing the first token in the buffer and able to /// Creates a cursor referencing the first token in the buffer and able to
/// traverse until the end of the buffer. /// traverse until the end of the buffer.
pub fn begin(&self) -> Cursor<'_> { pub fn begin(&self) -> Cursor<'_, Span> {
Cursor::create(self, EntryPtr(EntryId(0), 0)) Cursor::create(self, EntryPtr(EntryId(0), 0))
} }
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_>> { fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_, Span>> {
let id = ptr.0; let id = ptr.0;
self.buffers[id.0].get(ptr.1) self.buffers[id.0].get(ptr.1)
} }
} }
#[derive(Debug)] #[derive(Debug)]
pub enum TokenTreeRef<'a> { pub enum TokenTreeRef<'a, Span> {
Subtree(&'a Subtree, Option<&'a TokenTree>), Subtree(&'a Subtree<Span>, Option<&'a TokenTree<Span>>),
Leaf(&'a Leaf, &'a TokenTree), Leaf(&'a Leaf<Span>, &'a TokenTree<Span>),
} }
impl<'a> TokenTreeRef<'a> { impl<'a, Span: Clone> TokenTreeRef<'a, Span> {
pub fn cloned(&self) -> TokenTree { pub fn cloned(&self) -> TokenTree<Span> {
match &self { match self {
TokenTreeRef::Subtree(subtree, tt) => match tt { TokenTreeRef::Subtree(subtree, tt) => match tt {
Some(it) => (*it).clone(), Some(it) => (*it).clone(),
None => (*subtree).clone().into(), None => (*subtree).clone().into(),
@ -135,20 +143,20 @@ impl<'a> TokenTreeRef<'a> {
/// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125> /// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125>
#[derive(Copy, Clone, Debug)] #[derive(Copy, Clone, Debug)]
pub struct Cursor<'a> { pub struct Cursor<'a, Span> {
buffer: &'a TokenBuffer<'a>, buffer: &'a TokenBuffer<'a, Span>,
ptr: EntryPtr, ptr: EntryPtr,
} }
impl<'a> PartialEq for Cursor<'a> { impl<'a, Span> PartialEq for Cursor<'a, Span> {
fn eq(&self, other: &Cursor<'_>) -> bool { fn eq(&self, other: &Cursor<'_, Span>) -> bool {
self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer) self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer)
} }
} }
impl<'a> Eq for Cursor<'a> {} impl<'a, Span> Eq for Cursor<'a, Span> {}
impl<'a> Cursor<'a> { impl<'a, Span> Cursor<'a, Span> {
/// Check whether it is eof /// Check whether it is eof
pub fn eof(self) -> bool { pub fn eof(self) -> bool {
matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None))) matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None)))
@ -156,7 +164,7 @@ impl<'a> Cursor<'a> {
/// If the cursor is pointing at the end of a subtree, returns /// If the cursor is pointing at the end of a subtree, returns
/// the parent subtree /// the parent subtree
pub fn end(self) -> Option<&'a Subtree> { pub fn end(self) -> Option<&'a Subtree<Span>> {
match self.entry() { match self.entry() {
Some(Entry::End(Some(ptr))) => { Some(Entry::End(Some(ptr))) => {
let idx = ptr.1; let idx = ptr.1;
@ -171,13 +179,13 @@ impl<'a> Cursor<'a> {
} }
} }
fn entry(self) -> Option<&'a Entry<'a>> { fn entry(&self) -> Option<&'a Entry<'a, Span>> {
self.buffer.entry(&self.ptr) self.buffer.entry(&self.ptr)
} }
/// If the cursor is pointing at a `Subtree`, returns /// If the cursor is pointing at a `Subtree`, returns
/// a cursor into that subtree /// a cursor into that subtree
pub fn subtree(self) -> Option<Cursor<'a>> { pub fn subtree(self) -> Option<Cursor<'a, Span>> {
match self.entry() { match self.entry() {
Some(Entry::Subtree(_, _, entry_id)) => { Some(Entry::Subtree(_, _, entry_id)) => {
Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0))) Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
@ -187,7 +195,7 @@ impl<'a> Cursor<'a> {
} }
/// If the cursor is pointing at a `TokenTree`, returns it /// If the cursor is pointing at a `TokenTree`, returns it
pub fn token_tree(self) -> Option<TokenTreeRef<'a>> { pub fn token_tree(self) -> Option<TokenTreeRef<'a, Span>> {
match self.entry() { match self.entry() {
Some(Entry::Leaf(tt)) => match tt { Some(Entry::Leaf(tt)) => match tt {
TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)), TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)),
@ -198,12 +206,12 @@ impl<'a> Cursor<'a> {
} }
} }
fn create(buffer: &'a TokenBuffer<'_>, ptr: EntryPtr) -> Cursor<'a> { fn create(buffer: &'a TokenBuffer<'_, Span>, ptr: EntryPtr) -> Cursor<'a, Span> {
Cursor { buffer, ptr } Cursor { buffer, ptr }
} }
/// Bump the cursor /// Bump the cursor
pub fn bump(self) -> Cursor<'a> { pub fn bump(self) -> Cursor<'a, Span> {
if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) { if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) {
match exit { match exit {
Some(exit) => Cursor::create(self.buffer, *exit), Some(exit) => Cursor::create(self.buffer, *exit),
@ -216,7 +224,7 @@ impl<'a> Cursor<'a> {
/// Bump the cursor, if it is a subtree, returns /// Bump the cursor, if it is a subtree, returns
/// a cursor into that subtree /// a cursor into that subtree
pub fn bump_subtree(self) -> Cursor<'a> { pub fn bump_subtree(self) -> Cursor<'a, Span> {
match self.entry() { match self.entry() {
Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(), Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
_ => self.bump(), _ => self.bump(),

232
crates/tt/src/lib.rs
View file

@ -16,45 +16,106 @@ pub use smol_str::SmolStr;
/// which source tokens. We do it by assigning an distinct identity to each /// which source tokens. We do it by assigning an distinct identity to each
/// source token and making sure that identities are preserved during macro /// source token and making sure that identities are preserved during macro
/// expansion. /// expansion.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct TokenId(pub u32); pub struct TokenId(pub u32);
impl fmt::Debug for TokenId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl TokenId { impl TokenId {
pub const UNSPECIFIED: TokenId = TokenId(!0);
pub const fn unspecified() -> TokenId { pub const fn unspecified() -> TokenId {
TokenId(!0) Self::UNSPECIFIED
} }
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub mod token_id {
pub enum TokenTree { pub use crate::{DelimiterKind, Spacing, TokenId};
Leaf(Leaf), pub type Span = crate::TokenId;
Subtree(Subtree), pub type Subtree = crate::Subtree<Span>;
} pub type Punct = crate::Punct<Span>;
impl_from!(Leaf, Subtree for TokenTree); pub type Delimiter = crate::Delimiter<Span>;
pub type Leaf = crate::Leaf<Span>;
pub type Ident = crate::Ident<Span>;
pub type Literal = crate::Literal<Span>;
pub type TokenTree = crate::TokenTree<Span>;
pub mod buffer {
pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>;
pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>;
pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>;
}
impl TokenTree { impl Delimiter {
pub fn empty() -> Self { pub const UNSPECIFIED: Self = Self {
TokenTree::Subtree(Subtree::default()) open: TokenId::UNSPECIFIED,
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Invisible,
};
pub const fn unspecified() -> Self {
Self::UNSPECIFIED
}
}
impl Subtree {
pub const fn empty() -> Self {
Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
}
}
impl TokenTree {
pub const fn empty() -> Self {
Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] })
}
} }
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Leaf { pub struct SyntaxContext(pub u32);
Literal(Literal),
Punct(Punct),
Ident(Ident),
}
impl_from!(Literal, Punct, Ident for Leaf);
#[derive(Clone, PartialEq, Eq, Hash, Default)] // #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Subtree { // pub struct Span {
pub delimiter: Option<Delimiter>, // pub id: TokenId,
pub token_trees: Vec<TokenTree>, // pub ctx: SyntaxContext,
// }
// pub type Span = (TokenId, SyntaxContext);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree<Span> {
Leaf(Leaf<Span>),
Subtree(Subtree<Span>),
}
impl_from!(Leaf<Span>, Subtree<Span> for TokenTree);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf<Span> {
Literal(Literal<Span>),
Punct(Punct<Span>),
Ident(Ident<Span>),
}
impl<Span> Leaf<Span> {
pub fn span(&self) -> &Span {
match self {
Leaf::Literal(it) => &it.span,
Leaf::Punct(it) => &it.span,
Leaf::Ident(it) => &it.span,
}
}
}
impl_from!(Literal<Span>, Punct<Span>, Ident<Span> for Leaf);
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct Subtree<Span> {
// FIXME, this should not be Option
pub delimiter: Delimiter<Span>,
pub token_trees: Vec<TokenTree<Span>>,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Delimiter { pub struct Delimiter<Span> {
pub id: TokenId, pub open: Span,
pub close: Span,
pub kind: DelimiterKind, pub kind: DelimiterKind,
} }
@ -63,19 +124,20 @@ pub enum DelimiterKind {
Parenthesis, Parenthesis,
Brace, Brace,
Bracket, Bracket,
Invisible,
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Literal { pub struct Literal<Span> {
pub text: SmolStr, pub text: SmolStr,
pub id: TokenId, pub span: Span,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Punct { pub struct Punct<Span> {
pub char: char, pub char: char,
pub spacing: Spacing, pub spacing: Spacing,
pub id: TokenId, pub span: Span,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -85,39 +147,25 @@ pub enum Spacing {
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Ident { /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. pub struct Ident<Span> {
pub text: SmolStr, pub text: SmolStr,
pub id: TokenId, pub span: Span,
} }
impl Ident { fn print_debug_subtree<Span: fmt::Debug>(
/// Constructor intended to be used only by proc macro server. `text` should not contain raw f: &mut fmt::Formatter<'_>,
/// identifier prefix. subtree: &Subtree<Span>,
pub fn new_with_is_raw(text: SmolStr, id: TokenId, is_raw: bool) -> Self { level: usize,
let text = if is_raw { SmolStr::from_iter(["r#", &text]) } else { text }; ) -> fmt::Result {
Ident { text, id }
}
}
impl Leaf {
pub fn id(&self) -> TokenId {
match self {
Leaf::Literal(l) => l.id,
Leaf::Punct(p) => p.id,
Leaf::Ident(i) => i.id,
}
}
}
fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result {
let align = " ".repeat(level); let align = " ".repeat(level);
let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) { let Delimiter { kind, open, close } = &subtree.delimiter;
None => "$".to_string(), let aux = match kind {
Some((DelimiterKind::Parenthesis, id)) => format!("() {id}"), DelimiterKind::Invisible => format!("$$ {:?} {:?}", open, close),
Some((DelimiterKind::Brace, id)) => format!("{{}} {id}"), DelimiterKind::Parenthesis => format!("() {:?} {:?}", open, close),
Some((DelimiterKind::Bracket, id)) => format!("[] {id}"), DelimiterKind::Brace => format!("{{}} {:?} {:?}", open, close),
DelimiterKind::Bracket => format!("[] {:?} {:?}", open, close),
}; };
if subtree.token_trees.is_empty() { if subtree.token_trees.is_empty() {
@ -135,21 +183,25 @@ fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usi
Ok(()) Ok(())
} }
fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result { fn print_debug_token<Span: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
tkn: &TokenTree<Span>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level); let align = " ".repeat(level);
match tkn { match tkn {
TokenTree::Leaf(leaf) => match leaf { TokenTree::Leaf(leaf) => match leaf {
Leaf::Literal(lit) => write!(f, "{align}LITERAL {} {}", lit.text, lit.id.0)?, Leaf::Literal(lit) => write!(f, "{}LITERAL {} {:?}", align, lit.text, lit.span)?,
Leaf::Punct(punct) => write!( Leaf::Punct(punct) => write!(
f, f,
"{}PUNCH {} [{}] {}", "{}PUNCH {} [{}] {:?}",
align, align,
punct.char, punct.char,
if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
punct.id.0 punct.span
)?, )?,
Leaf::Ident(ident) => write!(f, "{align}IDENT {} {}", ident.text, ident.id.0)?, Leaf::Ident(ident) => write!(f, "{}IDENT {} {:?}", align, ident.text, ident.span)?,
}, },
TokenTree::Subtree(subtree) => { TokenTree::Subtree(subtree) => {
print_debug_subtree(f, subtree, level)?; print_debug_subtree(f, subtree, level)?;
@ -159,13 +211,13 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize)
Ok(()) Ok(())
} }
impl fmt::Debug for Subtree { impl<Span: fmt::Debug> fmt::Debug for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
print_debug_subtree(f, self, 0) print_debug_subtree(f, self, 0)
} }
} }
impl fmt::Display for TokenTree { impl<Span> fmt::Display for TokenTree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self { match self {
TokenTree::Leaf(it) => fmt::Display::fmt(it, f), TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
@ -174,13 +226,13 @@ impl fmt::Display for TokenTree {
} }
} }
impl fmt::Display for Subtree { impl<Span> fmt::Display for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (l, r) = match self.delimiter_kind() { let (l, r) = match self.delimiter.kind {
Some(DelimiterKind::Parenthesis) => ("(", ")"), DelimiterKind::Parenthesis => ("(", ")"),
Some(DelimiterKind::Brace) => ("{", "}"), DelimiterKind::Brace => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"), DelimiterKind::Bracket => ("[", "]"),
None => ("", ""), DelimiterKind::Invisible => ("", ""),
}; };
f.write_str(l)?; f.write_str(l)?;
let mut needs_space = false; let mut needs_space = false;
@ -202,7 +254,7 @@ impl fmt::Display for Subtree {
} }
} }
impl fmt::Display for Leaf { impl<Span> fmt::Display for Leaf<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self { match self {
Leaf::Ident(it) => fmt::Display::fmt(it, f), Leaf::Ident(it) => fmt::Display::fmt(it, f),
@ -212,25 +264,25 @@ impl fmt::Display for Leaf {
} }
} }
impl fmt::Display for Ident { impl<Span> fmt::Display for Ident<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f) fmt::Display::fmt(&self.text, f)
} }
} }
impl fmt::Display for Literal { impl<Span> fmt::Display for Literal<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f) fmt::Display::fmt(&self.text, f)
} }
} }
impl fmt::Display for Punct { impl<Span> fmt::Display for Punct<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.char, f) fmt::Display::fmt(&self.char, f)
} }
} }
impl Subtree { impl<Span> Subtree<Span> {
/// Count the number of tokens recursively /// Count the number of tokens recursively
pub fn count(&self) -> usize { pub fn count(&self) -> usize {
let children_count = self let children_count = self
@ -244,20 +296,16 @@ impl Subtree {
self.token_trees.len() + children_count self.token_trees.len() + children_count
} }
pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
self.delimiter.map(|it| it.kind)
}
} }
impl Subtree { impl<Span> Subtree<Span> {
/// A simple line string used for debugging /// A simple line string used for debugging
pub fn as_debug_string(&self) -> String { pub fn as_debug_string(&self) -> String {
let delim = match self.delimiter_kind() { let delim = match self.delimiter.kind {
Some(DelimiterKind::Brace) => ("{", "}"), DelimiterKind::Brace => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"), DelimiterKind::Bracket => ("[", "]"),
Some(DelimiterKind::Parenthesis) => ("(", ")"), DelimiterKind::Parenthesis => ("(", ")"),
None => (" ", " "), DelimiterKind::Invisible => ("$", "$"),
}; };
let mut res = String::new(); let mut res = String::new();
@ -275,7 +323,7 @@ impl Subtree {
(Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => { (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
" ".to_string() + &s " ".to_string() + &s
} }
(Leaf::Punct(_), Some(&TokenTree::Leaf(Leaf::Punct(punct)))) => { (Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => {
if punct.spacing == Spacing::Alone { if punct.spacing == Spacing::Alone {
" ".to_string() + &s " ".to_string() + &s
} else { } else {
@ -298,19 +346,19 @@ impl Subtree {
pub mod buffer; pub mod buffer;
pub fn pretty(tkns: &[TokenTree]) -> String { pub fn pretty<Span>(tkns: &[TokenTree<Span>]) -> String {
fn tokentree_to_text(tkn: &TokenTree) -> String { fn tokentree_to_text<Span>(tkn: &TokenTree<Span>) -> String {
match tkn { match tkn {
TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(), TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(),
TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(),
TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
TokenTree::Subtree(subtree) => { TokenTree::Subtree(subtree) => {
let content = pretty(&subtree.token_trees); let content = pretty(&subtree.token_trees);
let (open, close) = match subtree.delimiter.map(|it| it.kind) { let (open, close) = match subtree.delimiter.kind {
None => ("", ""), DelimiterKind::Brace => ("{", "}"),
Some(DelimiterKind::Brace) => ("{", "}"), DelimiterKind::Bracket => ("[", "]"),
Some(DelimiterKind::Parenthesis) => ("(", ")"), DelimiterKind::Parenthesis => ("(", ")"),
Some(DelimiterKind::Bracket) => ("[", "]"), DelimiterKind::Invisible => ("", ""),
}; };
format!("{open}{content}{close}") format!("{open}{content}{close}")
} }