Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-28 05:53:45 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #14061 - Veykril:generic-tt, r=Veykril

Browse source 
Make tt generic over the span data

This also fixes up our delimiter representation in tt, it is no longer optional (we use invisible delims in the same way as before, that is still incorrectly) and we now store two spans instead of one.

These changes should help with adjusting our token map. Though this will probably break proc-macros in some ways, will need to test that for now.
This commit is contained in:
bors 2023-01-31 14:45:45 +00:00
commit da249539e8
48 changed files with 806 additions and 569 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
crates/base-db/src/fixture.rs
View file

@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
use test_utils::{
extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
};
use tt::Subtree;
use tt::token_id::Subtree;
use vfs::{file_set::FileSet, VfsPath};
use crate::{
@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
_: &Env,
) -> Result<Subtree, ProcMacroExpansionError> {
fn traverse(input: &Subtree) -> Subtree {
let mut res = Subtree::default();
res.delimiter = input.delimiter;
let mut token_trees = vec![];
for tt in input.token_trees.iter().rev() {
let tt = match tt {
tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
};
res.token_trees.push(tt);
token_trees.push(tt);
}
res
Subtree { delimiter: input.delimiter, token_trees }
}
Ok(traverse(input))
}

2
crates/base-db/src/input.rs
View file

@ -12,7 +12,7 @@ use cfg::CfgOptions;
use rustc_hash::FxHashMap;
use stdx::hash::{NoHashHashMap, NoHashHashSet};
use syntax::SmolStr;
use tt::Subtree;
use tt::token_id::Subtree;
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
/// Files are grouped into source roots. A source root is a directory on the

4
crates/cfg/src/cfg_expr.rs
View file

@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
}
impl CfgExpr {
pub fn parse(tt: &tt::Subtree) -> CfgExpr {
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
}
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@ -85,7 +85,7 @@ impl CfgExpr {
}
}
fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> {
fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
let name = match it.next() {
None => return None,
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

4
crates/hir-def/src/adt.rs
View file

@ -2,6 +2,7 @@
use std::sync::Arc;
use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use base_db::CrateId;
use either::Either;
use hir_expand::{
@ -12,7 +13,6 @@ use intern::Interned;
use la_arena::{Arena, ArenaMap};
use rustc_abi::{Integer, IntegerType};
use syntax::ast::{self, HasName, HasVisibility};
use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use crate::{
body::{CfgExpander, LowerCtx},
@ -82,7 +82,7 @@ fn repr_from_value(
fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
match tt.delimiter {
Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {}
Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
_ => return None,
}

5
crates/hir-def/src/attr.rs
View file

@ -16,7 +16,6 @@ use syntax::{
ast::{self, HasAttrs, IsString},
AstPtr, AstToken, SmolStr, TextRange, TextSize,
};
use tt::Subtree;
use crate::{
db::DefDatabase,
@ -234,7 +233,7 @@ impl Attrs {
pub fn has_doc_hidden(&self) -> bool {
self.by_key("doc").tt_values().any(|tt| {
tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) &&
tt.delimiter.kind == DelimiterKind::Parenthesis &&
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
})
}
@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
}
impl<'attr> AttrQuery<'attr> {
pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> {
pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
self.attrs().filter_map(|attr| attr.token_tree_value())
}

2
crates/hir-def/src/data.rs
View file

@ -142,7 +142,7 @@ impl FunctionData {
}
}
fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> {
fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
let mut indices = Vec::new();
for args in tt.token_trees.chunks(2) {
match &args[0] {

23
crates/hir-def/src/lib.rs
View file

@ -79,6 +79,8 @@ use nameres::DefMap;
use stdx::impl_from;
use syntax::ast;
use ::tt::token_id as tt;
use crate::{
adt::VariantData,
builtin_type::BuiltinType,
@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
def: MacroDefId,
is_derive: bool,
) -> MacroCallId {
let mut arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()),
_ => Default::default(),
let arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt, map)) => (
{
let mut tt = tt.clone();
tt.delimiter = tt::Delimiter::UNSPECIFIED;
tt
},
map.clone(),
),
_ => (tt::Subtree::empty(), Default::default()),
};
// The parentheses are always disposed here.
arg.0.delimiter = None;
let res = def.as_lazy_macro(
def.as_lazy_macro(
db.upcast(),
krate,
MacroCallKind::Attr {
@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
invoc_attr_index: macro_attr.id,
is_derive,
},
);
res
)
}
intern::impl_internable!(
crate::type_ref::TypeRef,

8
crates/hir-def/src/macro_expansion_tests.rs
View file

@ -30,7 +30,7 @@ use syntax::{
SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
SyntaxNode, TextRange, T,
};
use tt::{Subtree, TokenId};
use tt::token_id::{Subtree, TokenId};
use crate::{
db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
tree.token_trees.iter().for_each(|tree| match tree {
tt::TokenTree::Leaf(leaf) => {
let id = match leaf {
tt::Leaf::Literal(it) => it.id,
tt::Leaf::Punct(it) => it.id,
tt::Leaf::Ident(it) => it.id,
tt::Leaf::Literal(it) => it.span,
tt::Leaf::Punct(it) => it.span,
tt::Leaf::Ident(it) => it.span,
};
ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
}

12
crates/hir-def/src/nameres/collector.rs
View file

@ -46,6 +46,7 @@ use crate::{
},
path::{ImportAlias, ModPath, PathKind},
per_ns::PerNs,
tt,
visibility::{RawVisibility, Visibility},
AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
.enumerate()
.map(|(idx, it)| {
// FIXME: a hacky way to create a Name from string.
let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() };
let name =
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
(
name.as_name(),
ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@ -451,7 +453,10 @@ impl DefCollector<'_> {
directive.module_id,
MacroCallKind::Attr {
ast_id: ast_id.ast_id,
attr_args: Default::default(),
attr_args: std::sync::Arc::new((
tt::Subtree::empty(),
Default::default(),
)),
invoc_attr_index: attr.id,
is_derive: false,
},
@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
let name = match attrs.by_key("rustc_builtin_macro").string_value() {
Some(it) => {
// FIXME: a hacky way to create a Name from string.
name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name();
name =
tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
&name
}
None => {

2
crates/hir-def/src/nameres/proc_macro.rs
View file

@ -1,9 +1,9 @@
//! Nameres-specific procedural macro data and helpers.
use hir_expand::name::{AsName, Name};
use tt::{Leaf, TokenTree};
use crate::attr::Attrs;
use crate::tt::{Leaf, TokenTree};
#[derive(Debug, PartialEq, Eq)]
pub struct ProcMacroDef {

9
crates/hir-expand/src/attrs.rs
View file

@ -8,13 +8,13 @@ use intern::Interned;
use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
use smallvec::{smallvec, SmallVec};
use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
use tt::Subtree;
use crate::{
db::AstDatabase,
hygiene::Hygiene,
mod_path::{ModPath, PathKind},
name::AsName,
tt::{self, Subtree},
InFile,
};
@ -117,7 +117,10 @@ impl RawAttrs {
let index = attr.id;
let attrs =
parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
let tree = Subtree { delimiter: None, token_trees: attr.to_vec() };
let tree = Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: attr.to_vec(),
};
// FIXME hygiene
let hygiene = Hygiene::new_unhygienic();
Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
@ -266,7 +269,7 @@ impl Attr {
pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
let args = self.token_tree_value()?;
if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) {
if args.delimiter.kind != DelimiterKind::Parenthesis {
return None;
}
let paths = args

6
crates/hir-expand/src/builtin_attr_macro.rs
View file

@ -1,6 +1,6 @@
//! Builtin attributes.
use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind};
use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
macro_rules! register_builtin {
( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@ -97,7 +97,7 @@ fn derive_attr_expand(
let loc = db.lookup_intern_macro_call(id);
let derives = match &loc.kind {
MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
_ => return ExpandResult::ok(Default::default()),
_ => return ExpandResult::ok(tt::Subtree::empty()),
};
pseudo_derive_attr_expansion(tt, derives)
}
@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
};

10
crates/hir-expand/src/builtin_derive_macro.rs
View file

@ -3,11 +3,11 @@
use base_db::{CrateOrigin, LangCrateOrigin};
use tracing::debug;
use crate::tt::{self, TokenId};
use syntax::{
ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
match_ast,
};
use tt::TokenId;
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
})?;
let name_token_id =
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
let name_token = tt::Ident { id: name_token_id, text: name.text().into() };
let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
let param_types = params
.into_iter()
.flat_map(|param_list| param_list.type_or_const_params())
@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
let ty = param
.ty()
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
.unwrap_or_default();
.unwrap_or_else(tt::Subtree::empty);
Some(ty)
} else {
None
@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
let info = match parse_adt(tt) {
Ok(info) => info,
Err(e) => return ExpandResult::only_err(e),
Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
};
let (params, args): (Vec<_>, Vec<_>) = info
.param_types
@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu
.enumerate()
.map(|(idx, param_ty)| {
let ident = tt::Leaf::Ident(tt::Ident {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
text: format!("T{idx}").into(),
});
let ident_ = ident.clone();

66
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -9,7 +9,9 @@ use syntax::{
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
use crate::{
db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
};
macro_rules! register_builtin {
( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => {
@ -61,7 +63,7 @@ macro_rules! register_builtin {
};
}
#[derive(Debug, Default)]
#[derive(Debug)]
pub struct ExpandedEager {
pub(crate) subtree: tt::Subtree,
/// The included file ID of the include macro.
@ -116,7 +118,7 @@ register_builtin! {
}
const DOLLAR_CRATE: tt::Ident =
tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() };
tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() };
fn module_path_expand(
_db: &dyn AstDatabase,
@ -162,7 +164,7 @@ fn stringify_expand(
_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<tt::Subtree> {
let pretty = tt::pretty(&tt.token_trees);
let pretty = ::tt::pretty(&tt.token_trees);
let expanded = quote! {
#pretty
@ -194,11 +196,11 @@ fn assert_expand(
let expanded = match &*args {
[cond, panic_args @ ..] => {
let comma = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char: ',',
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
};
let cond = cond.clone();
@ -247,7 +249,10 @@ fn format_args_expand(
let mut args = parse_exprs_with_sep(tt, ',');
if args.is_empty() {
return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into());
return ExpandResult::with_err(
tt::Subtree::empty(),
mbe::ExpandError::NoMatchingRule.into(),
);
}
for arg in &mut args {
// Remove `key =`.
@ -282,7 +287,7 @@ fn asm_expand(
for tt in tt.token_trees.chunks(2) {
match tt {
[tt::TokenTree::Leaf(tt::Leaf::Literal(lit))]
| [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] =>
| [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] =>
{
let krate = DOLLAR_CRATE.clone();
literals.push(quote!(#krate::format_args!(#lit);));
@ -400,7 +405,7 @@ fn concat_expand(
// FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses
// to ensure the right parsing order, so skip the parentheses here. Ideally we'd
// implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623
if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t {
if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t {
if let [tt] = &**token_trees {
if delim.kind == tt::DelimiterKind::Parenthesis {
t = tt;
@ -459,9 +464,7 @@ fn concat_bytes_expand(
}
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (),
tt::TokenTree::Subtree(tree)
if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) =>
{
tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => {
if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) {
err.get_or_insert(e);
break;
@ -473,7 +476,7 @@ fn concat_bytes_expand(
}
}
}
let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() };
let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
}
@ -521,7 +524,7 @@ fn concat_idents_expand(
}
}
}
let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() };
let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
}
@ -572,7 +575,10 @@ fn include_expand(
Ok((subtree, file_id)) => {
ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) })
}
Err(e) => ExpandResult::only_err(e),
Err(e) => ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
),
}
}
@ -582,15 +588,18 @@ fn include_bytes_expand(
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
if let Err(e) = parse_string(tt) {
return ExpandResult::only_err(e);
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
);
}
// FIXME: actually read the file here if the user asked for macro expansion
let res = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: r#"b"""#.into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
};
ExpandResult::ok(ExpandedEager::new(res))
@ -603,7 +612,12 @@ fn include_str_expand(
) -> ExpandResult<ExpandedEager> {
let path = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
// FIXME: we're not able to read excluded files (which is most of them because
@ -635,7 +649,12 @@ fn env_expand(
) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
let mut err = None;
@ -666,7 +685,12 @@ fn option_env_expand(
) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
let expanded = match get_env_inner(db, arg_id, &key) {

14
crates/hir-expand/src/db.rs
View file

@ -14,7 +14,7 @@ use syntax::{
use crate::{
ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup,
hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
};
@ -175,7 +175,7 @@ pub fn expand_speculative(
match attr.token_tree() {
Some(token_tree) => {
let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax());
tree.delimiter = None;
tree.delimiter = tt::Delimiter::unspecified();
let shift = mbe::Shift::new(&tt);
shift.shift_all(&mut tree);
@ -210,7 +210,7 @@ pub fn expand_speculative(
// Otherwise the expand query will fetch the non speculative attribute args and pass those instead.
let mut speculative_expansion = match loc.def.kind {
MacroDefKind::ProcMacro(expander, ..) => {
tt.delimiter = None;
tt.delimiter = tt::Delimiter::unspecified();
expander.expand(db, loc.krate, &tt, attr_arg.as_ref())
}
MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => {
@ -316,9 +316,8 @@ fn macro_arg(
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
tt.delimiter = None;
tt.delimiter = tt::Delimiter::unspecified();
}
Some(Arc::new((tt, tmap, fixups.undo_info)))
}
@ -479,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult<tt::
let macro_arg = match db.macro_arg(id) {
Some(it) => it,
None => {
return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into()))
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("No arguments for proc-macro".into()),
)
}
};

6
crates/hir-expand/src/eager.rs
View file

@ -108,7 +108,7 @@ pub fn expand_eager_macro(
.value
.token_tree()
.map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
.unwrap_or_default();
.unwrap_or_else(tt::Subtree::empty);
let ast_map = db.ast_id_map(macro_call.file_id);
let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@ -165,9 +165,9 @@ pub fn expand_eager_macro(
}
}
fn to_subtree(node: &SyntaxNode) -> tt::Subtree {
fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree {
let mut subtree = mbe::syntax_node_to_token_tree(node).0;
subtree.delimiter = None;
subtree.delimiter = crate::tt::Delimiter::unspecified();
subtree
}

18
crates/hir-expand/src/fixup.rs
View file

@ -9,7 +9,7 @@ use syntax::{
ast::{self, AstNode, HasLoopBody},
match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
};
use tt::Subtree;
use tt::token_id::Subtree;
/// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to
@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups(
tt.token_trees = tts
.into_iter()
.filter(|tt| match tt {
tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID),
tt::TokenTree::Leaf(leaf) => {
token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID)
}
tt::TokenTree::Subtree(st) => {
st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID))
token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID)
}
})
.flat_map(|tt| match tt {
@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups(
SmallVec::from_const([tt.into()])
}
tt::TokenTree::Leaf(leaf) => {
if let Some(id) = token_map.synthetic_token_id(leaf.id()) {
if let Some(id) = token_map.synthetic_token_id(*leaf.span()) {
let original = undo_info.original[id.0 as usize].clone();
if original.delimiter.is_none() {
if original.delimiter.kind == tt::DelimiterKind::Invisible {
original.token_trees.into()
} else {
SmallVec::from_const([original.into()])
@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups(
mod tests {
use expect_test::{expect, Expect};
use crate::tt;
use super::reverse_fixups;
// The following three functions are only meant to check partial structural equivalence of
@ -341,7 +345,7 @@ mod tests {
}
fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind)
a.delimiter.kind == b.delimiter.kind
&& a.token_trees.len() == b.token_trees.len()
&& a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
}
@ -386,7 +390,7 @@ mod tests {
let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
assert!(
check_subtree_eq(&tt, &original_as_tt),
"different token tree: {tt:?}, {original_as_tt:?}"
"different token tree: {tt:?},\n{original_as_tt:?}"
);
}

2
crates/hir-expand/src/hygiene.rs
View file

@ -128,7 +128,7 @@ struct HygieneInfo {
attr_input_or_mac_def_start: Option<InFile<TextSize>>,
macro_def: Arc<TokenExpander>,
macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
macro_arg_shift: mbe::Shift,
exp_map: Arc<mbe::TokenMap>,
}

2
crates/hir-expand/src/lib.rs
View file

@ -22,6 +22,8 @@ mod fixup;
pub use mbe::{Origin, ValueResult};
use ::tt::token_id as tt;
use std::{fmt, hash::Hash, iter, sync::Arc};
use base_db::{

2
crates/hir-expand/src/name.rs
View file

@ -191,7 +191,7 @@ impl AsName for ast::NameOrNameRef {
}
}
impl AsName for tt::Ident {
impl<Span> AsName for tt::Ident<Span> {
fn as_name(&self) -> Name {
Name::resolve(&self.text)
}

24
crates/hir-expand/src/proc_macro.rs
View file

@ -3,7 +3,7 @@
use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
use stdx::never;
use crate::{db::AstDatabase, ExpandError, ExpandResult};
use crate::{db::AstDatabase, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct ProcMacroExpander {
@ -39,7 +39,10 @@ impl ProcMacroExpander {
Ok(proc_macros) => proc_macros,
Err(_) => {
never!("Non-dummy expander even though there are no proc macros");
return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
}
};
let proc_macro = match proc_macros.get(id.0 as usize) {
@ -50,7 +53,10 @@ impl ProcMacroExpander {
proc_macros.len(),
id.0
);
return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
}
};
@ -69,13 +75,17 @@ impl ProcMacroExpander {
}
}
ProcMacroExpansionError::System(text)
| ProcMacroExpansionError::Panic(text) => {
ExpandResult::only_err(ExpandError::Other(text.into()))
}
| ProcMacroExpansionError::Panic(text) => ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other(text.into()),
),
},
}
}
None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)),
None => ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::UnresolvedProcMacro(self.krate),
),
}
}
}

112
crates/hir-expand/src/quote.rs
View file

@ -9,17 +9,18 @@
#[macro_export]
macro_rules! __quote {
() => {
Vec::<tt::TokenTree>::new()
Vec::<crate::tt::TokenTree>::new()
};
( @SUBTREE $delim:ident $($tt:tt)* ) => {
{
let children = $crate::__quote!($($tt)*);
tt::Subtree {
delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::$delim,
id: tt::TokenId::unspecified(),
}),
crate::tt::Subtree {
delimiter: crate::tt::Delimiter {
kind: crate::tt::DelimiterKind::$delim,
open: crate::tt::TokenId::unspecified(),
close: crate::tt::TokenId::unspecified(),
},
token_trees: $crate::quote::IntoTt::to_tokens(children),
}
}
@ -28,10 +29,10 @@ macro_rules! __quote {
( @PUNCT $first:literal ) => {
{
vec![
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Alone,
span: crate::tt::TokenId::unspecified(),
}).into()
]
}
@ -40,15 +41,15 @@ macro_rules! __quote {
( @PUNCT $first:literal, $sec:literal ) => {
{
vec![
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first,
spacing: tt::Spacing::Joint,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Joint,
span: crate::tt::TokenId::unspecified(),
}).into(),
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $sec,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Alone,
span: crate::tt::TokenId::unspecified(),
}).into()
]
}
@ -67,7 +68,7 @@ macro_rules! __quote {
( ## $first:ident $($tail:tt)* ) => {
{
let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<tt::TokenTree>>();
let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<crate::tt::TokenTree>>();
let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
tokens.append(&mut tail_tokens);
tokens
@ -86,9 +87,9 @@ macro_rules! __quote {
// Ident
( $tt:ident ) => {
vec![ {
tt::Leaf::Ident(tt::Ident {
crate::tt::Leaf::Ident(crate::tt::Ident {
text: stringify!($tt).into(),
id: tt::TokenId::unspecified(),
span: crate::tt::TokenId::unspecified(),
}).into()
}]
};
@ -127,42 +128,42 @@ macro_rules! quote {
}
pub(crate) trait IntoTt {
fn to_subtree(self) -> tt::Subtree;
fn to_tokens(self) -> Vec<tt::TokenTree>;
fn to_subtree(self) -> crate::tt::Subtree;
fn to_tokens(self) -> Vec<crate::tt::TokenTree>;
}
impl IntoTt for Vec<tt::TokenTree> {
fn to_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self }
impl IntoTt for Vec<crate::tt::TokenTree> {
fn to_subtree(self) -> crate::tt::Subtree {
crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self }
}
fn to_tokens(self) -> Vec<tt::TokenTree> {
fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
self
}
}
impl IntoTt for tt::Subtree {
fn to_subtree(self) -> tt::Subtree {
impl IntoTt for crate::tt::Subtree {
fn to_subtree(self) -> crate::tt::Subtree {
self
}
fn to_tokens(self) -> Vec<tt::TokenTree> {
vec![tt::TokenTree::Subtree(self)]
fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
vec![crate::tt::TokenTree::Subtree(self)]
}
}
pub(crate) trait ToTokenTree {
fn to_token(self) -> tt::TokenTree;
fn to_token(self) -> crate::tt::TokenTree;
}
impl ToTokenTree for tt::TokenTree {
fn to_token(self) -> tt::TokenTree {
impl ToTokenTree for crate::tt::TokenTree {
fn to_token(self) -> crate::tt::TokenTree {
self
}
}
impl ToTokenTree for tt::Subtree {
fn to_token(self) -> tt::TokenTree {
impl ToTokenTree for crate::tt::Subtree {
fn to_token(self) -> crate::tt::TokenTree {
self.into()
}
}
@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees {
($($ty:ty => $this:ident $im:block);*) => {
$(
impl ToTokenTree for $ty {
fn to_token($this) -> tt::TokenTree {
let leaf: tt::Leaf = $im.into();
fn to_token($this) -> crate::tt::TokenTree {
let leaf: crate::tt::Leaf = $im.into();
leaf.into()
}
}
impl ToTokenTree for &$ty {
fn to_token($this) -> tt::TokenTree {
let leaf: tt::Leaf = $im.clone().into();
fn to_token($this) -> crate::tt::TokenTree {
let leaf: crate::tt::Leaf = $im.clone().into();
leaf.into()
}
}
@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees {
}
impl_to_to_tokentrees! {
u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
tt::Leaf => self { self };
tt::Literal => self { self };
tt::Ident => self { self };
tt::Punct => self { self };
&str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}};
String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}
u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
crate::tt::Leaf => self { self };
crate::tt::Literal => self { self };
crate::tt::Ident => self { self };
crate::tt::Punct => self { self };
&str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}};
String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}
}
#[cfg(test)]
@ -223,8 +224,8 @@ mod tests {
assert_eq!(quote!(#s).to_string(), "\"hello\"");
}
fn mk_ident(name: &str) -> tt::Ident {
tt::Ident { text: name.into(), id: tt::TokenId::unspecified() }
fn mk_ident(name: &str) -> crate::tt::Ident {
crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() }
}
#[test]
@ -234,7 +235,7 @@ mod tests {
let quoted = quote!(#a);
assert_eq!(quoted.to_string(), "hello");
let t = format!("{quoted:?}");
assert_eq!(t, "SUBTREE $\n IDENT hello 4294967295");
assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295");
}
#[test]
@ -263,11 +264,12 @@ mod tests {
let fields = [mk_ident("name"), mk_ident("id")];
let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees);
let list = tt::Subtree {
delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::Brace,
id: tt::TokenId::unspecified(),
}),
let list = crate::tt::Subtree {
delimiter: crate::tt::Delimiter {
kind: crate::tt::DelimiterKind::Brace,
open: crate::tt::TokenId::unspecified(),
close: crate::tt::TokenId::unspecified(),
},
token_trees: fields.collect(),
};

28
crates/mbe/src/benchmark.rs
View file

@ -9,7 +9,7 @@ use test_utils::{bench, bench_fixture, skip_slow_tests};
use crate::{
parser::{MetaVarKind, Op, RepeatKind, Separator},
syntax_node_to_token_tree, DeclarativeMacro,
syntax_node_to_token_tree, tt, DeclarativeMacro,
};
#[test]
@ -91,7 +91,14 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
// So we just skip any error cases and try again
let mut try_cnt = 0;
loop {
let mut subtree = tt::Subtree::default();
let mut subtree = tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Invisible,
},
token_trees: vec![],
};
for op in rule.lhs.iter() {
collect_from_op(op, &mut subtree, &mut seed);
}
@ -196,12 +203,15 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
*seed
}
fn make_ident(ident: &str) -> tt::TokenTree {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) })
.into()
tt::Leaf::Ident(tt::Ident {
span: tt::TokenId::unspecified(),
text: SmolStr::new(ident),
})
.into()
}
fn make_punct(char: char) -> tt::TokenTree {
tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
char,
spacing: tt::Spacing::Alone,
})
@ -209,7 +219,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
}
fn make_literal(lit: &str) -> tt::TokenTree {
tt::Leaf::Literal(tt::Literal {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
text: SmolStr::new(lit),
})
.into()
@ -219,7 +229,11 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
token_trees: Option<Vec<tt::TokenTree>>,
) -> tt::TokenTree {
tt::Subtree {
delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind,
},
token_trees: token_trees.unwrap_or_default(),
}
.into()

7
crates/mbe/src/expander.rs
View file

@ -8,7 +8,7 @@ mod transcriber;
use rustc_hash::FxHashMap;
use syntax::SmolStr;
use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult};
pub(crate) fn expand_rules(
rules: &[crate::Rule],
@ -45,7 +45,10 @@ pub(crate) fn expand_rules(
transcriber::transcribe(&rule.rhs, &match_.bindings);
ExpandResult { value, err: match_.err.or(transcribe_err) }
} else {
ExpandResult::only_err(ExpandError::NoMatchingRule)
ExpandResult::with_err(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
ExpandError::NoMatchingRule,
)
}
}

41
crates/mbe/src/expander/matcher.rs
View file

@ -67,6 +67,7 @@ use syntax::SmolStr;
use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt,
tt_iter::TtIter,
ExpandError, MetaTemplate, ValueResult,
};
@ -75,7 +76,8 @@ impl Bindings {
fn push_optional(&mut self, name: &SmolStr) {
// FIXME: Do we have a better way to represent an empty token ?
// Insert an empty subtree for empty token
let tt = tt::Subtree::default().into();
let tt =
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into();
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
}
@ -462,9 +464,9 @@ fn match_loop_inner<'t>(
}
OpDelimited::Op(Op::Subtree { tokens, delimiter }) => {
if let Ok(subtree) = src.clone().expect_subtree() {
if subtree.delimiter_kind() == delimiter.map(|it| it.kind) {
if subtree.delimiter.kind == delimiter.kind {
item.stack.push(item.dot);
item.dot = tokens.iter_delimited(delimiter.as_ref());
item.dot = tokens.iter_delimited(Some(delimiter));
cur_items.push(item);
}
}
@ -663,8 +665,8 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
}
res.add_err(ExpandError::LeftoverTokens);
if let Some(error_reover_item) = error_recover_item {
res.bindings = bindings_builder.build(&error_reover_item);
if let Some(error_recover_item) = error_recover_item {
res.bindings = bindings_builder.build(&error_recover_item);
}
return res;
}
@ -782,7 +784,7 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Opt
match neg {
None => lit.into(),
Some(neg) => tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![neg, lit.into()],
}),
}
@ -810,7 +812,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
}
impl MetaTemplate {
fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> {
OpDelimitedIter { inner: &self.0, idx: 0, delimited }
OpDelimitedIter {
inner: &self.0,
idx: 0,
delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED),
}
}
}
@ -824,20 +830,21 @@ enum OpDelimited<'a> {
#[derive(Debug, Clone, Copy)]
struct OpDelimitedIter<'a> {
inner: &'a [Op],
delimited: Option<&'a tt::Delimiter>,
delimited: &'a tt::Delimiter,
idx: usize,
}
impl<'a> OpDelimitedIter<'a> {
fn is_eof(&self) -> bool {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
self.idx >= len
}
fn peek(&self) -> Option<OpDelimited<'a>> {
match self.delimited {
None => self.inner.get(self.idx).map(OpDelimited::Op),
Some(_) => match self.idx {
match self.delimited.kind {
tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
_ => match self.idx {
0 => Some(OpDelimited::Open),
i if i == self.inner.len() + 1 => Some(OpDelimited::Close),
i => self.inner.get(i - 1).map(OpDelimited::Op),
@ -860,7 +867,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> {
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
let remain = len.saturating_sub(self.idx);
(remain, Some(remain))
}
@ -904,7 +912,10 @@ impl<'a> TtIter<'a> {
} else {
let puncts = self.expect_glued_punct()?;
let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect();
Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees }))
Ok(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees,
}))
}
} else {
self.next().ok_or(()).cloned()
@ -919,7 +930,7 @@ impl<'a> TtIter<'a> {
let ident = self.expect_ident_or_underscore()?;
Ok(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![
tt::Leaf::Punct(*punct).into(),
tt::Leaf::Ident(ident.clone()).into(),

70
crates/mbe/src/expander/transcriber.rs
View file

@ -2,11 +2,11 @@
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use syntax::SmolStr;
use tt::{Delimiter, Subtree};
use crate::{
expander::{Binding, Bindings, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt::{self, Delimiter},
ExpandError, ExpandResult, MetaTemplate,
};
@ -44,22 +44,23 @@ impl Bindings {
Binding::Missing(it) => Ok(match it {
MetaVarKind::Stmt => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
char: ';',
spacing: tt::Spacing::Alone,
})))
}
MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
})),
// FIXME: Meta and Item should get proper defaults
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}))
}
@ -71,19 +72,19 @@ impl Bindings {
| MetaVarKind::Ident => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("missing"),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
MetaVarKind::Lifetime => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("'missing"),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
MetaVarKind::Literal => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("\"missing\""),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
}),
@ -143,7 +144,7 @@ fn expand_subtree(
}
Op::Subtree { tokens, delimiter } => {
let ExpandResult { value: tt, err: e } =
expand_subtree(ctx, tokens, *delimiter, arena);
expand_subtree(ctx, tokens, Some(*delimiter), arena);
err = err.or(e);
arena.push(tt.into());
}
@ -170,7 +171,7 @@ fn expand_subtree(
arena.push(
tt::Leaf::Literal(tt::Literal {
text: index.to_string().into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})
.into(),
);
@ -179,7 +180,13 @@ fn expand_subtree(
}
// drain the elements added in this instance of expand_subtree
let tts = arena.drain(start_elements..).collect();
ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err }
ExpandResult {
value: tt::Subtree {
delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified),
token_trees: tts,
},
err,
}
}
fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@ -201,17 +208,24 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
// ```
// We just treat it a normal tokens
let tt = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(),
tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(),
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id })
.into(),
tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(),
],
}
.into();
ExpandResult::ok(Fragment::Tokens(tt))
} else {
ctx.bindings.get(v, &mut ctx.nesting).map_or_else(
|e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) },
|e| ExpandResult {
value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![],
})),
err: Some(e),
},
ExpandResult::ok,
)
}
@ -249,7 +263,10 @@ fn expand_repeat(
ctx
);
return ExpandResult {
value: Fragment::Tokens(Subtree::default().into()),
value: Fragment::Tokens(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }
.into(),
),
err: Some(ExpandError::LimitExceeded),
};
}
@ -258,7 +275,7 @@ fn expand_repeat(
continue;
}
t.delimiter = None;
t.delimiter = tt::Delimiter::unspecified();
push_subtree(&mut buf, t);
if let Some(sep) = separator {
@ -292,7 +309,7 @@ fn expand_repeat(
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: None, token_trees: buf }.into();
let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into();
if RepeatKind::OneOrMore == kind && counter == 0 {
return ExpandResult {
@ -307,11 +324,12 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
match fragment {
Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => {
if tt.delimiter.is_none() {
tt.delimiter = Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
if tt.delimiter.kind == tt::DelimiterKind::Invisible {
tt.delimiter = tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Parenthesis,
})
};
}
buf.push(tt.into())
}
@ -320,8 +338,8 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
}
fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
match tt.delimiter {
None => buf.extend(tt.token_trees),
Some(_) => buf.push(tt.into()),
match tt.delimiter.kind {
tt::DelimiterKind::Invisible => buf.extend(tt.token_trees),
_ => buf.push(tt.into()),
}
}

55
crates/mbe/src/lib.rs
View file

@ -18,6 +18,8 @@ mod to_parser_input;
mod benchmark;
mod token_map;
use ::tt::token_id as tt;
use std::fmt;
use crate::{
@ -26,8 +28,8 @@ use crate::{
};
// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces
pub use self::tt::{Delimiter, DelimiterKind, Punct};
pub use ::parser::TopEntryPoint;
pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::{
syntax_bridge::{
@ -125,24 +127,26 @@ impl Shift {
// Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> {
let filter = |tt: &_| match tt {
tt::TokenTree::Subtree(subtree) => {
let tree_id = max_id(subtree);
match subtree.delimiter {
Some(it) if it.id != tt::TokenId::unspecified() => {
Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
let filter =
|tt: &_| match tt {
tt::TokenTree::Subtree(subtree) => {
let tree_id = max_id(subtree);
if subtree.delimiter.open != tt::TokenId::unspecified() {
Some(tree_id.map_or(subtree.delimiter.open.0, |t| {
t.max(subtree.delimiter.open.0)
}))
} else {
tree_id
}
_ => tree_id,
}
}
tt::TokenTree::Leaf(leaf) => {
let &(tt::Leaf::Ident(tt::Ident { id, .. })
| tt::Leaf::Punct(tt::Punct { id, .. })
| tt::Leaf::Literal(tt::Literal { id, .. })) = leaf;
tt::TokenTree::Leaf(leaf) => {
let &(tt::Leaf::Ident(tt::Ident { span, .. })
| tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Literal(tt::Literal { span, .. })) = leaf;
(id != tt::TokenId::unspecified()).then_some(id.0)
}
};
(span != tt::TokenId::unspecified()).then_some(span.0)
}
};
subtree.token_trees.iter().filter_map(filter).max()
}
}
@ -152,14 +156,13 @@ impl Shift {
for t in &mut tt.token_trees {
match t {
tt::TokenTree::Leaf(
tt::Leaf::Ident(tt::Ident { id, .. })
| tt::Leaf::Punct(tt::Punct { id, .. })
| tt::Leaf::Literal(tt::Literal { id, .. }),
) => *id = self.shift(*id),
tt::Leaf::Ident(tt::Ident { span, .. })
| tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Literal(tt::Literal { span, .. }),
) => *span = self.shift(*span),
tt::TokenTree::Subtree(tt) => {
if let Some(it) = tt.delimiter.as_mut() {
it.id = self.shift(it.id);
}
tt.delimiter.open = self.shift(tt.delimiter.open);
tt.delimiter.close = self.shift(tt.delimiter.close);
self.shift_all(tt)
}
}
@ -216,7 +219,7 @@ impl DeclarativeMacro {
let mut src = TtIter::new(tt);
let mut rules = Vec::new();
if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() {
if tt::DelimiterKind::Brace == tt.delimiter.kind {
cov_mark::hit!(parse_macro_def_rules);
while src.len() > 0 {
let rule = Rule::parse(&mut src, true)?;
@ -325,6 +328,10 @@ impl<T, E> ValueResult<T, E> {
Self { value, err: None }
}
pub fn with_err(value: T, err: E) -> Self {
Self { value, err: Some(err) }
}
pub fn only_err(err: E) -> Self
where
T: Default,

20
crates/mbe/src/parser.rs
View file

@ -4,7 +4,7 @@
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use crate::{tt_iter::TtIter, ParseError};
use crate::{tt, tt_iter::TtIter, ParseError};
/// Consider
///
@ -54,7 +54,7 @@ pub(crate) enum Op {
Ignore { name: SmolStr, id: tt::TokenId },
Index { depth: u32 },
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> },
Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter },
Literal(tt::Literal),
Punct(SmallVec<[tt::Punct; 3]>),
Ident(tt::Ident),
@ -130,13 +130,13 @@ fn next_op(
Some(it) => it,
};
match second {
tt::TokenTree::Subtree(subtree) => match subtree.delimiter_kind() {
Some(tt::DelimiterKind::Parenthesis) => {
tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => {
let (separator, kind) = parse_repeat(src)?;
let tokens = MetaTemplate::parse(subtree, mode)?;
Op::Repeat { tokens, separator, kind }
}
Some(tt::DelimiterKind::Brace) => match mode {
tt::DelimiterKind::Brace => match mode {
Mode::Template => {
parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| {
ParseError::unexpected("invalid metavariable expression")
@ -157,18 +157,18 @@ fn next_op(
tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Ident(ident) if ident.text == "crate" => {
// We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id })
Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span })
}
tt::Leaf::Ident(ident) => {
let kind = eat_fragment_kind(src, mode)?;
let name = ident.text.clone();
let id = ident.id;
let id = ident.span;
Op::Var { name, kind, id }
}
tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
let kind = eat_fragment_kind(src, mode)?;
let name = lit.text.clone();
let id = lit.id;
let id = lit.span;
Op::Var { name, kind, id }
}
tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode {
@ -284,7 +284,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let func = src.expect_ident()?;
let args = src.expect_subtree()?;
if args.delimiter_kind() != Some(tt::DelimiterKind::Parenthesis) {
if args.delimiter.kind != tt::DelimiterKind::Parenthesis {
return Err(());
}
@ -293,7 +293,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let op = match &*func.text {
"ignore" => {
let ident = args.expect_ident()?;
Op::Ignore { name: ident.text.clone(), id: ident.id }
Op::Ignore { name: ident.text.clone(), id: ident.span }
}
"index" => {
let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? };

126
crates/mbe/src/syntax_bridge.rs
View file

@ -8,9 +8,16 @@ use syntax::{
SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
use tt::buffer::{Cursor, TokenBuffer};
use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
use crate::{
to_parser_input::to_parser_input,
tt::{
self,
buffer::{Cursor, TokenBuffer},
},
tt_iter::TtIter,
TokenMap,
};
#[cfg(test)]
mod tests;
@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node(
entry_point: parser::TopEntryPoint,
) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt {
tt::Subtree { delimiter: None, token_trees } => {
TokenBuffer::from_tokens(token_trees.as_slice())
}
tt::Subtree {
delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. },
token_trees,
} => TokenBuffer::from_tokens(token_trees.as_slice()),
_ => TokenBuffer::from_subtree(tt),
};
let parser_input = to_parser_input(&buffer);
@ -92,8 +100,7 @@ pub fn token_tree_to_syntax_node(
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
}
}
let (parse, range_map) = tree_sink.finish();
(parse, range_map)
tree_sink.finish()
}
/// Convert a string to a `TokenTree`
@ -132,7 +139,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
res.push(match expanded.value {
None => break,
Some(tt @ tt::TokenTree::Leaf(_)) => {
tt::Subtree { delimiter: None, token_trees: vec![tt] }
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] }
}
Some(tt::TokenTree::Subtree(tt)) => tt,
});
@ -145,7 +152,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
}
if iter.peek_n(0).is_some() {
res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() });
res.push(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: iter.cloned().collect(),
});
}
res
@ -159,7 +169,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
}
let entry = StackEntry {
subtree: tt::Subtree { delimiter: None, ..Default::default() },
subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
// never used (delimiter is `None`)
idx: !0,
open_range: TextRange::empty(TextSize::of('.')),
@ -186,7 +196,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
sub.token_trees.get_mut(2)
{
lit.id = id
lit.span = id
}
}
tt
@ -199,13 +209,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
assert_eq!(range.len(), TextSize::of('.'));
}
if let Some(delim) = subtree.delimiter {
let expected = match delim.kind {
tt::DelimiterKind::Parenthesis => T![')'],
tt::DelimiterKind::Brace => T!['}'],
tt::DelimiterKind::Bracket => T![']'],
};
let expected = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
};
if let Some(expected) = expected {
if kind == expected {
if let Some(entry) = stack.pop() {
conv.id_alloc().close_delim(entry.idx, Some(range));
@ -223,9 +234,11 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
};
if let Some(kind) = delim {
let mut subtree = tt::Subtree::default();
let (id, idx) = conv.id_alloc().open_delim(range, synth_id);
subtree.delimiter = Some(tt::Delimiter { id, kind });
let subtree = tt::Subtree {
delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind },
token_trees: vec![],
};
stack.push(StackEntry { subtree, idx, open_range: range });
continue;
}
@ -240,13 +253,20 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
panic!("Token from lexer must be single char: token = {token:#?}");
}
};
tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) })
.into()
tt::Leaf::from(tt::Punct {
char,
spacing,
span: conv.id_alloc().alloc(range, synth_id),
})
.into()
} else {
macro_rules! make_leaf {
($i:ident) => {
tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) }
.into()
tt::$i {
span: conv.id_alloc().alloc(range, synth_id),
text: token.to_text(conv),
}
.into()
};
}
let leaf: tt::Leaf = match kind {
@ -261,14 +281,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
let apostrophe = tt::Leaf::from(tt::Punct {
char: '\'',
spacing: tt::Spacing::Joint,
id: conv.id_alloc().alloc(r, synth_id),
span: conv.id_alloc().alloc(r, synth_id),
});
result.push(apostrophe.into());
let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
let ident = tt::Leaf::from(tt::Ident {
text: SmolStr::new(&token.to_text(conv)[1..]),
id: conv.id_alloc().alloc(r, synth_id),
span: conv.id_alloc().alloc(r, synth_id),
});
result.push(ident.into());
continue;
@ -289,11 +309,12 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
conv.id_alloc().close_delim(entry.idx, None);
let leaf: tt::Leaf = tt::Punct {
id: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.unwrap().kind {
span: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => '(',
tt::DelimiterKind::Brace => '{',
tt::DelimiterKind::Bracket => '[',
tt::DelimiterKind::Invisible => '$',
},
spacing: tt::Spacing::Alone,
}
@ -373,10 +394,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
token_trees.push(mk_punct('!'));
}
token_trees.push(tt::TokenTree::from(tt::Subtree {
delimiter: Some(tt::Delimiter {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Bracket,
id: tt::TokenId::unspecified(),
}),
},
token_trees: meta_tkns,
}));
@ -386,7 +408,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
fn mk_ident(s: &str) -> tt::TokenTree {
tt::TokenTree::from(tt::Leaf::from(tt::Ident {
text: s.into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
}
@ -394,12 +416,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
tt::TokenTree::from(tt::Leaf::from(tt::Punct {
char: c,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
}
fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() };
tt::TokenTree::from(tt::Leaf::from(lit))
}
@ -761,15 +783,16 @@ impl<'a> TtTreeSink<'a> {
}
}
fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
let texts = match d {
tt::DelimiterKind::Parenthesis => "()",
tt::DelimiterKind::Brace => "{}",
tt::DelimiterKind::Bracket => "[]",
tt::DelimiterKind::Invisible => return None,
};
let idx = closing as usize;
&texts[idx..texts.len() - (1 - idx)]
Some(&texts[idx..texts.len() - (1 - idx)])
}
impl<'a> TtTreeSink<'a> {
@ -790,13 +813,16 @@ impl<'a> TtTreeSink<'a> {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
// Mark the range if needed
let (text, id) = match leaf {
tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span),
tt::Leaf::Punct(punct) => {
assert!(punct.char.is_ascii());
tmp = punct.char as u8;
(std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
(
std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(),
punct.span,
)
}
tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span),
};
let range = TextRange::at(self.text_pos, TextSize::of(text));
self.token_map.insert(id, range);
@ -805,10 +831,10 @@ impl<'a> TtTreeSink<'a> {
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cursor = self.cursor.subtree().unwrap();
match subtree.delimiter {
Some(d) => {
self.open_delims.insert(d.id, self.text_pos);
delim_to_str(d.kind, false)
match delim_to_str(subtree.delimiter.kind, false) {
Some(it) => {
self.open_delims.insert(subtree.delimiter.open, self.text_pos);
it
}
None => continue,
}
@ -816,15 +842,21 @@ impl<'a> TtTreeSink<'a> {
None => {
let parent = self.cursor.end().unwrap();
self.cursor = self.cursor.bump();
match parent.delimiter {
Some(d) => {
if let Some(open_delim) = self.open_delims.get(&d.id) {
match delim_to_str(parent.delimiter.kind, true) {
Some(it) => {
if let Some(open_delim) =
self.open_delims.get(&parent.delimiter.open)
{
let open_range = TextRange::at(*open_delim, TextSize::of('('));
let close_range =
TextRange::at(self.text_pos, TextSize::of('('));
self.token_map.insert_delim(d.id, open_range, close_range);
self.token_map.insert_delim(
parent.delimiter.open,
open_range,
close_range,
);
}
delim_to_str(d.kind, true)
it
}
None => continue,
}

4
crates/mbe/src/syntax_bridge/tests.rs
View file

@ -29,8 +29,8 @@ fn check_punct_spacing(fixture: &str) {
let mut cursor = buf.begin();
while !cursor.eof() {
while let Some(token_tree) = cursor.token_tree() {
if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree {
if let Some(expected) = annotations.remove(id) {
if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree {
if let Some(expected) = annotations.remove(span) {
assert_eq!(expected, *spacing);
}
}

29
crates/mbe/src/to_parser_input.rs
View file

@ -2,7 +2,8 @@
//! format that works for our parser.
use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
use crate::tt::buffer::TokenBuffer;
pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
let mut res = parser::Input::default();
@ -70,23 +71,25 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
cursor.bump()
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
if let Some(d) = subtree.delimiter_kind() {
res.push(match d {
tt::DelimiterKind::Parenthesis => T!['('],
tt::DelimiterKind::Brace => T!['{'],
tt::DelimiterKind::Bracket => T!['['],
});
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T!['(']),
tt::DelimiterKind::Brace => Some(T!['{']),
tt::DelimiterKind::Bracket => Some(T!['[']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.subtree().unwrap()
}
None => match cursor.end() {
Some(subtree) => {
if let Some(d) = subtree.delimiter_kind() {
res.push(match d {
tt::DelimiterKind::Parenthesis => T![')'],
tt::DelimiterKind::Brace => T!['}'],
tt::DelimiterKind::Bracket => T![']'],
})
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.bump()
}

7
crates/mbe/src/tt_iter.rs
View file

@ -3,9 +3,8 @@
use smallvec::{smallvec, SmallVec};
use syntax::SyntaxKind;
use tt::buffer::TokenBuffer;
use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult};
use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone)]
pub(crate) struct TtIter<'a> {
@ -135,7 +134,7 @@ impl<'a> TtIter<'a> {
&mut self,
entry_point: parser::PrefixEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> {
let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
let parser_input = to_parser_input(&buffer);
let tree_traversal = entry_point.parse(&parser_input);
@ -178,7 +177,7 @@ impl<'a> TtIter<'a> {
1 => Some(res[0].cloned()),
0 => None,
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
})),
};

9
crates/proc-macro-api/src/lib.rs
View file

@ -19,7 +19,8 @@ use std::{
};
use serde::{Deserialize, Serialize};
use tt::Subtree;
use ::tt::token_id as tt;
use crate::{
msg::{ExpandMacro, FlatTree, PanicMessage},
@ -151,10 +152,10 @@ impl ProcMacro {
pub fn expand(
&self,
subtree: &Subtree,
attr: Option<&Subtree>,
subtree: &tt::Subtree,
attr: Option<&tt::Subtree>,
env: Vec<(String, String)>,
) -> Result<Result<Subtree, PanicMessage>, ServerError> {
) -> Result<Result<tt::Subtree, PanicMessage>, ServerError> {
let current_dir = env
.iter()
.find(|(name, _)| name == "CARGO_MANIFEST_DIR")

18
crates/proc-macro-api/src/msg.rs
View file

@ -107,27 +107,31 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use tt::*;
use crate::tt::*;
fn fixture_token_tree() -> Subtree {
let mut subtree = Subtree::default();
let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() };
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into()));
.push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into()));
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into()));
.push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into()));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal {
text: "Foo".into(),
id: TokenId::unspecified(),
span: TokenId::unspecified(),
})));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct {
char: '@',
id: TokenId::unspecified(),
span: TokenId::unspecified(),
spacing: Spacing::Joint,
})));
subtree.token_trees.push(TokenTree::Subtree(Subtree {
delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }),
delimiter: Delimiter {
open: TokenId(2),
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Brace,
},
token_trees: vec![],
}));
subtree

43
crates/proc-macro-api/src/msg/flat.rs
View file

@ -38,7 +38,8 @@
use std::collections::{HashMap, VecDeque};
use serde::{Deserialize, Serialize};
use tt::TokenId;
use crate::tt::{self, TokenId};
#[derive(Serialize, Deserialize, Debug)]
pub struct FlatTree {
@ -52,7 +53,7 @@ pub struct FlatTree {
struct SubtreeRepr {
id: tt::TokenId,
kind: Option<tt::DelimiterKind>,
kind: tt::DelimiterKind,
tt: [u32; 2],
}
@ -124,19 +125,19 @@ impl FlatTree {
impl SubtreeRepr {
fn write(self) -> [u32; 4] {
let kind = match self.kind {
None => 0,
Some(tt::DelimiterKind::Parenthesis) => 1,
Some(tt::DelimiterKind::Brace) => 2,
Some(tt::DelimiterKind::Bracket) => 3,
tt::DelimiterKind::Invisible => 0,
tt::DelimiterKind::Parenthesis => 1,
tt::DelimiterKind::Brace => 2,
tt::DelimiterKind::Bracket => 3,
};
[self.id.0, kind, self.tt[0], self.tt[1]]
}
fn read([id, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
let kind = match kind {
0 => None,
1 => Some(tt::DelimiterKind::Parenthesis),
2 => Some(tt::DelimiterKind::Brace),
3 => Some(tt::DelimiterKind::Bracket),
0 => tt::DelimiterKind::Invisible,
1 => tt::DelimiterKind::Parenthesis,
2 => tt::DelimiterKind::Brace,
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr { id: TokenId(id), kind, tt: [lo, len] }
@ -216,7 +217,7 @@ impl<'a> Writer<'a> {
tt::Leaf::Literal(lit) => {
let idx = self.literal.len() as u32;
let text = self.intern(&lit.text);
self.literal.push(LiteralRepr { id: lit.id, text });
self.literal.push(LiteralRepr { id: lit.span, text });
idx << 2 | 0b01
}
tt::Leaf::Punct(punct) => {
@ -224,14 +225,14 @@ impl<'a> Writer<'a> {
self.punct.push(PunctRepr {
char: punct.char,
spacing: punct.spacing,
id: punct.id,
id: punct.span,
});
idx << 2 | 0b10
}
tt::Leaf::Ident(ident) => {
let idx = self.ident.len() as u32;
let text = self.intern(&ident.text);
self.ident.push(IdentRepr { id: ident.id, text });
self.ident.push(IdentRepr { id: ident.span, text });
idx << 2 | 0b11
}
},
@ -243,8 +244,8 @@ impl<'a> Writer<'a> {
fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 {
let idx = self.subtree.len();
let delimiter_id = subtree.delimiter.map_or(TokenId::unspecified(), |it| it.id);
let delimiter_kind = subtree.delimiter.map(|it| it.kind);
let delimiter_id = subtree.delimiter.open;
let delimiter_kind = subtree.delimiter.kind;
self.subtree.push(SubtreeRepr { id: delimiter_id, kind: delimiter_kind, tt: [!0, !0] });
self.work.push_back((idx, subtree));
idx as u32
@ -276,7 +277,11 @@ impl Reader {
let repr = &self.subtree[i];
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
let s = tt::Subtree {
delimiter: repr.kind.map(|kind| tt::Delimiter { id: repr.id, kind }),
delimiter: tt::Delimiter {
open: repr.id,
close: TokenId::UNSPECIFIED,
kind: repr.kind,
},
token_trees: token_trees
.iter()
.copied()
@ -291,7 +296,7 @@ impl Reader {
let repr = &self.literal[idx];
tt::Leaf::Literal(tt::Literal {
text: self.text[repr.text as usize].as_str().into(),
id: repr.id,
span: repr.id,
})
.into()
}
@ -300,7 +305,7 @@ impl Reader {
tt::Leaf::Punct(tt::Punct {
char: repr.char,
spacing: repr.spacing,
id: repr.id,
span: repr.id,
})
.into()
}
@ -308,7 +313,7 @@ impl Reader {
let repr = &self.ident[idx];
tt::Leaf::Ident(tt::Ident {
text: self.text[repr.text as usize].as_str().into(),
id: repr.id,
span: repr.id,
})
.into()
}

1
crates/proc-macro-srv/src/abis/abi_1_63/mod.rs
View file

@ -11,6 +11,7 @@ mod ra_server;
use libloading::Library;
use proc_macro_api::ProcMacroKind;
use super::tt;
use super::PanicMessage;
pub use ra_server::TokenStream;

119
crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs
View file

@ -15,6 +15,8 @@ use std::hash::Hash;
use std::ops::Bound;
use std::{ascii, vec::IntoIter};
use crate::tt;
type Group = tt::Subtree;
type TokenTree = tt::TokenTree;
type Punct = tt::Punct;
@ -33,7 +35,7 @@ impl TokenStream {
}
pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() {
if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else {
TokenStream { token_trees: subtree.token_trees }
@ -41,7 +43,7 @@ impl TokenStream {
}
pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees }
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
}
pub fn is_empty(&self) -> bool {
@ -84,7 +86,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams {
for tkn in item {
match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => {
tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind == tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees);
}
_ => {
@ -165,7 +169,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream {
use std::str::FromStr;
use super::{TokenStream, TokenTree};
use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -202,15 +206,17 @@ pub mod token_stream {
impl ToString for TokenStream {
fn to_string(&self) -> String {
tt::pretty(&self.token_trees)
::tt::pretty(&self.token_trees)
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree
.delimiter
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree
.token_trees
.into_iter()
@ -233,13 +239,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
}
}
}
@ -389,22 +395,22 @@ impl server::TokenStream for RustAnalyzer {
}
}
fn delim_to_internal(d: bridge::Delimiter) -> Option<tt::Delimiter> {
fn delim_to_internal(d: bridge::Delimiter) -> tt::Delimiter {
let kind = match d {
bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
bridge::Delimiter::Brace => tt::DelimiterKind::Brace,
bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket,
bridge::Delimiter::None => return None,
bridge::Delimiter::None => tt::DelimiterKind::Invisible,
};
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind })
tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
}
fn delim_to_external(d: Option<tt::Delimiter>) -> bridge::Delimiter {
match d.map(|it| it.kind) {
Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket,
None => bridge::Delimiter::None,
fn delim_to_external(d: tt::Delimiter) -> bridge::Delimiter {
match d.kind {
tt::DelimiterKind::Parenthesis => bridge::Delimiter::Parenthesis,
tt::DelimiterKind::Brace => bridge::Delimiter::Brace,
tt::DelimiterKind::Bracket => bridge::Delimiter::Bracket,
tt::DelimiterKind::Invisible => bridge::Delimiter::None,
}
}
@ -443,23 +449,19 @@ impl server::Group for RustAnalyzer {
}
fn span(&mut self, group: &Self::Group) -> Self::Span {
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.open
}
fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
if let Some(delim) = &mut group.delimiter {
delim.id = span;
}
group.delimiter.open = span;
}
fn span_open(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.open
}
fn span_close(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.close
}
}
@ -468,7 +470,7 @@ impl server::Punct for RustAnalyzer {
tt::Punct {
char: ch,
spacing: spacing_to_internal(spacing),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}
}
fn as_char(&mut self, punct: Self::Punct) -> char {
@ -478,28 +480,27 @@ impl server::Punct for RustAnalyzer {
spacing_to_external(punct.spacing)
}
fn span(&mut self, punct: Self::Punct) -> Self::Span {
punct.id
punct.span
}
fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
tt::Punct { id: span, ..punct }
tt::Punct { span: span, ..punct }
}
}
impl server::Ident for RustAnalyzer {
fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw(
string.into(),
IdentId(self.ident_interner.intern(&IdentData(tt::Ident {
text: if is_raw { ::tt::SmolStr::from_iter(["r#", string]) } else { string.into() },
span,
is_raw,
))))
})))
}
fn span(&mut self, ident: Self::Ident) -> Self::Span {
self.ident_interner.get(ident.0).0.id
self.ident_interner.get(ident.0).0.span
}
fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
let data = self.ident_interner.get(ident.0);
let new = IdentData(tt::Ident { id: span, ..data.0.clone() });
let new = IdentData(tt::Ident { span: span, ..data.0.clone() });
IdentId(self.ident_interner.intern(&new))
}
}
@ -511,7 +512,7 @@ impl server::Literal for RustAnalyzer {
"".to_owned()
}
fn from_str(&mut self, s: &str) -> Result<Self::Literal, ()> {
Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() })
Ok(Literal { text: s.into(), span: tt::TokenId::unspecified() })
}
fn symbol(&mut self, literal: &Self::Literal) -> String {
literal.text.to_string()
@ -529,7 +530,7 @@ impl server::Literal for RustAnalyzer {
Ok(n) => n.to_string(),
Err(_) => n.parse::<u128>().unwrap().to_string(),
};
Literal { text: n.into(), id: tt::TokenId::unspecified() }
Literal { text: n.into(), span: tt::TokenId::unspecified() }
}
fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
@ -549,7 +550,7 @@ impl server::Literal for RustAnalyzer {
let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize};
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn float(&mut self, n: &str) -> Self::Literal {
@ -558,19 +559,19 @@ impl server::Literal for RustAnalyzer {
if !text.contains('.') {
text += ".0"
}
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn f32(&mut self, n: &str) -> Self::Literal {
let n: f32 = n.parse().unwrap();
let text = format!("{n}f32");
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn f64(&mut self, n: &str) -> Self::Literal {
let n: f64 = n.parse().unwrap();
let text = format!("{n}f64");
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn string(&mut self, string: &str) -> Self::Literal {
@ -578,11 +579,11 @@ impl server::Literal for RustAnalyzer {
for ch in string.chars() {
escaped.extend(ch.escape_debug());
}
Literal { text: format!("\"{escaped}\"").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("\"{escaped}\"").into(), span: tt::TokenId::unspecified() }
}
fn character(&mut self, ch: char) -> Self::Literal {
Literal { text: format!("'{ch}'").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("'{ch}'").into(), span: tt::TokenId::unspecified() }
}
fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
@ -593,15 +594,15 @@ impl server::Literal for RustAnalyzer {
.map(Into::<char>::into)
.collect::<String>();
Literal { text: format!("b\"{string}\"").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("b\"{string}\"").into(), span: tt::TokenId::unspecified() }
}
fn span(&mut self, literal: &Self::Literal) -> Self::Span {
literal.id
literal.span
}
fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
literal.id = span;
literal.span = span;
}
fn subspan(
@ -784,17 +785,18 @@ mod tests {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
}),
],
@ -807,13 +809,14 @@ mod tests {
fn test_ra_server_from_str() {
use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis,
}),
},
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
});
@ -830,7 +833,7 @@ mod tests {
underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
);
}

2
crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs
View file

@ -9,7 +9,7 @@ mod ra_server;
use libloading::Library;
use proc_macro_api::ProcMacroKind;
use super::PanicMessage;
use super::{tt, PanicMessage};
pub use ra_server::TokenStream;

65
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs
View file

@ -22,6 +22,8 @@ pub use symbol::*;
use std::ops::Bound;
use crate::tt;
type Group = tt::Subtree;
type TokenTree = tt::TokenTree;
type Punct = tt::Punct;
@ -108,8 +110,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(ident) => {
let text = ident.sym.text();
let text = if ident.is_raw { tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, id: ident.span };
let text =
if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, span: ident.span };
let leaf = tt::Leaf::from(ident);
let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree])
@ -118,9 +121,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Literal(literal) => {
let literal = LiteralFormatter(literal);
let text = literal
.with_stringify_parts(|parts| tt::SmolStr::from_iter(parts.iter().copied()));
.with_stringify_parts(|parts| ::tt::SmolStr::from_iter(parts.iter().copied()));
let literal = tt::Literal { text, id: literal.0.span };
let literal = tt::Literal { text, span: literal.0.span };
let leaf = tt::Leaf::from(literal);
let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree])
@ -130,7 +133,7 @@ impl server::TokenStream for RustAnalyzer {
let punct = tt::Punct {
char: p.ch as char,
spacing: if p.joint { Spacing::Joint } else { Spacing::Alone },
id: p.span,
span: p.span,
};
let leaf = tt::Leaf::from(punct);
let tree = TokenTree::from(leaf);
@ -184,7 +187,7 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(bridge::Ident {
sym: Symbol::intern(ident.text.trim_start_matches("r#")),
is_raw: ident.text.starts_with("r#"),
span: ident.id,
span: ident.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
@ -194,14 +197,14 @@ impl server::TokenStream for RustAnalyzer {
symbol: Symbol::intern(&lit.text),
// FIXME: handle suffixes
suffix: None,
span: lit.id,
span: lit.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
bridge::TokenTree::Punct(bridge::Punct {
ch: punct.char as u8,
joint: punct.spacing == Spacing::Joint,
span: punct.id,
span: punct.span,
})
}
tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group {
@ -211,31 +214,29 @@ impl server::TokenStream for RustAnalyzer {
} else {
Some(subtree.token_trees.into_iter().collect())
},
span: bridge::DelimSpan::from_single(
subtree.delimiter.map_or(Span::unspecified(), |del| del.id),
),
span: bridge::DelimSpan::from_single(subtree.delimiter.open),
}),
})
.collect()
}
}
fn delim_to_internal(d: proc_macro::Delimiter) -> Option<tt::Delimiter> {
fn delim_to_internal(d: proc_macro::Delimiter) -> tt::Delimiter {
let kind = match d {
proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace,
proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket,
proc_macro::Delimiter::None => return None,
proc_macro::Delimiter::None => tt::DelimiterKind::Invisible,
};
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind })
tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
}
fn delim_to_external(d: Option<tt::Delimiter>) -> proc_macro::Delimiter {
match d.map(|it| it.kind) {
Some(tt::DelimiterKind::Parenthesis) => proc_macro::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => proc_macro::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => proc_macro::Delimiter::Bracket,
None => proc_macro::Delimiter::None,
fn delim_to_external(d: tt::Delimiter) -> proc_macro::Delimiter {
match d.kind {
tt::DelimiterKind::Parenthesis => proc_macro::Delimiter::Parenthesis,
tt::DelimiterKind::Brace => proc_macro::Delimiter::Brace,
tt::DelimiterKind::Bracket => proc_macro::Delimiter::Bracket,
tt::DelimiterKind::Invisible => proc_macro::Delimiter::None,
}
}
@ -349,7 +350,7 @@ impl server::Server for RustAnalyzer {
}
fn intern_symbol(ident: &str) -> Self::Symbol {
Symbol::intern(&tt::SmolStr::from(ident))
Symbol::intern(&::tt::SmolStr::from(ident))
}
fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
@ -413,17 +414,18 @@ mod tests {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
}),
],
@ -436,13 +438,14 @@ mod tests {
fn test_ra_server_from_str() {
use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis,
}),
},
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
});
@ -459,7 +462,7 @@ mod tests {
underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
);
}

28
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs
View file

@ -1,6 +1,6 @@
//! TokenStream implementation used by sysroot ABI
use tt::TokenTree;
use crate::tt::{self, TokenTree};
#[derive(Debug, Default, Clone)]
pub struct TokenStream {
@ -13,7 +13,7 @@ impl TokenStream {
}
pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() {
if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else {
TokenStream { token_trees: subtree.token_trees }
@ -21,7 +21,7 @@ impl TokenStream {
}
pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees }
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
}
pub fn is_empty(&self) -> bool {
@ -64,7 +64,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams {
for tkn in item {
match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => {
tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind != tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees);
}
_ => {
@ -84,7 +86,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream {
use std::str::FromStr;
use super::{TokenStream, TokenTree};
use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -121,15 +123,17 @@ pub mod token_stream {
impl ToString for TokenStream {
fn to_string(&self) -> String {
tt::pretty(&self.token_trees)
::tt::pretty(&self.token_trees)
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree
.delimiter
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree
.token_trees
.into_iter()
@ -152,13 +156,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
}
}
}

2
crates/proc-macro-srv/src/abis/mod.rs
View file

@ -41,6 +41,8 @@ pub(crate) use abi_sysroot::Abi as Abi_Sysroot;
use libloading::Library;
use proc_macro_api::{ProcMacroKind, RustCInfo};
use crate::tt;
pub struct PanicMessage {
message: Option<String>,
}

2
crates/proc-macro-srv/src/dylib.rs
View file

@ -13,6 +13,8 @@ use object::Object;
use paths::AbsPath;
use proc_macro_api::{read_dylib_info, ProcMacroKind};
use crate::tt;
use super::abis::Abi;
const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";

2
crates/proc-macro-srv/src/lib.rs
View file

@ -37,6 +37,8 @@ use proc_macro_api::{
ProcMacroKind,
};
use ::tt::token_id as tt;
#[derive(Default)]
pub(crate) struct ProcMacroSrv {
expanders: HashMap<(PathBuf, SystemTime), dylib::Expander>,

26
crates/proc-macro-srv/src/tests/mod.rs
View file

@ -8,7 +8,7 @@ use expect_test::expect;
#[test]
fn test_derive_empty() {
assert_expand("DeriveEmpty", r#"struct S;"#, expect![[r#"SUBTREE $"#]]);
assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 4294967295 4294967295"]);
}
#[test]
@ -17,10 +17,10 @@ fn test_derive_error() {
"DeriveError",
r#"struct S;"#,
expect![[r##"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295
SUBTREE () 4294967295
SUBTREE () 4294967295 4294967295
LITERAL "#[derive(DeriveError)] struct S ;" 4294967295
PUNCH ; [alone] 4294967295"##]],
);
@ -32,14 +32,14 @@ fn test_fn_like_macro_noop() {
"fn_like_noop",
r#"ident, 0, 1, []"#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295
PUNCH , [alone] 4294967295
LITERAL 0 4294967295
PUNCH , [alone] 4294967295
LITERAL 1 4294967295
PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]],
SUBTREE [] 4294967295 4294967295"#]],
);
}
@ -49,10 +49,10 @@ fn test_fn_like_macro_clone_ident_subtree() {
"fn_like_clone_tokens",
r#"ident, []"#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295
PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]],
SUBTREE [] 4294967295 4294967295"#]],
);
}
@ -62,7 +62,7 @@ fn test_fn_like_macro_clone_raw_ident() {
"fn_like_clone_tokens",
"r#async",
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT r#async 4294967295"#]],
);
}
@ -73,7 +73,7 @@ fn test_fn_like_mk_literals() {
"fn_like_mk_literals",
r#""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
LITERAL b"byte_string" 4294967295
LITERAL 'c' 4294967295
LITERAL "string" 4294967295
@ -90,7 +90,7 @@ fn test_fn_like_mk_idents() {
"fn_like_mk_idents",
r#""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT standard 4294967295
IDENT r#raw 4294967295"#]],
);
@ -102,7 +102,7 @@ fn test_fn_like_macro_clone_literals() {
"fn_like_clone_tokens",
r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
LITERAL 1u16 4294967295
PUNCH , [alone] 4294967295
LITERAL 2_u32 4294967295
@ -126,10 +126,10 @@ fn test_attr_macro() {
r#"mod m {}"#,
r#"some arguments"#,
expect![[r##"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295
SUBTREE () 4294967295
SUBTREE () 4294967295 4294967295
LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295
PUNCH ; [alone] 4294967295"##]],
);

4
crates/rust-analyzer/src/reload.rs
View file

@ -34,6 +34,8 @@ use crate::{
op_queue::Cause,
};
use ::tt::token_id as tt;
#[derive(Debug)]
pub(crate) enum ProjectWorkspaceProgress {
Begin,
@ -656,7 +658,7 @@ pub(crate) fn load_proc_macro(
_: Option<&tt::Subtree>,
_: &Env,
) -> Result<tt::Subtree, ProcMacroExpansionError> {
Ok(tt::Subtree::default())
Ok(tt::Subtree::empty())
}
}
}

9
crates/stdx/src/macros.rs
View file

@ -43,5 +43,14 @@ macro_rules! impl_from {
}
)*)?
)*
};
($($variant:ident$(<$V:ident>)?),* for $enum:ident) => {
$(
impl$(<$V>)? From<$variant$(<$V>)?> for $enum$(<$V>)? {
fn from(it: $variant$(<$V>)?) -> $enum$(<$V>)? {
$enum::$variant(it)
}
}
)*
}
}

84
crates/tt/src/buffer.rs
View file

@ -12,10 +12,10 @@ struct EntryPtr(EntryId, usize);
/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
#[derive(Debug)]
enum Entry<'t> {
enum Entry<'t, Span> {
// Mimicking types from proc-macro.
Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId),
Leaf(&'t TokenTree),
Subtree(Option<&'t TokenTree<Span>>, &'t Subtree<Span>, EntryId),
Leaf(&'t TokenTree<Span>),
// End entries contain a pointer to the entry from the containing
// token tree, or None if this is the outermost level.
End(Option<EntryPtr>),
@ -24,16 +24,21 @@ enum Entry<'t> {
/// A token tree buffer
/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
#[derive(Debug)]
pub struct TokenBuffer<'t> {
buffers: Vec<Box<[Entry<'t>]>>,
pub struct TokenBuffer<'t, Span> {
buffers: Vec<Box<[Entry<'t, Span>]>>,
}
trait TokenList<'a> {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>);
trait TokenList<'a, Span> {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>);
}
impl<'a> TokenList<'a> for &'a [TokenTree] {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
impl<'a, Span> TokenList<'a, Span> for &'a [TokenTree<Span>] {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End
let start_capacity = self.len() + 1;
let mut entries = Vec::with_capacity(start_capacity);
@ -53,8 +58,11 @@ impl<'a> TokenList<'a> for &'a [TokenTree] {
}
}
impl<'a> TokenList<'a> for &'a Subtree {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
impl<'a, Span> TokenList<'a, Span> for &'a Subtree<Span> {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End
let mut entries = vec![];
let mut children = vec![];
@ -64,25 +72,25 @@ impl<'a> TokenList<'a> for &'a Subtree {
}
}
impl<'t> TokenBuffer<'t> {
pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
impl<'t, Span> TokenBuffer<'t, Span> {
pub fn from_tokens(tokens: &'t [TokenTree<Span>]) -> TokenBuffer<'t, Span> {
Self::new(tokens)
}
pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> {
pub fn from_subtree(subtree: &'t Subtree<Span>) -> TokenBuffer<'t, Span> {
Self::new(subtree)
}
fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> {
fn new<T: TokenList<'t, Span>>(tokens: T) -> TokenBuffer<'t, Span> {
let mut buffers = vec![];
let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
assert_eq!(idx, 0);
TokenBuffer { buffers }
}
fn new_inner<T: TokenList<'t>>(
fn new_inner<T: TokenList<'t, Span>>(
tokens: T,
buffers: &mut Vec<Box<[Entry<'t>]>>,
buffers: &mut Vec<Box<[Entry<'t, Span>]>>,
next: Option<EntryPtr>,
) -> usize {
let (children, mut entries) = tokens.entries();
@ -105,25 +113,25 @@ impl<'t> TokenBuffer<'t> {
/// Creates a cursor referencing the first token in the buffer and able to
/// traverse until the end of the buffer.
pub fn begin(&self) -> Cursor<'_> {
pub fn begin(&self) -> Cursor<'_, Span> {
Cursor::create(self, EntryPtr(EntryId(0), 0))
}
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_>> {
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_, Span>> {
let id = ptr.0;
self.buffers[id.0].get(ptr.1)
}
}
#[derive(Debug)]
pub enum TokenTreeRef<'a> {
Subtree(&'a Subtree, Option<&'a TokenTree>),
Leaf(&'a Leaf, &'a TokenTree),
pub enum TokenTreeRef<'a, Span> {
Subtree(&'a Subtree<Span>, Option<&'a TokenTree<Span>>),
Leaf(&'a Leaf<Span>, &'a TokenTree<Span>),
}
impl<'a> TokenTreeRef<'a> {
pub fn cloned(&self) -> TokenTree {
match &self {
impl<'a, Span: Clone> TokenTreeRef<'a, Span> {
pub fn cloned(&self) -> TokenTree<Span> {
match self {
TokenTreeRef::Subtree(subtree, tt) => match tt {
Some(it) => (*it).clone(),
None => (*subtree).clone().into(),
@ -135,20 +143,20 @@ impl<'a> TokenTreeRef<'a> {
/// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125>
#[derive(Copy, Clone, Debug)]
pub struct Cursor<'a> {
buffer: &'a TokenBuffer<'a>,
pub struct Cursor<'a, Span> {
buffer: &'a TokenBuffer<'a, Span>,
ptr: EntryPtr,
}
impl<'a> PartialEq for Cursor<'a> {
fn eq(&self, other: &Cursor<'_>) -> bool {
impl<'a, Span> PartialEq for Cursor<'a, Span> {
fn eq(&self, other: &Cursor<'_, Span>) -> bool {
self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer)
}
}
impl<'a> Eq for Cursor<'a> {}
impl<'a, Span> Eq for Cursor<'a, Span> {}
impl<'a> Cursor<'a> {
impl<'a, Span> Cursor<'a, Span> {
/// Check whether it is eof
pub fn eof(self) -> bool {
matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None)))
@ -156,7 +164,7 @@ impl<'a> Cursor<'a> {
/// If the cursor is pointing at the end of a subtree, returns
/// the parent subtree
pub fn end(self) -> Option<&'a Subtree> {
pub fn end(self) -> Option<&'a Subtree<Span>> {
match self.entry() {
Some(Entry::End(Some(ptr))) => {
let idx = ptr.1;
@ -171,13 +179,13 @@ impl<'a> Cursor<'a> {
}
}
fn entry(self) -> Option<&'a Entry<'a>> {
fn entry(&self) -> Option<&'a Entry<'a, Span>> {
self.buffer.entry(&self.ptr)
}
/// If the cursor is pointing at a `Subtree`, returns
/// a cursor into that subtree
pub fn subtree(self) -> Option<Cursor<'a>> {
pub fn subtree(self) -> Option<Cursor<'a, Span>> {
match self.entry() {
Some(Entry::Subtree(_, _, entry_id)) => {
Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
@ -187,7 +195,7 @@ impl<'a> Cursor<'a> {
}
/// If the cursor is pointing at a `TokenTree`, returns it
pub fn token_tree(self) -> Option<TokenTreeRef<'a>> {
pub fn token_tree(self) -> Option<TokenTreeRef<'a, Span>> {
match self.entry() {
Some(Entry::Leaf(tt)) => match tt {
TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)),
@ -198,12 +206,12 @@ impl<'a> Cursor<'a> {
}
}
fn create(buffer: &'a TokenBuffer<'_>, ptr: EntryPtr) -> Cursor<'a> {
fn create(buffer: &'a TokenBuffer<'_, Span>, ptr: EntryPtr) -> Cursor<'a, Span> {
Cursor { buffer, ptr }
}
/// Bump the cursor
pub fn bump(self) -> Cursor<'a> {
pub fn bump(self) -> Cursor<'a, Span> {
if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) {
match exit {
Some(exit) => Cursor::create(self.buffer, *exit),
@ -216,7 +224,7 @@ impl<'a> Cursor<'a> {
/// Bump the cursor, if it is a subtree, returns
/// a cursor into that subtree
pub fn bump_subtree(self) -> Cursor<'a> {
pub fn bump_subtree(self) -> Cursor<'a, Span> {
match self.entry() {
Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
_ => self.bump(),

232
crates/tt/src/lib.rs
View file

@ -16,45 +16,106 @@ pub use smol_str::SmolStr;
/// which source tokens. We do it by assigning an distinct identity to each
/// source token and making sure that identities are preserved during macro
/// expansion.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct TokenId(pub u32);
impl fmt::Debug for TokenId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl TokenId {
pub const UNSPECIFIED: TokenId = TokenId(!0);
pub const fn unspecified() -> TokenId {
TokenId(!0)
Self::UNSPECIFIED
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree {
Leaf(Leaf),
Subtree(Subtree),
}
impl_from!(Leaf, Subtree for TokenTree);
pub mod token_id {
pub use crate::{DelimiterKind, Spacing, TokenId};
pub type Span = crate::TokenId;
pub type Subtree = crate::Subtree<Span>;
pub type Punct = crate::Punct<Span>;
pub type Delimiter = crate::Delimiter<Span>;
pub type Leaf = crate::Leaf<Span>;
pub type Ident = crate::Ident<Span>;
pub type Literal = crate::Literal<Span>;
pub type TokenTree = crate::TokenTree<Span>;
pub mod buffer {
pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>;
pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>;
pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>;
}
impl TokenTree {
pub fn empty() -> Self {
TokenTree::Subtree(Subtree::default())
impl Delimiter {
pub const UNSPECIFIED: Self = Self {
open: TokenId::UNSPECIFIED,
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Invisible,
};
pub const fn unspecified() -> Self {
Self::UNSPECIFIED
}
}
impl Subtree {
pub const fn empty() -> Self {
Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
}
}
impl TokenTree {
pub const fn empty() -> Self {
Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] })
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf {
Literal(Literal),
Punct(Punct),
Ident(Ident),
}
impl_from!(Literal, Punct, Ident for Leaf);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SyntaxContext(pub u32);
#[derive(Clone, PartialEq, Eq, Hash, Default)]
pub struct Subtree {
pub delimiter: Option<Delimiter>,
pub token_trees: Vec<TokenTree>,
// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
// pub struct Span {
// pub id: TokenId,
// pub ctx: SyntaxContext,
// }
// pub type Span = (TokenId, SyntaxContext);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree<Span> {
Leaf(Leaf<Span>),
Subtree(Subtree<Span>),
}
impl_from!(Leaf<Span>, Subtree<Span> for TokenTree);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf<Span> {
Literal(Literal<Span>),
Punct(Punct<Span>),
Ident(Ident<Span>),
}
impl<Span> Leaf<Span> {
pub fn span(&self) -> &Span {
match self {
Leaf::Literal(it) => &it.span,
Leaf::Punct(it) => &it.span,
Leaf::Ident(it) => &it.span,
}
}
}
impl_from!(Literal<Span>, Punct<Span>, Ident<Span> for Leaf);
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct Subtree<Span> {
// FIXME, this should not be Option
pub delimiter: Delimiter<Span>,
pub token_trees: Vec<TokenTree<Span>>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Delimiter {
pub id: TokenId,
pub struct Delimiter<Span> {
pub open: Span,
pub close: Span,
pub kind: DelimiterKind,
}
@ -63,19 +124,20 @@ pub enum DelimiterKind {
Parenthesis,
Brace,
Bracket,
Invisible,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Literal {
pub struct Literal<Span> {
pub text: SmolStr,
pub id: TokenId,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Punct {
pub struct Punct<Span> {
pub char: char,
pub spacing: Spacing,
pub id: TokenId,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -85,39 +147,25 @@ pub enum Spacing {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Ident {
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
pub struct Ident<Span> {
pub text: SmolStr,
pub id: TokenId,
pub span: Span,
}
impl Ident {
/// Constructor intended to be used only by proc macro server. `text` should not contain raw
/// identifier prefix.
pub fn new_with_is_raw(text: SmolStr, id: TokenId, is_raw: bool) -> Self {
let text = if is_raw { SmolStr::from_iter(["r#", &text]) } else { text };
Ident { text, id }
}
}
impl Leaf {
pub fn id(&self) -> TokenId {
match self {
Leaf::Literal(l) => l.id,
Leaf::Punct(p) => p.id,
Leaf::Ident(i) => i.id,
}
}
}
fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result {
fn print_debug_subtree<Span: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
subtree: &Subtree<Span>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) {
None => "$".to_string(),
Some((DelimiterKind::Parenthesis, id)) => format!("() {id}"),
Some((DelimiterKind::Brace, id)) => format!("{{}} {id}"),
Some((DelimiterKind::Bracket, id)) => format!("[] {id}"),
let Delimiter { kind, open, close } = &subtree.delimiter;
let aux = match kind {
DelimiterKind::Invisible => format!("$$ {:?} {:?}", open, close),
DelimiterKind::Parenthesis => format!("() {:?} {:?}", open, close),
DelimiterKind::Brace => format!("{{}} {:?} {:?}", open, close),
DelimiterKind::Bracket => format!("[] {:?} {:?}", open, close),
};
if subtree.token_trees.is_empty() {
@ -135,21 +183,25 @@ fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usi
Ok(())
}
fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result {
fn print_debug_token<Span: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
tkn: &TokenTree<Span>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
match tkn {
TokenTree::Leaf(leaf) => match leaf {
Leaf::Literal(lit) => write!(f, "{align}LITERAL {} {}", lit.text, lit.id.0)?,
Leaf::Literal(lit) => write!(f, "{}LITERAL {} {:?}", align, lit.text, lit.span)?,
Leaf::Punct(punct) => write!(
f,
"{}PUNCH {} [{}] {}",
"{}PUNCH {} [{}] {:?}",
align,
punct.char,
if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
punct.id.0
punct.span
)?,
Leaf::Ident(ident) => write!(f, "{align}IDENT {} {}", ident.text, ident.id.0)?,
Leaf::Ident(ident) => write!(f, "{}IDENT {} {:?}", align, ident.text, ident.span)?,
},
TokenTree::Subtree(subtree) => {
print_debug_subtree(f, subtree, level)?;
@ -159,13 +211,13 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize)
Ok(())
}
impl fmt::Debug for Subtree {
impl<Span: fmt::Debug> fmt::Debug for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
print_debug_subtree(f, self, 0)
}
}
impl fmt::Display for TokenTree {
impl<Span> fmt::Display for TokenTree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
@ -174,13 +226,13 @@ impl fmt::Display for TokenTree {
}
}
impl fmt::Display for Subtree {
impl<Span> fmt::Display for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (l, r) = match self.delimiter_kind() {
Some(DelimiterKind::Parenthesis) => ("(", ")"),
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"),
None => ("", ""),
let (l, r) = match self.delimiter.kind {
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Invisible => ("", ""),
};
f.write_str(l)?;
let mut needs_space = false;
@ -202,7 +254,7 @@ impl fmt::Display for Subtree {
}
}
impl fmt::Display for Leaf {
impl<Span> fmt::Display for Leaf<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Leaf::Ident(it) => fmt::Display::fmt(it, f),
@ -212,25 +264,25 @@ impl fmt::Display for Leaf {
}
}
impl fmt::Display for Ident {
impl<Span> fmt::Display for Ident<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl fmt::Display for Literal {
impl<Span> fmt::Display for Literal<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl fmt::Display for Punct {
impl<Span> fmt::Display for Punct<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.char, f)
}
}
impl Subtree {
impl<Span> Subtree<Span> {
/// Count the number of tokens recursively
pub fn count(&self) -> usize {
let children_count = self
@ -244,20 +296,16 @@ impl Subtree {
self.token_trees.len() + children_count
}
pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
self.delimiter.map(|it| it.kind)
}
}
impl Subtree {
impl<Span> Subtree<Span> {
/// A simple line string used for debugging
pub fn as_debug_string(&self) -> String {
let delim = match self.delimiter_kind() {
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"),
Some(DelimiterKind::Parenthesis) => ("(", ")"),
None => (" ", " "),
let delim = match self.delimiter.kind {
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Invisible => ("$", "$"),
};
let mut res = String::new();
@ -275,7 +323,7 @@ impl Subtree {
(Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
" ".to_string() + &s
}
(Leaf::Punct(_), Some(&TokenTree::Leaf(Leaf::Punct(punct)))) => {
(Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => {
if punct.spacing == Spacing::Alone {
" ".to_string() + &s
} else {
@ -298,19 +346,19 @@ impl Subtree {
pub mod buffer;
pub fn pretty(tkns: &[TokenTree]) -> String {
fn tokentree_to_text(tkn: &TokenTree) -> String {
pub fn pretty<Span>(tkns: &[TokenTree<Span>]) -> String {
fn tokentree_to_text<Span>(tkn: &TokenTree<Span>) -> String {
match tkn {
TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(),
TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(),
TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
TokenTree::Subtree(subtree) => {
let content = pretty(&subtree.token_trees);
let (open, close) = match subtree.delimiter.map(|it| it.kind) {
None => ("", ""),
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Parenthesis) => ("(", ")"),
Some(DelimiterKind::Bracket) => ("[", "]"),
let (open, close) = match subtree.delimiter.kind {
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Invisible => ("", ""),
};
format!("{open}{content}{close}")
}