Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-09-20 14:32:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

Make tt generic over the span data

Browse source
This commit is contained in:
Lukas Wirth 2023-01-31 11:49:49 +01:00
parent d805c74c51
commit 41a46a78f2
48 changed files with 806 additions and 569 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
crates/base-db/src/fixture.rs
View file

@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
use test_utils::{
extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
};
use tt::Subtree;
use tt::token_id::Subtree;
use vfs::{file_set::FileSet, VfsPath};
use crate::{
@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
_: &Env,
) -> Result<Subtree, ProcMacroExpansionError> {
fn traverse(input: &Subtree) -> Subtree {
let mut res = Subtree::default();
res.delimiter = input.delimiter;
let mut token_trees = vec![];
for tt in input.token_trees.iter().rev() {
let tt = match tt {
tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
};
res.token_trees.push(tt);
token_trees.push(tt);
}
res
Subtree { delimiter: input.delimiter, token_trees }
}
Ok(traverse(input))
}

2
crates/base-db/src/input.rs
View file

@ -12,7 +12,7 @@ use cfg::CfgOptions;
use rustc_hash::FxHashMap;
use stdx::hash::{NoHashHashMap, NoHashHashSet};
use syntax::SmolStr;
use tt::Subtree;
use tt::token_id::Subtree;
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
/// Files are grouped into source roots. A source root is a directory on the

4
crates/cfg/src/cfg_expr.rs
View file

@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
}
impl CfgExpr {
pub fn parse(tt: &tt::Subtree) -> CfgExpr {
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
}
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@ -85,7 +85,7 @@ impl CfgExpr {
}
}
fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> {
fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
let name = match it.next() {
None => return None,
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

4
crates/hir-def/src/adt.rs
View file

@ -2,6 +2,7 @@
use std::sync::Arc;
use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use base_db::CrateId;
use either::Either;
use hir_expand::{
@ -12,7 +13,6 @@ use intern::Interned;
use la_arena::{Arena, ArenaMap};
use rustc_abi::{Integer, IntegerType};
use syntax::ast::{self, HasName, HasVisibility};
use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
use crate::{
body::{CfgExpander, LowerCtx},
@ -82,7 +82,7 @@ fn repr_from_value(
fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
match tt.delimiter {
Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {}
Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
_ => return None,
}

5
crates/hir-def/src/attr.rs
View file

@ -16,7 +16,6 @@ use syntax::{
ast::{self, HasAttrs, IsString},
AstPtr, AstToken, SmolStr, TextRange, TextSize,
};
use tt::Subtree;
use crate::{
db::DefDatabase,
@ -234,7 +233,7 @@ impl Attrs {
pub fn has_doc_hidden(&self) -> bool {
self.by_key("doc").tt_values().any(|tt| {
tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) &&
tt.delimiter.kind == DelimiterKind::Parenthesis &&
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
})
}
@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
}
impl<'attr> AttrQuery<'attr> {
pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> {
pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
self.attrs().filter_map(|attr| attr.token_tree_value())
}

2
crates/hir-def/src/data.rs
View file

@ -142,7 +142,7 @@ impl FunctionData {
}
}
fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> {
fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
let mut indices = Vec::new();
for args in tt.token_trees.chunks(2) {
match &args[0] {

23
crates/hir-def/src/lib.rs
View file

@ -79,6 +79,8 @@ use nameres::DefMap;
use stdx::impl_from;
use syntax::ast;
use ::tt::token_id as tt;
use crate::{
adt::VariantData,
builtin_type::BuiltinType,
@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
def: MacroDefId,
is_derive: bool,
) -> MacroCallId {
let mut arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()),
_ => Default::default(),
let arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt, map)) => (
{
let mut tt = tt.clone();
tt.delimiter = tt::Delimiter::UNSPECIFIED;
tt
},
map.clone(),
),
_ => (tt::Subtree::empty(), Default::default()),
};
// The parentheses are always disposed here.
arg.0.delimiter = None;
let res = def.as_lazy_macro(
def.as_lazy_macro(
db.upcast(),
krate,
MacroCallKind::Attr {
@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
invoc_attr_index: macro_attr.id,
is_derive,
},
);
res
)
}
intern::impl_internable!(
crate::type_ref::TypeRef,

8
crates/hir-def/src/macro_expansion_tests.rs
View file

@ -30,7 +30,7 @@ use syntax::{
SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
SyntaxNode, TextRange, T,
};
use tt::{Subtree, TokenId};
use tt::token_id::{Subtree, TokenId};
use crate::{
db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
tree.token_trees.iter().for_each(|tree| match tree {
tt::TokenTree::Leaf(leaf) => {
let id = match leaf {
tt::Leaf::Literal(it) => it.id,
tt::Leaf::Punct(it) => it.id,
tt::Leaf::Ident(it) => it.id,
tt::Leaf::Literal(it) => it.span,
tt::Leaf::Punct(it) => it.span,
tt::Leaf::Ident(it) => it.span,
};
ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
}

12
crates/hir-def/src/nameres/collector.rs
View file

@ -46,6 +46,7 @@ use crate::{
},
path::{ImportAlias, ModPath, PathKind},
per_ns::PerNs,
tt,
visibility::{RawVisibility, Visibility},
AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
.enumerate()
.map(|(idx, it)| {
// FIXME: a hacky way to create a Name from string.
let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() };
let name =
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
(
name.as_name(),
ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@ -451,7 +453,10 @@ impl DefCollector<'_> {
directive.module_id,
MacroCallKind::Attr {
ast_id: ast_id.ast_id,
attr_args: Default::default(),
attr_args: std::sync::Arc::new((
tt::Subtree::empty(),
Default::default(),
)),
invoc_attr_index: attr.id,
is_derive: false,
},
@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
let name = match attrs.by_key("rustc_builtin_macro").string_value() {
Some(it) => {
// FIXME: a hacky way to create a Name from string.
name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name();
name =
tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
&name
}
None => {

2
crates/hir-def/src/nameres/proc_macro.rs
View file

@ -1,9 +1,9 @@
//! Nameres-specific procedural macro data and helpers.
use hir_expand::name::{AsName, Name};
use tt::{Leaf, TokenTree};
use crate::attr::Attrs;
use crate::tt::{Leaf, TokenTree};
#[derive(Debug, PartialEq, Eq)]
pub struct ProcMacroDef {

9
crates/hir-expand/src/attrs.rs
View file

@ -8,13 +8,13 @@ use intern::Interned;
use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
use smallvec::{smallvec, SmallVec};
use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
use tt::Subtree;
use crate::{
db::AstDatabase,
hygiene::Hygiene,
mod_path::{ModPath, PathKind},
name::AsName,
tt::{self, Subtree},
InFile,
};
@ -117,7 +117,10 @@ impl RawAttrs {
let index = attr.id;
let attrs =
parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
let tree = Subtree { delimiter: None, token_trees: attr.to_vec() };
let tree = Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: attr.to_vec(),
};
// FIXME hygiene
let hygiene = Hygiene::new_unhygienic();
Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
@ -266,7 +269,7 @@ impl Attr {
pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
let args = self.token_tree_value()?;
if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) {
if args.delimiter.kind != DelimiterKind::Parenthesis {
return None;
}
let paths = args

6
crates/hir-expand/src/builtin_attr_macro.rs
View file

@ -1,6 +1,6 @@
//! Builtin attributes.
use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind};
use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
macro_rules! register_builtin {
( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@ -97,7 +97,7 @@ fn derive_attr_expand(
let loc = db.lookup_intern_macro_call(id);
let derives = match &loc.kind {
MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
_ => return ExpandResult::ok(Default::default()),
_ => return ExpandResult::ok(tt::Subtree::empty()),
};
pseudo_derive_attr_expansion(tt, derives)
}
@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
};

10
crates/hir-expand/src/builtin_derive_macro.rs
View file

@ -3,11 +3,11 @@
use base_db::{CrateOrigin, LangCrateOrigin};
use tracing::debug;
use crate::tt::{self, TokenId};
use syntax::{
ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
match_ast,
};
use tt::TokenId;
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
})?;
let name_token_id =
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
let name_token = tt::Ident { id: name_token_id, text: name.text().into() };
let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
let param_types = params
.into_iter()
.flat_map(|param_list| param_list.type_or_const_params())
@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
let ty = param
.ty()
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
.unwrap_or_default();
.unwrap_or_else(tt::Subtree::empty);
Some(ty)
} else {
None
@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
let info = match parse_adt(tt) {
Ok(info) => info,
Err(e) => return ExpandResult::only_err(e),
Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
};
let (params, args): (Vec<_>, Vec<_>) = info
.param_types
@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu
.enumerate()
.map(|(idx, param_ty)| {
let ident = tt::Leaf::Ident(tt::Ident {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
text: format!("T{idx}").into(),
});
let ident_ = ident.clone();

66
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -9,7 +9,9 @@ use syntax::{
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
use crate::{
db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
};
macro_rules! register_builtin {
( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => {
@ -61,7 +63,7 @@ macro_rules! register_builtin {
};
}
#[derive(Debug, Default)]
#[derive(Debug)]
pub struct ExpandedEager {
pub(crate) subtree: tt::Subtree,
/// The included file ID of the include macro.
@ -116,7 +118,7 @@ register_builtin! {
}
const DOLLAR_CRATE: tt::Ident =
tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() };
tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() };
fn module_path_expand(
_db: &dyn AstDatabase,
@ -162,7 +164,7 @@ fn stringify_expand(
_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<tt::Subtree> {
let pretty = tt::pretty(&tt.token_trees);
let pretty = ::tt::pretty(&tt.token_trees);
let expanded = quote! {
#pretty
@ -194,11 +196,11 @@ fn assert_expand(
let expanded = match &*args {
[cond, panic_args @ ..] => {
let comma = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char: ',',
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
};
let cond = cond.clone();
@ -247,7 +249,10 @@ fn format_args_expand(
let mut args = parse_exprs_with_sep(tt, ',');
if args.is_empty() {
return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into());
return ExpandResult::with_err(
tt::Subtree::empty(),
mbe::ExpandError::NoMatchingRule.into(),
);
}
for arg in &mut args {
// Remove `key =`.
@ -282,7 +287,7 @@ fn asm_expand(
for tt in tt.token_trees.chunks(2) {
match tt {
[tt::TokenTree::Leaf(tt::Leaf::Literal(lit))]
| [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] =>
| [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] =>
{
let krate = DOLLAR_CRATE.clone();
literals.push(quote!(#krate::format_args!(#lit);));
@ -400,7 +405,7 @@ fn concat_expand(
// FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses
// to ensure the right parsing order, so skip the parentheses here. Ideally we'd
// implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623
if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t {
if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t {
if let [tt] = &**token_trees {
if delim.kind == tt::DelimiterKind::Parenthesis {
t = tt;
@ -459,9 +464,7 @@ fn concat_bytes_expand(
}
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (),
tt::TokenTree::Subtree(tree)
if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) =>
{
tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => {
if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) {
err.get_or_insert(e);
break;
@ -473,7 +476,7 @@ fn concat_bytes_expand(
}
}
}
let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() };
let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
}
@ -521,7 +524,7 @@ fn concat_idents_expand(
}
}
}
let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() };
let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
}
@ -572,7 +575,10 @@ fn include_expand(
Ok((subtree, file_id)) => {
ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) })
}
Err(e) => ExpandResult::only_err(e),
Err(e) => ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
),
}
}
@ -582,15 +588,18 @@ fn include_bytes_expand(
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
if let Err(e) = parse_string(tt) {
return ExpandResult::only_err(e);
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
);
}
// FIXME: actually read the file here if the user asked for macro expansion
let res = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: r#"b"""#.into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
};
ExpandResult::ok(ExpandedEager::new(res))
@ -603,7 +612,12 @@ fn include_str_expand(
) -> ExpandResult<ExpandedEager> {
let path = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
// FIXME: we're not able to read excluded files (which is most of them because
@ -635,7 +649,12 @@ fn env_expand(
) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
let mut err = None;
@ -666,7 +685,12 @@ fn option_env_expand(
) -> ExpandResult<ExpandedEager> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => return ExpandResult::only_err(e),
Err(e) => {
return ExpandResult::with_err(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
};
let expanded = match get_env_inner(db, arg_id, &key) {

14
crates/hir-expand/src/db.rs
View file

@ -14,7 +14,7 @@ use syntax::{
use crate::{
ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup,
hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander,
ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
};
@ -175,7 +175,7 @@ pub fn expand_speculative(
match attr.token_tree() {
Some(token_tree) => {
let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax());
tree.delimiter = None;
tree.delimiter = tt::Delimiter::unspecified();
let shift = mbe::Shift::new(&tt);
shift.shift_all(&mut tree);
@ -210,7 +210,7 @@ pub fn expand_speculative(
// Otherwise the expand query will fetch the non speculative attribute args and pass those instead.
let mut speculative_expansion = match loc.def.kind {
MacroDefKind::ProcMacro(expander, ..) => {
tt.delimiter = None;
tt.delimiter = tt::Delimiter::unspecified();
expander.expand(db, loc.krate, &tt, attr_arg.as_ref())
}
MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => {
@ -316,9 +316,8 @@ fn macro_arg(
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
tt.delimiter = None;
tt.delimiter = tt::Delimiter::unspecified();
}
Some(Arc::new((tt, tmap, fixups.undo_info)))
}
@ -479,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult<tt::
let macro_arg = match db.macro_arg(id) {
Some(it) => it,
None => {
return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into()))
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("No arguments for proc-macro".into()),
)
}
};

6
crates/hir-expand/src/eager.rs
View file

@ -108,7 +108,7 @@ pub fn expand_eager_macro(
.value
.token_tree()
.map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
.unwrap_or_default();
.unwrap_or_else(tt::Subtree::empty);
let ast_map = db.ast_id_map(macro_call.file_id);
let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@ -165,9 +165,9 @@ pub fn expand_eager_macro(
}
}
fn to_subtree(node: &SyntaxNode) -> tt::Subtree {
fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree {
let mut subtree = mbe::syntax_node_to_token_tree(node).0;
subtree.delimiter = None;
subtree.delimiter = crate::tt::Delimiter::unspecified();
subtree
}

18
crates/hir-expand/src/fixup.rs
View file

@ -9,7 +9,7 @@ use syntax::{
ast::{self, AstNode, HasLoopBody},
match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
};
use tt::Subtree;
use tt::token_id::Subtree;
/// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to
@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups(
tt.token_trees = tts
.into_iter()
.filter(|tt| match tt {
tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID),
tt::TokenTree::Leaf(leaf) => {
token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID)
}
tt::TokenTree::Subtree(st) => {
st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID))
token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID)
}
})
.flat_map(|tt| match tt {
@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups(
SmallVec::from_const([tt.into()])
}
tt::TokenTree::Leaf(leaf) => {
if let Some(id) = token_map.synthetic_token_id(leaf.id()) {
if let Some(id) = token_map.synthetic_token_id(*leaf.span()) {
let original = undo_info.original[id.0 as usize].clone();
if original.delimiter.is_none() {
if original.delimiter.kind == tt::DelimiterKind::Invisible {
original.token_trees.into()
} else {
SmallVec::from_const([original.into()])
@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups(
mod tests {
use expect_test::{expect, Expect};
use crate::tt;
use super::reverse_fixups;
// The following three functions are only meant to check partial structural equivalence of
@ -341,7 +345,7 @@ mod tests {
}
fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind)
a.delimiter.kind == b.delimiter.kind
&& a.token_trees.len() == b.token_trees.len()
&& a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
}
@ -386,7 +390,7 @@ mod tests {
let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
assert!(
check_subtree_eq(&tt, &original_as_tt),
"different token tree: {tt:?}, {original_as_tt:?}"
"different token tree: {tt:?},\n{original_as_tt:?}"
);
}

2
crates/hir-expand/src/hygiene.rs
View file

@ -128,7 +128,7 @@ struct HygieneInfo {
attr_input_or_mac_def_start: Option<InFile<TextSize>>,
macro_def: Arc<TokenExpander>,
macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>,
macro_arg_shift: mbe::Shift,
exp_map: Arc<mbe::TokenMap>,
}

2
crates/hir-expand/src/lib.rs
View file

@ -22,6 +22,8 @@ mod fixup;
pub use mbe::{Origin, ValueResult};
use ::tt::token_id as tt;
use std::{fmt, hash::Hash, iter, sync::Arc};
use base_db::{

2
crates/hir-expand/src/name.rs
View file

@ -191,7 +191,7 @@ impl AsName for ast::NameOrNameRef {
}
}
impl AsName for tt::Ident {
impl<Span> AsName for tt::Ident<Span> {
fn as_name(&self) -> Name {
Name::resolve(&self.text)
}

24
crates/hir-expand/src/proc_macro.rs
View file

@ -3,7 +3,7 @@
use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind};
use stdx::never;
use crate::{db::AstDatabase, ExpandError, ExpandResult};
use crate::{db::AstDatabase, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct ProcMacroExpander {
@ -39,7 +39,10 @@ impl ProcMacroExpander {
Ok(proc_macros) => proc_macros,
Err(_) => {
never!("Non-dummy expander even though there are no proc macros");
return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
}
};
let proc_macro = match proc_macros.get(id.0 as usize) {
@ -50,7 +53,10 @@ impl ProcMacroExpander {
proc_macros.len(),
id.0
);
return ExpandResult::only_err(ExpandError::Other("Internal error".into()));
return ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
);
}
};
@ -69,13 +75,17 @@ impl ProcMacroExpander {
}
}
ProcMacroExpansionError::System(text)
| ProcMacroExpansionError::Panic(text) => {
ExpandResult::only_err(ExpandError::Other(text.into()))
}
| ProcMacroExpansionError::Panic(text) => ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::Other(text.into()),
),
},
}
}
None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)),
None => ExpandResult::with_err(
tt::Subtree::empty(),
ExpandError::UnresolvedProcMacro(self.krate),
),
}
}
}

112
crates/hir-expand/src/quote.rs
View file

@ -9,17 +9,18 @@
#[macro_export]
macro_rules! __quote {
() => {
Vec::<tt::TokenTree>::new()
Vec::<crate::tt::TokenTree>::new()
};
( @SUBTREE $delim:ident $($tt:tt)* ) => {
{
let children = $crate::__quote!($($tt)*);
tt::Subtree {
delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::$delim,
id: tt::TokenId::unspecified(),
}),
crate::tt::Subtree {
delimiter: crate::tt::Delimiter {
kind: crate::tt::DelimiterKind::$delim,
open: crate::tt::TokenId::unspecified(),
close: crate::tt::TokenId::unspecified(),
},
token_trees: $crate::quote::IntoTt::to_tokens(children),
}
}
@ -28,10 +29,10 @@ macro_rules! __quote {
( @PUNCT $first:literal ) => {
{
vec![
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Alone,
span: crate::tt::TokenId::unspecified(),
}).into()
]
}
@ -40,15 +41,15 @@ macro_rules! __quote {
( @PUNCT $first:literal, $sec:literal ) => {
{
vec![
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $first,
spacing: tt::Spacing::Joint,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Joint,
span: crate::tt::TokenId::unspecified(),
}).into(),
tt::Leaf::Punct(tt::Punct {
crate::tt::Leaf::Punct(crate::tt::Punct {
char: $sec,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
spacing: crate::tt::Spacing::Alone,
span: crate::tt::TokenId::unspecified(),
}).into()
]
}
@ -67,7 +68,7 @@ macro_rules! __quote {
( ## $first:ident $($tail:tt)* ) => {
{
let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<tt::TokenTree>>();
let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::<Vec<crate::tt::TokenTree>>();
let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
tokens.append(&mut tail_tokens);
tokens
@ -86,9 +87,9 @@ macro_rules! __quote {
// Ident
( $tt:ident ) => {
vec![ {
tt::Leaf::Ident(tt::Ident {
crate::tt::Leaf::Ident(crate::tt::Ident {
text: stringify!($tt).into(),
id: tt::TokenId::unspecified(),
span: crate::tt::TokenId::unspecified(),
}).into()
}]
};
@ -127,42 +128,42 @@ macro_rules! quote {
}
pub(crate) trait IntoTt {
fn to_subtree(self) -> tt::Subtree;
fn to_tokens(self) -> Vec<tt::TokenTree>;
fn to_subtree(self) -> crate::tt::Subtree;
fn to_tokens(self) -> Vec<crate::tt::TokenTree>;
}
impl IntoTt for Vec<tt::TokenTree> {
fn to_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self }
impl IntoTt for Vec<crate::tt::TokenTree> {
fn to_subtree(self) -> crate::tt::Subtree {
crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self }
}
fn to_tokens(self) -> Vec<tt::TokenTree> {
fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
self
}
}
impl IntoTt for tt::Subtree {
fn to_subtree(self) -> tt::Subtree {
impl IntoTt for crate::tt::Subtree {
fn to_subtree(self) -> crate::tt::Subtree {
self
}
fn to_tokens(self) -> Vec<tt::TokenTree> {
vec![tt::TokenTree::Subtree(self)]
fn to_tokens(self) -> Vec<crate::tt::TokenTree> {
vec![crate::tt::TokenTree::Subtree(self)]
}
}
pub(crate) trait ToTokenTree {
fn to_token(self) -> tt::TokenTree;
fn to_token(self) -> crate::tt::TokenTree;
}
impl ToTokenTree for tt::TokenTree {
fn to_token(self) -> tt::TokenTree {
impl ToTokenTree for crate::tt::TokenTree {
fn to_token(self) -> crate::tt::TokenTree {
self
}
}
impl ToTokenTree for tt::Subtree {
fn to_token(self) -> tt::TokenTree {
impl ToTokenTree for crate::tt::Subtree {
fn to_token(self) -> crate::tt::TokenTree {
self.into()
}
}
@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees {
($($ty:ty => $this:ident $im:block);*) => {
$(
impl ToTokenTree for $ty {
fn to_token($this) -> tt::TokenTree {
let leaf: tt::Leaf = $im.into();
fn to_token($this) -> crate::tt::TokenTree {
let leaf: crate::tt::Leaf = $im.into();
leaf.into()
}
}
impl ToTokenTree for &$ty {
fn to_token($this) -> tt::TokenTree {
let leaf: tt::Leaf = $im.clone().into();
fn to_token($this) -> crate::tt::TokenTree {
let leaf: crate::tt::Leaf = $im.clone().into();
leaf.into()
}
}
@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees {
}
impl_to_to_tokentrees! {
u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
tt::Leaf => self { self };
tt::Literal => self { self };
tt::Ident => self { self };
tt::Punct => self { self };
&str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}};
String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}
u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} };
crate::tt::Leaf => self { self };
crate::tt::Literal => self { self };
crate::tt::Ident => self { self };
crate::tt::Punct => self { self };
&str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}};
String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}
}
#[cfg(test)]
@ -223,8 +224,8 @@ mod tests {
assert_eq!(quote!(#s).to_string(), "\"hello\"");
}
fn mk_ident(name: &str) -> tt::Ident {
tt::Ident { text: name.into(), id: tt::TokenId::unspecified() }
fn mk_ident(name: &str) -> crate::tt::Ident {
crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() }
}
#[test]
@ -234,7 +235,7 @@ mod tests {
let quoted = quote!(#a);
assert_eq!(quoted.to_string(), "hello");
let t = format!("{quoted:?}");
assert_eq!(t, "SUBTREE $\n IDENT hello 4294967295");
assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295");
}
#[test]
@ -263,11 +264,12 @@ mod tests {
let fields = [mk_ident("name"), mk_ident("id")];
let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees);
let list = tt::Subtree {
delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::Brace,
id: tt::TokenId::unspecified(),
}),
let list = crate::tt::Subtree {
delimiter: crate::tt::Delimiter {
kind: crate::tt::DelimiterKind::Brace,
open: crate::tt::TokenId::unspecified(),
close: crate::tt::TokenId::unspecified(),
},
token_trees: fields.collect(),
};

28
crates/mbe/src/benchmark.rs
View file

@ -9,7 +9,7 @@ use test_utils::{bench, bench_fixture, skip_slow_tests};
use crate::{
parser::{MetaVarKind, Op, RepeatKind, Separator},
syntax_node_to_token_tree, DeclarativeMacro,
syntax_node_to_token_tree, tt, DeclarativeMacro,
};
#[test]
@ -91,7 +91,14 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
// So we just skip any error cases and try again
let mut try_cnt = 0;
loop {
let mut subtree = tt::Subtree::default();
let mut subtree = tt::Subtree {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Invisible,
},
token_trees: vec![],
};
for op in rule.lhs.iter() {
collect_from_op(op, &mut subtree, &mut seed);
}
@ -196,12 +203,15 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
*seed
}
fn make_ident(ident: &str) -> tt::TokenTree {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) })
.into()
tt::Leaf::Ident(tt::Ident {
span: tt::TokenId::unspecified(),
text: SmolStr::new(ident),
})
.into()
}
fn make_punct(char: char) -> tt::TokenTree {
tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
char,
spacing: tt::Spacing::Alone,
})
@ -209,7 +219,7 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
}
fn make_literal(lit: &str) -> tt::TokenTree {
tt::Leaf::Literal(tt::Literal {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
text: SmolStr::new(lit),
})
.into()
@ -219,7 +229,11 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
token_trees: Option<Vec<tt::TokenTree>>,
) -> tt::TokenTree {
tt::Subtree {
delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind,
},
token_trees: token_trees.unwrap_or_default(),
}
.into()

7
crates/mbe/src/expander.rs
View file

@ -8,7 +8,7 @@ mod transcriber;
use rustc_hash::FxHashMap;
use syntax::SmolStr;
use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult};
pub(crate) fn expand_rules(
rules: &[crate::Rule],
@ -45,7 +45,10 @@ pub(crate) fn expand_rules(
transcriber::transcribe(&rule.rhs, &match_.bindings);
ExpandResult { value, err: match_.err.or(transcribe_err) }
} else {
ExpandResult::only_err(ExpandError::NoMatchingRule)
ExpandResult::with_err(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
ExpandError::NoMatchingRule,
)
}
}

41
crates/mbe/src/expander/matcher.rs
View file

@ -67,6 +67,7 @@ use syntax::SmolStr;
use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt,
tt_iter::TtIter,
ExpandError, MetaTemplate, ValueResult,
};
@ -75,7 +76,8 @@ impl Bindings {
fn push_optional(&mut self, name: &SmolStr) {
// FIXME: Do we have a better way to represent an empty token ?
// Insert an empty subtree for empty token
let tt = tt::Subtree::default().into();
let tt =
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into();
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
}
@ -462,9 +464,9 @@ fn match_loop_inner<'t>(
}
OpDelimited::Op(Op::Subtree { tokens, delimiter }) => {
if let Ok(subtree) = src.clone().expect_subtree() {
if subtree.delimiter_kind() == delimiter.map(|it| it.kind) {
if subtree.delimiter.kind == delimiter.kind {
item.stack.push(item.dot);
item.dot = tokens.iter_delimited(delimiter.as_ref());
item.dot = tokens.iter_delimited(Some(delimiter));
cur_items.push(item);
}
}
@ -663,8 +665,8 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
}
res.add_err(ExpandError::LeftoverTokens);
if let Some(error_reover_item) = error_recover_item {
res.bindings = bindings_builder.build(&error_reover_item);
if let Some(error_recover_item) = error_recover_item {
res.bindings = bindings_builder.build(&error_recover_item);
}
return res;
}
@ -782,7 +784,7 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Opt
match neg {
None => lit.into(),
Some(neg) => tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![neg, lit.into()],
}),
}
@ -810,7 +812,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate)
}
impl MetaTemplate {
fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> {
OpDelimitedIter { inner: &self.0, idx: 0, delimited }
OpDelimitedIter {
inner: &self.0,
idx: 0,
delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED),
}
}
}
@ -824,20 +830,21 @@ enum OpDelimited<'a> {
#[derive(Debug, Clone, Copy)]
struct OpDelimitedIter<'a> {
inner: &'a [Op],
delimited: Option<&'a tt::Delimiter>,
delimited: &'a tt::Delimiter,
idx: usize,
}
impl<'a> OpDelimitedIter<'a> {
fn is_eof(&self) -> bool {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
self.idx >= len
}
fn peek(&self) -> Option<OpDelimited<'a>> {
match self.delimited {
None => self.inner.get(self.idx).map(OpDelimited::Op),
Some(_) => match self.idx {
match self.delimited.kind {
tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
_ => match self.idx {
0 => Some(OpDelimited::Open),
i if i == self.inner.len() + 1 => Some(OpDelimited::Close),
i => self.inner.get(i - 1).map(OpDelimited::Op),
@ -860,7 +867,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> {
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
let remain = len.saturating_sub(self.idx);
(remain, Some(remain))
}
@ -904,7 +912,10 @@ impl<'a> TtIter<'a> {
} else {
let puncts = self.expect_glued_punct()?;
let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect();
Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees }))
Ok(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees,
}))
}
} else {
self.next().ok_or(()).cloned()
@ -919,7 +930,7 @@ impl<'a> TtIter<'a> {
let ident = self.expect_ident_or_underscore()?;
Ok(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![
tt::Leaf::Punct(*punct).into(),
tt::Leaf::Ident(ident.clone()).into(),

70
crates/mbe/src/expander/transcriber.rs
View file

@ -2,11 +2,11 @@
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use syntax::SmolStr;
use tt::{Delimiter, Subtree};
use crate::{
expander::{Binding, Bindings, Fragment},
parser::{MetaVarKind, Op, RepeatKind, Separator},
tt::{self, Delimiter},
ExpandError, ExpandResult, MetaTemplate,
};
@ -44,22 +44,23 @@ impl Bindings {
Binding::Missing(it) => Ok(match it {
MetaVarKind::Stmt => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
char: ';',
spacing: tt::Spacing::Alone,
})))
}
MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
})),
// FIXME: Meta and Item should get proper defaults
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}))
}
@ -71,19 +72,19 @@ impl Bindings {
| MetaVarKind::Ident => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("missing"),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
MetaVarKind::Lifetime => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("'missing"),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
MetaVarKind::Literal => {
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: SmolStr::new_inline("\"missing\""),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})))
}
}),
@ -143,7 +144,7 @@ fn expand_subtree(
}
Op::Subtree { tokens, delimiter } => {
let ExpandResult { value: tt, err: e } =
expand_subtree(ctx, tokens, *delimiter, arena);
expand_subtree(ctx, tokens, Some(*delimiter), arena);
err = err.or(e);
arena.push(tt.into());
}
@ -170,7 +171,7 @@ fn expand_subtree(
arena.push(
tt::Leaf::Literal(tt::Literal {
text: index.to_string().into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})
.into(),
);
@ -179,7 +180,13 @@ fn expand_subtree(
}
// drain the elements added in this instance of expand_subtree
let tts = arena.drain(start_elements..).collect();
ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err }
ExpandResult {
value: tt::Subtree {
delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified),
token_trees: tts,
},
err,
}
}
fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@ -201,17 +208,24 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
// ```
// We just treat it a normal tokens
let tt = tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(),
tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(),
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id })
.into(),
tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(),
],
}
.into();
ExpandResult::ok(Fragment::Tokens(tt))
} else {
ctx.bindings.get(v, &mut ctx.nesting).map_or_else(
|e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) },
|e| ExpandResult {
value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: vec![],
})),
err: Some(e),
},
ExpandResult::ok,
)
}
@ -249,7 +263,10 @@ fn expand_repeat(
ctx
);
return ExpandResult {
value: Fragment::Tokens(Subtree::default().into()),
value: Fragment::Tokens(
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }
.into(),
),
err: Some(ExpandError::LimitExceeded),
};
}
@ -258,7 +275,7 @@ fn expand_repeat(
continue;
}
t.delimiter = None;
t.delimiter = tt::Delimiter::unspecified();
push_subtree(&mut buf, t);
if let Some(sep) = separator {
@ -292,7 +309,7 @@ fn expand_repeat(
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: None, token_trees: buf }.into();
let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into();
if RepeatKind::OneOrMore == kind && counter == 0 {
return ExpandResult {
@ -307,11 +324,12 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
match fragment {
Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => {
if tt.delimiter.is_none() {
tt.delimiter = Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
if tt.delimiter.kind == tt::DelimiterKind::Invisible {
tt.delimiter = tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Parenthesis,
})
};
}
buf.push(tt.into())
}
@ -320,8 +338,8 @@ fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
}
fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
match tt.delimiter {
None => buf.extend(tt.token_trees),
Some(_) => buf.push(tt.into()),
match tt.delimiter.kind {
tt::DelimiterKind::Invisible => buf.extend(tt.token_trees),
_ => buf.push(tt.into()),
}
}

55
crates/mbe/src/lib.rs
View file

@ -18,6 +18,8 @@ mod to_parser_input;
mod benchmark;
mod token_map;
use ::tt::token_id as tt;
use std::fmt;
use crate::{
@ -26,8 +28,8 @@ use crate::{
};
// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces
pub use self::tt::{Delimiter, DelimiterKind, Punct};
pub use ::parser::TopEntryPoint;
pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::{
syntax_bridge::{
@ -125,24 +127,26 @@ impl Shift {
// Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> {
let filter = |tt: &_| match tt {
tt::TokenTree::Subtree(subtree) => {
let tree_id = max_id(subtree);
match subtree.delimiter {
Some(it) if it.id != tt::TokenId::unspecified() => {
Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
let filter =
|tt: &_| match tt {
tt::TokenTree::Subtree(subtree) => {
let tree_id = max_id(subtree);
if subtree.delimiter.open != tt::TokenId::unspecified() {
Some(tree_id.map_or(subtree.delimiter.open.0, |t| {
t.max(subtree.delimiter.open.0)
}))
} else {
tree_id
}
_ => tree_id,
}
}
tt::TokenTree::Leaf(leaf) => {
let &(tt::Leaf::Ident(tt::Ident { id, .. })
| tt::Leaf::Punct(tt::Punct { id, .. })
| tt::Leaf::Literal(tt::Literal { id, .. })) = leaf;
tt::TokenTree::Leaf(leaf) => {
let &(tt::Leaf::Ident(tt::Ident { span, .. })
| tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Literal(tt::Literal { span, .. })) = leaf;
(id != tt::TokenId::unspecified()).then_some(id.0)
}
};
(span != tt::TokenId::unspecified()).then_some(span.0)
}
};
subtree.token_trees.iter().filter_map(filter).max()
}
}
@ -152,14 +156,13 @@ impl Shift {
for t in &mut tt.token_trees {
match t {
tt::TokenTree::Leaf(
tt::Leaf::Ident(tt::Ident { id, .. })
| tt::Leaf::Punct(tt::Punct { id, .. })
| tt::Leaf::Literal(tt::Literal { id, .. }),
) => *id = self.shift(*id),
tt::Leaf::Ident(tt::Ident { span, .. })
| tt::Leaf::Punct(tt::Punct { span, .. })
| tt::Leaf::Literal(tt::Literal { span, .. }),
) => *span = self.shift(*span),
tt::TokenTree::Subtree(tt) => {
if let Some(it) = tt.delimiter.as_mut() {
it.id = self.shift(it.id);
}
tt.delimiter.open = self.shift(tt.delimiter.open);
tt.delimiter.close = self.shift(tt.delimiter.close);
self.shift_all(tt)
}
}
@ -216,7 +219,7 @@ impl DeclarativeMacro {
let mut src = TtIter::new(tt);
let mut rules = Vec::new();
if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() {
if tt::DelimiterKind::Brace == tt.delimiter.kind {
cov_mark::hit!(parse_macro_def_rules);
while src.len() > 0 {
let rule = Rule::parse(&mut src, true)?;
@ -325,6 +328,10 @@ impl<T, E> ValueResult<T, E> {
Self { value, err: None }
}
pub fn with_err(value: T, err: E) -> Self {
Self { value, err: Some(err) }
}
pub fn only_err(err: E) -> Self
where
T: Default,

20
crates/mbe/src/parser.rs
View file

@ -4,7 +4,7 @@
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use crate::{tt_iter::TtIter, ParseError};
use crate::{tt, tt_iter::TtIter, ParseError};
/// Consider
///
@ -54,7 +54,7 @@ pub(crate) enum Op {
Ignore { name: SmolStr, id: tt::TokenId },
Index { depth: u32 },
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> },
Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter },
Literal(tt::Literal),
Punct(SmallVec<[tt::Punct; 3]>),
Ident(tt::Ident),
@ -130,13 +130,13 @@ fn next_op(
Some(it) => it,
};
match second {
tt::TokenTree::Subtree(subtree) => match subtree.delimiter_kind() {
Some(tt::DelimiterKind::Parenthesis) => {
tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => {
let (separator, kind) = parse_repeat(src)?;
let tokens = MetaTemplate::parse(subtree, mode)?;
Op::Repeat { tokens, separator, kind }
}
Some(tt::DelimiterKind::Brace) => match mode {
tt::DelimiterKind::Brace => match mode {
Mode::Template => {
parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| {
ParseError::unexpected("invalid metavariable expression")
@ -157,18 +157,18 @@ fn next_op(
tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Ident(ident) if ident.text == "crate" => {
// We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id })
Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span })
}
tt::Leaf::Ident(ident) => {
let kind = eat_fragment_kind(src, mode)?;
let name = ident.text.clone();
let id = ident.id;
let id = ident.span;
Op::Var { name, kind, id }
}
tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
let kind = eat_fragment_kind(src, mode)?;
let name = lit.text.clone();
let id = lit.id;
let id = lit.span;
Op::Var { name, kind, id }
}
tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode {
@ -284,7 +284,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let func = src.expect_ident()?;
let args = src.expect_subtree()?;
if args.delimiter_kind() != Some(tt::DelimiterKind::Parenthesis) {
if args.delimiter.kind != tt::DelimiterKind::Parenthesis {
return Err(());
}
@ -293,7 +293,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
let op = match &*func.text {
"ignore" => {
let ident = args.expect_ident()?;
Op::Ignore { name: ident.text.clone(), id: ident.id }
Op::Ignore { name: ident.text.clone(), id: ident.span }
}
"index" => {
let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? };

126
crates/mbe/src/syntax_bridge.rs
View file

@ -8,9 +8,16 @@ use syntax::{
SyntaxKind::*,
SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
use tt::buffer::{Cursor, TokenBuffer};
use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
use crate::{
to_parser_input::to_parser_input,
tt::{
self,
buffer::{Cursor, TokenBuffer},
},
tt_iter::TtIter,
TokenMap,
};
#[cfg(test)]
mod tests;
@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node(
entry_point: parser::TopEntryPoint,
) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt {
tt::Subtree { delimiter: None, token_trees } => {
TokenBuffer::from_tokens(token_trees.as_slice())
}
tt::Subtree {
delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. },
token_trees,
} => TokenBuffer::from_tokens(token_trees.as_slice()),
_ => TokenBuffer::from_subtree(tt),
};
let parser_input = to_parser_input(&buffer);
@ -92,8 +100,7 @@ pub fn token_tree_to_syntax_node(
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
}
}
let (parse, range_map) = tree_sink.finish();
(parse, range_map)
tree_sink.finish()
}
/// Convert a string to a `TokenTree`
@ -132,7 +139,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
res.push(match expanded.value {
None => break,
Some(tt @ tt::TokenTree::Leaf(_)) => {
tt::Subtree { delimiter: None, token_trees: vec![tt] }
tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] }
}
Some(tt::TokenTree::Subtree(tt)) => tt,
});
@ -145,7 +152,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
}
if iter.peek_n(0).is_some() {
res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() });
res.push(tt::Subtree {
delimiter: tt::Delimiter::unspecified(),
token_trees: iter.cloned().collect(),
});
}
res
@ -159,7 +169,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
}
let entry = StackEntry {
subtree: tt::Subtree { delimiter: None, ..Default::default() },
subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] },
// never used (delimiter is `None`)
idx: !0,
open_range: TextRange::empty(TextSize::of('.')),
@ -186,7 +196,7 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
sub.token_trees.get_mut(2)
{
lit.id = id
lit.span = id
}
}
tt
@ -199,13 +209,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
assert_eq!(range.len(), TextSize::of('.'));
}
if let Some(delim) = subtree.delimiter {
let expected = match delim.kind {
tt::DelimiterKind::Parenthesis => T![')'],
tt::DelimiterKind::Brace => T!['}'],
tt::DelimiterKind::Bracket => T![']'],
};
let expected = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
};
if let Some(expected) = expected {
if kind == expected {
if let Some(entry) = stack.pop() {
conv.id_alloc().close_delim(entry.idx, Some(range));
@ -223,9 +234,11 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
};
if let Some(kind) = delim {
let mut subtree = tt::Subtree::default();
let (id, idx) = conv.id_alloc().open_delim(range, synth_id);
subtree.delimiter = Some(tt::Delimiter { id, kind });
let subtree = tt::Subtree {
delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind },
token_trees: vec![],
};
stack.push(StackEntry { subtree, idx, open_range: range });
continue;
}
@ -240,13 +253,20 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
panic!("Token from lexer must be single char: token = {token:#?}");
}
};
tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) })
.into()
tt::Leaf::from(tt::Punct {
char,
spacing,
span: conv.id_alloc().alloc(range, synth_id),
})
.into()
} else {
macro_rules! make_leaf {
($i:ident) => {
tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) }
.into()
tt::$i {
span: conv.id_alloc().alloc(range, synth_id),
text: token.to_text(conv),
}
.into()
};
}
let leaf: tt::Leaf = match kind {
@ -261,14 +281,14 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
let apostrophe = tt::Leaf::from(tt::Punct {
char: '\'',
spacing: tt::Spacing::Joint,
id: conv.id_alloc().alloc(r, synth_id),
span: conv.id_alloc().alloc(r, synth_id),
});
result.push(apostrophe.into());
let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
let ident = tt::Leaf::from(tt::Ident {
text: SmolStr::new(&token.to_text(conv)[1..]),
id: conv.id_alloc().alloc(r, synth_id),
span: conv.id_alloc().alloc(r, synth_id),
});
result.push(ident.into());
continue;
@ -289,11 +309,12 @@ fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
conv.id_alloc().close_delim(entry.idx, None);
let leaf: tt::Leaf = tt::Punct {
id: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.unwrap().kind {
span: conv.id_alloc().alloc(entry.open_range, None),
char: match entry.subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => '(',
tt::DelimiterKind::Brace => '{',
tt::DelimiterKind::Bracket => '[',
tt::DelimiterKind::Invisible => '$',
},
spacing: tt::Spacing::Alone,
}
@ -373,10 +394,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
token_trees.push(mk_punct('!'));
}
token_trees.push(tt::TokenTree::from(tt::Subtree {
delimiter: Some(tt::Delimiter {
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
kind: tt::DelimiterKind::Bracket,
id: tt::TokenId::unspecified(),
}),
},
token_trees: meta_tkns,
}));
@ -386,7 +408,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
fn mk_ident(s: &str) -> tt::TokenTree {
tt::TokenTree::from(tt::Leaf::from(tt::Ident {
text: s.into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
}
@ -394,12 +416,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>
tt::TokenTree::from(tt::Leaf::from(tt::Punct {
char: c,
spacing: tt::Spacing::Alone,
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
}
fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() };
tt::TokenTree::from(tt::Leaf::from(lit))
}
@ -761,15 +783,16 @@ impl<'a> TtTreeSink<'a> {
}
}
fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> {
let texts = match d {
tt::DelimiterKind::Parenthesis => "()",
tt::DelimiterKind::Brace => "{}",
tt::DelimiterKind::Bracket => "[]",
tt::DelimiterKind::Invisible => return None,
};
let idx = closing as usize;
&texts[idx..texts.len() - (1 - idx)]
Some(&texts[idx..texts.len() - (1 - idx)])
}
impl<'a> TtTreeSink<'a> {
@ -790,13 +813,16 @@ impl<'a> TtTreeSink<'a> {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
// Mark the range if needed
let (text, id) = match leaf {
tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span),
tt::Leaf::Punct(punct) => {
assert!(punct.char.is_ascii());
tmp = punct.char as u8;
(std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
(
std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(),
punct.span,
)
}
tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span),
};
let range = TextRange::at(self.text_pos, TextSize::of(text));
self.token_map.insert(id, range);
@ -805,10 +831,10 @@ impl<'a> TtTreeSink<'a> {
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cursor = self.cursor.subtree().unwrap();
match subtree.delimiter {
Some(d) => {
self.open_delims.insert(d.id, self.text_pos);
delim_to_str(d.kind, false)
match delim_to_str(subtree.delimiter.kind, false) {
Some(it) => {
self.open_delims.insert(subtree.delimiter.open, self.text_pos);
it
}
None => continue,
}
@ -816,15 +842,21 @@ impl<'a> TtTreeSink<'a> {
None => {
let parent = self.cursor.end().unwrap();
self.cursor = self.cursor.bump();
match parent.delimiter {
Some(d) => {
if let Some(open_delim) = self.open_delims.get(&d.id) {
match delim_to_str(parent.delimiter.kind, true) {
Some(it) => {
if let Some(open_delim) =
self.open_delims.get(&parent.delimiter.open)
{
let open_range = TextRange::at(*open_delim, TextSize::of('('));
let close_range =
TextRange::at(self.text_pos, TextSize::of('('));
self.token_map.insert_delim(d.id, open_range, close_range);
self.token_map.insert_delim(
parent.delimiter.open,
open_range,
close_range,
);
}
delim_to_str(d.kind, true)
it
}
None => continue,
}

4
crates/mbe/src/syntax_bridge/tests.rs
View file

@ -29,8 +29,8 @@ fn check_punct_spacing(fixture: &str) {
let mut cursor = buf.begin();
while !cursor.eof() {
while let Some(token_tree) = cursor.token_tree() {
if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree {
if let Some(expected) = annotations.remove(id) {
if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree {
if let Some(expected) = annotations.remove(span) {
assert_eq!(expected, *spacing);
}
}

29
crates/mbe/src/to_parser_input.rs
View file

@ -2,7 +2,8 @@
//! format that works for our parser.
use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
use crate::tt::buffer::TokenBuffer;
pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
let mut res = parser::Input::default();
@ -70,23 +71,25 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input {
cursor.bump()
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
if let Some(d) = subtree.delimiter_kind() {
res.push(match d {
tt::DelimiterKind::Parenthesis => T!['('],
tt::DelimiterKind::Brace => T!['{'],
tt::DelimiterKind::Bracket => T!['['],
});
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T!['(']),
tt::DelimiterKind::Brace => Some(T!['{']),
tt::DelimiterKind::Bracket => Some(T!['[']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.subtree().unwrap()
}
None => match cursor.end() {
Some(subtree) => {
if let Some(d) = subtree.delimiter_kind() {
res.push(match d {
tt::DelimiterKind::Parenthesis => T![')'],
tt::DelimiterKind::Brace => T!['}'],
tt::DelimiterKind::Bracket => T![']'],
})
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.bump()
}

7
crates/mbe/src/tt_iter.rs
View file

@ -3,9 +3,8 @@
use smallvec::{smallvec, SmallVec};
use syntax::SyntaxKind;
use tt::buffer::TokenBuffer;
use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult};
use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult};
#[derive(Debug, Clone)]
pub(crate) struct TtIter<'a> {
@ -135,7 +134,7 @@ impl<'a> TtIter<'a> {
&mut self,
entry_point: parser::PrefixEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> {
let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
let parser_input = to_parser_input(&buffer);
let tree_traversal = entry_point.parse(&parser_input);
@ -178,7 +177,7 @@ impl<'a> TtIter<'a> {
1 => Some(res[0].cloned()),
0 => None,
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: None,
delimiter: tt::Delimiter::unspecified(),
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
})),
};

9
crates/proc-macro-api/src/lib.rs
View file

@ -19,7 +19,8 @@ use std::{
};
use serde::{Deserialize, Serialize};
use tt::Subtree;
use ::tt::token_id as tt;
use crate::{
msg::{ExpandMacro, FlatTree, PanicMessage},
@ -151,10 +152,10 @@ impl ProcMacro {
pub fn expand(
&self,
subtree: &Subtree,
attr: Option<&Subtree>,
subtree: &tt::Subtree,
attr: Option<&tt::Subtree>,
env: Vec<(String, String)>,
) -> Result<Result<Subtree, PanicMessage>, ServerError> {
) -> Result<Result<tt::Subtree, PanicMessage>, ServerError> {
let current_dir = env
.iter()
.find(|(name, _)| name == "CARGO_MANIFEST_DIR")

18
crates/proc-macro-api/src/msg.rs
View file

@ -107,27 +107,31 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use tt::*;
use crate::tt::*;
fn fixture_token_tree() -> Subtree {
let mut subtree = Subtree::default();
let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() };
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into()));
.push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into()));
subtree
.token_trees
.push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into()));
.push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into()));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal {
text: "Foo".into(),
id: TokenId::unspecified(),
span: TokenId::unspecified(),
})));
subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct {
char: '@',
id: TokenId::unspecified(),
span: TokenId::unspecified(),
spacing: Spacing::Joint,
})));
subtree.token_trees.push(TokenTree::Subtree(Subtree {
delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }),
delimiter: Delimiter {
open: TokenId(2),
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Brace,
},
token_trees: vec![],
}));
subtree

43
crates/proc-macro-api/src/msg/flat.rs
View file

@ -38,7 +38,8 @@
use std::collections::{HashMap, VecDeque};
use serde::{Deserialize, Serialize};
use tt::TokenId;
use crate::tt::{self, TokenId};
#[derive(Serialize, Deserialize, Debug)]
pub struct FlatTree {
@ -52,7 +53,7 @@ pub struct FlatTree {
struct SubtreeRepr {
id: tt::TokenId,
kind: Option<tt::DelimiterKind>,
kind: tt::DelimiterKind,
tt: [u32; 2],
}
@ -124,19 +125,19 @@ impl FlatTree {
impl SubtreeRepr {
fn write(self) -> [u32; 4] {
let kind = match self.kind {
None => 0,
Some(tt::DelimiterKind::Parenthesis) => 1,
Some(tt::DelimiterKind::Brace) => 2,
Some(tt::DelimiterKind::Bracket) => 3,
tt::DelimiterKind::Invisible => 0,
tt::DelimiterKind::Parenthesis => 1,
tt::DelimiterKind::Brace => 2,
tt::DelimiterKind::Bracket => 3,
};
[self.id.0, kind, self.tt[0], self.tt[1]]
}
fn read([id, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
let kind = match kind {
0 => None,
1 => Some(tt::DelimiterKind::Parenthesis),
2 => Some(tt::DelimiterKind::Brace),
3 => Some(tt::DelimiterKind::Bracket),
0 => tt::DelimiterKind::Invisible,
1 => tt::DelimiterKind::Parenthesis,
2 => tt::DelimiterKind::Brace,
3 => tt::DelimiterKind::Bracket,
other => panic!("bad kind {other}"),
};
SubtreeRepr { id: TokenId(id), kind, tt: [lo, len] }
@ -216,7 +217,7 @@ impl<'a> Writer<'a> {
tt::Leaf::Literal(lit) => {
let idx = self.literal.len() as u32;
let text = self.intern(&lit.text);
self.literal.push(LiteralRepr { id: lit.id, text });
self.literal.push(LiteralRepr { id: lit.span, text });
idx << 2 | 0b01
}
tt::Leaf::Punct(punct) => {
@ -224,14 +225,14 @@ impl<'a> Writer<'a> {
self.punct.push(PunctRepr {
char: punct.char,
spacing: punct.spacing,
id: punct.id,
id: punct.span,
});
idx << 2 | 0b10
}
tt::Leaf::Ident(ident) => {
let idx = self.ident.len() as u32;
let text = self.intern(&ident.text);
self.ident.push(IdentRepr { id: ident.id, text });
self.ident.push(IdentRepr { id: ident.span, text });
idx << 2 | 0b11
}
},
@ -243,8 +244,8 @@ impl<'a> Writer<'a> {
fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 {
let idx = self.subtree.len();
let delimiter_id = subtree.delimiter.map_or(TokenId::unspecified(), |it| it.id);
let delimiter_kind = subtree.delimiter.map(|it| it.kind);
let delimiter_id = subtree.delimiter.open;
let delimiter_kind = subtree.delimiter.kind;
self.subtree.push(SubtreeRepr { id: delimiter_id, kind: delimiter_kind, tt: [!0, !0] });
self.work.push_back((idx, subtree));
idx as u32
@ -276,7 +277,11 @@ impl Reader {
let repr = &self.subtree[i];
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
let s = tt::Subtree {
delimiter: repr.kind.map(|kind| tt::Delimiter { id: repr.id, kind }),
delimiter: tt::Delimiter {
open: repr.id,
close: TokenId::UNSPECIFIED,
kind: repr.kind,
},
token_trees: token_trees
.iter()
.copied()
@ -291,7 +296,7 @@ impl Reader {
let repr = &self.literal[idx];
tt::Leaf::Literal(tt::Literal {
text: self.text[repr.text as usize].as_str().into(),
id: repr.id,
span: repr.id,
})
.into()
}
@ -300,7 +305,7 @@ impl Reader {
tt::Leaf::Punct(tt::Punct {
char: repr.char,
spacing: repr.spacing,
id: repr.id,
span: repr.id,
})
.into()
}
@ -308,7 +313,7 @@ impl Reader {
let repr = &self.ident[idx];
tt::Leaf::Ident(tt::Ident {
text: self.text[repr.text as usize].as_str().into(),
id: repr.id,
span: repr.id,
})
.into()
}

1
crates/proc-macro-srv/src/abis/abi_1_63/mod.rs
View file

@ -11,6 +11,7 @@ mod ra_server;
use libloading::Library;
use proc_macro_api::ProcMacroKind;
use super::tt;
use super::PanicMessage;
pub use ra_server::TokenStream;

119
crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs
View file

@ -15,6 +15,8 @@ use std::hash::Hash;
use std::ops::Bound;
use std::{ascii, vec::IntoIter};
use crate::tt;
type Group = tt::Subtree;
type TokenTree = tt::TokenTree;
type Punct = tt::Punct;
@ -33,7 +35,7 @@ impl TokenStream {
}
pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() {
if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else {
TokenStream { token_trees: subtree.token_trees }
@ -41,7 +43,7 @@ impl TokenStream {
}
pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees }
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
}
pub fn is_empty(&self) -> bool {
@ -84,7 +86,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams {
for tkn in item {
match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => {
tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind == tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees);
}
_ => {
@ -165,7 +169,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream {
use std::str::FromStr;
use super::{TokenStream, TokenTree};
use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -202,15 +206,17 @@ pub mod token_stream {
impl ToString for TokenStream {
fn to_string(&self) -> String {
tt::pretty(&self.token_trees)
::tt::pretty(&self.token_trees)
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree
.delimiter
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree
.token_trees
.into_iter()
@ -233,13 +239,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
}
}
}
@ -389,22 +395,22 @@ impl server::TokenStream for RustAnalyzer {
}
}
fn delim_to_internal(d: bridge::Delimiter) -> Option<tt::Delimiter> {
fn delim_to_internal(d: bridge::Delimiter) -> tt::Delimiter {
let kind = match d {
bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
bridge::Delimiter::Brace => tt::DelimiterKind::Brace,
bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket,
bridge::Delimiter::None => return None,
bridge::Delimiter::None => tt::DelimiterKind::Invisible,
};
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind })
tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
}
fn delim_to_external(d: Option<tt::Delimiter>) -> bridge::Delimiter {
match d.map(|it| it.kind) {
Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket,
None => bridge::Delimiter::None,
fn delim_to_external(d: tt::Delimiter) -> bridge::Delimiter {
match d.kind {
tt::DelimiterKind::Parenthesis => bridge::Delimiter::Parenthesis,
tt::DelimiterKind::Brace => bridge::Delimiter::Brace,
tt::DelimiterKind::Bracket => bridge::Delimiter::Bracket,
tt::DelimiterKind::Invisible => bridge::Delimiter::None,
}
}
@ -443,23 +449,19 @@ impl server::Group for RustAnalyzer {
}
fn span(&mut self, group: &Self::Group) -> Self::Span {
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.open
}
fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
if let Some(delim) = &mut group.delimiter {
delim.id = span;
}
group.delimiter.open = span;
}
fn span_open(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.open
}
fn span_close(&mut self, group: &Self::Group) -> Self::Span {
// FIXME we only store one `TokenId` for the delimiters
group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified)
group.delimiter.close
}
}
@ -468,7 +470,7 @@ impl server::Punct for RustAnalyzer {
tt::Punct {
char: ch,
spacing: spacing_to_internal(spacing),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}
}
fn as_char(&mut self, punct: Self::Punct) -> char {
@ -478,28 +480,27 @@ impl server::Punct for RustAnalyzer {
spacing_to_external(punct.spacing)
}
fn span(&mut self, punct: Self::Punct) -> Self::Span {
punct.id
punct.span
}
fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
tt::Punct { id: span, ..punct }
tt::Punct { span: span, ..punct }
}
}
impl server::Ident for RustAnalyzer {
fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw(
string.into(),
IdentId(self.ident_interner.intern(&IdentData(tt::Ident {
text: if is_raw { ::tt::SmolStr::from_iter(["r#", string]) } else { string.into() },
span,
is_raw,
))))
})))
}
fn span(&mut self, ident: Self::Ident) -> Self::Span {
self.ident_interner.get(ident.0).0.id
self.ident_interner.get(ident.0).0.span
}
fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
let data = self.ident_interner.get(ident.0);
let new = IdentData(tt::Ident { id: span, ..data.0.clone() });
let new = IdentData(tt::Ident { span: span, ..data.0.clone() });
IdentId(self.ident_interner.intern(&new))
}
}
@ -511,7 +512,7 @@ impl server::Literal for RustAnalyzer {
"".to_owned()
}
fn from_str(&mut self, s: &str) -> Result<Self::Literal, ()> {
Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() })
Ok(Literal { text: s.into(), span: tt::TokenId::unspecified() })
}
fn symbol(&mut self, literal: &Self::Literal) -> String {
literal.text.to_string()
@ -529,7 +530,7 @@ impl server::Literal for RustAnalyzer {
Ok(n) => n.to_string(),
Err(_) => n.parse::<u128>().unwrap().to_string(),
};
Literal { text: n.into(), id: tt::TokenId::unspecified() }
Literal { text: n.into(), span: tt::TokenId::unspecified() }
}
fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
@ -549,7 +550,7 @@ impl server::Literal for RustAnalyzer {
let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize};
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn float(&mut self, n: &str) -> Self::Literal {
@ -558,19 +559,19 @@ impl server::Literal for RustAnalyzer {
if !text.contains('.') {
text += ".0"
}
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn f32(&mut self, n: &str) -> Self::Literal {
let n: f32 = n.parse().unwrap();
let text = format!("{n}f32");
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn f64(&mut self, n: &str) -> Self::Literal {
let n: f64 = n.parse().unwrap();
let text = format!("{n}f64");
Literal { text: text.into(), id: tt::TokenId::unspecified() }
Literal { text: text.into(), span: tt::TokenId::unspecified() }
}
fn string(&mut self, string: &str) -> Self::Literal {
@ -578,11 +579,11 @@ impl server::Literal for RustAnalyzer {
for ch in string.chars() {
escaped.extend(ch.escape_debug());
}
Literal { text: format!("\"{escaped}\"").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("\"{escaped}\"").into(), span: tt::TokenId::unspecified() }
}
fn character(&mut self, ch: char) -> Self::Literal {
Literal { text: format!("'{ch}'").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("'{ch}'").into(), span: tt::TokenId::unspecified() }
}
fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
@ -593,15 +594,15 @@ impl server::Literal for RustAnalyzer {
.map(Into::<char>::into)
.collect::<String>();
Literal { text: format!("b\"{string}\"").into(), id: tt::TokenId::unspecified() }
Literal { text: format!("b\"{string}\"").into(), span: tt::TokenId::unspecified() }
}
fn span(&mut self, literal: &Self::Literal) -> Self::Span {
literal.id
literal.span
}
fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
literal.id = span;
literal.span = span;
}
fn subspan(
@ -784,17 +785,18 @@ mod tests {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
}),
],
@ -807,13 +809,14 @@ mod tests {
fn test_ra_server_from_str() {
use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis,
}),
},
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
});
@ -830,7 +833,7 @@ mod tests {
underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
);
}

2
crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs
View file

@ -9,7 +9,7 @@ mod ra_server;
use libloading::Library;
use proc_macro_api::ProcMacroKind;
use super::PanicMessage;
use super::{tt, PanicMessage};
pub use ra_server::TokenStream;

65
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs
View file

@ -22,6 +22,8 @@ pub use symbol::*;
use std::ops::Bound;
use crate::tt;
type Group = tt::Subtree;
type TokenTree = tt::TokenTree;
type Punct = tt::Punct;
@ -108,8 +110,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(ident) => {
let text = ident.sym.text();
let text = if ident.is_raw { tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, id: ident.span };
let text =
if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, span: ident.span };
let leaf = tt::Leaf::from(ident);
let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree])
@ -118,9 +121,9 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Literal(literal) => {
let literal = LiteralFormatter(literal);
let text = literal
.with_stringify_parts(|parts| tt::SmolStr::from_iter(parts.iter().copied()));
.with_stringify_parts(|parts| ::tt::SmolStr::from_iter(parts.iter().copied()));
let literal = tt::Literal { text, id: literal.0.span };
let literal = tt::Literal { text, span: literal.0.span };
let leaf = tt::Leaf::from(literal);
let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(vec![tree])
@ -130,7 +133,7 @@ impl server::TokenStream for RustAnalyzer {
let punct = tt::Punct {
char: p.ch as char,
spacing: if p.joint { Spacing::Joint } else { Spacing::Alone },
id: p.span,
span: p.span,
};
let leaf = tt::Leaf::from(punct);
let tree = TokenTree::from(leaf);
@ -184,7 +187,7 @@ impl server::TokenStream for RustAnalyzer {
bridge::TokenTree::Ident(bridge::Ident {
sym: Symbol::intern(ident.text.trim_start_matches("r#")),
is_raw: ident.text.starts_with("r#"),
span: ident.id,
span: ident.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
@ -194,14 +197,14 @@ impl server::TokenStream for RustAnalyzer {
symbol: Symbol::intern(&lit.text),
// FIXME: handle suffixes
suffix: None,
span: lit.id,
span: lit.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
bridge::TokenTree::Punct(bridge::Punct {
ch: punct.char as u8,
joint: punct.spacing == Spacing::Joint,
span: punct.id,
span: punct.span,
})
}
tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group {
@ -211,31 +214,29 @@ impl server::TokenStream for RustAnalyzer {
} else {
Some(subtree.token_trees.into_iter().collect())
},
span: bridge::DelimSpan::from_single(
subtree.delimiter.map_or(Span::unspecified(), |del| del.id),
),
span: bridge::DelimSpan::from_single(subtree.delimiter.open),
}),
})
.collect()
}
}
fn delim_to_internal(d: proc_macro::Delimiter) -> Option<tt::Delimiter> {
fn delim_to_internal(d: proc_macro::Delimiter) -> tt::Delimiter {
let kind = match d {
proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis,
proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace,
proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket,
proc_macro::Delimiter::None => return None,
proc_macro::Delimiter::None => tt::DelimiterKind::Invisible,
};
Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind })
tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind }
}
fn delim_to_external(d: Option<tt::Delimiter>) -> proc_macro::Delimiter {
match d.map(|it| it.kind) {
Some(tt::DelimiterKind::Parenthesis) => proc_macro::Delimiter::Parenthesis,
Some(tt::DelimiterKind::Brace) => proc_macro::Delimiter::Brace,
Some(tt::DelimiterKind::Bracket) => proc_macro::Delimiter::Bracket,
None => proc_macro::Delimiter::None,
fn delim_to_external(d: tt::Delimiter) -> proc_macro::Delimiter {
match d.kind {
tt::DelimiterKind::Parenthesis => proc_macro::Delimiter::Parenthesis,
tt::DelimiterKind::Brace => proc_macro::Delimiter::Brace,
tt::DelimiterKind::Bracket => proc_macro::Delimiter::Bracket,
tt::DelimiterKind::Invisible => proc_macro::Delimiter::None,
}
}
@ -349,7 +350,7 @@ impl server::Server for RustAnalyzer {
}
fn intern_symbol(ident: &str) -> Self::Symbol {
Symbol::intern(&tt::SmolStr::from(ident))
Symbol::intern(&::tt::SmolStr::from(ident))
}
fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
@ -413,17 +414,18 @@ mod tests {
token_trees: vec![
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Brace,
}),
},
token_trees: vec![],
}),
],
@ -436,13 +438,14 @@ mod tests {
fn test_ra_server_from_str() {
use std::str::FromStr;
let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
delimiter: Some(tt::Delimiter {
id: tt::TokenId::unspecified(),
delimiter: tt::Delimiter {
open: tt::TokenId::unspecified(),
close: tt::TokenId::unspecified(),
kind: tt::DelimiterKind::Parenthesis,
}),
},
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "a".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))],
});
@ -459,7 +462,7 @@ mod tests {
underscore.token_trees[0],
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
id: tt::TokenId::unspecified(),
span: tt::TokenId::unspecified(),
}))
);
}

28
crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs
View file

@ -1,6 +1,6 @@
//! TokenStream implementation used by sysroot ABI
use tt::TokenTree;
use crate::tt::{self, TokenTree};
#[derive(Debug, Default, Clone)]
pub struct TokenStream {
@ -13,7 +13,7 @@ impl TokenStream {
}
pub fn with_subtree(subtree: tt::Subtree) -> Self {
if subtree.delimiter.is_some() {
if subtree.delimiter.kind != tt::DelimiterKind::Invisible {
TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
} else {
TokenStream { token_trees: subtree.token_trees }
@ -21,7 +21,7 @@ impl TokenStream {
}
pub fn into_subtree(self) -> tt::Subtree {
tt::Subtree { delimiter: None, token_trees: self.token_trees }
tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees }
}
pub fn is_empty(&self) -> bool {
@ -64,7 +64,9 @@ impl Extend<TokenStream> for TokenStream {
for item in streams {
for tkn in item {
match tkn {
tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => {
tt::TokenTree::Subtree(subtree)
if subtree.delimiter.kind != tt::DelimiterKind::Invisible =>
{
self.token_trees.extend(subtree.token_trees);
}
_ => {
@ -84,7 +86,7 @@ pub struct TokenStreamBuilder {
pub mod token_stream {
use std::str::FromStr;
use super::{TokenStream, TokenTree};
use super::{tt, TokenStream, TokenTree};
/// An iterator over `TokenStream`'s `TokenTree`s.
/// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
@ -121,15 +123,17 @@ pub mod token_stream {
impl ToString for TokenStream {
fn to_string(&self) -> String {
tt::pretty(&self.token_trees)
::tt::pretty(&self.token_trees)
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree
.delimiter
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
delimiter: tt::Delimiter {
open: tt::TokenId::UNSPECIFIED,
close: tt::TokenId::UNSPECIFIED,
..subtree.delimiter
},
token_trees: subtree
.token_trees
.into_iter()
@ -152,13 +156,13 @@ pub mod token_stream {
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident })
}
}
}

2
crates/proc-macro-srv/src/abis/mod.rs
View file

@ -41,6 +41,8 @@ pub(crate) use abi_sysroot::Abi as Abi_Sysroot;
use libloading::Library;
use proc_macro_api::{ProcMacroKind, RustCInfo};
use crate::tt;
pub struct PanicMessage {
message: Option<String>,
}

2
crates/proc-macro-srv/src/dylib.rs
View file

@ -13,6 +13,8 @@ use object::Object;
use paths::AbsPath;
use proc_macro_api::{read_dylib_info, ProcMacroKind};
use crate::tt;
use super::abis::Abi;
const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";

2
crates/proc-macro-srv/src/lib.rs
View file

@ -37,6 +37,8 @@ use proc_macro_api::{
ProcMacroKind,
};
use ::tt::token_id as tt;
#[derive(Default)]
pub(crate) struct ProcMacroSrv {
expanders: HashMap<(PathBuf, SystemTime), dylib::Expander>,

26
crates/proc-macro-srv/src/tests/mod.rs
View file

@ -8,7 +8,7 @@ use expect_test::expect;
#[test]
fn test_derive_empty() {
assert_expand("DeriveEmpty", r#"struct S;"#, expect![[r#"SUBTREE $"#]]);
assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 4294967295 4294967295"]);
}
#[test]
@ -17,10 +17,10 @@ fn test_derive_error() {
"DeriveError",
r#"struct S;"#,
expect![[r##"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295
SUBTREE () 4294967295
SUBTREE () 4294967295 4294967295
LITERAL "#[derive(DeriveError)] struct S ;" 4294967295
PUNCH ; [alone] 4294967295"##]],
);
@ -32,14 +32,14 @@ fn test_fn_like_macro_noop() {
"fn_like_noop",
r#"ident, 0, 1, []"#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295
PUNCH , [alone] 4294967295
LITERAL 0 4294967295
PUNCH , [alone] 4294967295
LITERAL 1 4294967295
PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]],
SUBTREE [] 4294967295 4294967295"#]],
);
}
@ -49,10 +49,10 @@ fn test_fn_like_macro_clone_ident_subtree() {
"fn_like_clone_tokens",
r#"ident, []"#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT ident 4294967295
PUNCH , [alone] 4294967295
SUBTREE [] 4294967295"#]],
SUBTREE [] 4294967295 4294967295"#]],
);
}
@ -62,7 +62,7 @@ fn test_fn_like_macro_clone_raw_ident() {
"fn_like_clone_tokens",
"r#async",
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT r#async 4294967295"#]],
);
}
@ -73,7 +73,7 @@ fn test_fn_like_mk_literals() {
"fn_like_mk_literals",
r#""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
LITERAL b"byte_string" 4294967295
LITERAL 'c' 4294967295
LITERAL "string" 4294967295
@ -90,7 +90,7 @@ fn test_fn_like_mk_idents() {
"fn_like_mk_idents",
r#""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT standard 4294967295
IDENT r#raw 4294967295"#]],
);
@ -102,7 +102,7 @@ fn test_fn_like_macro_clone_literals() {
"fn_like_clone_tokens",
r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#,
expect![[r#"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
LITERAL 1u16 4294967295
PUNCH , [alone] 4294967295
LITERAL 2_u32 4294967295
@ -126,10 +126,10 @@ fn test_attr_macro() {
r#"mod m {}"#,
r#"some arguments"#,
expect![[r##"
SUBTREE $
SUBTREE $$ 4294967295 4294967295
IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295
SUBTREE () 4294967295
SUBTREE () 4294967295 4294967295
LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295
PUNCH ; [alone] 4294967295"##]],
);

4
crates/rust-analyzer/src/reload.rs
View file

@ -34,6 +34,8 @@ use crate::{
op_queue::Cause,
};
use ::tt::token_id as tt;
#[derive(Debug)]
pub(crate) enum ProjectWorkspaceProgress {
Begin,
@ -656,7 +658,7 @@ pub(crate) fn load_proc_macro(
_: Option<&tt::Subtree>,
_: &Env,
) -> Result<tt::Subtree, ProcMacroExpansionError> {
Ok(tt::Subtree::default())
Ok(tt::Subtree::empty())
}
}
}

9
crates/stdx/src/macros.rs
View file

@ -43,5 +43,14 @@ macro_rules! impl_from {
}
)*)?
)*
};
($($variant:ident$(<$V:ident>)?),* for $enum:ident) => {
$(
impl$(<$V>)? From<$variant$(<$V>)?> for $enum$(<$V>)? {
fn from(it: $variant$(<$V>)?) -> $enum$(<$V>)? {
$enum::$variant(it)
}
}
)*
}
}

84
crates/tt/src/buffer.rs
View file

@ -12,10 +12,10 @@ struct EntryPtr(EntryId, usize);
/// Internal type which is used instead of `TokenTree` to represent a token tree
/// within a `TokenBuffer`.
#[derive(Debug)]
enum Entry<'t> {
enum Entry<'t, Span> {
// Mimicking types from proc-macro.
Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId),
Leaf(&'t TokenTree),
Subtree(Option<&'t TokenTree<Span>>, &'t Subtree<Span>, EntryId),
Leaf(&'t TokenTree<Span>),
// End entries contain a pointer to the entry from the containing
// token tree, or None if this is the outermost level.
End(Option<EntryPtr>),
@ -24,16 +24,21 @@ enum Entry<'t> {
/// A token tree buffer
/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
#[derive(Debug)]
pub struct TokenBuffer<'t> {
buffers: Vec<Box<[Entry<'t>]>>,
pub struct TokenBuffer<'t, Span> {
buffers: Vec<Box<[Entry<'t, Span>]>>,
}
trait TokenList<'a> {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>);
trait TokenList<'a, Span> {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>);
}
impl<'a> TokenList<'a> for &'a [TokenTree] {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
impl<'a, Span> TokenList<'a, Span> for &'a [TokenTree<Span>] {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End
let start_capacity = self.len() + 1;
let mut entries = Vec::with_capacity(start_capacity);
@ -53,8 +58,11 @@ impl<'a> TokenList<'a> for &'a [TokenTree] {
}
}
impl<'a> TokenList<'a> for &'a Subtree {
fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
impl<'a, Span> TokenList<'a, Span> for &'a Subtree<Span> {
fn entries(
&self,
) -> (Vec<(usize, (&'a Subtree<Span>, Option<&'a TokenTree<Span>>))>, Vec<Entry<'a, Span>>)
{
// Must contain everything in tokens and then the Entry::End
let mut entries = vec![];
let mut children = vec![];
@ -64,25 +72,25 @@ impl<'a> TokenList<'a> for &'a Subtree {
}
}
impl<'t> TokenBuffer<'t> {
pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
impl<'t, Span> TokenBuffer<'t, Span> {
pub fn from_tokens(tokens: &'t [TokenTree<Span>]) -> TokenBuffer<'t, Span> {
Self::new(tokens)
}
pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> {
pub fn from_subtree(subtree: &'t Subtree<Span>) -> TokenBuffer<'t, Span> {
Self::new(subtree)
}
fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> {
fn new<T: TokenList<'t, Span>>(tokens: T) -> TokenBuffer<'t, Span> {
let mut buffers = vec![];
let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
assert_eq!(idx, 0);
TokenBuffer { buffers }
}
fn new_inner<T: TokenList<'t>>(
fn new_inner<T: TokenList<'t, Span>>(
tokens: T,
buffers: &mut Vec<Box<[Entry<'t>]>>,
buffers: &mut Vec<Box<[Entry<'t, Span>]>>,
next: Option<EntryPtr>,
) -> usize {
let (children, mut entries) = tokens.entries();
@ -105,25 +113,25 @@ impl<'t> TokenBuffer<'t> {
/// Creates a cursor referencing the first token in the buffer and able to
/// traverse until the end of the buffer.
pub fn begin(&self) -> Cursor<'_> {
pub fn begin(&self) -> Cursor<'_, Span> {
Cursor::create(self, EntryPtr(EntryId(0), 0))
}
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_>> {
fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_, Span>> {
let id = ptr.0;
self.buffers[id.0].get(ptr.1)
}
}
#[derive(Debug)]
pub enum TokenTreeRef<'a> {
Subtree(&'a Subtree, Option<&'a TokenTree>),
Leaf(&'a Leaf, &'a TokenTree),
pub enum TokenTreeRef<'a, Span> {
Subtree(&'a Subtree<Span>, Option<&'a TokenTree<Span>>),
Leaf(&'a Leaf<Span>, &'a TokenTree<Span>),
}
impl<'a> TokenTreeRef<'a> {
pub fn cloned(&self) -> TokenTree {
match &self {
impl<'a, Span: Clone> TokenTreeRef<'a, Span> {
pub fn cloned(&self) -> TokenTree<Span> {
match self {
TokenTreeRef::Subtree(subtree, tt) => match tt {
Some(it) => (*it).clone(),
None => (*subtree).clone().into(),
@ -135,20 +143,20 @@ impl<'a> TokenTreeRef<'a> {
/// A safe version of `Cursor` from `syn` crate <https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125>
#[derive(Copy, Clone, Debug)]
pub struct Cursor<'a> {
buffer: &'a TokenBuffer<'a>,
pub struct Cursor<'a, Span> {
buffer: &'a TokenBuffer<'a, Span>,
ptr: EntryPtr,
}
impl<'a> PartialEq for Cursor<'a> {
fn eq(&self, other: &Cursor<'_>) -> bool {
impl<'a, Span> PartialEq for Cursor<'a, Span> {
fn eq(&self, other: &Cursor<'_, Span>) -> bool {
self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer)
}
}
impl<'a> Eq for Cursor<'a> {}
impl<'a, Span> Eq for Cursor<'a, Span> {}
impl<'a> Cursor<'a> {
impl<'a, Span> Cursor<'a, Span> {
/// Check whether it is eof
pub fn eof(self) -> bool {
matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None)))
@ -156,7 +164,7 @@ impl<'a> Cursor<'a> {
/// If the cursor is pointing at the end of a subtree, returns
/// the parent subtree
pub fn end(self) -> Option<&'a Subtree> {
pub fn end(self) -> Option<&'a Subtree<Span>> {
match self.entry() {
Some(Entry::End(Some(ptr))) => {
let idx = ptr.1;
@ -171,13 +179,13 @@ impl<'a> Cursor<'a> {
}
}
fn entry(self) -> Option<&'a Entry<'a>> {
fn entry(&self) -> Option<&'a Entry<'a, Span>> {
self.buffer.entry(&self.ptr)
}
/// If the cursor is pointing at a `Subtree`, returns
/// a cursor into that subtree
pub fn subtree(self) -> Option<Cursor<'a>> {
pub fn subtree(self) -> Option<Cursor<'a, Span>> {
match self.entry() {
Some(Entry::Subtree(_, _, entry_id)) => {
Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
@ -187,7 +195,7 @@ impl<'a> Cursor<'a> {
}
/// If the cursor is pointing at a `TokenTree`, returns it
pub fn token_tree(self) -> Option<TokenTreeRef<'a>> {
pub fn token_tree(self) -> Option<TokenTreeRef<'a, Span>> {
match self.entry() {
Some(Entry::Leaf(tt)) => match tt {
TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)),
@ -198,12 +206,12 @@ impl<'a> Cursor<'a> {
}
}
fn create(buffer: &'a TokenBuffer<'_>, ptr: EntryPtr) -> Cursor<'a> {
fn create(buffer: &'a TokenBuffer<'_, Span>, ptr: EntryPtr) -> Cursor<'a, Span> {
Cursor { buffer, ptr }
}
/// Bump the cursor
pub fn bump(self) -> Cursor<'a> {
pub fn bump(self) -> Cursor<'a, Span> {
if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) {
match exit {
Some(exit) => Cursor::create(self.buffer, *exit),
@ -216,7 +224,7 @@ impl<'a> Cursor<'a> {
/// Bump the cursor, if it is a subtree, returns
/// a cursor into that subtree
pub fn bump_subtree(self) -> Cursor<'a> {
pub fn bump_subtree(self) -> Cursor<'a, Span> {
match self.entry() {
Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
_ => self.bump(),

232
crates/tt/src/lib.rs
View file

@ -16,45 +16,106 @@ pub use smol_str::SmolStr;
/// which source tokens. We do it by assigning an distinct identity to each
/// source token and making sure that identities are preserved during macro
/// expansion.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct TokenId(pub u32);
impl fmt::Debug for TokenId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl TokenId {
pub const UNSPECIFIED: TokenId = TokenId(!0);
pub const fn unspecified() -> TokenId {
TokenId(!0)
Self::UNSPECIFIED
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree {
Leaf(Leaf),
Subtree(Subtree),
}
impl_from!(Leaf, Subtree for TokenTree);
pub mod token_id {
pub use crate::{DelimiterKind, Spacing, TokenId};
pub type Span = crate::TokenId;
pub type Subtree = crate::Subtree<Span>;
pub type Punct = crate::Punct<Span>;
pub type Delimiter = crate::Delimiter<Span>;
pub type Leaf = crate::Leaf<Span>;
pub type Ident = crate::Ident<Span>;
pub type Literal = crate::Literal<Span>;
pub type TokenTree = crate::TokenTree<Span>;
pub mod buffer {
pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>;
pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>;
pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>;
}
impl TokenTree {
pub fn empty() -> Self {
TokenTree::Subtree(Subtree::default())
impl Delimiter {
pub const UNSPECIFIED: Self = Self {
open: TokenId::UNSPECIFIED,
close: TokenId::UNSPECIFIED,
kind: DelimiterKind::Invisible,
};
pub const fn unspecified() -> Self {
Self::UNSPECIFIED
}
}
impl Subtree {
pub const fn empty() -> Self {
Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }
}
}
impl TokenTree {
pub const fn empty() -> Self {
Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] })
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf {
Literal(Literal),
Punct(Punct),
Ident(Ident),
}
impl_from!(Literal, Punct, Ident for Leaf);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct SyntaxContext(pub u32);
#[derive(Clone, PartialEq, Eq, Hash, Default)]
pub struct Subtree {
pub delimiter: Option<Delimiter>,
pub token_trees: Vec<TokenTree>,
// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
// pub struct Span {
// pub id: TokenId,
// pub ctx: SyntaxContext,
// }
// pub type Span = (TokenId, SyntaxContext);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TokenTree<Span> {
Leaf(Leaf<Span>),
Subtree(Subtree<Span>),
}
impl_from!(Leaf<Span>, Subtree<Span> for TokenTree);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Leaf<Span> {
Literal(Literal<Span>),
Punct(Punct<Span>),
Ident(Ident<Span>),
}
impl<Span> Leaf<Span> {
pub fn span(&self) -> &Span {
match self {
Leaf::Literal(it) => &it.span,
Leaf::Punct(it) => &it.span,
Leaf::Ident(it) => &it.span,
}
}
}
impl_from!(Literal<Span>, Punct<Span>, Ident<Span> for Leaf);
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct Subtree<Span> {
// FIXME, this should not be Option
pub delimiter: Delimiter<Span>,
pub token_trees: Vec<TokenTree<Span>>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Delimiter {
pub id: TokenId,
pub struct Delimiter<Span> {
pub open: Span,
pub close: Span,
pub kind: DelimiterKind,
}
@ -63,19 +124,20 @@ pub enum DelimiterKind {
Parenthesis,
Brace,
Bracket,
Invisible,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Literal {
pub struct Literal<Span> {
pub text: SmolStr,
pub id: TokenId,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Punct {
pub struct Punct<Span> {
pub char: char,
pub spacing: Spacing,
pub id: TokenId,
pub span: Span,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -85,39 +147,25 @@ pub enum Spacing {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Ident {
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier.
pub struct Ident<Span> {
pub text: SmolStr,
pub id: TokenId,
pub span: Span,
}
impl Ident {
/// Constructor intended to be used only by proc macro server. `text` should not contain raw
/// identifier prefix.
pub fn new_with_is_raw(text: SmolStr, id: TokenId, is_raw: bool) -> Self {
let text = if is_raw { SmolStr::from_iter(["r#", &text]) } else { text };
Ident { text, id }
}
}
impl Leaf {
pub fn id(&self) -> TokenId {
match self {
Leaf::Literal(l) => l.id,
Leaf::Punct(p) => p.id,
Leaf::Ident(i) => i.id,
}
}
}
fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result {
fn print_debug_subtree<Span: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
subtree: &Subtree<Span>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) {
None => "$".to_string(),
Some((DelimiterKind::Parenthesis, id)) => format!("() {id}"),
Some((DelimiterKind::Brace, id)) => format!("{{}} {id}"),
Some((DelimiterKind::Bracket, id)) => format!("[] {id}"),
let Delimiter { kind, open, close } = &subtree.delimiter;
let aux = match kind {
DelimiterKind::Invisible => format!("$$ {:?} {:?}", open, close),
DelimiterKind::Parenthesis => format!("() {:?} {:?}", open, close),
DelimiterKind::Brace => format!("{{}} {:?} {:?}", open, close),
DelimiterKind::Bracket => format!("[] {:?} {:?}", open, close),
};
if subtree.token_trees.is_empty() {
@ -135,21 +183,25 @@ fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usi
Ok(())
}
fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result {
fn print_debug_token<Span: fmt::Debug>(
f: &mut fmt::Formatter<'_>,
tkn: &TokenTree<Span>,
level: usize,
) -> fmt::Result {
let align = " ".repeat(level);
match tkn {
TokenTree::Leaf(leaf) => match leaf {
Leaf::Literal(lit) => write!(f, "{align}LITERAL {} {}", lit.text, lit.id.0)?,
Leaf::Literal(lit) => write!(f, "{}LITERAL {} {:?}", align, lit.text, lit.span)?,
Leaf::Punct(punct) => write!(
f,
"{}PUNCH {} [{}] {}",
"{}PUNCH {} [{}] {:?}",
align,
punct.char,
if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
punct.id.0
punct.span
)?,
Leaf::Ident(ident) => write!(f, "{align}IDENT {} {}", ident.text, ident.id.0)?,
Leaf::Ident(ident) => write!(f, "{}IDENT {} {:?}", align, ident.text, ident.span)?,
},
TokenTree::Subtree(subtree) => {
print_debug_subtree(f, subtree, level)?;
@ -159,13 +211,13 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize)
Ok(())
}
impl fmt::Debug for Subtree {
impl<Span: fmt::Debug> fmt::Debug for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
print_debug_subtree(f, self, 0)
}
}
impl fmt::Display for TokenTree {
impl<Span> fmt::Display for TokenTree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
@ -174,13 +226,13 @@ impl fmt::Display for TokenTree {
}
}
impl fmt::Display for Subtree {
impl<Span> fmt::Display for Subtree<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (l, r) = match self.delimiter_kind() {
Some(DelimiterKind::Parenthesis) => ("(", ")"),
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"),
None => ("", ""),
let (l, r) = match self.delimiter.kind {
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Invisible => ("", ""),
};
f.write_str(l)?;
let mut needs_space = false;
@ -202,7 +254,7 @@ impl fmt::Display for Subtree {
}
}
impl fmt::Display for Leaf {
impl<Span> fmt::Display for Leaf<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Leaf::Ident(it) => fmt::Display::fmt(it, f),
@ -212,25 +264,25 @@ impl fmt::Display for Leaf {
}
}
impl fmt::Display for Ident {
impl<Span> fmt::Display for Ident<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl fmt::Display for Literal {
impl<Span> fmt::Display for Literal<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.text, f)
}
}
impl fmt::Display for Punct {
impl<Span> fmt::Display for Punct<Span> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.char, f)
}
}
impl Subtree {
impl<Span> Subtree<Span> {
/// Count the number of tokens recursively
pub fn count(&self) -> usize {
let children_count = self
@ -244,20 +296,16 @@ impl Subtree {
self.token_trees.len() + children_count
}
pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
self.delimiter.map(|it| it.kind)
}
}
impl Subtree {
impl<Span> Subtree<Span> {
/// A simple line string used for debugging
pub fn as_debug_string(&self) -> String {
let delim = match self.delimiter_kind() {
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Bracket) => ("[", "]"),
Some(DelimiterKind::Parenthesis) => ("(", ")"),
None => (" ", " "),
let delim = match self.delimiter.kind {
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Invisible => ("$", "$"),
};
let mut res = String::new();
@ -275,7 +323,7 @@ impl Subtree {
(Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
" ".to_string() + &s
}
(Leaf::Punct(_), Some(&TokenTree::Leaf(Leaf::Punct(punct)))) => {
(Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => {
if punct.spacing == Spacing::Alone {
" ".to_string() + &s
} else {
@ -298,19 +346,19 @@ impl Subtree {
pub mod buffer;
pub fn pretty(tkns: &[TokenTree]) -> String {
fn tokentree_to_text(tkn: &TokenTree) -> String {
pub fn pretty<Span>(tkns: &[TokenTree<Span>]) -> String {
fn tokentree_to_text<Span>(tkn: &TokenTree<Span>) -> String {
match tkn {
TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(),
TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(),
TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
TokenTree::Subtree(subtree) => {
let content = pretty(&subtree.token_trees);
let (open, close) = match subtree.delimiter.map(|it| it.kind) {
None => ("", ""),
Some(DelimiterKind::Brace) => ("{", "}"),
Some(DelimiterKind::Parenthesis) => ("(", ")"),
Some(DelimiterKind::Bracket) => ("[", "]"),
let (open, close) = match subtree.delimiter.kind {
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Parenthesis => ("(", ")"),
DelimiterKind::Invisible => ("", ""),
};
format!("{open}{content}{close}")
}