diff --git a/crates/base-db/src/fixture.rs b/crates/base-db/src/fixture.rs index 5b0ed1648d..60d1e488d8 100644 --- a/crates/base-db/src/fixture.rs +++ b/crates/base-db/src/fixture.rs @@ -6,7 +6,7 @@ use rustc_hash::FxHashMap; use test_utils::{ extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER, }; -use tt::Subtree; +use tt::token_id::Subtree; use vfs::{file_set::FileSet, VfsPath}; use crate::{ @@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander { _: &Env, ) -> Result { fn traverse(input: &Subtree) -> Subtree { - let mut res = Subtree::default(); - res.delimiter = input.delimiter; + let mut token_trees = vec![]; for tt in input.token_trees.iter().rev() { let tt = match tt { tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()), tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)), }; - res.token_trees.push(tt); + token_trees.push(tt); } - res + Subtree { delimiter: input.delimiter, token_trees } } Ok(traverse(input)) } diff --git a/crates/base-db/src/input.rs b/crates/base-db/src/input.rs index db9589ca18..43388e915b 100644 --- a/crates/base-db/src/input.rs +++ b/crates/base-db/src/input.rs @@ -12,7 +12,7 @@ use cfg::CfgOptions; use rustc_hash::FxHashMap; use stdx::hash::{NoHashHashMap, NoHashHashSet}; use syntax::SmolStr; -use tt::Subtree; +use tt::token_id::Subtree; use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath}; /// Files are grouped into source roots. A source root is a directory on the diff --git a/crates/cfg/src/cfg_expr.rs b/crates/cfg/src/cfg_expr.rs index 5f4eefa836..fb7505ba2d 100644 --- a/crates/cfg/src/cfg_expr.rs +++ b/crates/cfg/src/cfg_expr.rs @@ -66,7 +66,7 @@ impl From for CfgExpr { } impl CfgExpr { - pub fn parse(tt: &tt::Subtree) -> CfgExpr { + pub fn parse(tt: &tt::Subtree) -> CfgExpr { next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid) } /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates. @@ -85,7 +85,7 @@ impl CfgExpr { } } -fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option { +fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option { let name = match it.next() { None => return None, Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), diff --git a/crates/hir-def/src/adt.rs b/crates/hir-def/src/adt.rs index cd35ba00f6..dcea679567 100644 --- a/crates/hir-def/src/adt.rs +++ b/crates/hir-def/src/adt.rs @@ -2,6 +2,7 @@ use std::sync::Arc; +use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree}; use base_db::CrateId; use either::Either; use hir_expand::{ @@ -12,7 +13,6 @@ use intern::Interned; use la_arena::{Arena, ArenaMap}; use rustc_abi::{Integer, IntegerType}; use syntax::ast::{self, HasName, HasVisibility}; -use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree}; use crate::{ body::{CfgExpander, LowerCtx}, @@ -82,7 +82,7 @@ fn repr_from_value( fn parse_repr_tt(tt: &Subtree) -> Option { match tt.delimiter { - Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {} + Delimiter { kind: DelimiterKind::Parenthesis, .. } => {} _ => return None, } diff --git a/crates/hir-def/src/attr.rs b/crates/hir-def/src/attr.rs index 4e49217329..fcd92ad338 100644 --- a/crates/hir-def/src/attr.rs +++ b/crates/hir-def/src/attr.rs @@ -16,7 +16,6 @@ use syntax::{ ast::{self, HasAttrs, IsString}, AstPtr, AstToken, SmolStr, TextRange, TextSize, }; -use tt::Subtree; use crate::{ db::DefDatabase, @@ -234,7 +233,7 @@ impl Attrs { pub fn has_doc_hidden(&self) -> bool { self.by_key("doc").tt_values().any(|tt| { - tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) && + tt.delimiter.kind == DelimiterKind::Parenthesis && matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden") }) } @@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> { } impl<'attr> AttrQuery<'attr> { - pub fn tt_values(self) -> impl Iterator { + pub fn tt_values(self) -> impl Iterator { self.attrs().filter_map(|attr| attr.token_tree_value()) } diff --git a/crates/hir-def/src/data.rs b/crates/hir-def/src/data.rs index f461e85b01..c3c1dfd39a 100644 --- a/crates/hir-def/src/data.rs +++ b/crates/hir-def/src/data.rs @@ -142,7 +142,7 @@ impl FunctionData { } } -fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> { +fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> { let mut indices = Vec::new(); for args in tt.token_trees.chunks(2) { match &args[0] { diff --git a/crates/hir-def/src/lib.rs b/crates/hir-def/src/lib.rs index aabd694820..d07c5fb67c 100644 --- a/crates/hir-def/src/lib.rs +++ b/crates/hir-def/src/lib.rs @@ -79,6 +79,8 @@ use nameres::DefMap; use stdx::impl_from; use syntax::ast; +use ::tt::token_id as tt; + use crate::{ adt::VariantData, builtin_type::BuiltinType, @@ -973,15 +975,19 @@ fn attr_macro_as_call_id( def: MacroDefId, is_derive: bool, ) -> MacroCallId { - let mut arg = match macro_attr.input.as_deref() { - Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()), - _ => Default::default(), + let arg = match macro_attr.input.as_deref() { + Some(AttrInput::TokenTree(tt, map)) => ( + { + let mut tt = tt.clone(); + tt.delimiter = tt::Delimiter::UNSPECIFIED; + tt + }, + map.clone(), + ), + _ => (tt::Subtree::empty(), Default::default()), }; - // The parentheses are always disposed here. - arg.0.delimiter = None; - - let res = def.as_lazy_macro( + def.as_lazy_macro( db.upcast(), krate, MacroCallKind::Attr { @@ -990,8 +996,7 @@ fn attr_macro_as_call_id( invoc_attr_index: macro_attr.id, is_derive, }, - ); - res + ) } intern::impl_internable!( crate::type_ref::TypeRef, diff --git a/crates/hir-def/src/macro_expansion_tests.rs b/crates/hir-def/src/macro_expansion_tests.rs index 4907f237f6..5ab90d92d9 100644 --- a/crates/hir-def/src/macro_expansion_tests.rs +++ b/crates/hir-def/src/macro_expansion_tests.rs @@ -30,7 +30,7 @@ use syntax::{ SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT}, SyntaxNode, TextRange, T, }; -use tt::{Subtree, TokenId}; +use tt::token_id::{Subtree, TokenId}; use crate::{ db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver, @@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre tree.token_trees.iter().for_each(|tree| match tree { tt::TokenTree::Leaf(leaf) => { let id = match leaf { - tt::Leaf::Literal(it) => it.id, - tt::Leaf::Punct(it) => it.id, - tt::Leaf::Ident(it) => it.id, + tt::Leaf::Literal(it) => it.span, + tt::Leaf::Punct(it) => it.span, + tt::Leaf::Ident(it) => it.span, }; ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id))); } diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index 6c2bbc74d6..4b39a20d86 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -46,6 +46,7 @@ use crate::{ }, path::{ImportAlias, ModPath, PathKind}, per_ns::PerNs, + tt, visibility::{RawVisibility, Visibility}, AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId, FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc, @@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T .enumerate() .map(|(idx, it)| { // FIXME: a hacky way to create a Name from string. - let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() }; + let name = + tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() }; ( name.as_name(), ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)), @@ -451,7 +453,10 @@ impl DefCollector<'_> { directive.module_id, MacroCallKind::Attr { ast_id: ast_id.ast_id, - attr_args: Default::default(), + attr_args: std::sync::Arc::new(( + tt::Subtree::empty(), + Default::default(), + )), invoc_attr_index: attr.id, is_derive: false, }, @@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> { let name = match attrs.by_key("rustc_builtin_macro").string_value() { Some(it) => { // FIXME: a hacky way to create a Name from string. - name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name(); + name = + tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name(); &name } None => { diff --git a/crates/hir-def/src/nameres/proc_macro.rs b/crates/hir-def/src/nameres/proc_macro.rs index 06b23392cf..caad4a1f38 100644 --- a/crates/hir-def/src/nameres/proc_macro.rs +++ b/crates/hir-def/src/nameres/proc_macro.rs @@ -1,9 +1,9 @@ //! Nameres-specific procedural macro data and helpers. use hir_expand::name::{AsName, Name}; -use tt::{Leaf, TokenTree}; use crate::attr::Attrs; +use crate::tt::{Leaf, TokenTree}; #[derive(Debug, PartialEq, Eq)] pub struct ProcMacroDef { diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index c7135732b8..5c04f8e8b8 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -8,13 +8,13 @@ use intern::Interned; use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct}; use smallvec::{smallvec, SmallVec}; use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode}; -use tt::Subtree; use crate::{ db::AstDatabase, hygiene::Hygiene, mod_path::{ModPath, PathKind}, name::AsName, + tt::{self, Subtree}, InFile, }; @@ -117,7 +117,10 @@ impl RawAttrs { let index = attr.id; let attrs = parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| { - let tree = Subtree { delimiter: None, token_trees: attr.to_vec() }; + let tree = Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: attr.to_vec(), + }; // FIXME hygiene let hygiene = Hygiene::new_unhygienic(); Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx)) @@ -266,7 +269,7 @@ impl Attr { pub fn parse_path_comma_token_tree(&self) -> Option + '_> { let args = self.token_tree_value()?; - if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) { + if args.delimiter.kind != DelimiterKind::Parenthesis { return None; } let paths = args diff --git a/crates/hir-expand/src/builtin_attr_macro.rs b/crates/hir-expand/src/builtin_attr_macro.rs index 58d192f9fe..906ca991d7 100644 --- a/crates/hir-expand/src/builtin_attr_macro.rs +++ b/crates/hir-expand/src/builtin_attr_macro.rs @@ -1,6 +1,6 @@ //! Builtin attributes. -use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind}; +use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind}; macro_rules! register_builtin { ( $(($name:ident, $variant:ident) => $expand:ident),* ) => { @@ -97,7 +97,7 @@ fn derive_attr_expand( let loc = db.lookup_intern_macro_call(id); let derives = match &loc.kind { MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0, - _ => return ExpandResult::ok(Default::default()), + _ => return ExpandResult::ok(tt::Subtree::empty()), }; pseudo_derive_attr_expansion(tt, derives) } @@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion( tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char, spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) }; diff --git a/crates/hir-expand/src/builtin_derive_macro.rs b/crates/hir-expand/src/builtin_derive_macro.rs index 8966047c9b..060a680542 100644 --- a/crates/hir-expand/src/builtin_derive_macro.rs +++ b/crates/hir-expand/src/builtin_derive_macro.rs @@ -3,11 +3,11 @@ use base_db::{CrateOrigin, LangCrateOrigin}; use tracing::debug; +use crate::tt::{self, TokenId}; use syntax::{ ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName}, match_ast, }; -use tt::TokenId; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId}; @@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result { })?; let name_token_id = token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified); - let name_token = tt::Ident { id: name_token_id, text: name.text().into() }; + let name_token = tt::Ident { span: name_token_id, text: name.text().into() }; let param_types = params .into_iter() .flat_map(|param_list| param_list.type_or_const_params()) @@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result { let ty = param .ty() .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0) - .unwrap_or_default(); + .unwrap_or_else(tt::Subtree::empty); Some(ty) } else { None @@ -114,7 +114,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result { fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult { let info = match parse_adt(tt) { Ok(info) => info, - Err(e) => return ExpandResult::only_err(e), + Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e), }; let (params, args): (Vec<_>, Vec<_>) = info .param_types @@ -122,7 +122,7 @@ fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResu .enumerate() .map(|(idx, param_ty)| { let ident = tt::Leaf::Ident(tt::Ident { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), text: format!("T{idx}").into(), }); let ident_ = ident.clone(); diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 5522bdf3b3..9f3fa73d4e 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -9,7 +9,9 @@ use syntax::{ SmolStr, }; -use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; +use crate::{ + db::AstDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc, +}; macro_rules! register_builtin { ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),* ) => { @@ -61,7 +63,7 @@ macro_rules! register_builtin { }; } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct ExpandedEager { pub(crate) subtree: tt::Subtree, /// The included file ID of the include macro. @@ -116,7 +118,7 @@ register_builtin! { } const DOLLAR_CRATE: tt::Ident = - tt::Ident { text: SmolStr::new_inline("$crate"), id: tt::TokenId::unspecified() }; + tt::Ident { text: SmolStr::new_inline("$crate"), span: tt::TokenId::unspecified() }; fn module_path_expand( _db: &dyn AstDatabase, @@ -162,7 +164,7 @@ fn stringify_expand( _id: MacroCallId, tt: &tt::Subtree, ) -> ExpandResult { - let pretty = tt::pretty(&tt.token_trees); + let pretty = ::tt::pretty(&tt.token_trees); let expanded = quote! { #pretty @@ -194,11 +196,11 @@ fn assert_expand( let expanded = match &*args { [cond, panic_args @ ..] => { let comma = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }; let cond = cond.clone(); @@ -247,7 +249,10 @@ fn format_args_expand( let mut args = parse_exprs_with_sep(tt, ','); if args.is_empty() { - return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule.into()); + return ExpandResult::with_err( + tt::Subtree::empty(), + mbe::ExpandError::NoMatchingRule.into(), + ); } for arg in &mut args { // Remove `key =`. @@ -282,7 +287,7 @@ fn asm_expand( for tt in tt.token_trees.chunks(2) { match tt { [tt::TokenTree::Leaf(tt::Leaf::Literal(lit))] - | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', id: _, spacing: _ }))] => + | [tt::TokenTree::Leaf(tt::Leaf::Literal(lit)), tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', span: _, spacing: _ }))] => { let krate = DOLLAR_CRATE.clone(); literals.push(quote!(#krate::format_args!(#lit);)); @@ -400,7 +405,7 @@ fn concat_expand( // FIXME: hack on top of a hack: `$e:expr` captures get surrounded in parentheses // to ensure the right parsing order, so skip the parentheses here. Ideally we'd // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623 - if let tt::TokenTree::Subtree(tt::Subtree { delimiter: Some(delim), token_trees }) = t { + if let tt::TokenTree::Subtree(tt::Subtree { delimiter: delim, token_trees }) = t { if let [tt] = &**token_trees { if delim.kind == tt::DelimiterKind::Parenthesis { t = tt; @@ -459,9 +464,7 @@ fn concat_bytes_expand( } } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (), - tt::TokenTree::Subtree(tree) - if tree.delimiter_kind() == Some(tt::DelimiterKind::Bracket) => - { + tt::TokenTree::Subtree(tree) if tree.delimiter.kind == tt::DelimiterKind::Bracket => { if let Err(e) = concat_bytes_expand_subtree(tree, &mut bytes) { err.get_or_insert(e); break; @@ -473,7 +476,7 @@ fn concat_bytes_expand( } } } - let ident = tt::Ident { text: bytes.join(", ").into(), id: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() }; ExpandResult { value: ExpandedEager::new(quote!([#ident])), err } } @@ -521,7 +524,7 @@ fn concat_idents_expand( } } } - let ident = tt::Ident { text: ident.into(), id: tt::TokenId::unspecified() }; + let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() }; ExpandResult { value: ExpandedEager::new(quote!(#ident)), err } } @@ -572,7 +575,10 @@ fn include_expand( Ok((subtree, file_id)) => { ExpandResult::ok(ExpandedEager { subtree, included_file: Some(file_id) }) } - Err(e) => ExpandResult::only_err(e), + Err(e) => ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ), } } @@ -582,15 +588,18 @@ fn include_bytes_expand( tt: &tt::Subtree, ) -> ExpandResult { if let Err(e) = parse_string(tt) { - return ExpandResult::only_err(e); + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ); } // FIXME: actually read the file here if the user asked for macro expansion let res = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text: r#"b"""#.into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }; ExpandResult::ok(ExpandedEager::new(res)) @@ -603,7 +612,12 @@ fn include_str_expand( ) -> ExpandResult { let path = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; // FIXME: we're not able to read excluded files (which is most of them because @@ -635,7 +649,12 @@ fn env_expand( ) -> ExpandResult { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; let mut err = None; @@ -666,7 +685,12 @@ fn option_env_expand( ) -> ExpandResult { let key = match parse_string(tt) { Ok(it) => it, - Err(e) => return ExpandResult::only_err(e), + Err(e) => { + return ExpandResult::with_err( + ExpandedEager { subtree: tt::Subtree::empty(), included_file: None }, + e, + ) + } }; let expanded = match get_env_inner(db, arg_id, &key) { diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 2ab78c3287..76016274f0 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -14,7 +14,7 @@ use syntax::{ use crate::{ ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion, fixup, - hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, + hygiene::HygieneFrame, tt, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, }; @@ -175,7 +175,7 @@ pub fn expand_speculative( match attr.token_tree() { Some(token_tree) => { let (mut tree, map) = syntax_node_to_token_tree(attr.token_tree()?.syntax()); - tree.delimiter = None; + tree.delimiter = tt::Delimiter::unspecified(); let shift = mbe::Shift::new(&tt); shift.shift_all(&mut tree); @@ -210,7 +210,7 @@ pub fn expand_speculative( // Otherwise the expand query will fetch the non speculative attribute args and pass those instead. let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(expander, ..) => { - tt.delimiter = None; + tt.delimiter = tt::Delimiter::unspecified(); expander.expand(db, loc.krate, &tt, attr_arg.as_ref()) } MacroDefKind::BuiltInAttr(BuiltinAttrExpander::Derive, _) => { @@ -316,9 +316,8 @@ fn macro_arg( if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.delimiter = None; + tt.delimiter = tt::Delimiter::unspecified(); } - Some(Arc::new((tt, tmap, fixups.undo_info))) } @@ -479,7 +478,10 @@ fn expand_proc_macro(db: &dyn AstDatabase, id: MacroCallId) -> ExpandResult it, None => { - return ExpandResult::only_err(ExpandError::Other("No arguments for proc-macro".into())) + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("No arguments for proc-macro".into()), + ) } }; diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index a1474c44e6..dfab7ec92c 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -108,7 +108,7 @@ pub fn expand_eager_macro( .value .token_tree() .map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0) - .unwrap_or_default(); + .unwrap_or_else(tt::Subtree::empty); let ast_map = db.ast_id_map(macro_call.file_id); let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(¯o_call.value)); @@ -165,9 +165,9 @@ pub fn expand_eager_macro( } } -fn to_subtree(node: &SyntaxNode) -> tt::Subtree { +fn to_subtree(node: &SyntaxNode) -> crate::tt::Subtree { let mut subtree = mbe::syntax_node_to_token_tree(node).0; - subtree.delimiter = None; + subtree.delimiter = crate::tt::Delimiter::unspecified(); subtree } diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs index 75d364d5f8..c811d1c66a 100644 --- a/crates/hir-expand/src/fixup.rs +++ b/crates/hir-expand/src/fixup.rs @@ -9,7 +9,7 @@ use syntax::{ ast::{self, AstNode, HasLoopBody}, match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, }; -use tt::Subtree; +use tt::token_id::Subtree; /// The result of calculating fixes for a syntax node -- a bunch of changes /// (appending to and replacing nodes), the information that is needed to @@ -297,9 +297,11 @@ pub(crate) fn reverse_fixups( tt.token_trees = tts .into_iter() .filter(|tt| match tt { - tt::TokenTree::Leaf(leaf) => token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID), + tt::TokenTree::Leaf(leaf) => { + token_map.synthetic_token_id(*leaf.span()) != Some(EMPTY_ID) + } tt::TokenTree::Subtree(st) => { - st.delimiter.map_or(true, |d| token_map.synthetic_token_id(d.id) != Some(EMPTY_ID)) + token_map.synthetic_token_id(st.delimiter.open) != Some(EMPTY_ID) } }) .flat_map(|tt| match tt { @@ -308,9 +310,9 @@ pub(crate) fn reverse_fixups( SmallVec::from_const([tt.into()]) } tt::TokenTree::Leaf(leaf) => { - if let Some(id) = token_map.synthetic_token_id(leaf.id()) { + if let Some(id) = token_map.synthetic_token_id(*leaf.span()) { let original = undo_info.original[id.0 as usize].clone(); - if original.delimiter.is_none() { + if original.delimiter.kind == tt::DelimiterKind::Invisible { original.token_trees.into() } else { SmallVec::from_const([original.into()]) @@ -327,6 +329,8 @@ pub(crate) fn reverse_fixups( mod tests { use expect_test::{expect, Expect}; + use crate::tt; + use super::reverse_fixups; // The following three functions are only meant to check partial structural equivalence of @@ -341,7 +345,7 @@ mod tests { } fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool { - a.delimiter.map(|it| it.kind) == b.delimiter.map(|it| it.kind) + a.delimiter.kind == b.delimiter.kind && a.token_trees.len() == b.token_trees.len() && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b)) } @@ -386,7 +390,7 @@ mod tests { let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node()); assert!( check_subtree_eq(&tt, &original_as_tt), - "different token tree: {tt:?}, {original_as_tt:?}" + "different token tree: {tt:?},\n{original_as_tt:?}" ); } diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index 5a55dc5c9e..2300ee9d08 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -128,7 +128,7 @@ struct HygieneInfo { attr_input_or_mac_def_start: Option>, macro_def: Arc, - macro_arg: Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, + macro_arg: Arc<(crate::tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>, macro_arg_shift: mbe::Shift, exp_map: Arc, } diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 967a8fedbd..bc941b5417 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -22,6 +22,8 @@ mod fixup; pub use mbe::{Origin, ValueResult}; +use ::tt::token_id as tt; + use std::{fmt, hash::Hash, iter, sync::Arc}; use base_db::{ diff --git a/crates/hir-expand/src/name.rs b/crates/hir-expand/src/name.rs index b62f4fe770..b515472501 100644 --- a/crates/hir-expand/src/name.rs +++ b/crates/hir-expand/src/name.rs @@ -191,7 +191,7 @@ impl AsName for ast::NameOrNameRef { } } -impl AsName for tt::Ident { +impl AsName for tt::Ident { fn as_name(&self) -> Name { Name::resolve(&self.text) } diff --git a/crates/hir-expand/src/proc_macro.rs b/crates/hir-expand/src/proc_macro.rs index 5afdcc0e66..3f4d2540c0 100644 --- a/crates/hir-expand/src/proc_macro.rs +++ b/crates/hir-expand/src/proc_macro.rs @@ -3,7 +3,7 @@ use base_db::{CrateId, ProcMacroExpansionError, ProcMacroId, ProcMacroKind}; use stdx::never; -use crate::{db::AstDatabase, ExpandError, ExpandResult}; +use crate::{db::AstDatabase, tt, ExpandError, ExpandResult}; #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] pub struct ProcMacroExpander { @@ -39,7 +39,10 @@ impl ProcMacroExpander { Ok(proc_macros) => proc_macros, Err(_) => { never!("Non-dummy expander even though there are no proc macros"); - return ExpandResult::only_err(ExpandError::Other("Internal error".into())); + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("Internal error".into()), + ); } }; let proc_macro = match proc_macros.get(id.0 as usize) { @@ -50,7 +53,10 @@ impl ProcMacroExpander { proc_macros.len(), id.0 ); - return ExpandResult::only_err(ExpandError::Other("Internal error".into())); + return ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other("Internal error".into()), + ); } }; @@ -69,13 +75,17 @@ impl ProcMacroExpander { } } ProcMacroExpansionError::System(text) - | ProcMacroExpansionError::Panic(text) => { - ExpandResult::only_err(ExpandError::Other(text.into())) - } + | ProcMacroExpansionError::Panic(text) => ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::Other(text.into()), + ), }, } } - None => ExpandResult::only_err(ExpandError::UnresolvedProcMacro(self.krate)), + None => ExpandResult::with_err( + tt::Subtree::empty(), + ExpandError::UnresolvedProcMacro(self.krate), + ), } } } diff --git a/crates/hir-expand/src/quote.rs b/crates/hir-expand/src/quote.rs index c0a7bc7ca8..63586f9daf 100644 --- a/crates/hir-expand/src/quote.rs +++ b/crates/hir-expand/src/quote.rs @@ -9,17 +9,18 @@ #[macro_export] macro_rules! __quote { () => { - Vec::::new() + Vec::::new() }; ( @SUBTREE $delim:ident $($tt:tt)* ) => { { let children = $crate::__quote!($($tt)*); - tt::Subtree { - delimiter: Some(tt::Delimiter { - kind: tt::DelimiterKind::$delim, - id: tt::TokenId::unspecified(), - }), + crate::tt::Subtree { + delimiter: crate::tt::Delimiter { + kind: crate::tt::DelimiterKind::$delim, + open: crate::tt::TokenId::unspecified(), + close: crate::tt::TokenId::unspecified(), + }, token_trees: $crate::quote::IntoTt::to_tokens(children), } } @@ -28,10 +29,10 @@ macro_rules! __quote { ( @PUNCT $first:literal ) => { { vec![ - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Alone, + span: crate::tt::TokenId::unspecified(), }).into() ] } @@ -40,15 +41,15 @@ macro_rules! __quote { ( @PUNCT $first:literal, $sec:literal ) => { { vec![ - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $first, - spacing: tt::Spacing::Joint, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Joint, + span: crate::tt::TokenId::unspecified(), }).into(), - tt::Leaf::Punct(tt::Punct { + crate::tt::Leaf::Punct(crate::tt::Punct { char: $sec, - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + spacing: crate::tt::Spacing::Alone, + span: crate::tt::TokenId::unspecified(), }).into() ] } @@ -67,7 +68,7 @@ macro_rules! __quote { ( ## $first:ident $($tail:tt)* ) => { { - let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::>(); + let mut tokens = $first.into_iter().map($crate::quote::ToTokenTree::to_token).collect::>(); let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*)); tokens.append(&mut tail_tokens); tokens @@ -86,9 +87,9 @@ macro_rules! __quote { // Ident ( $tt:ident ) => { vec![ { - tt::Leaf::Ident(tt::Ident { + crate::tt::Leaf::Ident(crate::tt::Ident { text: stringify!($tt).into(), - id: tt::TokenId::unspecified(), + span: crate::tt::TokenId::unspecified(), }).into() }] }; @@ -127,42 +128,42 @@ macro_rules! quote { } pub(crate) trait IntoTt { - fn to_subtree(self) -> tt::Subtree; - fn to_tokens(self) -> Vec; + fn to_subtree(self) -> crate::tt::Subtree; + fn to_tokens(self) -> Vec; } -impl IntoTt for Vec { - fn to_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: None, token_trees: self } +impl IntoTt for Vec { + fn to_subtree(self) -> crate::tt::Subtree { + crate::tt::Subtree { delimiter: crate::tt::Delimiter::unspecified(), token_trees: self } } - fn to_tokens(self) -> Vec { + fn to_tokens(self) -> Vec { self } } -impl IntoTt for tt::Subtree { - fn to_subtree(self) -> tt::Subtree { +impl IntoTt for crate::tt::Subtree { + fn to_subtree(self) -> crate::tt::Subtree { self } - fn to_tokens(self) -> Vec { - vec![tt::TokenTree::Subtree(self)] + fn to_tokens(self) -> Vec { + vec![crate::tt::TokenTree::Subtree(self)] } } pub(crate) trait ToTokenTree { - fn to_token(self) -> tt::TokenTree; + fn to_token(self) -> crate::tt::TokenTree; } -impl ToTokenTree for tt::TokenTree { - fn to_token(self) -> tt::TokenTree { +impl ToTokenTree for crate::tt::TokenTree { + fn to_token(self) -> crate::tt::TokenTree { self } } -impl ToTokenTree for tt::Subtree { - fn to_token(self) -> tt::TokenTree { +impl ToTokenTree for crate::tt::Subtree { + fn to_token(self) -> crate::tt::TokenTree { self.into() } } @@ -171,15 +172,15 @@ macro_rules! impl_to_to_tokentrees { ($($ty:ty => $this:ident $im:block);*) => { $( impl ToTokenTree for $ty { - fn to_token($this) -> tt::TokenTree { - let leaf: tt::Leaf = $im.into(); + fn to_token($this) -> crate::tt::TokenTree { + let leaf: crate::tt::Leaf = $im.into(); leaf.into() } } impl ToTokenTree for &$ty { - fn to_token($this) -> tt::TokenTree { - let leaf: tt::Leaf = $im.clone().into(); + fn to_token($this) -> crate::tt::TokenTree { + let leaf: crate::tt::Leaf = $im.clone().into(); leaf.into() } } @@ -188,16 +189,16 @@ macro_rules! impl_to_to_tokentrees { } impl_to_to_tokentrees! { - u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - bool => self { tt::Ident{text: self.to_string().into(), id: tt::TokenId::unspecified()} }; - tt::Leaf => self { self }; - tt::Literal => self { self }; - tt::Ident => self { self }; - tt::Punct => self { self }; - &str => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}}; - String => self { tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), id: tt::TokenId::unspecified()}} + u32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + usize => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + i32 => self { crate::tt::Literal{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + bool => self { crate::tt::Ident{text: self.to_string().into(), span: crate::tt::TokenId::unspecified()} }; + crate::tt::Leaf => self { self }; + crate::tt::Literal => self { self }; + crate::tt::Ident => self { self }; + crate::tt::Punct => self { self }; + &str => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}}; + String => self { crate::tt::Literal{text: format!("\"{}\"", self.escape_default()).into(), span: crate::tt::TokenId::unspecified()}} } #[cfg(test)] @@ -223,8 +224,8 @@ mod tests { assert_eq!(quote!(#s).to_string(), "\"hello\""); } - fn mk_ident(name: &str) -> tt::Ident { - tt::Ident { text: name.into(), id: tt::TokenId::unspecified() } + fn mk_ident(name: &str) -> crate::tt::Ident { + crate::tt::Ident { text: name.into(), span: crate::tt::TokenId::unspecified() } } #[test] @@ -234,7 +235,7 @@ mod tests { let quoted = quote!(#a); assert_eq!(quoted.to_string(), "hello"); let t = format!("{quoted:?}"); - assert_eq!(t, "SUBTREE $\n IDENT hello 4294967295"); + assert_eq!(t, "SUBTREE $$ 4294967295 4294967295\n IDENT hello 4294967295"); } #[test] @@ -263,11 +264,12 @@ mod tests { let fields = [mk_ident("name"), mk_ident("id")]; let fields = fields.iter().flat_map(|it| quote!(#it: self.#it.clone(), ).token_trees); - let list = tt::Subtree { - delimiter: Some(tt::Delimiter { - kind: tt::DelimiterKind::Brace, - id: tt::TokenId::unspecified(), - }), + let list = crate::tt::Subtree { + delimiter: crate::tt::Delimiter { + kind: crate::tt::DelimiterKind::Brace, + open: crate::tt::TokenId::unspecified(), + close: crate::tt::TokenId::unspecified(), + }, token_trees: fields.collect(), }; diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index 0fee6dfe43..894355fcbc 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -9,7 +9,7 @@ use test_utils::{bench, bench_fixture, skip_slow_tests}; use crate::{ parser::{MetaVarKind, Op, RepeatKind, Separator}, - syntax_node_to_token_tree, DeclarativeMacro, + syntax_node_to_token_tree, tt, DeclarativeMacro, }; #[test] @@ -91,7 +91,14 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri // So we just skip any error cases and try again let mut try_cnt = 0; loop { - let mut subtree = tt::Subtree::default(); + let mut subtree = tt::Subtree { + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, + kind: tt::DelimiterKind::Invisible, + }, + token_trees: vec![], + }; for op in rule.lhs.iter() { collect_from_op(op, &mut subtree, &mut seed); } @@ -196,12 +203,15 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri *seed } fn make_ident(ident: &str) -> tt::TokenTree { - tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) }) - .into() + tt::Leaf::Ident(tt::Ident { + span: tt::TokenId::unspecified(), + text: SmolStr::new(ident), + }) + .into() } fn make_punct(char: char) -> tt::TokenTree { tt::Leaf::Punct(tt::Punct { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), char, spacing: tt::Spacing::Alone, }) @@ -209,7 +219,7 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri } fn make_literal(lit: &str) -> tt::TokenTree { tt::Leaf::Literal(tt::Literal { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), text: SmolStr::new(lit), }) .into() @@ -219,7 +229,11 @@ fn invocation_fixtures(rules: &FxHashMap) -> Vec<(Stri token_trees: Option>, ) -> tt::TokenTree { tt::Subtree { - delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), + kind, + }, token_trees: token_trees.unwrap_or_default(), } .into() diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 100ec6bfb9..7537dc3226 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -8,7 +8,7 @@ mod transcriber; use rustc_hash::FxHashMap; use syntax::SmolStr; -use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; +use crate::{parser::MetaVarKind, tt, ExpandError, ExpandResult}; pub(crate) fn expand_rules( rules: &[crate::Rule], @@ -45,7 +45,10 @@ pub(crate) fn expand_rules( transcriber::transcribe(&rule.rhs, &match_.bindings); ExpandResult { value, err: match_.err.or(transcribe_err) } } else { - ExpandResult::only_err(ExpandError::NoMatchingRule) + ExpandResult::with_err( + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, + ExpandError::NoMatchingRule, + ) } } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 88eae136f7..f4ea9e5c81 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -67,6 +67,7 @@ use syntax::SmolStr; use crate::{ expander::{Binding, Bindings, ExpandResult, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, + tt, tt_iter::TtIter, ExpandError, MetaTemplate, ValueResult, }; @@ -75,7 +76,8 @@ impl Bindings { fn push_optional(&mut self, name: &SmolStr) { // FIXME: Do we have a better way to represent an empty token ? // Insert an empty subtree for empty token - let tt = tt::Subtree::default().into(); + let tt = + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }.into(); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); } @@ -462,9 +464,9 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Subtree { tokens, delimiter }) => { if let Ok(subtree) = src.clone().expect_subtree() { - if subtree.delimiter_kind() == delimiter.map(|it| it.kind) { + if subtree.delimiter.kind == delimiter.kind { item.stack.push(item.dot); - item.dot = tokens.iter_delimited(delimiter.as_ref()); + item.dot = tokens.iter_delimited(Some(delimiter)); cur_items.push(item); } } @@ -663,8 +665,8 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { } res.add_err(ExpandError::LeftoverTokens); - if let Some(error_reover_item) = error_recover_item { - res.bindings = bindings_builder.build(&error_reover_item); + if let Some(error_recover_item) = error_recover_item { + res.bindings = bindings_builder.build(&error_recover_item); } return res; } @@ -782,7 +784,7 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult lit.into(), Some(neg) => tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![neg, lit.into()], }), } @@ -810,7 +812,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) } impl MetaTemplate { fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> { - OpDelimitedIter { inner: &self.0, idx: 0, delimited } + OpDelimitedIter { + inner: &self.0, + idx: 0, + delimited: delimited.unwrap_or(&tt::Delimiter::UNSPECIFIED), + } } } @@ -824,20 +830,21 @@ enum OpDelimited<'a> { #[derive(Debug, Clone, Copy)] struct OpDelimitedIter<'a> { inner: &'a [Op], - delimited: Option<&'a tt::Delimiter>, + delimited: &'a tt::Delimiter, idx: usize, } impl<'a> OpDelimitedIter<'a> { fn is_eof(&self) -> bool { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + let len = self.inner.len() + + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; self.idx >= len } fn peek(&self) -> Option> { - match self.delimited { - None => self.inner.get(self.idx).map(OpDelimited::Op), - Some(_) => match self.idx { + match self.delimited.kind { + tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op), + _ => match self.idx { 0 => Some(OpDelimited::Open), i if i == self.inner.len() + 1 => Some(OpDelimited::Close), i => self.inner.get(i - 1).map(OpDelimited::Op), @@ -860,7 +867,8 @@ impl<'a> Iterator for OpDelimitedIter<'a> { } fn size_hint(&self) -> (usize, Option) { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + let len = self.inner.len() + + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; let remain = len.saturating_sub(self.idx); (remain, Some(remain)) } @@ -904,7 +912,10 @@ impl<'a> TtIter<'a> { } else { let puncts = self.expect_glued_punct()?; let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect(); - Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees })) + Ok(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees, + })) } } else { self.next().ok_or(()).cloned() @@ -919,7 +930,7 @@ impl<'a> TtIter<'a> { let ident = self.expect_ident_or_underscore()?; Ok(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: vec![ tt::Leaf::Punct(*punct).into(), tt::Leaf::Ident(ident.clone()).into(), diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index c956770896..dffb40d4bc 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -2,11 +2,11 @@ //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` use syntax::SmolStr; -use tt::{Delimiter, Subtree}; use crate::{ expander::{Binding, Bindings, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, + tt::{self, Delimiter}, ExpandError, ExpandResult, MetaTemplate, }; @@ -44,22 +44,23 @@ impl Bindings { Binding::Missing(it) => Ok(match it { MetaVarKind::Stmt => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), char: ';', spacing: tt::Spacing::Alone, }))) } MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Brace, - }), + }, token_trees: vec![], })), // FIXME: Meta and Item should get proper defaults MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![], })) } @@ -71,19 +72,19 @@ impl Bindings { | MetaVarKind::Ident => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("missing"), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } MetaVarKind::Lifetime => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("'missing"), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } MetaVarKind::Literal => { Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: SmolStr::new_inline("\"missing\""), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))) } }), @@ -143,7 +144,7 @@ fn expand_subtree( } Op::Subtree { tokens, delimiter } => { let ExpandResult { value: tt, err: e } = - expand_subtree(ctx, tokens, *delimiter, arena); + expand_subtree(ctx, tokens, Some(*delimiter), arena); err = err.or(e); arena.push(tt.into()); } @@ -170,7 +171,7 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: index.to_string().into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }) .into(), ); @@ -179,7 +180,13 @@ fn expand_subtree( } // drain the elements added in this instance of expand_subtree let tts = arena.drain(start_elements..).collect(); - ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err } + ExpandResult { + value: tt::Subtree { + delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified), + token_trees: tts, + }, + err, + } } fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult { @@ -201,17 +208,24 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe // ``` // We just treat it a normal tokens let tt = tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::UNSPECIFIED, token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(), - tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(), + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) + .into(), + tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), ], } .into(); ExpandResult::ok(Fragment::Tokens(tt)) } else { ctx.bindings.get(v, &mut ctx.nesting).map_or_else( - |e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, + |e| ExpandResult { + value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: vec![], + })), + err: Some(e), + }, ExpandResult::ok, ) } @@ -249,7 +263,10 @@ fn expand_repeat( ctx ); return ExpandResult { - value: Fragment::Tokens(Subtree::default().into()), + value: Fragment::Tokens( + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] } + .into(), + ), err: Some(ExpandError::LimitExceeded), }; } @@ -258,7 +275,7 @@ fn expand_repeat( continue; } - t.delimiter = None; + t.delimiter = tt::Delimiter::unspecified(); push_subtree(&mut buf, t); if let Some(sep) = separator { @@ -292,7 +309,7 @@ fn expand_repeat( // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { return ExpandResult { @@ -307,11 +324,12 @@ fn push_fragment(buf: &mut Vec, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { - if tt.delimiter.is_none() { - tt.delimiter = Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + if tt.delimiter.kind == tt::DelimiterKind::Invisible { + tt.delimiter = tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, kind: tt::DelimiterKind::Parenthesis, - }) + }; } buf.push(tt.into()) } @@ -320,8 +338,8 @@ fn push_fragment(buf: &mut Vec, fragment: Fragment) { } fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { - match tt.delimiter { - None => buf.extend(tt.token_trees), - Some(_) => buf.push(tt.into()), + match tt.delimiter.kind { + tt::DelimiterKind::Invisible => buf.extend(tt.token_trees), + _ => buf.push(tt.into()), } } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 2373db97a3..ac107a0d6d 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -18,6 +18,8 @@ mod to_parser_input; mod benchmark; mod token_map; +use ::tt::token_id as tt; + use std::fmt; use crate::{ @@ -26,8 +28,8 @@ use crate::{ }; // FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces +pub use self::tt::{Delimiter, DelimiterKind, Punct}; pub use ::parser::TopEntryPoint; -pub use tt::{Delimiter, DelimiterKind, Punct}; pub use crate::{ syntax_bridge::{ @@ -125,24 +127,26 @@ impl Shift { // Find the max token id inside a subtree fn max_id(subtree: &tt::Subtree) -> Option { - let filter = |tt: &_| match tt { - tt::TokenTree::Subtree(subtree) => { - let tree_id = max_id(subtree); - match subtree.delimiter { - Some(it) if it.id != tt::TokenId::unspecified() => { - Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) + let filter = + |tt: &_| match tt { + tt::TokenTree::Subtree(subtree) => { + let tree_id = max_id(subtree); + if subtree.delimiter.open != tt::TokenId::unspecified() { + Some(tree_id.map_or(subtree.delimiter.open.0, |t| { + t.max(subtree.delimiter.open.0) + })) + } else { + tree_id } - _ => tree_id, } - } - tt::TokenTree::Leaf(leaf) => { - let &(tt::Leaf::Ident(tt::Ident { id, .. }) - | tt::Leaf::Punct(tt::Punct { id, .. }) - | tt::Leaf::Literal(tt::Literal { id, .. })) = leaf; + tt::TokenTree::Leaf(leaf) => { + let &(tt::Leaf::Ident(tt::Ident { span, .. }) + | tt::Leaf::Punct(tt::Punct { span, .. }) + | tt::Leaf::Literal(tt::Literal { span, .. })) = leaf; - (id != tt::TokenId::unspecified()).then_some(id.0) - } - }; + (span != tt::TokenId::unspecified()).then_some(span.0) + } + }; subtree.token_trees.iter().filter_map(filter).max() } } @@ -152,14 +156,13 @@ impl Shift { for t in &mut tt.token_trees { match t { tt::TokenTree::Leaf( - tt::Leaf::Ident(tt::Ident { id, .. }) - | tt::Leaf::Punct(tt::Punct { id, .. }) - | tt::Leaf::Literal(tt::Literal { id, .. }), - ) => *id = self.shift(*id), + tt::Leaf::Ident(tt::Ident { span, .. }) + | tt::Leaf::Punct(tt::Punct { span, .. }) + | tt::Leaf::Literal(tt::Literal { span, .. }), + ) => *span = self.shift(*span), tt::TokenTree::Subtree(tt) => { - if let Some(it) = tt.delimiter.as_mut() { - it.id = self.shift(it.id); - } + tt.delimiter.open = self.shift(tt.delimiter.open); + tt.delimiter.close = self.shift(tt.delimiter.close); self.shift_all(tt) } } @@ -216,7 +219,7 @@ impl DeclarativeMacro { let mut src = TtIter::new(tt); let mut rules = Vec::new(); - if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() { + if tt::DelimiterKind::Brace == tt.delimiter.kind { cov_mark::hit!(parse_macro_def_rules); while src.len() > 0 { let rule = Rule::parse(&mut src, true)?; @@ -325,6 +328,10 @@ impl ValueResult { Self { value, err: None } } + pub fn with_err(value: T, err: E) -> Self { + Self { value, err: Some(err) } + } + pub fn only_err(err: E) -> Self where T: Default, diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 875dc1ad87..fd3d64719a 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -4,7 +4,7 @@ use smallvec::{smallvec, SmallVec}; use syntax::SmolStr; -use crate::{tt_iter::TtIter, ParseError}; +use crate::{tt, tt_iter::TtIter, ParseError}; /// Consider /// @@ -54,7 +54,7 @@ pub(crate) enum Op { Ignore { name: SmolStr, id: tt::TokenId }, Index { depth: u32 }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option }, - Subtree { tokens: MetaTemplate, delimiter: Option }, + Subtree { tokens: MetaTemplate, delimiter: tt::Delimiter }, Literal(tt::Literal), Punct(SmallVec<[tt::Punct; 3]>), Ident(tt::Ident), @@ -130,13 +130,13 @@ fn next_op( Some(it) => it, }; match second { - tt::TokenTree::Subtree(subtree) => match subtree.delimiter_kind() { - Some(tt::DelimiterKind::Parenthesis) => { + tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => { let (separator, kind) = parse_repeat(src)?; let tokens = MetaTemplate::parse(subtree, mode)?; Op::Repeat { tokens, separator, kind } } - Some(tt::DelimiterKind::Brace) => match mode { + tt::DelimiterKind::Brace => match mode { Mode::Template => { parse_metavar_expr(&mut TtIter::new(subtree)).map_err(|()| { ParseError::unexpected("invalid metavariable expression") @@ -157,18 +157,18 @@ fn next_op( tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) if ident.text == "crate" => { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. - Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id }) + Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span }) } tt::Leaf::Ident(ident) => { let kind = eat_fragment_kind(src, mode)?; let name = ident.text.clone(); - let id = ident.id; + let id = ident.span; Op::Var { name, kind, id } } tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { let kind = eat_fragment_kind(src, mode)?; let name = lit.text.clone(); - let id = lit.id; + let id = lit.span; Op::Var { name, kind, id } } tt::Leaf::Punct(punct @ tt::Punct { char: '$', .. }) => match mode { @@ -284,7 +284,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result { let func = src.expect_ident()?; let args = src.expect_subtree()?; - if args.delimiter_kind() != Some(tt::DelimiterKind::Parenthesis) { + if args.delimiter.kind != tt::DelimiterKind::Parenthesis { return Err(()); } @@ -293,7 +293,7 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result { let op = match &*func.text { "ignore" => { let ident = args.expect_ident()?; - Op::Ignore { name: ident.text.clone(), id: ident.id } + Op::Ignore { name: ident.text.clone(), id: ident.span } } "index" => { let depth = if args.len() == 0 { 0 } else { args.expect_u32_literal()? }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 5c96505563..fbf6b53006 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -8,9 +8,16 @@ use syntax::{ SyntaxKind::*, SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; -use tt::buffer::{Cursor, TokenBuffer}; -use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; +use crate::{ + to_parser_input::to_parser_input, + tt::{ + self, + buffer::{Cursor, TokenBuffer}, + }, + tt_iter::TtIter, + TokenMap, +}; #[cfg(test)] mod tests; @@ -74,9 +81,10 @@ pub fn token_tree_to_syntax_node( entry_point: parser::TopEntryPoint, ) -> (Parse, TokenMap) { let buffer = match tt { - tt::Subtree { delimiter: None, token_trees } => { - TokenBuffer::from_tokens(token_trees.as_slice()) - } + tt::Subtree { + delimiter: tt::Delimiter { kind: tt::DelimiterKind::Invisible, .. }, + token_trees, + } => TokenBuffer::from_tokens(token_trees.as_slice()), _ => TokenBuffer::from_subtree(tt), }; let parser_input = to_parser_input(&buffer); @@ -92,8 +100,7 @@ pub fn token_tree_to_syntax_node( parser::Step::Error { msg } => tree_sink.error(msg.to_string()), } } - let (parse, range_map) = tree_sink.finish(); - (parse, range_map) + tree_sink.finish() } /// Convert a string to a `TokenTree` @@ -132,7 +139,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec { res.push(match expanded.value { None => break, Some(tt @ tt::TokenTree::Leaf(_)) => { - tt::Subtree { delimiter: None, token_trees: vec![tt] } + tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] } } Some(tt::TokenTree::Subtree(tt)) => tt, }); @@ -145,7 +152,10 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec { } if iter.peek_n(0).is_some() { - res.push(tt::Subtree { delimiter: None, token_trees: iter.cloned().collect() }); + res.push(tt::Subtree { + delimiter: tt::Delimiter::unspecified(), + token_trees: iter.cloned().collect(), + }); } res @@ -159,7 +169,7 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { } let entry = StackEntry { - subtree: tt::Subtree { delimiter: None, ..Default::default() }, + subtree: tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }, // never used (delimiter is `None`) idx: !0, open_range: TextRange::empty(TextSize::of('.')), @@ -186,7 +196,7 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = sub.token_trees.get_mut(2) { - lit.id = id + lit.span = id } } tt @@ -199,13 +209,14 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { assert_eq!(range.len(), TextSize::of('.')); } - if let Some(delim) = subtree.delimiter { - let expected = match delim.kind { - tt::DelimiterKind::Parenthesis => T![')'], - tt::DelimiterKind::Brace => T!['}'], - tt::DelimiterKind::Bracket => T![']'], - }; + let expected = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + }; + if let Some(expected) = expected { if kind == expected { if let Some(entry) = stack.pop() { conv.id_alloc().close_delim(entry.idx, Some(range)); @@ -223,9 +234,11 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { }; if let Some(kind) = delim { - let mut subtree = tt::Subtree::default(); let (id, idx) = conv.id_alloc().open_delim(range, synth_id); - subtree.delimiter = Some(tt::Delimiter { id, kind }); + let subtree = tt::Subtree { + delimiter: tt::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind }, + token_trees: vec![], + }; stack.push(StackEntry { subtree, idx, open_range: range }); continue; } @@ -240,13 +253,20 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { panic!("Token from lexer must be single char: token = {token:#?}"); } }; - tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) }) - .into() + tt::Leaf::from(tt::Punct { + char, + spacing, + span: conv.id_alloc().alloc(range, synth_id), + }) + .into() } else { macro_rules! make_leaf { ($i:ident) => { - tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) } - .into() + tt::$i { + span: conv.id_alloc().alloc(range, synth_id), + text: token.to_text(conv), + } + .into() }; } let leaf: tt::Leaf = match kind { @@ -261,14 +281,14 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { let apostrophe = tt::Leaf::from(tt::Punct { char: '\'', spacing: tt::Spacing::Joint, - id: conv.id_alloc().alloc(r, synth_id), + span: conv.id_alloc().alloc(r, synth_id), }); result.push(apostrophe.into()); let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); let ident = tt::Leaf::from(tt::Ident { text: SmolStr::new(&token.to_text(conv)[1..]), - id: conv.id_alloc().alloc(r, synth_id), + span: conv.id_alloc().alloc(r, synth_id), }); result.push(ident.into()); continue; @@ -289,11 +309,12 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { conv.id_alloc().close_delim(entry.idx, None); let leaf: tt::Leaf = tt::Punct { - id: conv.id_alloc().alloc(entry.open_range, None), - char: match entry.subtree.delimiter.unwrap().kind { + span: conv.id_alloc().alloc(entry.open_range, None), + char: match entry.subtree.delimiter.kind { tt::DelimiterKind::Parenthesis => '(', tt::DelimiterKind::Brace => '{', tt::DelimiterKind::Bracket => '[', + tt::DelimiterKind::Invisible => '$', }, spacing: tt::Spacing::Alone, } @@ -373,10 +394,11 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option token_trees.push(mk_punct('!')); } token_trees.push(tt::TokenTree::from(tt::Subtree { - delimiter: Some(tt::Delimiter { + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, kind: tt::DelimiterKind::Bracket, - id: tt::TokenId::unspecified(), - }), + }, token_trees: meta_tkns, })); @@ -386,7 +408,7 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option fn mk_ident(s: &str) -> tt::TokenTree { tt::TokenTree::from(tt::Leaf::from(tt::Ident { text: s.into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) } @@ -394,12 +416,12 @@ fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) } fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { - let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; + let lit = tt::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() }; tt::TokenTree::from(tt::Leaf::from(lit)) } @@ -761,15 +783,16 @@ impl<'a> TtTreeSink<'a> { } } -fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { +fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { let texts = match d { tt::DelimiterKind::Parenthesis => "()", tt::DelimiterKind::Brace => "{}", tt::DelimiterKind::Bracket => "[]", + tt::DelimiterKind::Invisible => return None, }; let idx = closing as usize; - &texts[idx..texts.len() - (1 - idx)] + Some(&texts[idx..texts.len() - (1 - idx)]) } impl<'a> TtTreeSink<'a> { @@ -790,13 +813,16 @@ impl<'a> TtTreeSink<'a> { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { // Mark the range if needed let (text, id) = match leaf { - tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id), + tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.span), tt::Leaf::Punct(punct) => { assert!(punct.char.is_ascii()); tmp = punct.char as u8; - (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id) + ( + std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), + punct.span, + ) } - tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id), + tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.span), }; let range = TextRange::at(self.text_pos, TextSize::of(text)); self.token_map.insert(id, range); @@ -805,10 +831,10 @@ impl<'a> TtTreeSink<'a> { } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { self.cursor = self.cursor.subtree().unwrap(); - match subtree.delimiter { - Some(d) => { - self.open_delims.insert(d.id, self.text_pos); - delim_to_str(d.kind, false) + match delim_to_str(subtree.delimiter.kind, false) { + Some(it) => { + self.open_delims.insert(subtree.delimiter.open, self.text_pos); + it } None => continue, } @@ -816,15 +842,21 @@ impl<'a> TtTreeSink<'a> { None => { let parent = self.cursor.end().unwrap(); self.cursor = self.cursor.bump(); - match parent.delimiter { - Some(d) => { - if let Some(open_delim) = self.open_delims.get(&d.id) { + match delim_to_str(parent.delimiter.kind, true) { + Some(it) => { + if let Some(open_delim) = + self.open_delims.get(&parent.delimiter.open) + { let open_range = TextRange::at(*open_delim, TextSize::of('(')); let close_range = TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim(d.id, open_range, close_range); + self.token_map.insert_delim( + parent.delimiter.open, + open_range, + close_range, + ); } - delim_to_str(d.kind, true) + it } None => continue, } diff --git a/crates/mbe/src/syntax_bridge/tests.rs b/crates/mbe/src/syntax_bridge/tests.rs index c1a6083655..fa0125f3e9 100644 --- a/crates/mbe/src/syntax_bridge/tests.rs +++ b/crates/mbe/src/syntax_bridge/tests.rs @@ -29,8 +29,8 @@ fn check_punct_spacing(fixture: &str) { let mut cursor = buf.begin(); while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { - if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, id, .. }), _) = token_tree { - if let Some(expected) = annotations.remove(id) { + if let TokenTreeRef::Leaf(Leaf::Punct(Punct { spacing, span, .. }), _) = token_tree { + if let Some(expected) = annotations.remove(span) { assert_eq!(expected, *spacing); } } diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 7013aa58b5..d4c19b3ab8 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -2,7 +2,8 @@ //! format that works for our parser. use syntax::{SyntaxKind, SyntaxKind::*, T}; -use tt::buffer::TokenBuffer; + +use crate::tt::buffer::TokenBuffer; pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { let mut res = parser::Input::default(); @@ -70,23 +71,25 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer<'_>) -> parser::Input { cursor.bump() } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - if let Some(d) = subtree.delimiter_kind() { - res.push(match d { - tt::DelimiterKind::Parenthesis => T!['('], - tt::DelimiterKind::Brace => T!['{'], - tt::DelimiterKind::Bracket => T!['['], - }); + if let Some(kind) = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T!['(']), + tt::DelimiterKind::Brace => Some(T!['{']), + tt::DelimiterKind::Bracket => Some(T!['[']), + tt::DelimiterKind::Invisible => None, + } { + res.push(kind); } cursor.subtree().unwrap() } None => match cursor.end() { Some(subtree) => { - if let Some(d) = subtree.delimiter_kind() { - res.push(match d { - tt::DelimiterKind::Parenthesis => T![')'], - tt::DelimiterKind::Brace => T!['}'], - tt::DelimiterKind::Bracket => T![']'], - }) + if let Some(kind) = match subtree.delimiter.kind { + tt::DelimiterKind::Parenthesis => Some(T![')']), + tt::DelimiterKind::Brace => Some(T!['}']), + tt::DelimiterKind::Bracket => Some(T![']']), + tt::DelimiterKind::Invisible => None, + } { + res.push(kind); } cursor.bump() } diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index 7787c74da8..e5f6b13722 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -3,9 +3,8 @@ use smallvec::{smallvec, SmallVec}; use syntax::SyntaxKind; -use tt::buffer::TokenBuffer; -use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult}; +use crate::{to_parser_input::to_parser_input, tt, ExpandError, ExpandResult}; #[derive(Debug, Clone)] pub(crate) struct TtIter<'a> { @@ -135,7 +134,7 @@ impl<'a> TtIter<'a> { &mut self, entry_point: parser::PrefixEntryPoint, ) -> ExpandResult> { - let buffer = TokenBuffer::from_tokens(self.inner.as_slice()); + let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice()); let parser_input = to_parser_input(&buffer); let tree_traversal = entry_point.parse(&parser_input); @@ -178,7 +177,7 @@ impl<'a> TtIter<'a> { 1 => Some(res[0].cloned()), 0 => None, _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, + delimiter: tt::Delimiter::unspecified(), token_trees: res.into_iter().map(|it| it.cloned()).collect(), })), }; diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 52f976e457..bb381c4d44 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -19,7 +19,8 @@ use std::{ }; use serde::{Deserialize, Serialize}; -use tt::Subtree; + +use ::tt::token_id as tt; use crate::{ msg::{ExpandMacro, FlatTree, PanicMessage}, @@ -151,10 +152,10 @@ impl ProcMacro { pub fn expand( &self, - subtree: &Subtree, - attr: Option<&Subtree>, + subtree: &tt::Subtree, + attr: Option<&tt::Subtree>, env: Vec<(String, String)>, - ) -> Result, ServerError> { + ) -> Result, ServerError> { let current_dir = env .iter() .find(|(name, _)| name == "CARGO_MANIFEST_DIR") diff --git a/crates/proc-macro-api/src/msg.rs b/crates/proc-macro-api/src/msg.rs index f9c2b9fda3..9b7bcaeffe 100644 --- a/crates/proc-macro-api/src/msg.rs +++ b/crates/proc-macro-api/src/msg.rs @@ -107,27 +107,31 @@ fn write_json(out: &mut impl Write, msg: &str) -> io::Result<()> { #[cfg(test)] mod tests { use super::*; - use tt::*; + use crate::tt::*; fn fixture_token_tree() -> Subtree { - let mut subtree = Subtree::default(); + let mut subtree = Subtree { delimiter: Delimiter::unspecified(), token_trees: Vec::new() }; subtree .token_trees - .push(TokenTree::Leaf(Ident { text: "struct".into(), id: TokenId(0) }.into())); + .push(TokenTree::Leaf(Ident { text: "struct".into(), span: TokenId(0) }.into())); subtree .token_trees - .push(TokenTree::Leaf(Ident { text: "Foo".into(), id: TokenId(1) }.into())); + .push(TokenTree::Leaf(Ident { text: "Foo".into(), span: TokenId(1) }.into())); subtree.token_trees.push(TokenTree::Leaf(Leaf::Literal(Literal { text: "Foo".into(), - id: TokenId::unspecified(), + span: TokenId::unspecified(), }))); subtree.token_trees.push(TokenTree::Leaf(Leaf::Punct(Punct { char: '@', - id: TokenId::unspecified(), + span: TokenId::unspecified(), spacing: Spacing::Joint, }))); subtree.token_trees.push(TokenTree::Subtree(Subtree { - delimiter: Some(Delimiter { id: TokenId(2), kind: DelimiterKind::Brace }), + delimiter: Delimiter { + open: TokenId(2), + close: TokenId::UNSPECIFIED, + kind: DelimiterKind::Brace, + }, token_trees: vec![], })); subtree diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index b178c46263..fd3202e0b2 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -38,7 +38,8 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; -use tt::TokenId; + +use crate::tt::{self, TokenId}; #[derive(Serialize, Deserialize, Debug)] pub struct FlatTree { @@ -52,7 +53,7 @@ pub struct FlatTree { struct SubtreeRepr { id: tt::TokenId, - kind: Option, + kind: tt::DelimiterKind, tt: [u32; 2], } @@ -124,19 +125,19 @@ impl FlatTree { impl SubtreeRepr { fn write(self) -> [u32; 4] { let kind = match self.kind { - None => 0, - Some(tt::DelimiterKind::Parenthesis) => 1, - Some(tt::DelimiterKind::Brace) => 2, - Some(tt::DelimiterKind::Bracket) => 3, + tt::DelimiterKind::Invisible => 0, + tt::DelimiterKind::Parenthesis => 1, + tt::DelimiterKind::Brace => 2, + tt::DelimiterKind::Bracket => 3, }; [self.id.0, kind, self.tt[0], self.tt[1]] } fn read([id, kind, lo, len]: [u32; 4]) -> SubtreeRepr { let kind = match kind { - 0 => None, - 1 => Some(tt::DelimiterKind::Parenthesis), - 2 => Some(tt::DelimiterKind::Brace), - 3 => Some(tt::DelimiterKind::Bracket), + 0 => tt::DelimiterKind::Invisible, + 1 => tt::DelimiterKind::Parenthesis, + 2 => tt::DelimiterKind::Brace, + 3 => tt::DelimiterKind::Bracket, other => panic!("bad kind {other}"), }; SubtreeRepr { id: TokenId(id), kind, tt: [lo, len] } @@ -216,7 +217,7 @@ impl<'a> Writer<'a> { tt::Leaf::Literal(lit) => { let idx = self.literal.len() as u32; let text = self.intern(&lit.text); - self.literal.push(LiteralRepr { id: lit.id, text }); + self.literal.push(LiteralRepr { id: lit.span, text }); idx << 2 | 0b01 } tt::Leaf::Punct(punct) => { @@ -224,14 +225,14 @@ impl<'a> Writer<'a> { self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, - id: punct.id, + id: punct.span, }); idx << 2 | 0b10 } tt::Leaf::Ident(ident) => { let idx = self.ident.len() as u32; let text = self.intern(&ident.text); - self.ident.push(IdentRepr { id: ident.id, text }); + self.ident.push(IdentRepr { id: ident.span, text }); idx << 2 | 0b11 } }, @@ -243,8 +244,8 @@ impl<'a> Writer<'a> { fn enqueue(&mut self, subtree: &'a tt::Subtree) -> u32 { let idx = self.subtree.len(); - let delimiter_id = subtree.delimiter.map_or(TokenId::unspecified(), |it| it.id); - let delimiter_kind = subtree.delimiter.map(|it| it.kind); + let delimiter_id = subtree.delimiter.open; + let delimiter_kind = subtree.delimiter.kind; self.subtree.push(SubtreeRepr { id: delimiter_id, kind: delimiter_kind, tt: [!0, !0] }); self.work.push_back((idx, subtree)); idx as u32 @@ -276,7 +277,11 @@ impl Reader { let repr = &self.subtree[i]; let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; let s = tt::Subtree { - delimiter: repr.kind.map(|kind| tt::Delimiter { id: repr.id, kind }), + delimiter: tt::Delimiter { + open: repr.id, + close: TokenId::UNSPECIFIED, + kind: repr.kind, + }, token_trees: token_trees .iter() .copied() @@ -291,7 +296,7 @@ impl Reader { let repr = &self.literal[idx]; tt::Leaf::Literal(tt::Literal { text: self.text[repr.text as usize].as_str().into(), - id: repr.id, + span: repr.id, }) .into() } @@ -300,7 +305,7 @@ impl Reader { tt::Leaf::Punct(tt::Punct { char: repr.char, spacing: repr.spacing, - id: repr.id, + span: repr.id, }) .into() } @@ -308,7 +313,7 @@ impl Reader { let repr = &self.ident[idx]; tt::Leaf::Ident(tt::Ident { text: self.text[repr.text as usize].as_str().into(), - id: repr.id, + span: repr.id, }) .into() } diff --git a/crates/proc-macro-srv/src/abis/abi_1_63/mod.rs b/crates/proc-macro-srv/src/abis/abi_1_63/mod.rs index 76e89e3191..93805c8935 100644 --- a/crates/proc-macro-srv/src/abis/abi_1_63/mod.rs +++ b/crates/proc-macro-srv/src/abis/abi_1_63/mod.rs @@ -11,6 +11,7 @@ mod ra_server; use libloading::Library; use proc_macro_api::ProcMacroKind; +use super::tt; use super::PanicMessage; pub use ra_server::TokenStream; diff --git a/crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs index f82f20c37b..30baf3a13f 100644 --- a/crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_1_63/ra_server.rs @@ -15,6 +15,8 @@ use std::hash::Hash; use std::ops::Bound; use std::{ascii, vec::IntoIter}; +use crate::tt; + type Group = tt::Subtree; type TokenTree = tt::TokenTree; type Punct = tt::Punct; @@ -33,7 +35,7 @@ impl TokenStream { } pub fn with_subtree(subtree: tt::Subtree) -> Self { - if subtree.delimiter.is_some() { + if subtree.delimiter.kind != tt::DelimiterKind::Invisible { TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } } else { TokenStream { token_trees: subtree.token_trees } @@ -41,7 +43,7 @@ impl TokenStream { } pub fn into_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: None, token_trees: self.token_trees } + tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees } } pub fn is_empty(&self) -> bool { @@ -84,7 +86,9 @@ impl Extend for TokenStream { for item in streams { for tkn in item { match tkn { - tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { + tt::TokenTree::Subtree(subtree) + if subtree.delimiter.kind == tt::DelimiterKind::Invisible => + { self.token_trees.extend(subtree.token_trees); } _ => { @@ -165,7 +169,7 @@ pub struct TokenStreamBuilder { pub mod token_stream { use std::str::FromStr; - use super::{TokenStream, TokenTree}; + use super::{tt, TokenStream, TokenTree}; /// An iterator over `TokenStream`'s `TokenTree`s. /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, @@ -202,15 +206,17 @@ pub mod token_stream { impl ToString for TokenStream { fn to_string(&self) -> String { - tt::pretty(&self.token_trees) + ::tt::pretty(&self.token_trees) } } fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { tt::Subtree { - delimiter: subtree - .delimiter - .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, + ..subtree.delimiter + }, token_trees: subtree .token_trees .into_iter() @@ -233,13 +239,13 @@ pub mod token_stream { fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { match leaf { tt::Leaf::Literal(lit) => { - tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) + tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit }) } tt::Leaf::Punct(punct) => { - tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) + tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct }) } tt::Leaf::Ident(ident) => { - tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) + tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident }) } } } @@ -389,22 +395,22 @@ impl server::TokenStream for RustAnalyzer { } } -fn delim_to_internal(d: bridge::Delimiter) -> Option { +fn delim_to_internal(d: bridge::Delimiter) -> tt::Delimiter { let kind = match d { bridge::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, bridge::Delimiter::Brace => tt::DelimiterKind::Brace, bridge::Delimiter::Bracket => tt::DelimiterKind::Bracket, - bridge::Delimiter::None => return None, + bridge::Delimiter::None => tt::DelimiterKind::Invisible, }; - Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) + tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind } } -fn delim_to_external(d: Option) -> bridge::Delimiter { - match d.map(|it| it.kind) { - Some(tt::DelimiterKind::Parenthesis) => bridge::Delimiter::Parenthesis, - Some(tt::DelimiterKind::Brace) => bridge::Delimiter::Brace, - Some(tt::DelimiterKind::Bracket) => bridge::Delimiter::Bracket, - None => bridge::Delimiter::None, +fn delim_to_external(d: tt::Delimiter) -> bridge::Delimiter { + match d.kind { + tt::DelimiterKind::Parenthesis => bridge::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => bridge::Delimiter::Brace, + tt::DelimiterKind::Bracket => bridge::Delimiter::Bracket, + tt::DelimiterKind::Invisible => bridge::Delimiter::None, } } @@ -443,23 +449,19 @@ impl server::Group for RustAnalyzer { } fn span(&mut self, group: &Self::Group) -> Self::Span { - group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified) + group.delimiter.open } fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - if let Some(delim) = &mut group.delimiter { - delim.id = span; - } + group.delimiter.open = span; } fn span_open(&mut self, group: &Self::Group) -> Self::Span { - // FIXME we only store one `TokenId` for the delimiters - group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified) + group.delimiter.open } fn span_close(&mut self, group: &Self::Group) -> Self::Span { - // FIXME we only store one `TokenId` for the delimiters - group.delimiter.map(|it| it.id).unwrap_or_else(tt::TokenId::unspecified) + group.delimiter.close } } @@ -468,7 +470,7 @@ impl server::Punct for RustAnalyzer { tt::Punct { char: ch, spacing: spacing_to_internal(spacing), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), } } fn as_char(&mut self, punct: Self::Punct) -> char { @@ -478,28 +480,27 @@ impl server::Punct for RustAnalyzer { spacing_to_external(punct.spacing) } fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.id + punct.span } fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - tt::Punct { id: span, ..punct } + tt::Punct { span: span, ..punct } } } impl server::Ident for RustAnalyzer { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - IdentId(self.ident_interner.intern(&IdentData(tt::Ident::new_with_is_raw( - string.into(), + IdentId(self.ident_interner.intern(&IdentData(tt::Ident { + text: if is_raw { ::tt::SmolStr::from_iter(["r#", string]) } else { string.into() }, span, - is_raw, - )))) + }))) } fn span(&mut self, ident: Self::Ident) -> Self::Span { - self.ident_interner.get(ident.0).0.id + self.ident_interner.get(ident.0).0.span } fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { let data = self.ident_interner.get(ident.0); - let new = IdentData(tt::Ident { id: span, ..data.0.clone() }); + let new = IdentData(tt::Ident { span: span, ..data.0.clone() }); IdentId(self.ident_interner.intern(&new)) } } @@ -511,7 +512,7 @@ impl server::Literal for RustAnalyzer { "".to_owned() } fn from_str(&mut self, s: &str) -> Result { - Ok(Literal { text: s.into(), id: tt::TokenId::unspecified() }) + Ok(Literal { text: s.into(), span: tt::TokenId::unspecified() }) } fn symbol(&mut self, literal: &Self::Literal) -> String { literal.text.to_string() @@ -529,7 +530,7 @@ impl server::Literal for RustAnalyzer { Ok(n) => n.to_string(), Err(_) => n.parse::().unwrap().to_string(), }; - Literal { text: n.into(), id: tt::TokenId::unspecified() } + Literal { text: n.into(), span: tt::TokenId::unspecified() } } fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { @@ -549,7 +550,7 @@ impl server::Literal for RustAnalyzer { let text = def_suffixed_integer! {kind, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize}; - Literal { text: text.into(), id: tt::TokenId::unspecified() } + Literal { text: text.into(), span: tt::TokenId::unspecified() } } fn float(&mut self, n: &str) -> Self::Literal { @@ -558,19 +559,19 @@ impl server::Literal for RustAnalyzer { if !text.contains('.') { text += ".0" } - Literal { text: text.into(), id: tt::TokenId::unspecified() } + Literal { text: text.into(), span: tt::TokenId::unspecified() } } fn f32(&mut self, n: &str) -> Self::Literal { let n: f32 = n.parse().unwrap(); let text = format!("{n}f32"); - Literal { text: text.into(), id: tt::TokenId::unspecified() } + Literal { text: text.into(), span: tt::TokenId::unspecified() } } fn f64(&mut self, n: &str) -> Self::Literal { let n: f64 = n.parse().unwrap(); let text = format!("{n}f64"); - Literal { text: text.into(), id: tt::TokenId::unspecified() } + Literal { text: text.into(), span: tt::TokenId::unspecified() } } fn string(&mut self, string: &str) -> Self::Literal { @@ -578,11 +579,11 @@ impl server::Literal for RustAnalyzer { for ch in string.chars() { escaped.extend(ch.escape_debug()); } - Literal { text: format!("\"{escaped}\"").into(), id: tt::TokenId::unspecified() } + Literal { text: format!("\"{escaped}\"").into(), span: tt::TokenId::unspecified() } } fn character(&mut self, ch: char) -> Self::Literal { - Literal { text: format!("'{ch}'").into(), id: tt::TokenId::unspecified() } + Literal { text: format!("'{ch}'").into(), span: tt::TokenId::unspecified() } } fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { @@ -593,15 +594,15 @@ impl server::Literal for RustAnalyzer { .map(Into::::into) .collect::(); - Literal { text: format!("b\"{string}\"").into(), id: tt::TokenId::unspecified() } + Literal { text: format!("b\"{string}\"").into(), span: tt::TokenId::unspecified() } } fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.id + literal.span } fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.id = span; + literal.span = span; } fn subspan( @@ -784,17 +785,18 @@ mod tests { token_trees: vec![ tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "struct".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "T".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })), tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Brace, - }), + }, token_trees: vec![], }), ], @@ -807,13 +809,14 @@ mod tests { fn test_ra_server_from_str() { use std::str::FromStr; let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Parenthesis, - }), + }, token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "a".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }); @@ -830,7 +833,7 @@ mod tests { underscore.token_trees[0], tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "_".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) ); } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs index 243972b049..0a3b8866a7 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/mod.rs @@ -9,7 +9,7 @@ mod ra_server; use libloading::Library; use proc_macro_api::ProcMacroKind; -use super::PanicMessage; +use super::{tt, PanicMessage}; pub use ra_server::TokenStream; diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs index 068f79f824..d258a02472 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server.rs @@ -22,6 +22,8 @@ pub use symbol::*; use std::ops::Bound; +use crate::tt; + type Group = tt::Subtree; type TokenTree = tt::TokenTree; type Punct = tt::Punct; @@ -108,8 +110,9 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Ident(ident) => { let text = ident.sym.text(); - let text = if ident.is_raw { tt::SmolStr::from_iter(["r#", &text]) } else { text }; - let ident: tt::Ident = tt::Ident { text, id: ident.span }; + let text = + if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text }; + let ident: tt::Ident = tt::Ident { text, span: ident.span }; let leaf = tt::Leaf::from(ident); let tree = TokenTree::from(leaf); Self::TokenStream::from_iter(vec![tree]) @@ -118,9 +121,9 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Literal(literal) => { let literal = LiteralFormatter(literal); let text = literal - .with_stringify_parts(|parts| tt::SmolStr::from_iter(parts.iter().copied())); + .with_stringify_parts(|parts| ::tt::SmolStr::from_iter(parts.iter().copied())); - let literal = tt::Literal { text, id: literal.0.span }; + let literal = tt::Literal { text, span: literal.0.span }; let leaf = tt::Leaf::from(literal); let tree = TokenTree::from(leaf); Self::TokenStream::from_iter(vec![tree]) @@ -130,7 +133,7 @@ impl server::TokenStream for RustAnalyzer { let punct = tt::Punct { char: p.ch as char, spacing: if p.joint { Spacing::Joint } else { Spacing::Alone }, - id: p.span, + span: p.span, }; let leaf = tt::Leaf::from(punct); let tree = TokenTree::from(leaf); @@ -184,7 +187,7 @@ impl server::TokenStream for RustAnalyzer { bridge::TokenTree::Ident(bridge::Ident { sym: Symbol::intern(ident.text.trim_start_matches("r#")), is_raw: ident.text.starts_with("r#"), - span: ident.id, + span: ident.span, }) } tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { @@ -194,14 +197,14 @@ impl server::TokenStream for RustAnalyzer { symbol: Symbol::intern(&lit.text), // FIXME: handle suffixes suffix: None, - span: lit.id, + span: lit.span, }) } tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { bridge::TokenTree::Punct(bridge::Punct { ch: punct.char as u8, joint: punct.spacing == Spacing::Joint, - span: punct.id, + span: punct.span, }) } tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(bridge::Group { @@ -211,31 +214,29 @@ impl server::TokenStream for RustAnalyzer { } else { Some(subtree.token_trees.into_iter().collect()) }, - span: bridge::DelimSpan::from_single( - subtree.delimiter.map_or(Span::unspecified(), |del| del.id), - ), + span: bridge::DelimSpan::from_single(subtree.delimiter.open), }), }) .collect() } } -fn delim_to_internal(d: proc_macro::Delimiter) -> Option { +fn delim_to_internal(d: proc_macro::Delimiter) -> tt::Delimiter { let kind = match d { proc_macro::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, proc_macro::Delimiter::Brace => tt::DelimiterKind::Brace, proc_macro::Delimiter::Bracket => tt::DelimiterKind::Bracket, - proc_macro::Delimiter::None => return None, + proc_macro::Delimiter::None => tt::DelimiterKind::Invisible, }; - Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }) + tt::Delimiter { open: tt::TokenId::unspecified(), close: tt::TokenId::unspecified(), kind } } -fn delim_to_external(d: Option) -> proc_macro::Delimiter { - match d.map(|it| it.kind) { - Some(tt::DelimiterKind::Parenthesis) => proc_macro::Delimiter::Parenthesis, - Some(tt::DelimiterKind::Brace) => proc_macro::Delimiter::Brace, - Some(tt::DelimiterKind::Bracket) => proc_macro::Delimiter::Bracket, - None => proc_macro::Delimiter::None, +fn delim_to_external(d: tt::Delimiter) -> proc_macro::Delimiter { + match d.kind { + tt::DelimiterKind::Parenthesis => proc_macro::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => proc_macro::Delimiter::Brace, + tt::DelimiterKind::Bracket => proc_macro::Delimiter::Bracket, + tt::DelimiterKind::Invisible => proc_macro::Delimiter::None, } } @@ -349,7 +350,7 @@ impl server::Server for RustAnalyzer { } fn intern_symbol(ident: &str) -> Self::Symbol { - Symbol::intern(&tt::SmolStr::from(ident)) + Symbol::intern(&::tt::SmolStr::from(ident)) } fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { @@ -413,17 +414,18 @@ mod tests { token_trees: vec![ tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "struct".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })), tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "T".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })), tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Brace, - }), + }, token_trees: vec![], }), ], @@ -436,13 +438,14 @@ mod tests { fn test_ra_server_from_str() { use std::str::FromStr; let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree { - delimiter: Some(tt::Delimiter { - id: tt::TokenId::unspecified(), + delimiter: tt::Delimiter { + open: tt::TokenId::unspecified(), + close: tt::TokenId::unspecified(), kind: tt::DelimiterKind::Parenthesis, - }), + }, token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "a".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), }))], }); @@ -459,7 +462,7 @@ mod tests { underscore.token_trees[0], tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text: "_".into(), - id: tt::TokenId::unspecified(), + span: tt::TokenId::unspecified(), })) ); } diff --git a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs index 113bb52c1a..7de30e7392 100644 --- a/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs +++ b/crates/proc-macro-srv/src/abis/abi_sysroot/ra_server/token_stream.rs @@ -1,6 +1,6 @@ //! TokenStream implementation used by sysroot ABI -use tt::TokenTree; +use crate::tt::{self, TokenTree}; #[derive(Debug, Default, Clone)] pub struct TokenStream { @@ -13,7 +13,7 @@ impl TokenStream { } pub fn with_subtree(subtree: tt::Subtree) -> Self { - if subtree.delimiter.is_some() { + if subtree.delimiter.kind != tt::DelimiterKind::Invisible { TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } } else { TokenStream { token_trees: subtree.token_trees } @@ -21,7 +21,7 @@ impl TokenStream { } pub fn into_subtree(self) -> tt::Subtree { - tt::Subtree { delimiter: None, token_trees: self.token_trees } + tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: self.token_trees } } pub fn is_empty(&self) -> bool { @@ -64,7 +64,9 @@ impl Extend for TokenStream { for item in streams { for tkn in item { match tkn { - tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { + tt::TokenTree::Subtree(subtree) + if subtree.delimiter.kind != tt::DelimiterKind::Invisible => + { self.token_trees.extend(subtree.token_trees); } _ => { @@ -84,7 +86,7 @@ pub struct TokenStreamBuilder { pub mod token_stream { use std::str::FromStr; - use super::{TokenStream, TokenTree}; + use super::{tt, TokenStream, TokenTree}; /// An iterator over `TokenStream`'s `TokenTree`s. /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, @@ -121,15 +123,17 @@ pub mod token_stream { impl ToString for TokenStream { fn to_string(&self) -> String { - tt::pretty(&self.token_trees) + ::tt::pretty(&self.token_trees) } } fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree { tt::Subtree { - delimiter: subtree - .delimiter - .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }), + delimiter: tt::Delimiter { + open: tt::TokenId::UNSPECIFIED, + close: tt::TokenId::UNSPECIFIED, + ..subtree.delimiter + }, token_trees: subtree .token_trees .into_iter() @@ -152,13 +156,13 @@ pub mod token_stream { fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf { match leaf { tt::Leaf::Literal(lit) => { - tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit }) + tt::Leaf::Literal(tt::Literal { span: tt::TokenId::unspecified(), ..lit }) } tt::Leaf::Punct(punct) => { - tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct }) + tt::Leaf::Punct(tt::Punct { span: tt::TokenId::unspecified(), ..punct }) } tt::Leaf::Ident(ident) => { - tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident }) + tt::Leaf::Ident(tt::Ident { span: tt::TokenId::unspecified(), ..ident }) } } } diff --git a/crates/proc-macro-srv/src/abis/mod.rs b/crates/proc-macro-srv/src/abis/mod.rs index 9f874fb02b..04be39cffa 100644 --- a/crates/proc-macro-srv/src/abis/mod.rs +++ b/crates/proc-macro-srv/src/abis/mod.rs @@ -41,6 +41,8 @@ pub(crate) use abi_sysroot::Abi as Abi_Sysroot; use libloading::Library; use proc_macro_api::{ProcMacroKind, RustCInfo}; +use crate::tt; + pub struct PanicMessage { message: Option, } diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 0722cd89d7..89ffd1f493 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -13,6 +13,8 @@ use object::Object; use paths::AbsPath; use proc_macro_api::{read_dylib_info, ProcMacroKind}; +use crate::tt; + use super::abis::Abi; const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_"; diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 2eb939a7ce..ee70fe7d4f 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -37,6 +37,8 @@ use proc_macro_api::{ ProcMacroKind, }; +use ::tt::token_id as tt; + #[derive(Default)] pub(crate) struct ProcMacroSrv { expanders: HashMap<(PathBuf, SystemTime), dylib::Expander>, diff --git a/crates/proc-macro-srv/src/tests/mod.rs b/crates/proc-macro-srv/src/tests/mod.rs index 1ccc170f42..04a0ae7bc7 100644 --- a/crates/proc-macro-srv/src/tests/mod.rs +++ b/crates/proc-macro-srv/src/tests/mod.rs @@ -8,7 +8,7 @@ use expect_test::expect; #[test] fn test_derive_empty() { - assert_expand("DeriveEmpty", r#"struct S;"#, expect![[r#"SUBTREE $"#]]); + assert_expand("DeriveEmpty", r#"struct S;"#, expect!["SUBTREE $$ 4294967295 4294967295"]); } #[test] @@ -17,10 +17,10 @@ fn test_derive_error() { "DeriveError", r#"struct S;"#, expect![[r##" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT compile_error 4294967295 PUNCH ! [alone] 4294967295 - SUBTREE () 4294967295 + SUBTREE () 4294967295 4294967295 LITERAL "#[derive(DeriveError)] struct S ;" 4294967295 PUNCH ; [alone] 4294967295"##]], ); @@ -32,14 +32,14 @@ fn test_fn_like_macro_noop() { "fn_like_noop", r#"ident, 0, 1, []"#, expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT ident 4294967295 PUNCH , [alone] 4294967295 LITERAL 0 4294967295 PUNCH , [alone] 4294967295 LITERAL 1 4294967295 PUNCH , [alone] 4294967295 - SUBTREE [] 4294967295"#]], + SUBTREE [] 4294967295 4294967295"#]], ); } @@ -49,10 +49,10 @@ fn test_fn_like_macro_clone_ident_subtree() { "fn_like_clone_tokens", r#"ident, []"#, expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT ident 4294967295 PUNCH , [alone] 4294967295 - SUBTREE [] 4294967295"#]], + SUBTREE [] 4294967295 4294967295"#]], ); } @@ -62,7 +62,7 @@ fn test_fn_like_macro_clone_raw_ident() { "fn_like_clone_tokens", "r#async", expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT r#async 4294967295"#]], ); } @@ -73,7 +73,7 @@ fn test_fn_like_mk_literals() { "fn_like_mk_literals", r#""#, expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 LITERAL b"byte_string" 4294967295 LITERAL 'c' 4294967295 LITERAL "string" 4294967295 @@ -90,7 +90,7 @@ fn test_fn_like_mk_idents() { "fn_like_mk_idents", r#""#, expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT standard 4294967295 IDENT r#raw 4294967295"#]], ); @@ -102,7 +102,7 @@ fn test_fn_like_macro_clone_literals() { "fn_like_clone_tokens", r#"1u16, 2_u32, -4i64, 3.14f32, "hello bridge""#, expect![[r#" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 LITERAL 1u16 4294967295 PUNCH , [alone] 4294967295 LITERAL 2_u32 4294967295 @@ -126,10 +126,10 @@ fn test_attr_macro() { r#"mod m {}"#, r#"some arguments"#, expect![[r##" - SUBTREE $ + SUBTREE $$ 4294967295 4294967295 IDENT compile_error 4294967295 PUNCH ! [alone] 4294967295 - SUBTREE () 4294967295 + SUBTREE () 4294967295 4294967295 LITERAL "#[attr_error(some arguments)] mod m {}" 4294967295 PUNCH ; [alone] 4294967295"##]], ); diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs index a33b8e14cf..5ac5af94f5 100644 --- a/crates/rust-analyzer/src/reload.rs +++ b/crates/rust-analyzer/src/reload.rs @@ -34,6 +34,8 @@ use crate::{ op_queue::Cause, }; +use ::tt::token_id as tt; + #[derive(Debug)] pub(crate) enum ProjectWorkspaceProgress { Begin, @@ -656,7 +658,7 @@ pub(crate) fn load_proc_macro( _: Option<&tt::Subtree>, _: &Env, ) -> Result { - Ok(tt::Subtree::default()) + Ok(tt::Subtree::empty()) } } } diff --git a/crates/stdx/src/macros.rs b/crates/stdx/src/macros.rs index d91fc690cb..1a9982fa8b 100644 --- a/crates/stdx/src/macros.rs +++ b/crates/stdx/src/macros.rs @@ -43,5 +43,14 @@ macro_rules! impl_from { } )*)? )* + }; + ($($variant:ident$(<$V:ident>)?),* for $enum:ident) => { + $( + impl$(<$V>)? From<$variant$(<$V>)?> for $enum$(<$V>)? { + fn from(it: $variant$(<$V>)?) -> $enum$(<$V>)? { + $enum::$variant(it) + } + } + )* } } diff --git a/crates/tt/src/buffer.rs b/crates/tt/src/buffer.rs index d27a7aa0d4..4484431124 100644 --- a/crates/tt/src/buffer.rs +++ b/crates/tt/src/buffer.rs @@ -12,10 +12,10 @@ struct EntryPtr(EntryId, usize); /// Internal type which is used instead of `TokenTree` to represent a token tree /// within a `TokenBuffer`. #[derive(Debug)] -enum Entry<'t> { +enum Entry<'t, Span> { // Mimicking types from proc-macro. - Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId), - Leaf(&'t TokenTree), + Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId), + Leaf(&'t TokenTree), // End entries contain a pointer to the entry from the containing // token tree, or None if this is the outermost level. End(Option), @@ -24,16 +24,21 @@ enum Entry<'t> { /// A token tree buffer /// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41) #[derive(Debug)] -pub struct TokenBuffer<'t> { - buffers: Vec]>>, +pub struct TokenBuffer<'t, Span> { + buffers: Vec]>>, } -trait TokenList<'a> { - fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>); +trait TokenList<'a, Span> { + fn entries( + &self, + ) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>); } -impl<'a> TokenList<'a> for &'a [TokenTree] { - fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>) { +impl<'a, Span> TokenList<'a, Span> for &'a [TokenTree] { + fn entries( + &self, + ) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>) + { // Must contain everything in tokens and then the Entry::End let start_capacity = self.len() + 1; let mut entries = Vec::with_capacity(start_capacity); @@ -53,8 +58,11 @@ impl<'a> TokenList<'a> for &'a [TokenTree] { } } -impl<'a> TokenList<'a> for &'a Subtree { - fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>) { +impl<'a, Span> TokenList<'a, Span> for &'a Subtree { + fn entries( + &self, + ) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec>) + { // Must contain everything in tokens and then the Entry::End let mut entries = vec![]; let mut children = vec![]; @@ -64,25 +72,25 @@ impl<'a> TokenList<'a> for &'a Subtree { } } -impl<'t> TokenBuffer<'t> { - pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> { +impl<'t, Span> TokenBuffer<'t, Span> { + pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t, Span> { Self::new(tokens) } - pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> { + pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t, Span> { Self::new(subtree) } - fn new>(tokens: T) -> TokenBuffer<'t> { + fn new>(tokens: T) -> TokenBuffer<'t, Span> { let mut buffers = vec![]; let idx = TokenBuffer::new_inner(tokens, &mut buffers, None); assert_eq!(idx, 0); TokenBuffer { buffers } } - fn new_inner>( + fn new_inner>( tokens: T, - buffers: &mut Vec]>>, + buffers: &mut Vec]>>, next: Option, ) -> usize { let (children, mut entries) = tokens.entries(); @@ -105,25 +113,25 @@ impl<'t> TokenBuffer<'t> { /// Creates a cursor referencing the first token in the buffer and able to /// traverse until the end of the buffer. - pub fn begin(&self) -> Cursor<'_> { + pub fn begin(&self) -> Cursor<'_, Span> { Cursor::create(self, EntryPtr(EntryId(0), 0)) } - fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_>> { + fn entry(&self, ptr: &EntryPtr) -> Option<&Entry<'_, Span>> { let id = ptr.0; self.buffers[id.0].get(ptr.1) } } #[derive(Debug)] -pub enum TokenTreeRef<'a> { - Subtree(&'a Subtree, Option<&'a TokenTree>), - Leaf(&'a Leaf, &'a TokenTree), +pub enum TokenTreeRef<'a, Span> { + Subtree(&'a Subtree, Option<&'a TokenTree>), + Leaf(&'a Leaf, &'a TokenTree), } -impl<'a> TokenTreeRef<'a> { - pub fn cloned(&self) -> TokenTree { - match &self { +impl<'a, Span: Clone> TokenTreeRef<'a, Span> { + pub fn cloned(&self) -> TokenTree { + match self { TokenTreeRef::Subtree(subtree, tt) => match tt { Some(it) => (*it).clone(), None => (*subtree).clone().into(), @@ -135,20 +143,20 @@ impl<'a> TokenTreeRef<'a> { /// A safe version of `Cursor` from `syn` crate #[derive(Copy, Clone, Debug)] -pub struct Cursor<'a> { - buffer: &'a TokenBuffer<'a>, +pub struct Cursor<'a, Span> { + buffer: &'a TokenBuffer<'a, Span>, ptr: EntryPtr, } -impl<'a> PartialEq for Cursor<'a> { - fn eq(&self, other: &Cursor<'_>) -> bool { +impl<'a, Span> PartialEq for Cursor<'a, Span> { + fn eq(&self, other: &Cursor<'_, Span>) -> bool { self.ptr == other.ptr && std::ptr::eq(self.buffer, other.buffer) } } -impl<'a> Eq for Cursor<'a> {} +impl<'a, Span> Eq for Cursor<'a, Span> {} -impl<'a> Cursor<'a> { +impl<'a, Span> Cursor<'a, Span> { /// Check whether it is eof pub fn eof(self) -> bool { matches!(self.buffer.entry(&self.ptr), None | Some(Entry::End(None))) @@ -156,7 +164,7 @@ impl<'a> Cursor<'a> { /// If the cursor is pointing at the end of a subtree, returns /// the parent subtree - pub fn end(self) -> Option<&'a Subtree> { + pub fn end(self) -> Option<&'a Subtree> { match self.entry() { Some(Entry::End(Some(ptr))) => { let idx = ptr.1; @@ -171,13 +179,13 @@ impl<'a> Cursor<'a> { } } - fn entry(self) -> Option<&'a Entry<'a>> { + fn entry(&self) -> Option<&'a Entry<'a, Span>> { self.buffer.entry(&self.ptr) } /// If the cursor is pointing at a `Subtree`, returns /// a cursor into that subtree - pub fn subtree(self) -> Option> { + pub fn subtree(self) -> Option> { match self.entry() { Some(Entry::Subtree(_, _, entry_id)) => { Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0))) @@ -187,7 +195,7 @@ impl<'a> Cursor<'a> { } /// If the cursor is pointing at a `TokenTree`, returns it - pub fn token_tree(self) -> Option> { + pub fn token_tree(self) -> Option> { match self.entry() { Some(Entry::Leaf(tt)) => match tt { TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, tt)), @@ -198,12 +206,12 @@ impl<'a> Cursor<'a> { } } - fn create(buffer: &'a TokenBuffer<'_>, ptr: EntryPtr) -> Cursor<'a> { + fn create(buffer: &'a TokenBuffer<'_, Span>, ptr: EntryPtr) -> Cursor<'a, Span> { Cursor { buffer, ptr } } /// Bump the cursor - pub fn bump(self) -> Cursor<'a> { + pub fn bump(self) -> Cursor<'a, Span> { if let Some(Entry::End(exit)) = self.buffer.entry(&self.ptr) { match exit { Some(exit) => Cursor::create(self.buffer, *exit), @@ -216,7 +224,7 @@ impl<'a> Cursor<'a> { /// Bump the cursor, if it is a subtree, returns /// a cursor into that subtree - pub fn bump_subtree(self) -> Cursor<'a> { + pub fn bump_subtree(self) -> Cursor<'a, Span> { match self.entry() { Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(), _ => self.bump(), diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 353b09fd8c..b7dbc82e1d 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -16,45 +16,106 @@ pub use smol_str::SmolStr; /// which source tokens. We do it by assigning an distinct identity to each /// source token and making sure that identities are preserved during macro /// expansion. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct TokenId(pub u32); +impl fmt::Debug for TokenId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + impl TokenId { + pub const UNSPECIFIED: TokenId = TokenId(!0); pub const fn unspecified() -> TokenId { - TokenId(!0) + Self::UNSPECIFIED } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), -} -impl_from!(Leaf, Subtree for TokenTree); +pub mod token_id { + pub use crate::{DelimiterKind, Spacing, TokenId}; + pub type Span = crate::TokenId; + pub type Subtree = crate::Subtree; + pub type Punct = crate::Punct; + pub type Delimiter = crate::Delimiter; + pub type Leaf = crate::Leaf; + pub type Ident = crate::Ident; + pub type Literal = crate::Literal; + pub type TokenTree = crate::TokenTree; + pub mod buffer { + pub type TokenBuffer<'a> = crate::buffer::TokenBuffer<'a, super::Span>; + pub type Cursor<'a> = crate::buffer::Cursor<'a, super::Span>; + pub type TokenTreeRef<'a> = crate::buffer::TokenTreeRef<'a, super::Span>; + } -impl TokenTree { - pub fn empty() -> Self { - TokenTree::Subtree(Subtree::default()) + impl Delimiter { + pub const UNSPECIFIED: Self = Self { + open: TokenId::UNSPECIFIED, + close: TokenId::UNSPECIFIED, + kind: DelimiterKind::Invisible, + }; + pub const fn unspecified() -> Self { + Self::UNSPECIFIED + } + } + impl Subtree { + pub const fn empty() -> Self { + Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] } + } + } + impl TokenTree { + pub const fn empty() -> Self { + Self::Subtree(Subtree { delimiter: Delimiter::unspecified(), token_trees: vec![] }) + } } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), -} -impl_from!(Literal, Punct, Ident for Leaf); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SyntaxContext(pub u32); -#[derive(Clone, PartialEq, Eq, Hash, Default)] -pub struct Subtree { - pub delimiter: Option, - pub token_trees: Vec, +// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +// pub struct Span { +// pub id: TokenId, +// pub ctx: SyntaxContext, +// } +// pub type Span = (TokenId, SyntaxContext); + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), +} +impl_from!(Leaf, Subtree for TokenTree); + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), +} + +impl Leaf { + pub fn span(&self) -> &Span { + match self { + Leaf::Literal(it) => &it.span, + Leaf::Punct(it) => &it.span, + Leaf::Ident(it) => &it.span, + } + } +} +impl_from!(Literal, Punct, Ident for Leaf); + +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Subtree { + // FIXME, this should not be Option + pub delimiter: Delimiter, + pub token_trees: Vec>, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Delimiter { - pub id: TokenId, +pub struct Delimiter { + pub open: Span, + pub close: Span, pub kind: DelimiterKind, } @@ -63,19 +124,20 @@ pub enum DelimiterKind { Parenthesis, Brace, Bracket, + Invisible, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Literal { +pub struct Literal { pub text: SmolStr, - pub id: TokenId, + pub span: Span, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Punct { +pub struct Punct { pub char: char, pub spacing: Spacing, - pub id: TokenId, + pub span: Span, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -85,39 +147,25 @@ pub enum Spacing { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - /// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. +/// Identifier or keyword. Unlike rustc, we keep "r#" prefix when it represents a raw identifier. +pub struct Ident { pub text: SmolStr, - pub id: TokenId, + pub span: Span, } -impl Ident { - /// Constructor intended to be used only by proc macro server. `text` should not contain raw - /// identifier prefix. - pub fn new_with_is_raw(text: SmolStr, id: TokenId, is_raw: bool) -> Self { - let text = if is_raw { SmolStr::from_iter(["r#", &text]) } else { text }; - Ident { text, id } - } -} - -impl Leaf { - pub fn id(&self) -> TokenId { - match self { - Leaf::Literal(l) => l.id, - Leaf::Punct(p) => p.id, - Leaf::Ident(i) => i.id, - } - } -} - -fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result { +fn print_debug_subtree( + f: &mut fmt::Formatter<'_>, + subtree: &Subtree, + level: usize, +) -> fmt::Result { let align = " ".repeat(level); - let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) { - None => "$".to_string(), - Some((DelimiterKind::Parenthesis, id)) => format!("() {id}"), - Some((DelimiterKind::Brace, id)) => format!("{{}} {id}"), - Some((DelimiterKind::Bracket, id)) => format!("[] {id}"), + let Delimiter { kind, open, close } = &subtree.delimiter; + let aux = match kind { + DelimiterKind::Invisible => format!("$$ {:?} {:?}", open, close), + DelimiterKind::Parenthesis => format!("() {:?} {:?}", open, close), + DelimiterKind::Brace => format!("{{}} {:?} {:?}", open, close), + DelimiterKind::Bracket => format!("[] {:?} {:?}", open, close), }; if subtree.token_trees.is_empty() { @@ -135,21 +183,25 @@ fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usi Ok(()) } -fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result { +fn print_debug_token( + f: &mut fmt::Formatter<'_>, + tkn: &TokenTree, + level: usize, +) -> fmt::Result { let align = " ".repeat(level); match tkn { TokenTree::Leaf(leaf) => match leaf { - Leaf::Literal(lit) => write!(f, "{align}LITERAL {} {}", lit.text, lit.id.0)?, + Leaf::Literal(lit) => write!(f, "{}LITERAL {} {:?}", align, lit.text, lit.span)?, Leaf::Punct(punct) => write!( f, - "{}PUNCH {} [{}] {}", + "{}PUNCH {} [{}] {:?}", align, punct.char, if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, - punct.id.0 + punct.span )?, - Leaf::Ident(ident) => write!(f, "{align}IDENT {} {}", ident.text, ident.id.0)?, + Leaf::Ident(ident) => write!(f, "{}IDENT {} {:?}", align, ident.text, ident.span)?, }, TokenTree::Subtree(subtree) => { print_debug_subtree(f, subtree, level)?; @@ -159,13 +211,13 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) Ok(()) } -impl fmt::Debug for Subtree { +impl fmt::Debug for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { print_debug_subtree(f, self, 0) } } -impl fmt::Display for TokenTree { +impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { TokenTree::Leaf(it) => fmt::Display::fmt(it, f), @@ -174,13 +226,13 @@ impl fmt::Display for TokenTree { } } -impl fmt::Display for Subtree { +impl fmt::Display for Subtree { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let (l, r) = match self.delimiter_kind() { - Some(DelimiterKind::Parenthesis) => ("(", ")"), - Some(DelimiterKind::Brace) => ("{", "}"), - Some(DelimiterKind::Bracket) => ("[", "]"), - None => ("", ""), + let (l, r) = match self.delimiter.kind { + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Invisible => ("", ""), }; f.write_str(l)?; let mut needs_space = false; @@ -202,7 +254,7 @@ impl fmt::Display for Subtree { } } -impl fmt::Display for Leaf { +impl fmt::Display for Leaf { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Leaf::Ident(it) => fmt::Display::fmt(it, f), @@ -212,25 +264,25 @@ impl fmt::Display for Leaf { } } -impl fmt::Display for Ident { +impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Literal { +impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.text, f) } } -impl fmt::Display for Punct { +impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&self.char, f) } } -impl Subtree { +impl Subtree { /// Count the number of tokens recursively pub fn count(&self) -> usize { let children_count = self @@ -244,20 +296,16 @@ impl Subtree { self.token_trees.len() + children_count } - - pub fn delimiter_kind(&self) -> Option { - self.delimiter.map(|it| it.kind) - } } -impl Subtree { +impl Subtree { /// A simple line string used for debugging pub fn as_debug_string(&self) -> String { - let delim = match self.delimiter_kind() { - Some(DelimiterKind::Brace) => ("{", "}"), - Some(DelimiterKind::Bracket) => ("[", "]"), - Some(DelimiterKind::Parenthesis) => ("(", ")"), - None => (" ", " "), + let delim = match self.delimiter.kind { + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Invisible => ("$", "$"), }; let mut res = String::new(); @@ -275,7 +323,7 @@ impl Subtree { (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => { " ".to_string() + &s } - (Leaf::Punct(_), Some(&TokenTree::Leaf(Leaf::Punct(punct)))) => { + (Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => { if punct.spacing == Spacing::Alone { " ".to_string() + &s } else { @@ -298,19 +346,19 @@ impl Subtree { pub mod buffer; -pub fn pretty(tkns: &[TokenTree]) -> String { - fn tokentree_to_text(tkn: &TokenTree) -> String { +pub fn pretty(tkns: &[TokenTree]) -> String { + fn tokentree_to_text(tkn: &TokenTree) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => ident.text.clone().into(), TokenTree::Leaf(Leaf::Literal(literal)) => literal.text.clone().into(), TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Subtree(subtree) => { let content = pretty(&subtree.token_trees); - let (open, close) = match subtree.delimiter.map(|it| it.kind) { - None => ("", ""), - Some(DelimiterKind::Brace) => ("{", "}"), - Some(DelimiterKind::Parenthesis) => ("(", ")"), - Some(DelimiterKind::Bracket) => ("[", "]"), + let (open, close) = match subtree.delimiter.kind { + DelimiterKind::Brace => ("{", "}"), + DelimiterKind::Bracket => ("[", "]"), + DelimiterKind::Parenthesis => ("(", ")"), + DelimiterKind::Invisible => ("", ""), }; format!("{open}{content}{close}") }