internal: Lazy eager macros

This commit is contained in:
Lukas Wirth 2023-06-07 11:20:10 +02:00
parent 9973b11218
commit a02b9b279e
13 changed files with 355 additions and 276 deletions

View file

@ -113,10 +113,10 @@ impl Expander {
call_id: MacroCallId,
error: Option<ExpandError>,
) -> ExpandResult<Option<InFile<Parse<SyntaxNode>>>> {
let file_id = call_id.as_file();
let ExpandResult { value, err } = db.parse_or_expand_with_err(file_id);
let macro_file = call_id.as_macro_file();
let ExpandResult { value, err } = db.parse_macro_expansion(macro_file);
ExpandResult { value: Some(InFile::new(file_id, value)), err: error.or(err) }
ExpandResult { value: Some(InFile::new(macro_file.into(), value.0)), err: error.or(err) }
}
pub fn exit(&mut self, db: &dyn DefDatabase, mut mark: Mark) {
@ -179,8 +179,8 @@ impl Expander {
} else if self.recursion_limit.check(self.recursion_depth as usize + 1).is_err() {
self.recursion_depth = u32::MAX;
cov_mark::hit!(your_stack_belongs_to_me);
return ExpandResult::only_err(ExpandError::Other(
"reached recursion limit during macro expansion".into(),
return ExpandResult::only_err(ExpandError::other(
"reached recursion limit during macro expansion",
));
}

View file

@ -71,7 +71,7 @@ use hir_expand::{
builtin_derive_macro::BuiltinDeriveExpander,
builtin_fn_macro::{BuiltinFnLikeExpander, EagerExpander},
db::ExpandDatabase,
eager::expand_eager_macro,
eager::expand_eager_macro_input,
hygiene::Hygiene,
proc_macro::ProcMacroExpander,
AstId, ExpandError, ExpandResult, ExpandTo, HirFileId, InFile, MacroCallId, MacroCallKind,
@ -865,7 +865,7 @@ impl AsMacroCall for InFile<&ast::MacroCall> {
let path = self.value.path().and_then(|path| path::ModPath::from_src(db, path, &h));
let Some(path) = path else {
return Ok(ExpandResult::only_err(ExpandError::Other("malformed macro invocation".into())));
return Ok(ExpandResult::only_err(ExpandError::other("malformed macro invocation")));
};
macro_call_as_call_id_(
@ -913,7 +913,7 @@ fn macro_call_as_call_id_(
let res = if let MacroDefKind::BuiltInEager(..) = def.kind {
let macro_call = InFile::new(call.ast_id.file_id, call.ast_id.to_node(db));
expand_eager_macro(db, krate, macro_call, def, &resolver)?
expand_eager_macro_input(db, krate, macro_call, def, &resolver)?
} else {
ExpandResult {
value: Some(def.as_lazy_macro(

View file

@ -79,7 +79,7 @@ fn main() { env!("TEST_ENV_VAR"); }
#[rustc_builtin_macro]
macro_rules! env {() => {}}
fn main() { "__RA_UNIMPLEMENTED__"; }
fn main() { "UNRESOLVED_ENV_VAR"; }
"##]],
);
}
@ -442,10 +442,6 @@ macro_rules! surprise {
() => { "s" };
}
macro_rules! stuff {
($string:expr) => { concat!($string) };
}
fn main() { concat!(surprise!()); }
"##,
expect![[r##"
@ -456,10 +452,6 @@ macro_rules! surprise {
() => { "s" };
}
macro_rules! stuff {
($string:expr) => { concat!($string) };
}
fn main() { "s"; }
"##]],
);

View file

@ -194,15 +194,15 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MacroItems);
let macro_items = ast::MacroItems::cast(parsed.syntax_node()).ok_or_else(|| {
debug!("derive node didn't parse");
ExpandError::Other("invalid item definition".into())
ExpandError::other("invalid item definition")
})?;
let item = macro_items.items().next().ok_or_else(|| {
debug!("no module item parsed");
ExpandError::Other("no item found".into())
ExpandError::other("no item found")
})?;
let adt = ast::Adt::cast(item.syntax().clone()).ok_or_else(|| {
debug!("expected adt, found: {:?}", item);
ExpandError::Other("expected struct, enum or union".into())
ExpandError::other("expected struct, enum or union")
})?;
let (name, generic_param_list, shape) = match &adt {
ast::Adt::Struct(it) => (
@ -305,7 +305,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
fn name_to_token(token_map: &TokenMap, name: Option<ast::Name>) -> Result<tt::Ident, ExpandError> {
let name = name.ok_or_else(|| {
debug!("parsed item has no name");
ExpandError::Other("missing name".into())
ExpandError::other("missing name")
})?;
let name_token_id =
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);

View file

@ -14,7 +14,8 @@ use syntax::{
};
use crate::{
db::ExpandDatabase, name, quote, tt, ExpandError, ExpandResult, MacroCallId, MacroCallLoc,
db::ExpandDatabase, name, quote, tt, EagerCallInfo, ExpandError, ExpandResult, MacroCallId,
MacroCallLoc,
};
macro_rules! register_builtin {
@ -49,7 +50,7 @@ macro_rules! register_builtin {
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let expander = match *self {
$( EagerExpander::$e_kind => $e_expand, )*
};
@ -67,16 +68,9 @@ macro_rules! register_builtin {
};
}
#[derive(Debug)]
pub struct ExpandedEager {
pub(crate) subtree: tt::Subtree,
/// The included file ID of the include macro.
pub(crate) included_file: Option<(FileId, TokenMap)>,
}
impl ExpandedEager {
fn new(subtree: tt::Subtree) -> Self {
ExpandedEager { subtree, included_file: None }
impl EagerExpander {
pub fn is_include(&self) -> bool {
matches!(self, EagerExpander::Include)
}
}
@ -237,18 +231,16 @@ fn format_args_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
format_args_expand_general(db, id, tt, "")
.map(|x| ExpandedEager { subtree: x, included_file: None })
}
fn format_args_nl_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
format_args_expand_general(db, id, tt, "\\n")
.map(|x| ExpandedEager { subtree: x, included_file: None })
}
fn format_args_expand_general(
@ -509,23 +501,23 @@ fn compile_error_expand(
_db: &dyn ExpandDatabase,
_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let err = match &*tt.token_trees {
[tt::TokenTree::Leaf(tt::Leaf::Literal(it))] => match unquote_str(it) {
Some(unquoted) => ExpandError::Other(unquoted.into()),
None => ExpandError::Other("`compile_error!` argument must be a string".into()),
Some(unquoted) => ExpandError::other(unquoted),
None => ExpandError::other("`compile_error!` argument must be a string"),
},
_ => ExpandError::Other("`compile_error!` argument must be a string".into()),
_ => ExpandError::other("`compile_error!` argument must be a string"),
};
ExpandResult { value: ExpandedEager::new(quote! {}), err: Some(err) }
ExpandResult { value: quote! {}, err: Some(err) }
}
fn concat_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut err = None;
let mut text = String::new();
for (i, mut t) in tt.token_trees.iter().enumerate() {
@ -564,14 +556,14 @@ fn concat_expand(
}
}
}
ExpandResult { value: ExpandedEager::new(quote!(#text)), err }
ExpandResult { value: quote!(#text), err }
}
fn concat_bytes_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut bytes = Vec::new();
let mut err = None;
for (i, t) in tt.token_trees.iter().enumerate() {
@ -604,7 +596,7 @@ fn concat_bytes_expand(
}
}
let ident = tt::Ident { text: bytes.join(", ").into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!([#ident])), err }
ExpandResult { value: quote!([#ident]), err }
}
fn concat_bytes_expand_subtree(
@ -637,7 +629,7 @@ fn concat_idents_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let mut err = None;
let mut ident = String::new();
for (i, t) in tt.token_trees.iter().enumerate() {
@ -652,7 +644,7 @@ fn concat_idents_expand(
}
}
let ident = tt::Ident { text: ident.into(), span: tt::TokenId::unspecified() };
ExpandResult { value: ExpandedEager::new(quote!(#ident)), err }
ExpandResult { value: quote!(#ident), err }
}
fn relative_file(
@ -665,10 +657,10 @@ fn relative_file(
let path = AnchoredPath { anchor: call_site, path: path_str };
let res = db
.resolve_path(path)
.ok_or_else(|| ExpandError::Other(format!("failed to load file `{path_str}`").into()))?;
.ok_or_else(|| ExpandError::other(format!("failed to load file `{path_str}`")))?;
// Prevent include itself
if res == call_site && !allow_recursion {
Err(ExpandError::Other(format!("recursive inclusion of `{path_str}`").into()))
Err(ExpandError::other(format!("recursive inclusion of `{path_str}`")))
} else {
Ok(res)
}
@ -687,38 +679,37 @@ fn parse_string(tt: &tt::Subtree) -> Result<String, ExpandError> {
fn include_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
let res = (|| {
let path = parse_string(tt)?;
let file_id = relative_file(db, arg_id, &path, false)?;
let (subtree, map) =
parse_to_token_tree(&db.file_text(file_id)).ok_or(mbe::ExpandError::ConversionError)?;
Ok((subtree, map, file_id))
})();
match res {
Ok((subtree, map, file_id)) => {
ExpandResult::ok(ExpandedEager { subtree, included_file: Some((file_id, map)) })
}
Err(e) => ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
),
_tt: &tt::Subtree,
) -> ExpandResult<tt::Subtree> {
match db.include_expand(arg_id) {
Ok((res, _)) => ExpandResult::ok(res.0.clone()),
Err(e) => ExpandResult::new(tt::Subtree::empty(), e),
}
}
pub(crate) fn include_arg_to_tt(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
) -> Result<(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, TokenMap)>, FileId), ExpandError> {
let loc = db.lookup_intern_macro_call(arg_id);
let Some(EagerCallInfo {arg, arg_id: Some(arg_id), .. }) = loc.eager.as_deref() else {
panic!("include_arg_to_tt called on non include macro call: {:?}", &loc.eager);
};
let path = parse_string(&arg.0)?;
let file_id = relative_file(db, *arg_id, &path, false)?;
let (subtree, map) =
parse_to_token_tree(&db.file_text(file_id)).ok_or(mbe::ExpandError::ConversionError)?;
Ok((triomphe::Arc::new((subtree, map)), file_id))
}
fn include_bytes_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
if let Err(e) = parse_string(tt) {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
);
return ExpandResult::new(tt::Subtree::empty(), e);
}
// FIXME: actually read the file here if the user asked for macro expansion
@ -729,22 +720,17 @@ fn include_bytes_expand(
span: tt::TokenId::unspecified(),
}))],
};
ExpandResult::ok(ExpandedEager::new(res))
ExpandResult::ok(res)
}
fn include_str_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let path = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
// FIXME: we're not able to read excluded files (which is most of them because
@ -754,14 +740,14 @@ fn include_str_expand(
let file_id = match relative_file(db, arg_id, &path, true) {
Ok(file_id) => file_id,
Err(_) => {
return ExpandResult::ok(ExpandedEager::new(quote!("")));
return ExpandResult::ok(quote!(""));
}
};
let text = db.file_text(file_id);
let text = &*text;
ExpandResult::ok(ExpandedEager::new(quote!(#text)))
ExpandResult::ok(quote!(#text))
}
fn get_env_inner(db: &dyn ExpandDatabase, arg_id: MacroCallId, key: &str) -> Option<String> {
@ -773,15 +759,10 @@ fn env_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
let mut err = None;
@ -789,35 +770,28 @@ fn env_expand(
// The only variable rust-analyzer ever sets is `OUT_DIR`, so only diagnose that to avoid
// unnecessary diagnostics for eg. `CARGO_PKG_NAME`.
if key == "OUT_DIR" {
err = Some(ExpandError::Other(
r#"`OUT_DIR` not set, enable "build scripts" to fix"#.into(),
));
err = Some(ExpandError::other(r#"`OUT_DIR` not set, enable "build scripts" to fix"#));
}
// If the variable is unset, still return a dummy string to help type inference along.
// We cannot use an empty string here, because for
// `include!(concat!(env!("OUT_DIR"), "/foo.rs"))` will become
// `include!("foo.rs"), which might go to infinite loop
"__RA_UNIMPLEMENTED__".to_string()
"UNRESOLVED_ENV_VAR".to_string()
});
let expanded = quote! { #s };
ExpandResult { value: ExpandedEager::new(expanded), err }
ExpandResult { value: expanded, err }
}
fn option_env_expand(
db: &dyn ExpandDatabase,
arg_id: MacroCallId,
tt: &tt::Subtree,
) -> ExpandResult<ExpandedEager> {
) -> ExpandResult<tt::Subtree> {
let key = match parse_string(tt) {
Ok(it) => it,
Err(e) => {
return ExpandResult::new(
ExpandedEager { subtree: tt::Subtree::empty(), included_file: None },
e,
)
}
Err(e) => return ExpandResult::new(tt::Subtree::empty(), e),
};
// FIXME: Use `DOLLAR_CRATE` when that works in eager macros.
let expanded = match get_env_inner(db, arg_id, &key) {
@ -825,5 +799,5 @@ fn option_env_expand(
Some(s) => quote! { ::core::option::Option::Some(#s) },
};
ExpandResult::ok(ExpandedEager::new(expanded))
ExpandResult::ok(expanded)
}

View file

@ -14,9 +14,9 @@ use triomphe::Arc;
use crate::{
ast_id_map::AstIdMap, builtin_attr_macro::pseudo_derive_attr_expansion,
builtin_fn_macro::EagerExpander, fixup, hygiene::HygieneFrame, tt, BuiltinAttrExpander,
BuiltinDeriveExpander, BuiltinFnLikeExpander, ExpandError, ExpandResult, ExpandTo, HirFileId,
HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile,
ProcMacroExpander,
BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo, ExpandError, ExpandResult,
ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, MacroDefId,
MacroDefKind, MacroFile, ProcMacroExpander,
};
/// Total limit on the number of tokens produced by any macro invocation.
@ -53,9 +53,7 @@ impl TokenExpander {
match self {
TokenExpander::DeclarativeMacro { mac, .. } => mac.expand(tt).map_err(Into::into),
TokenExpander::Builtin(it) => it.expand(db, id, tt).map_err(Into::into),
TokenExpander::BuiltinEager(it) => {
it.expand(db, id, tt).map_err(Into::into).map(|res| res.subtree)
}
TokenExpander::BuiltinEager(it) => it.expand(db, id, tt).map_err(Into::into),
TokenExpander::BuiltinAttr(it) => it.expand(db, id, tt),
TokenExpander::BuiltinDerive(it) => it.expand(db, id, tt),
TokenExpander::ProcMacro(_) => {
@ -132,6 +130,14 @@ pub trait ExpandDatabase: SourceDatabase {
/// Expand macro call to a token tree.
// This query is LRU cached
fn macro_expand(&self, macro_call: MacroCallId) -> ExpandResult<Arc<tt::Subtree>>;
#[salsa::invoke(crate::builtin_fn_macro::include_arg_to_tt)]
fn include_expand(
&self,
arg_id: MacroCallId,
) -> Result<
(triomphe::Arc<(::tt::Subtree<::tt::TokenId>, mbe::TokenMap)>, base_db::FileId),
ExpandError,
>;
/// Special case of the previous query for procedural macros. We can't LRU
/// proc macros, since they are not deterministic in general, and
/// non-determinism breaks salsa in a very, very, very bad way.
@ -281,31 +287,6 @@ fn parse_macro_expansion(
let _p = profile::span("parse_macro_expansion");
let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id);
if let Some(err) = &err {
if tracing::enabled!(tracing::Level::DEBUG) {
// Note:
// The final goal we would like to make all parse_macro success,
// such that the following log will not call anyway.
let loc = db.lookup_intern_macro_call(macro_file.macro_call_id);
let node = loc.to_node(db);
// collect parent information for warning log
let parents = std::iter::successors(loc.kind.file_id().call_node(db), |it| {
it.file_id.call_node(db)
})
.map(|n| format!("{:#}", n.value))
.collect::<Vec<_>>()
.join("\n");
tracing::debug!(
"fail on macro_parse: (reason: {:?} macro_call: {:#}) parents: {}",
err,
node.value,
parents
);
}
}
let expand_to = macro_expand_to(db, macro_file.macro_call_id);
tracing::debug!("expanded = {}", tt.as_debug_string());
@ -320,9 +301,14 @@ fn macro_arg(
db: &dyn ExpandDatabase,
id: MacroCallId,
) -> Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>> {
let arg = db.macro_arg_text(id)?;
let loc = db.lookup_intern_macro_call(id);
if let Some(EagerCallInfo { arg, arg_id: Some(_), error: _ }) = loc.eager.as_deref() {
return Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default())));
}
let arg = db.macro_arg_text(id)?;
let node = SyntaxNode::new_root(arg);
let censor = censor_for_macro_input(&loc, &node);
let mut fixups = fixup::fixup_syntax(&node);
@ -398,7 +384,17 @@ fn macro_arg_text(db: &dyn ExpandDatabase, id: MacroCallId) -> Option<GreenNode>
return None;
}
}
Some(arg.green().into())
if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() {
Some(
mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::Expr)
.0
.syntax_node()
.green()
.into(),
)
} else {
Some(arg.green().into())
}
}
fn macro_def(
@ -445,23 +441,21 @@ fn macro_def(
fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt::Subtree>> {
let _p = profile::span("macro_expand");
let loc = db.lookup_intern_macro_call(id);
if let Some(eager) = &loc.eager {
return ExpandResult { value: eager.arg_or_expansion.clone(), err: eager.error.clone() };
if let Some(EagerCallInfo { arg, arg_id: None, error }) = loc.eager.as_deref() {
// This is an input expansion for an eager macro. These are already pre-expanded
return ExpandResult { value: Arc::new(arg.0.clone()), err: error.clone() };
}
let expander = match db.macro_def(loc.def) {
Ok(it) => it,
// FIXME: This is weird -- we effectively report macro *definition*
// errors lazily, when we try to expand the macro. Instead, they should
// be reported at the definition site when we construct a def map.
// (Note we do report them also at the definition site in the late diagnostic pass)
// FIXME: We should make sure to enforce a variant that invalid macro
// definitions do not get expanders that could reach this call path!
Err(err) => {
return ExpandResult {
value: Arc::new(tt::Subtree {
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}),
err: Some(ExpandError::Other(format!("invalid macro definition: {err}").into())),
err: Some(ExpandError::other(format!("invalid macro definition: {err}"))),
}
}
};
@ -473,13 +467,21 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
token_trees: Vec::new(),
},
),
err: Some(ExpandError::Other(
// FIXME: We should make sure to enforce a variant that invalid macro
// calls do not reach this call path!
err: Some(ExpandError::other(
"invalid token tree"
.into(),
)),
};
};
let ExpandResult { value: mut tt, err } = expander.expand(db, id, &macro_arg.0);
let (arg_tt, arg_tm, undo_info) = &*macro_arg;
let ExpandResult { value: mut tt, mut err } = expander.expand(db, id, arg_tt);
if let Some(EagerCallInfo { error, .. }) = loc.eager.as_deref() {
// FIXME: We should report both errors!
err = error.clone().or(err);
}
// Set a hard limit for the expanded tt
let count = tt.count();
if TOKEN_LIMIT.check(count).is_err() {
@ -488,18 +490,15 @@ fn macro_expand(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<Arc<tt
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: vec![],
}),
err: Some(ExpandError::Other(
format!(
"macro invocation exceeds token limit: produced {} tokens, limit is {}",
count,
TOKEN_LIMIT.inner(),
)
.into(),
)),
err: Some(ExpandError::other(format!(
"macro invocation exceeds token limit: produced {} tokens, limit is {}",
count,
TOKEN_LIMIT.inner(),
))),
};
}
fixup::reverse_fixups(&mut tt, &macro_arg.1, &macro_arg.2);
fixup::reverse_fixups(&mut tt, arg_tm, undo_info);
ExpandResult { value: Arc::new(tt), err }
}
@ -520,9 +519,8 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<t
delimiter: tt::Delimiter::UNSPECIFIED,
token_trees: Vec::new(),
},
err: Some(ExpandError::Other(
err: Some(ExpandError::other(
"invalid token tree"
.into(),
)),
};
};

View file

@ -31,22 +31,24 @@ use crate::{
MacroCallLoc, MacroDefId, MacroDefKind, UnresolvedMacro,
};
pub fn expand_eager_macro(
pub fn expand_eager_macro_input(
db: &dyn ExpandDatabase,
krate: CrateId,
macro_call: InFile<ast::MacroCall>,
def: MacroDefId,
resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
) -> Result<ExpandResult<Option<MacroCallId>>, UnresolvedMacro> {
let MacroDefKind::BuiltInEager(eager, _) = def.kind else {
panic!("called `expand_eager_macro` on non-eager macro def {def:?}")
assert!(matches!(def.kind, MacroDefKind::BuiltInEager(..)));
let token_tree = macro_call.value.token_tree();
let Some(token_tree) = token_tree else {
return Ok(ExpandResult { value: None, err:
Some(ExpandError::other(
"invalid token tree"
)),
});
};
let hygiene = Hygiene::new(db, macro_call.file_id);
let parsed_args = macro_call
.value
.token_tree()
.map(|tt| mbe::syntax_node_to_token_tree(tt.syntax()).0)
.unwrap_or_else(tt::Subtree::empty);
let (parsed_args, arg_token_map) = mbe::syntax_node_to_token_tree(token_tree.syntax());
let ast_map = db.ast_id_map(macro_call.file_id);
let call_id = InFile::new(macro_call.file_id, ast_map.ast_id(&macro_call.value));
@ -60,41 +62,40 @@ pub fn expand_eager_macro(
def,
krate,
eager: Some(Box::new(EagerCallInfo {
arg_or_expansion: Arc::new(parsed_args.clone()),
included_file: None,
arg: Arc::new((parsed_args, arg_token_map)),
arg_id: None,
error: None,
})),
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr },
});
let parsed_args = mbe::token_tree_to_syntax_node(&parsed_args, mbe::TopEntryPoint::Expr).0;
let ExpandResult { value, mut err } = eager_macro_recur(
let arg_as_expr = match db.macro_arg_text(arg_id) {
Some(it) => it,
None => {
return Ok(ExpandResult {
value: None,
err: Some(ExpandError::other("invalid token tree")),
})
}
};
let ExpandResult { value: expanded_eager_input, err } = eager_macro_recur(
db,
&hygiene,
InFile::new(arg_id.as_file(), parsed_args.syntax_node()),
&Hygiene::new(db, macro_call.file_id),
InFile::new(arg_id.as_file(), SyntaxNode::new_root(arg_as_expr)),
krate,
resolver,
)?;
let Some(value ) = value else {
let Some(expanded_eager_input) = expanded_eager_input else {
return Ok(ExpandResult { value: None, err })
};
let subtree = {
let mut subtree = mbe::syntax_node_to_token_tree(&value).0;
subtree.delimiter = crate::tt::Delimiter::unspecified();
subtree
};
let res = eager.expand(db, arg_id, &subtree);
if err.is_none() {
err = res.err;
}
let (mut subtree, token_map) = mbe::syntax_node_to_token_tree(&expanded_eager_input);
subtree.delimiter = crate::tt::Delimiter::unspecified();
let loc = MacroCallLoc {
def,
krate,
eager: Some(Box::new(EagerCallInfo {
arg_or_expansion: Arc::new(res.value.subtree),
included_file: res.value.included_file,
arg: Arc::new((subtree, token_map)),
arg_id: Some(arg_id),
error: err.clone(),
})),
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to },
@ -118,8 +119,9 @@ fn lazy_expand(
MacroCallKind::FnLike { ast_id: macro_call.with_value(ast_id), expand_to },
);
let file_id = id.as_file();
db.parse_or_expand_with_err(file_id).map(|parse| InFile::new(file_id, parse))
let macro_file = id.as_macro_file();
db.parse_macro_expansion(macro_file).map(|parse| InFile::new(macro_file.into(), parse.0))
}
fn eager_macro_recur(
@ -142,13 +144,13 @@ fn eager_macro_recur(
let def = match child.path().and_then(|path| ModPath::from_src(db, path, hygiene)) {
Some(path) => macro_resolver(path.clone()).ok_or(UnresolvedMacro { path })?,
None => {
error = Some(ExpandError::Other("malformed macro invocation".into()));
error = Some(ExpandError::other("malformed macro invocation"));
continue;
}
};
let ExpandResult { value, err } = match def.kind {
MacroDefKind::BuiltInEager(..) => {
let id = match expand_eager_macro(
let ExpandResult { value, err } = match expand_eager_macro_input(
db,
krate,
curr.with_value(child.clone()),
@ -158,9 +160,17 @@ fn eager_macro_recur(
Ok(it) => it,
Err(err) => return Err(err),
};
id.map(|call| {
call.map(|call| db.parse_or_expand(call.as_file()).clone_for_update())
})
match value {
Some(call) => {
let ExpandResult { value, err: err2 } =
db.parse_macro_expansion(call.as_macro_file());
ExpandResult {
value: Some(value.0.syntax_node().clone_for_update()),
err: err.or(err2),
}
}
None => ExpandResult { value: None, err },
}
}
MacroDefKind::Declarative(_)
| MacroDefKind::BuiltIn(..)
@ -180,7 +190,7 @@ fn eager_macro_recur(
krate,
macro_resolver,
)?;
let err = if err.is_none() { error } else { err };
let err = err.or(error);
ExpandResult { value, err }
}
};

View file

@ -58,7 +58,13 @@ pub enum ExpandError {
UnresolvedProcMacro(CrateId),
Mbe(mbe::ExpandError),
RecursionOverflowPoisoned,
Other(Box<str>),
Other(Box<Box<str>>),
}
impl ExpandError {
pub fn other(msg: impl Into<Box<str>>) -> Self {
ExpandError::Other(Box::new(msg.into()))
}
}
impl From<mbe::ExpandError> for ExpandError {
@ -97,9 +103,15 @@ impl fmt::Display for ExpandError {
/// The two variants are encoded in a single u32 which are differentiated by the MSB.
/// If the MSB is 0, the value represents a `FileId`, otherwise the remaining 31 bits represent a
/// `MacroCallId`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct HirFileId(u32);
impl fmt::Debug for HirFileId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.repr().fmt(f)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct MacroFile {
pub macro_call_id: MacroCallId,
@ -115,6 +127,7 @@ impl_intern_key!(MacroCallId);
pub struct MacroCallLoc {
pub def: MacroDefId,
pub(crate) krate: CrateId,
/// Some if `def` is a builtin eager macro.
eager: Option<Box<EagerCallInfo>>,
pub kind: MacroCallKind,
}
@ -140,8 +153,10 @@ pub enum MacroDefKind {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct EagerCallInfo {
/// NOTE: This can be *either* the expansion result, *or* the argument to the eager macro!
arg_or_expansion: Arc<tt::Subtree>,
included_file: Option<(FileId, TokenMap)>,
arg: Arc<(tt::Subtree, TokenMap)>,
/// call id of the eager macro's input file. If this is none, macro call containing this call info
/// is an eager macro's input, otherwise it is its output.
arg_id: Option<MacroCallId>,
error: Option<ExpandError>,
}
@ -206,10 +221,15 @@ impl HirFileId {
HirFileIdRepr::FileId(id) => break id,
HirFileIdRepr::MacroFile(MacroFile { macro_call_id }) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_call_id);
file_id = match loc.eager.as_deref() {
Some(&EagerCallInfo { included_file: Some((file, _)), .. }) => file.into(),
let is_include_expansion = loc.def.is_include()
&& matches!(
loc.eager.as_deref(),
Some(EagerCallInfo { arg_id: Some(_), .. })
);
file_id = match is_include_expansion.then(|| db.include_expand(macro_call_id)) {
Some(Ok((_, file))) => file.into(),
_ => loc.kind.file_id(),
};
}
}
}
}
@ -325,7 +345,17 @@ impl HirFileId {
match self.macro_file() {
Some(macro_file) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id);
matches!(loc.eager.as_deref(), Some(EagerCallInfo { included_file: Some(..), .. }))
loc.def.is_include()
}
_ => false,
}
}
pub fn is_eager(&self, db: &dyn db::ExpandDatabase) -> bool {
match self.macro_file() {
Some(macro_file) => {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id);
matches!(loc.eager.as_deref(), Some(EagerCallInfo { .. }))
}
_ => false,
}
@ -423,6 +453,10 @@ impl MacroDefId {
pub fn is_attribute_derive(&self) -> bool {
matches!(self.kind, MacroDefKind::BuiltInAttr(expander, ..) if expander.is_derive())
}
pub fn is_include(&self) -> bool {
matches!(self.kind, MacroDefKind::BuiltInEager(expander, ..) if expander.is_include())
}
}
impl MacroCallLoc {
@ -569,6 +603,10 @@ impl MacroCallId {
pub fn as_file(self) -> HirFileId {
MacroFile { macro_call_id: self }.into()
}
pub fn as_macro_file(self) -> MacroFile {
MacroFile { macro_call_id: self }
}
}
/// ExpansionInfo mainly describes how to map text range between src and expanded macro
@ -662,7 +700,7 @@ impl ExpansionInfo {
let token_id = match token_id_in_attr_input {
Some(token_id) => token_id,
// the token is not inside an attribute's input so do the lookup in the macro_arg as usual
// the token is not inside `an attribute's input so do the lookup in the macro_arg as usual
None => {
let relative_range =
token.value.text_range().checked_sub(self.arg.value.text_range().start())?;
@ -694,14 +732,18 @@ impl ExpansionInfo {
let call_id = self.expanded.file_id.macro_file()?.macro_call_id;
let loc = db.lookup_intern_macro_call(call_id);
if let Some((file, map)) = loc.eager.and_then(|e| e.included_file) {
// Special case: map tokens from `include!` expansions to the included file
let range = map.first_range_by_token(token_id, token.value.kind())?;
let source = db.parse(file);
// Special case: map tokens from `include!` expansions to the included file
if loc.def.is_include()
&& matches!(loc.eager.as_deref(), Some(EagerCallInfo { arg_id: Some(_), .. }))
{
if let Ok((tt_and_map, file_id)) = db.include_expand(call_id) {
let range = tt_and_map.1.first_range_by_token(token_id, token.value.kind())?;
let source = db.parse(file_id);
let token = source.syntax_node().covering_element(range).into_token()?;
let token = source.syntax_node().covering_element(range).into_token()?;
return Some((InFile::new(file.into(), token), Origin::Call));
return Some((InFile::new(file_id.into(), token), Origin::Call));
}
}
// Attributes are a bit special for us, they have two inputs, the input tokentree and the annotated item.

View file

@ -46,7 +46,7 @@ impl ProcMacroExpander {
never!("Non-dummy expander even though there are no proc macros");
return ExpandResult::new(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
ExpandError::other("Internal error"),
);
}
};
@ -60,7 +60,7 @@ impl ProcMacroExpander {
);
return ExpandResult::new(
tt::Subtree::empty(),
ExpandError::Other("Internal error".into()),
ExpandError::other("Internal error"),
);
}
};
@ -75,14 +75,11 @@ impl ProcMacroExpander {
ProcMacroExpansionError::System(text)
if proc_macro.kind == ProcMacroKind::Attr =>
{
ExpandResult {
value: tt.clone(),
err: Some(ExpandError::Other(text.into())),
}
ExpandResult { value: tt.clone(), err: Some(ExpandError::other(text)) }
}
ProcMacroExpansionError::System(text)
| ProcMacroExpansionError::Panic(text) => {
ExpandResult::new(tt::Subtree::empty(), ExpandError::Other(text.into()))
ExpandResult::new(tt::Subtree::empty(), ExpandError::other(text))
}
},
}

View file

@ -20,8 +20,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -47,8 +49,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -74,8 +78,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -101,8 +107,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -128,8 +136,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -155,8 +165,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -182,8 +194,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,

View file

@ -18,8 +18,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: TYPE_ALIAS,
@ -43,8 +45,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: CONST,
@ -68,8 +72,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: CONST,
@ -95,8 +101,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: ENUM,
@ -122,8 +130,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_DEF,
@ -147,8 +157,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STATIC,
@ -174,8 +186,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -201,8 +215,12 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
2147483648,
hir_file_id: MacroFile(
MacroFile {
macro_call_id: MacroCallId(
0,
),
},
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -228,8 +246,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -257,8 +277,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -286,8 +308,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -311,8 +335,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: TRAIT,
@ -338,8 +364,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: UNION,
@ -365,8 +393,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MODULE,
@ -392,8 +422,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MODULE,
@ -419,8 +451,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_RULES,
@ -444,8 +478,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -471,8 +507,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: MACRO_RULES,
@ -496,8 +534,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -521,8 +561,10 @@
},
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: FN,
@ -561,8 +603,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
0,
hir_file_id: FileId(
FileId(
0,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,
@ -599,8 +643,10 @@
),
),
loc: DeclarationLocation {
hir_file_id: HirFileId(
1,
hir_file_id: FileId(
FileId(
1,
),
),
ptr: SyntaxNodePtr {
kind: STRUCT,

View file

@ -86,6 +86,8 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">assert</span> <span class="brace">{</span><span class="brace">}</span>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">asm</span> <span class="brace">{</span><span class="brace">}</span>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">rustc_builtin_macro</span><span class="attribute_bracket attribute">]</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">concat</span> <span class="brace">{</span><span class="brace">}</span>
<span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">toho</span> <span class="brace">{</span>
<span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="operator">=</span><span class="angle">&gt;</span> <span class="parenthesis">(</span><span class="punctuation">$</span>crate<span class="colon">:</span><span class="colon">:</span>panic<span class="punctuation">!</span><span class="parenthesis">(</span><span class="string_literal">"not yet implemented"</span><span class="parenthesis">)</span><span class="parenthesis">)</span><span class="semicolon">;</span>
@ -172,4 +174,5 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">toho</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">fmt"</span><span class="comma macro">,</span> <span class="numeric_literal macro">0</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro unsafe">asm</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"mov eax, </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="none macro">concat</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="comma macro">,</span> <span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="none macro">format_args</span><span class="operator macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="numeric_literal macro">0</span><span class="parenthesis macro">)</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="brace">}</span></code></pre>

View file

@ -432,6 +432,8 @@ macro_rules! panic {}
macro_rules! assert {}
#[rustc_builtin_macro]
macro_rules! asm {}
#[rustc_builtin_macro]
macro_rules! concat {}
macro_rules! toho {
() => ($crate::panic!("not yet implemented"));
@ -518,6 +520,7 @@ fn main() {
toho!("{}fmt", 0);
asm!("mov eax, {0}");
format_args!(concat!("{}"), "{}");
format_args!("{}", format_args!("{}", 0));
}"#,
expect_file!["./test_data/highlight_strings.html"],
false,