From 24d65bb7cf8e24d651be1859efc6e2dea029fb3b Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 4 Oct 2024 11:08:11 +0200 Subject: [PATCH] internal: Filter out opaque tokens in some of IDE feature macro descensions --- crates/hir-expand/src/db.rs | 17 +++-- crates/hir-expand/src/hygiene.rs | 4 + crates/hir-expand/src/lib.rs | 24 +++--- crates/hir/src/semantics.rs | 100 ++++++++++++++----------- crates/ide-db/src/search.rs | 10 ++- crates/ide/src/goto_declaration.rs | 2 +- crates/ide/src/goto_definition.rs | 2 +- crates/ide/src/goto_type_definition.rs | 2 +- crates/ide/src/syntax_highlighting.rs | 3 +- crates/span/src/hygiene.rs | 9 +++ crates/span/src/map.rs | 11 ++- 11 files changed, 114 insertions(+), 70 deletions(-) diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 584f9631e3..484a8662eb 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -16,7 +16,10 @@ use crate::{ cfg_process, declarative::DeclarativeMacroExpander, fixup::{self, SyntaxFixupUndoInfo}, - hygiene::{span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt}, + hygiene::{ + span_with_call_site_ctxt, span_with_def_site_ctxt, span_with_mixed_site_ctxt, + SyntaxContextExt as _, + }, proc_macro::ProcMacros, span_map::{RealSpanMap, SpanMap, SpanMapRef}, tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, @@ -300,14 +303,16 @@ pub fn expand_speculative( token_tree_to_syntax_node(&speculative_expansion.value, expand_to, loc.def.edition); let syntax_node = node.syntax_node(); - let token = rev_tmap + let (token, _) = rev_tmap .ranges_with_span(span_map.span_for_range(token_to_map.text_range())) - .filter_map(|range| syntax_node.covering_element(range).into_token()) - .min_by_key(|t| { - // prefer tokens of the same kind and text + .filter_map(|(range, ctx)| syntax_node.covering_element(range).into_token().zip(Some(ctx))) + .min_by_key(|(t, ctx)| { + // prefer tokens of the same kind and text, as well as non opaque marked ones // Note the inversion of the score here, as we want to prefer the first token in case // of all tokens having the same score - (t.kind() != token_to_map.kind()) as u8 + 2 * ((t.text() != token_to_map.text()) as u8) + ctx.is_opaque(db) as u8 + + 2 * (t.kind() != token_to_map.kind()) as u8 + + 4 * ((t.text() != token_to_map.text()) as u8) })?; Some((node.syntax_node(), token)) } diff --git a/crates/hir-expand/src/hygiene.rs b/crates/hir-expand/src/hygiene.rs index cc02332207..5e1448f795 100644 --- a/crates/hir-expand/src/hygiene.rs +++ b/crates/hir-expand/src/hygiene.rs @@ -151,6 +151,7 @@ pub trait SyntaxContextExt { fn remove_mark(&mut self, db: &dyn ExpandDatabase) -> (Option, Transparency); fn outer_mark(self, db: &dyn ExpandDatabase) -> (Option, Transparency); fn marks(self, db: &dyn ExpandDatabase) -> Vec<(MacroCallId, Transparency)>; + fn is_opaque(self, db: &dyn ExpandDatabase) -> bool; } impl SyntaxContextExt for SyntaxContextId { @@ -177,6 +178,9 @@ impl SyntaxContextExt for SyntaxContextId { marks.reverse(); marks } + fn is_opaque(self, db: &dyn ExpandDatabase) -> bool { + !self.is_root() && db.lookup_intern_syntax_context(self).outer_transparency.is_opaque() + } } // FIXME: Make this a SyntaxContextExt method once we have RPIT diff --git a/crates/hir-expand/src/lib.rs b/crates/hir-expand/src/lib.rs index 9538097949..56cb5fd375 100644 --- a/crates/hir-expand/src/lib.rs +++ b/crates/hir-expand/src/lib.rs @@ -25,6 +25,7 @@ mod prettify_macro_expansion_; use attrs::collect_attrs; use rustc_hash::FxHashMap; +use stdx::TupleExt; use triomphe::Arc; use std::hash::Hash; @@ -772,14 +773,15 @@ impl ExpansionInfo { /// Maps the passed in file range down into a macro expansion if it is the input to a macro call. /// /// Note this does a linear search through the entire backing vector of the spanmap. + // FIXME: Consider adding a reverse map to ExpansionInfo to get rid of the linear search which + // potentially results in quadratic look ups (notably this might improve semantic highlighting perf) pub fn map_range_down_exact( &self, span: Span, - ) -> Option + '_>> { - let tokens = self - .exp_map - .ranges_with_span_exact(span) - .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); + ) -> Option + '_>> { + let tokens = self.exp_map.ranges_with_span_exact(span).flat_map(move |(range, ctx)| { + self.expanded.value.covering_element(range).into_token().zip(Some(ctx)) + }); Some(InMacroFile::new(self.expanded.file_id, tokens)) } @@ -791,11 +793,10 @@ impl ExpansionInfo { pub fn map_range_down( &self, span: Span, - ) -> Option + '_>> { - let tokens = self - .exp_map - .ranges_with_span(span) - .flat_map(move |range| self.expanded.value.covering_element(range).into_token()); + ) -> Option + '_>> { + let tokens = self.exp_map.ranges_with_span(span).flat_map(move |(range, ctx)| { + self.expanded.value.covering_element(range).into_token().zip(Some(ctx)) + }); Some(InMacroFile::new(self.expanded.file_id, tokens)) } @@ -845,7 +846,8 @@ impl ExpansionInfo { self.arg.file_id, arg_map .ranges_with_span_exact(span) - .filter(|range| range.intersect(arg_range).is_some()) + .filter(|(range, _)| range.intersect(arg_range).is_some()) + .map(TupleExt::head) .collect(), ) } diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index fa14b53dbc..bae0687e6a 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -24,6 +24,7 @@ use hir_expand::{ builtin::{BuiltinFnLikeExpander, EagerExpander}, db::ExpandDatabase, files::InRealFile, + hygiene::SyntaxContextExt as _, inert_attr_macro::find_builtin_attr_idx, name::AsName, FileRange, InMacroFile, MacroCallId, MacroFileId, MacroFileIdExt, @@ -32,7 +33,7 @@ use intern::Symbol; use itertools::Itertools; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::{smallvec, SmallVec}; -use span::{EditionedFileId, FileId, HirFileIdRepr}; +use span::{EditionedFileId, FileId, HirFileIdRepr, SyntaxContextId}; use stdx::TupleExt; use syntax::{ algo::skip_trivia_token, @@ -608,7 +609,7 @@ impl<'db> SemanticsImpl<'db> { let quote = string.open_quote_text_range()?; let token = self.wrap_token_infile(string.syntax().clone()).into_real_file().ok()?; - self.descend_into_macros_breakable(token, |token| { + self.descend_into_macros_breakable(token, |token, _| { (|| { let token = token.value; let string = ast::String::cast(token)?; @@ -655,7 +656,7 @@ impl<'db> SemanticsImpl<'db> { let original_string = ast::String::cast(original_token.clone())?; let original_token = self.wrap_token_infile(original_token).into_real_file().ok()?; let quote = original_string.open_quote_text_range()?; - self.descend_into_macros_breakable(original_token, |token| { + self.descend_into_macros_breakable(original_token, |token, _| { (|| { let token = token.value; self.resolve_offset_in_format_args( @@ -718,7 +719,7 @@ impl<'db> SemanticsImpl<'db> { // node is just the token, so descend the token self.descend_into_macros_impl( InRealFile::new(file_id, first), - &mut |InFile { value, .. }| { + &mut |InFile { value, .. }, _ctx| { if let Some(node) = value .parent_ancestors() .take_while(|it| it.text_range() == value.text_range()) @@ -732,7 +733,7 @@ impl<'db> SemanticsImpl<'db> { } else { // Descend first and last token, then zip them to look for the node they belong to let mut scratch: SmallVec<[_; 1]> = smallvec![]; - self.descend_into_macros_impl(InRealFile::new(file_id, first), &mut |token| { + self.descend_into_macros_impl(InRealFile::new(file_id, first), &mut |token, _ctx| { scratch.push(token); CONTINUE_NO_BREAKS }); @@ -740,7 +741,7 @@ impl<'db> SemanticsImpl<'db> { let mut scratch = scratch.into_iter(); self.descend_into_macros_impl( InRealFile::new(file_id, last), - &mut |InFile { value: last, file_id: last_fid }| { + &mut |InFile { value: last, file_id: last_fid }, _ctx| { if let Some(InFile { value: first, file_id: first_fid }) = scratch.next() { if first_fid == last_fid { if let Some(p) = first.parent() { @@ -763,7 +764,9 @@ impl<'db> SemanticsImpl<'db> { res } - fn is_inside_macro_call(token: &SyntaxToken) -> bool { + // FIXME: This isn't quite right wrt to inner attributes + /// Does a syntactic traversal to check whether this token might be inside a macro call + pub fn might_be_inside_macro_call(&self, token: &SyntaxToken) -> bool { token.parent_ancestors().any(|ancestor| { if ast::MacroCall::can_cast(ancestor.kind()) { return true; @@ -781,25 +784,14 @@ impl<'db> SemanticsImpl<'db> { }) } - pub fn descend_into_macros_exact_if_in_macro( - &self, - token: SyntaxToken, - ) -> SmallVec<[SyntaxToken; 1]> { - if Self::is_inside_macro_call(&token) { - self.descend_into_macros_exact(token) - } else { - smallvec![token] - } - } - pub fn descend_into_macros_cb( &self, token: SyntaxToken, - mut cb: impl FnMut(InFile), + mut cb: impl FnMut(InFile, SyntaxContextId), ) { if let Ok(token) = self.wrap_token_infile(token).into_real_file() { - self.descend_into_macros_impl(token, &mut |t| { - cb(t); + self.descend_into_macros_impl(token, &mut |t, ctx| { + cb(t, ctx); CONTINUE_NO_BREAKS }); } @@ -808,7 +800,7 @@ impl<'db> SemanticsImpl<'db> { pub fn descend_into_macros(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> { let mut res = smallvec![]; if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() { - self.descend_into_macros_impl(token, &mut |t| { + self.descend_into_macros_impl(token, &mut |t, _ctx| { res.push(t.value); CONTINUE_NO_BREAKS }); @@ -819,10 +811,27 @@ impl<'db> SemanticsImpl<'db> { res } + pub fn descend_into_macros_no_opaque(&self, token: SyntaxToken) -> SmallVec<[SyntaxToken; 1]> { + let mut res = smallvec![]; + if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() { + self.descend_into_macros_impl(token, &mut |t, ctx| { + if !ctx.is_opaque(self.db.upcast()) { + // Don't descend into opaque contexts + res.push(t.value); + } + CONTINUE_NO_BREAKS + }); + } + if res.is_empty() { + res.push(token); + } + res + } + pub fn descend_into_macros_breakable( &self, token: InRealFile, - mut cb: impl FnMut(InFile) -> ControlFlow, + mut cb: impl FnMut(InFile, SyntaxContextId) -> ControlFlow, ) -> Option { self.descend_into_macros_impl(token.clone(), &mut cb) } @@ -834,10 +843,12 @@ impl<'db> SemanticsImpl<'db> { let text = token.text(); let kind = token.kind(); - self.descend_into_macros_cb(token.clone(), |InFile { value, file_id: _ }| { + self.descend_into_macros_cb(token.clone(), |InFile { value, file_id: _ }, ctx| { let mapped_kind = value.kind(); let any_ident_match = || kind.is_any_identifier() && value.kind().is_any_identifier(); - let matches = (kind == mapped_kind || any_ident_match()) && text == value.text(); + let matches = (kind == mapped_kind || any_ident_match()) + && text == value.text() + && !ctx.is_opaque(self.db.upcast()); if matches { r.push(value); } @@ -854,17 +865,21 @@ impl<'db> SemanticsImpl<'db> { let text = token.text(); let kind = token.kind(); if let Ok(token) = self.wrap_token_infile(token.clone()).into_real_file() { - self.descend_into_macros_breakable(token.clone(), |InFile { value, file_id: _ }| { - let mapped_kind = value.kind(); - let any_ident_match = - || kind.is_any_identifier() && value.kind().is_any_identifier(); - let matches = (kind == mapped_kind || any_ident_match()) && text == value.text(); - if matches { - ControlFlow::Break(value) - } else { - ControlFlow::Continue(()) - } - }) + self.descend_into_macros_breakable( + token.clone(), + |InFile { value, file_id: _ }, _ctx| { + let mapped_kind = value.kind(); + let any_ident_match = + || kind.is_any_identifier() && value.kind().is_any_identifier(); + let matches = + (kind == mapped_kind || any_ident_match()) && text == value.text(); + if matches { + ControlFlow::Break(value) + } else { + ControlFlow::Continue(()) + } + }, + ) } else { None } @@ -874,7 +889,7 @@ impl<'db> SemanticsImpl<'db> { fn descend_into_macros_impl( &self, InRealFile { value: token, file_id }: InRealFile, - f: &mut dyn FnMut(InFile) -> ControlFlow, + f: &mut dyn FnMut(InFile, SyntaxContextId) -> ControlFlow, ) -> Option { let _p = tracing::info_span!("descend_into_macros_impl").entered(); let (sa, span, file_id) = token @@ -898,7 +913,8 @@ impl<'db> SemanticsImpl<'db> { // These are tracked to know which macro calls we still have to look into // the tokens themselves aren't that interesting as the span that is being used to map // things down never changes. - let mut stack: Vec<(_, SmallVec<[_; 2]>)> = vec![(file_id, smallvec![token])]; + let mut stack: Vec<(_, SmallVec<[_; 2]>)> = + vec![(file_id, smallvec![(token, SyntaxContextId::ROOT)])]; // Process the expansion of a call, pushing all tokens with our span in the expansion back onto our stack let process_expansion_for_token = |stack: &mut Vec<_>, macro_file| { @@ -921,11 +937,11 @@ impl<'db> SemanticsImpl<'db> { // Filters out all tokens that contain the given range (usually the macro call), any such // token is redundant as the corresponding macro call has already been processed let filter_duplicates = |tokens: &mut SmallVec<_>, range: TextRange| { - tokens.retain(|t: &mut SyntaxToken| !range.contains_range(t.text_range())) + tokens.retain(|(t, _): &mut (SyntaxToken, _)| !range.contains_range(t.text_range())) }; while let Some((expansion, ref mut tokens)) = stack.pop() { - while let Some(token) = tokens.pop() { + while let Some((token, ctx)) = tokens.pop() { let was_not_remapped = (|| { // First expand into attribute invocations let containing_attribute_macro_call = self.with_ctx(|ctx| { @@ -1036,7 +1052,7 @@ impl<'db> SemanticsImpl<'db> { let text_range = attr.syntax().text_range(); // remove any other token in this macro input, all their mappings are the // same as this - tokens.retain(|t| { + tokens.retain(|(t, _)| { !text_range.contains_range(t.text_range()) }); return process_expansion_for_token( @@ -1093,7 +1109,7 @@ impl<'db> SemanticsImpl<'db> { .is_none(); if was_not_remapped { - if let ControlFlow::Break(b) = f(InFile::new(expansion, token)) { + if let ControlFlow::Break(b) = f(InFile::new(expansion, token), ctx) { return Some(b); } } diff --git a/crates/ide-db/src/search.rs b/crates/ide-db/src/search.rs index 4166b08339..852ee595be 100644 --- a/crates/ide-db/src/search.rs +++ b/crates/ide-db/src/search.rs @@ -529,9 +529,13 @@ impl<'a> FindUsages<'a> { }) .into_iter() .flat_map(move |token| { - sema.descend_into_macros_exact_if_in_macro(token) - .into_iter() - .filter_map(|it| it.parent()) + if sema.might_be_inside_macro_call(&token) { + sema.descend_into_macros_exact(token) + } else { + <_>::from([token]) + } + .into_iter() + .filter_map(|it| it.parent()) }) } diff --git a/crates/ide/src/goto_declaration.rs b/crates/ide/src/goto_declaration.rs index 6ae9dde84b..9dacbd8bad 100644 --- a/crates/ide/src/goto_declaration.rs +++ b/crates/ide/src/goto_declaration.rs @@ -29,7 +29,7 @@ pub(crate) fn goto_declaration( .find(|it| matches!(it.kind(), IDENT | T![self] | T![super] | T![crate] | T![Self]))?; let range = original_token.text_range(); let info: Vec = sema - .descend_into_macros(original_token) + .descend_into_macros_no_opaque(original_token) .iter() .filter_map(|token| { let parent = token.parent()?; diff --git a/crates/ide/src/goto_definition.rs b/crates/ide/src/goto_definition.rs index 8836166d96..e85e6094fc 100644 --- a/crates/ide/src/goto_definition.rs +++ b/crates/ide/src/goto_definition.rs @@ -83,7 +83,7 @@ pub(crate) fn goto_definition( } let navs = sema - .descend_into_macros(original_token.clone()) + .descend_into_macros_no_opaque(original_token.clone()) .into_iter() .filter_map(|token| { let parent = token.parent()?; diff --git a/crates/ide/src/goto_type_definition.rs b/crates/ide/src/goto_type_definition.rs index ca04b7bb5a..c7ebd9a353 100644 --- a/crates/ide/src/goto_type_definition.rs +++ b/crates/ide/src/goto_type_definition.rs @@ -69,7 +69,7 @@ pub(crate) fn goto_type_definition( } let range = token.text_range(); - sema.descend_into_macros(token) + sema.descend_into_macros_no_opaque(token) .into_iter() .filter_map(|token| { let ty = sema diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 927fdaa178..961b2a4c93 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -409,7 +409,8 @@ fn traverse( let mut r = 0; sema.descend_into_macros_breakable( InRealFile::new(file_id, token.clone()), - |tok| { + |tok, _ctx| { + // FIXME: Consider checking ctx transparency for being opaque? let tok = tok.value; let tok_kind = tok.kind(); diff --git a/crates/span/src/hygiene.rs b/crates/span/src/hygiene.rs index cb9c092f5f..3863b3e809 100644 --- a/crates/span/src/hygiene.rs +++ b/crates/span/src/hygiene.rs @@ -142,3 +142,12 @@ pub enum Transparency { /// Def-site spans in procedural macros, identifiers from `macro` by default use this. Opaque, } + +impl Transparency { + /// Returns `true` if the transparency is [`Opaque`]. + /// + /// [`Opaque`]: Transparency::Opaque + pub fn is_opaque(&self) -> bool { + matches!(self, Self::Opaque) + } +} diff --git a/crates/span/src/map.rs b/crates/span/src/map.rs index f80de05ec6..66bbce1859 100644 --- a/crates/span/src/map.rs +++ b/crates/span/src/map.rs @@ -55,7 +55,10 @@ where /// Returns all [`TextRange`]s that correspond to the given span. /// /// Note this does a linear search through the entire backing vector. - pub fn ranges_with_span_exact(&self, span: SpanData) -> impl Iterator + '_ + pub fn ranges_with_span_exact( + &self, + span: SpanData, + ) -> impl Iterator + '_ where S: Copy, { @@ -64,14 +67,14 @@ where return None; } let start = idx.checked_sub(1).map_or(TextSize::new(0), |prev| self.spans[prev].0); - Some(TextRange::new(start, end)) + Some((TextRange::new(start, end), s.ctx)) }) } /// Returns all [`TextRange`]s whose spans contain the given span. /// /// Note this does a linear search through the entire backing vector. - pub fn ranges_with_span(&self, span: SpanData) -> impl Iterator + '_ + pub fn ranges_with_span(&self, span: SpanData) -> impl Iterator + '_ where S: Copy, { @@ -83,7 +86,7 @@ where return None; } let start = idx.checked_sub(1).map_or(TextSize::new(0), |prev| self.spans[prev].0); - Some(TextRange::new(start, end)) + Some((TextRange::new(start, end), s.ctx)) }) }