From ca3699bd506f9f196f08940a04d765f90493065d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 25 Oct 2024 12:02:48 +0200 Subject: [PATCH 1/2] Factor out token ranking --- crates/ide-db/src/lib.rs | 30 +++++++++++++++++++++++++++ crates/ide/src/hover.rs | 22 ++++---------------- crates/ide/src/syntax_highlighting.rs | 20 +++--------------- rust-bors.toml | 1 - 4 files changed, 37 insertions(+), 36 deletions(-) delete mode 100644 rust-bors.toml diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index aed093f0eb..b7b133ac10 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -293,3 +293,33 @@ impl SnippetCap { } } } + +pub struct Ranker<'a> { + pub kind: parser::SyntaxKind, + pub text: &'a str, + pub ident_kind: bool, +} + +impl<'a> Ranker<'a> { + pub fn from_token(token: &'a syntax::SyntaxToken) -> Self { + let kind = token.kind(); + Ranker { kind, text: token.text(), ident_kind: kind.is_any_identifier() } + } + + /// A utility function that ranks a token again a given kind and text, returning a number that + /// represents how close the token is to the given kind and text. + pub fn rank_token(&self, tok: &syntax::SyntaxToken) -> usize { + let tok_kind = tok.kind(); + + let exact_same_kind = tok_kind == self.kind; + let both_idents = exact_same_kind || (tok_kind.is_any_identifier() && self.ident_kind); + let same_text = tok.text() == self.text; + // anything that mapped into a token tree has likely no semantic information + let no_tt_parent = + tok.parent().map_or(false, |it| it.kind() != parser::SyntaxKind::TOKEN_TREE); + !((both_idents as usize) + | ((exact_same_kind as usize) << 1) + | ((same_text as usize) << 2) + | ((no_tt_parent as usize) << 3)) + } +} diff --git a/crates/ide/src/hover.rs b/crates/ide/src/hover.rs index 124db2985b..91ec672feb 100644 --- a/crates/ide/src/hover.rs +++ b/crates/ide/src/hover.rs @@ -11,7 +11,7 @@ use ide_db::{ defs::{Definition, IdentClass, NameRefClass, OperatorClass}, famous_defs::FamousDefs, helpers::pick_best_token, - FileRange, FxIndexSet, RootDatabase, + FileRange, FxIndexSet, Ranker, RootDatabase, }; use itertools::{multizip, Itertools}; use span::Edition; @@ -182,27 +182,13 @@ fn hover_offset( // equivalency is more important let mut descended = sema.descend_into_macros(original_token.clone()); - let kind = original_token.kind(); - let text = original_token.text(); - let ident_kind = kind.is_any_identifier(); + let ranker = Ranker::from_token(&original_token); - descended.sort_by_cached_key(|tok| { - let tok_kind = tok.kind(); - - let exact_same_kind = tok_kind == kind; - let both_idents = exact_same_kind || (tok_kind.is_any_identifier() && ident_kind); - let same_text = tok.text() == text; - // anything that mapped into a token tree has likely no semantic information - let no_tt_parent = tok.parent().map_or(false, |it| it.kind() != TOKEN_TREE); - !((both_idents as usize) - | ((exact_same_kind as usize) << 1) - | ((same_text as usize) << 2) - | ((no_tt_parent as usize) << 3)) - }); + descended.sort_by_cached_key(|tok| ranker.rank_token(tok)); let mut res = vec![]; for token in descended { - let is_same_kind = token.kind() == kind; + let is_same_kind = token.kind() == ranker.kind; let lint_hover = (|| { // FIXME: Definition should include known lints and the like instead of having this special case here let attr = token.parent_ancestors().find_map(ast::Attr::cast)?; diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 961b2a4c93..54d5307adf 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -16,7 +16,7 @@ mod tests; use std::ops::ControlFlow; use hir::{InRealFile, Name, Semantics}; -use ide_db::{FxHashMap, RootDatabase, SymbolKind}; +use ide_db::{FxHashMap, Ranker, RootDatabase, SymbolKind}; use span::EditionedFileId; use syntax::{ ast::{self, IsString}, @@ -401,9 +401,7 @@ fn traverse( // Attempt to descend tokens into macro-calls. let res = match element { NodeOrToken::Token(token) if token.kind() != COMMENT => { - let kind = token.kind(); - let text = token.text(); - let ident_kind = kind.is_any_identifier(); + let ranker = Ranker::from_token(&token); let mut t = None; let mut r = 0; @@ -412,19 +410,7 @@ fn traverse( |tok, _ctx| { // FIXME: Consider checking ctx transparency for being opaque? let tok = tok.value; - let tok_kind = tok.kind(); - - let exact_same_kind = tok_kind == kind; - let both_idents = - exact_same_kind || (tok_kind.is_any_identifier() && ident_kind); - let same_text = tok.text() == text; - // anything that mapped into a token tree has likely no semantic information - let no_tt_parent = - tok.parent().map_or(false, |it| it.kind() != TOKEN_TREE); - let my_rank = (both_idents as usize) - | ((exact_same_kind as usize) << 1) - | ((same_text as usize) << 2) - | ((no_tt_parent as usize) << 3); + let my_rank = ranker.rank_token(&tok); if my_rank > 0b1110 { // a rank of 0b1110 means that we have found a maximally interesting diff --git a/rust-bors.toml b/rust-bors.toml deleted file mode 100644 index c31ba66c50..0000000000 --- a/rust-bors.toml +++ /dev/null @@ -1 +0,0 @@ -timeout = 3600 From 20ac30fb75994389e0dd3b6b6a150bd228f2a352 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 25 Oct 2024 13:29:07 +0200 Subject: [PATCH 2/2] Invert token iteration order in macro mapping --- crates/hir/src/semantics.rs | 4 +++ crates/ide-db/src/lib.rs | 6 +++-- crates/ide/src/call_hierarchy.rs | 5 ++-- crates/ide/src/hover.rs | 2 +- crates/ide/src/hover/tests.rs | 26 +++++++++---------- crates/ide/src/references.rs | 10 +++---- crates/ide/src/syntax_highlighting.rs | 3 ++- .../test_data/highlight_issue_18089.html | 2 +- 8 files changed, 33 insertions(+), 25 deletions(-) diff --git a/crates/hir/src/semantics.rs b/crates/hir/src/semantics.rs index 860754d5e7..592a4b4853 100644 --- a/crates/hir/src/semantics.rs +++ b/crates/hir/src/semantics.rs @@ -954,6 +954,10 @@ impl<'db> SemanticsImpl<'db> { }; while let Some((expansion, ref mut tokens)) = stack.pop() { + // Reverse the tokens so we prefer first tokens (to accommodate for popping from the + // back) + // alternatively we could pop from the front but that would shift the content on every pop + tokens.reverse(); while let Some((token, ctx)) = tokens.pop() { let was_not_remapped = (|| { // First expand into attribute invocations diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index b7b133ac10..b764f852f0 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -301,6 +301,8 @@ pub struct Ranker<'a> { } impl<'a> Ranker<'a> { + pub const MAX_RANK: usize = 0b1110; + pub fn from_token(token: &'a syntax::SyntaxToken) -> Self { let kind = token.kind(); Ranker { kind, text: token.text(), ident_kind: kind.is_any_identifier() } @@ -317,9 +319,9 @@ impl<'a> Ranker<'a> { // anything that mapped into a token tree has likely no semantic information let no_tt_parent = tok.parent().map_or(false, |it| it.kind() != parser::SyntaxKind::TOKEN_TREE); - !((both_idents as usize) + (both_idents as usize) | ((exact_same_kind as usize) << 1) | ((same_text as usize) << 2) - | ((no_tt_parent as usize) << 3)) + | ((no_tt_parent as usize) << 3) } } diff --git a/crates/ide/src/call_hierarchy.rs b/crates/ide/src/call_hierarchy.rs index 1b82c00d1d..e5b4ed17b2 100644 --- a/crates/ide/src/call_hierarchy.rs +++ b/crates/ide/src/call_hierarchy.rs @@ -510,6 +510,7 @@ fn caller$0() { expect![[]], ); } + #[test] fn test_call_hierarchy_in_macros_incoming_different_files() { check_hierarchy( @@ -591,9 +592,9 @@ macro_rules! call { "#, expect!["callee Function FileId(0) 22..37 30..36"], expect![[r#" - callee Function FileId(0) 38..52 44..50 : FileId(0):44..50 caller Function FileId(0) 38..52 : FileId(0):44..50 - caller Function FileId(1) 130..136 130..136 : FileId(0):44..50"#]], + caller Function FileId(1) 130..136 130..136 : FileId(0):44..50 + callee Function FileId(0) 38..52 44..50 : FileId(0):44..50"#]], expect![[]], ); } diff --git a/crates/ide/src/hover.rs b/crates/ide/src/hover.rs index 91ec672feb..6cac4f1ee4 100644 --- a/crates/ide/src/hover.rs +++ b/crates/ide/src/hover.rs @@ -184,7 +184,7 @@ fn hover_offset( let ranker = Ranker::from_token(&original_token); - descended.sort_by_cached_key(|tok| ranker.rank_token(tok)); + descended.sort_by_cached_key(|tok| !ranker.rank_token(tok)); let mut res = vec![]; for token in descended { diff --git a/crates/ide/src/hover/tests.rs b/crates/ide/src/hover/tests.rs index ef835f5bef..3e40263041 100644 --- a/crates/ide/src/hover/tests.rs +++ b/crates/ide/src/hover/tests.rs @@ -288,19 +288,6 @@ m!(ab$0c); expect![[r#" *abc* - ```rust - test::module - ``` - - ```rust - fn abc() - ``` - - --- - - Inner - --- - ```rust test ``` @@ -312,6 +299,19 @@ m!(ab$0c); --- Outer + --- + + ```rust + test::module + ``` + + ```rust + fn abc() + ``` + + --- + + Inner "#]], ); } diff --git a/crates/ide/src/references.rs b/crates/ide/src/references.rs index e7cb8a253f..339315db57 100644 --- a/crates/ide/src/references.rs +++ b/crates/ide/src/references.rs @@ -1701,14 +1701,14 @@ fn f() { } "#, expect![[r#" - func Function FileId(0) 137..146 140..144 - - FileId(0) 161..165 - - func Function FileId(0) 137..146 140..144 module FileId(0) 181..185 + + + func Function FileId(0) 137..146 140..144 + + FileId(0) 161..165 "#]], ) } diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 54d5307adf..0747d1b404 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -397,6 +397,7 @@ fn traverse( Some(AttrOrDerive::Derive(_)) => inside_attribute, None => false, }; + let descended_element = if in_macro { // Attempt to descend tokens into macro-calls. let res = match element { @@ -412,7 +413,7 @@ fn traverse( let tok = tok.value; let my_rank = ranker.rank_token(&tok); - if my_rank > 0b1110 { + if my_rank >= Ranker::MAX_RANK { // a rank of 0b1110 means that we have found a maximally interesting // token so stop early. t = Some(tok); diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html b/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html index d07ba74db2..361dcd1bc3 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_issue_18089.html @@ -50,4 +50,4 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd } #[proc_macros::issue_18089] -fn template() {} \ No newline at end of file +fn template() {} \ No newline at end of file