From 97076c074d72bb2787bb48093b1607902d8bc894 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sun, 6 Mar 2022 03:49:54 +0100 Subject: [PATCH] internal: Simplify and optimize syntax_highlighting --- crates/ide/src/syntax_highlighting.rs | 125 ++++++++++-------- .../ide/src/syntax_highlighting/highlight.rs | 64 ++++----- crates/ide/src/syntax_highlighting/inject.rs | 77 ++++++----- .../ide/src/syntax_highlighting/injector.rs | 4 +- crates/ide/src/syntax_highlighting/macro_.rs | 15 +-- .../test_data/highlight_doctest.html | 56 ++++---- crates/ide_db/src/defs.rs | 10 ++ 7 files changed, 184 insertions(+), 167 deletions(-) diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 968aa33116..0119ba5cb7 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -241,19 +241,26 @@ fn traverse( current_macro = Some(mac.into()); continue; } - Some(item) if sema.is_attr_macro_call(&item) => current_attr_call = Some(item), - Some(item) if current_attr_call.is_none() => { - let adt = match item { - ast::Item::Enum(it) => Some(ast::Adt::Enum(it)), - ast::Item::Struct(it) => Some(ast::Adt::Struct(it)), - ast::Item::Union(it) => Some(ast::Adt::Union(it)), - _ => None, - }; - match adt { - Some(adt) if sema.is_derive_annotated(&adt) => { - current_derive_call = Some(ast::Item::from(adt)); + Some(item) => { + if matches!(node.kind(), FN | CONST | STATIC) { + bindings_shadow_count.clear(); + } + + if sema.is_attr_macro_call(&item) { + current_attr_call = Some(item); + } else if current_attr_call.is_none() { + let adt = match item { + ast::Item::Enum(it) => Some(ast::Adt::Enum(it)), + ast::Item::Struct(it) => Some(ast::Adt::Struct(it)), + ast::Item::Union(it) => Some(ast::Adt::Union(it)), + _ => None, + }; + match adt { + Some(adt) if sema.is_derive_annotated(&adt) => { + current_derive_call = Some(ast::Item::from(adt)); + } + _ => (), } - _ => (), } } None if ast::Attr::can_cast(node.kind()) => inside_attribute = true, @@ -291,6 +298,8 @@ fn traverse( WalkEvent::Enter(it) => it, WalkEvent::Leave(NodeOrToken::Token(_)) => continue, WalkEvent::Leave(NodeOrToken::Node(node)) => { + // Doc comment highlighting injection, we do this when leaving the node + // so that we overwrite the highlighting of the doc comment itself. inject::doc_comment(hl, sema, InFile::new(file_id.into(), &node)); continue; } @@ -302,57 +311,68 @@ fn traverse( } } - // only attempt to descend if we are inside a macro call or attribute - // as calling `descend_into_macros_single` gets rather expensive if done for every single token - // additionally, do not descend into comments, descending maps down to doc attributes which get - // tagged as string literals. - let descend_token = (current_macro_call.is_some() - || current_attr_call.is_some() - || current_derive_call.is_some()) - && element.kind() != COMMENT; - let element_to_highlight = if descend_token { - let token = match &element { - NodeOrToken::Node(_) => continue, - NodeOrToken::Token(tok) => tok.clone(), - }; - let in_mcall_outside_tt = current_attr_call.is_none() - && token.parent().as_ref().map(SyntaxNode::kind) != Some(TOKEN_TREE); - let token = match in_mcall_outside_tt { - // not in the macros/derives token tree, don't attempt to descend - true => token, - false => sema.descend_into_macros_single(token), - }; - match token.parent() { - Some(parent) => { - // Names and NameRefs have special semantics, use them instead of the tokens - // as otherwise we won't ever visit them - match (token.kind(), parent.kind()) { - (T![ident], NAME | NAME_REF) => parent.into(), - (T![self] | T![super] | T![crate] | T![Self], NAME_REF) => parent.into(), - (INT_NUMBER, NAME_REF) => parent.into(), - (LIFETIME_IDENT, LIFETIME) => parent.into(), - _ => token.into(), + let element = match element.clone() { + NodeOrToken::Node(n) => match ast::NameLike::cast(n) { + Some(n) => NodeOrToken::Node(n), + None => continue, + }, + NodeOrToken::Token(t) => NodeOrToken::Token(t), + }; + let token = element.as_token().cloned(); + + // Descending tokens into macros is expensive even if no descending occurs, so make sure + // that we actually are in a position where descending is possible. + let in_macro = current_macro_call.is_some() + || current_derive_call.is_some() + || current_attr_call.is_some(); + let descended_element = if in_macro { + // Attempt to descend tokens into macro-calls. + match element { + NodeOrToken::Token(token) if token.kind() != COMMENT => { + // For function-like macro calls and derive attributes, only attempt to descend if + // we are inside their token-trees. + let in_tt = current_attr_call.is_some() + || token.parent().as_ref().map(SyntaxNode::kind) == Some(TOKEN_TREE); + + if in_tt { + let token = sema.descend_into_macros_single(token); + match token.parent().and_then(ast::NameLike::cast) { + // Remap the token into the wrapping single token nodes + // FIXME: if the node doesn't resolve, we also won't do token based highlighting! + Some(parent) => match (token.kind(), parent.syntax().kind()) { + (T![self] | T![ident], NAME | NAME_REF) => { + NodeOrToken::Node(parent) + } + (T![self] | T![super] | T![crate] | T![Self], NAME_REF) => { + NodeOrToken::Node(parent) + } + (INT_NUMBER, NAME_REF) => NodeOrToken::Node(parent), + (LIFETIME_IDENT, LIFETIME) => NodeOrToken::Node(parent), + _ => NodeOrToken::Token(token), + }, + None => NodeOrToken::Token(token), + } + } else { + NodeOrToken::Token(token) } } - None => token.into(), + e => e, } } else { - element.clone() + element }; // FIXME: do proper macro def highlighting https://github.com/rust-analyzer/rust-analyzer/issues/6232 // Skip metavariables from being highlighted to prevent keyword highlighting in them - if macro_highlighter.highlight(&element_to_highlight).is_some() { + if descended_element.as_token().and_then(|t| macro_highlighter.highlight(t)).is_some() { continue; } // string highlight injections, note this does not use the descended element as proc-macros // can rewrite string literals which invalidates our indices - if let (Some(token), Some(token_to_highlight)) = - (element.into_token(), element_to_highlight.as_token()) - { + if let (Some(token), Some(descended_token)) = (token, descended_element.as_token()) { let string = ast::String::cast(token); - let string_to_highlight = ast::String::cast(token_to_highlight.clone()); + let string_to_highlight = ast::String::cast(descended_token.clone()); if let Some((string, expanded_string)) = string.zip(string_to_highlight) { if string.is_raw() { if inject::ra_fixture(hl, sema, &string, &expanded_string).is_some() { @@ -377,14 +397,13 @@ fn traverse( } } - // do the normal highlighting - let element = match element_to_highlight { - NodeOrToken::Node(node) => highlight::node( + let element = match descended_element { + NodeOrToken::Node(name_like) => highlight::name_like( sema, krate, &mut bindings_shadow_count, syntactic_name_ref_highlighting, - node, + name_like, ), NodeOrToken::Token(token) => highlight::token(sema, token).zip(Some(None)), }; diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index 43a3fb2d71..b98fa1ab39 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -2,7 +2,7 @@ use hir::{AsAssocItem, HasVisibility, Semantics}; use ide_db::{ - defs::{Definition, NameClass, NameRefClass}, + defs::{Definition, IdentClass, NameClass, NameRefClass}, helpers::FamousDefs, RootDatabase, SymbolKind, }; @@ -47,52 +47,36 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio Some(highlight) } -pub(super) fn node( +pub(super) fn name_like( sema: &Semantics, krate: Option, bindings_shadow_count: &mut FxHashMap, syntactic_name_ref_highlighting: bool, - node: SyntaxNode, + name_like: ast::NameLike, ) -> Option<(Highlight, Option)> { let mut binding_hash = None; - let highlight = match_ast! { - match node { - ast::NameRef(name_ref) => { - highlight_name_ref( - sema, - krate, - bindings_shadow_count, - &mut binding_hash, - syntactic_name_ref_highlighting, - name_ref, - ) - }, - ast::Name(name) => { - highlight_name(sema, bindings_shadow_count, &mut binding_hash, krate, name) - }, - ast::Lifetime(lifetime) => { - match NameClass::classify_lifetime(sema, &lifetime) { - Some(NameClass::Definition(def)) => { - highlight_def(sema, krate, def) | HlMod::Definition - } - None => match NameRefClass::classify_lifetime(sema, &lifetime) { - Some(NameRefClass::Definition(def)) => highlight_def(sema, krate, def), - _ => SymbolKind::LifetimeParam.into(), - }, - _ => Highlight::from(SymbolKind::LifetimeParam) | HlMod::Definition, - } - }, - ast::Fn(_) => { - bindings_shadow_count.clear(); - return None; - }, - _ => { - if [FN, CONST, STATIC].contains(&node.kind()) { - bindings_shadow_count.clear(); - } - return None - }, + let highlight = match name_like { + ast::NameLike::NameRef(name_ref) => highlight_name_ref( + sema, + krate, + bindings_shadow_count, + &mut binding_hash, + syntactic_name_ref_highlighting, + name_ref, + ), + ast::NameLike::Name(name) => { + highlight_name(sema, bindings_shadow_count, &mut binding_hash, krate, name) } + ast::NameLike::Lifetime(lifetime) => match IdentClass::classify_lifetime(sema, &lifetime) { + Some(IdentClass::NameClass(NameClass::Definition(def))) => { + highlight_def(sema, krate, def) | HlMod::Definition + } + Some(IdentClass::NameRefClass(NameRefClass::Definition(def))) => { + highlight_def(sema, krate, def) + } + // FIXME: Fallback for 'static and '_, as we do not resolve these yet + _ => SymbolKind::LifetimeParam.into(), + }, }; Some((highlight, binding_hash)) } diff --git a/crates/ide/src/syntax_highlighting/inject.rs b/crates/ide/src/syntax_highlighting/inject.rs index ec1df6d1de..774934a249 100644 --- a/crates/ide/src/syntax_highlighting/inject.rs +++ b/crates/ide/src/syntax_highlighting/inject.rs @@ -9,7 +9,7 @@ use ide_db::{ SymbolKind, }; use syntax::{ - ast::{self, AstNode, IsString}, + ast::{self, AstNode, IsString, QuoteOffsets}, AstToken, NodeOrToken, SyntaxNode, TextRange, TextSize, }; @@ -61,7 +61,7 @@ pub(super) fn ra_fixture( } } - let (analysis, tmp_file_id) = Analysis::from_single_file(inj.text().to_string()); + let (analysis, tmp_file_id) = Analysis::from_single_file(inj.take_text()); for mut hl_range in analysis.highlight(tmp_file_id).unwrap() { for range in inj.map_range_up(hl_range.range) { @@ -85,31 +85,19 @@ const RUSTDOC_FENCE: &'static str = "```"; pub(super) fn doc_comment( hl: &mut Highlights, sema: &Semantics, - node: InFile<&SyntaxNode>, + InFile { file_id: src_file_id, value: node }: InFile<&SyntaxNode>, ) { - let (attributes, def) = match doc_attributes(sema, node.value) { + let (attributes, def) = match doc_attributes(sema, node) { Some(it) => it, None => return, }; - let mut inj = Injector::default(); - inj.add_unmapped("fn doctest() {\n"); - - let attrs_source_map = attributes.source_map(sema.db); - - let mut is_codeblock = false; - let mut is_doctest = false; - - // Replace the original, line-spanning comment ranges by new, only comment-prefix - // spanning comment ranges. - let mut new_comments = Vec::new(); - let mut string; - + // Extract intra-doc links and emit highlights for them. if let Some((docs, doc_mapping)) = attributes.docs_with_rangemap(sema.db) { extract_definitions_from_docs(&docs) .into_iter() .filter_map(|(range, link, ns)| { - doc_mapping.map(range).filter(|mapping| mapping.file_id == node.file_id).and_then( + doc_mapping.map(range).filter(|mapping| mapping.file_id == src_file_id).and_then( |InFile { value: mapped_range, .. }| { Some(mapped_range).zip(resolve_doc_path_for_def(sema.db, def, &link, ns)) }, @@ -127,31 +115,49 @@ pub(super) fn doc_comment( }); } + // Extract doc-test sources from the docs and calculate highlighting for them. + + let mut inj = Injector::default(); + inj.add_unmapped("fn doctest() {\n"); + + let attrs_source_map = attributes.source_map(sema.db); + + let mut is_codeblock = false; + let mut is_doctest = false; + + let mut new_comments = Vec::new(); + let mut string; + for attr in attributes.by_key("doc").attrs() { let InFile { file_id, value: src } = attrs_source_map.source_of(attr); - if file_id != node.file_id { + if file_id != src_file_id { continue; } - let (line, range, prefix) = match &src { + let (line, range) = match &src { Either::Left(it) => { string = match find_doc_string_in_attr(attr, it) { Some(it) => it, None => continue, }; - let text_range = string.syntax().text_range(); - let text_range = TextRange::new( - text_range.start() + TextSize::from(1), - text_range.end() - TextSize::from(1), - ); let text = string.text(); - (&text[1..text.len() - 1], text_range, "") + let text_range = string.syntax().text_range(); + match string.quote_offsets() { + Some(QuoteOffsets { contents, .. }) => { + (&text[contents - text_range.start()], contents) + } + None => (text, text_range), + } } Either::Right(comment) => { - (comment.text(), comment.syntax().text_range(), comment.prefix()) + let value = comment.prefix().len(); + let range = comment.syntax().text_range(); + ( + &comment.text()[value..], + TextRange::new(range.start() + TextSize::try_from(value).unwrap(), range.end()), + ) } }; - let mut pos = TextSize::from(prefix.len() as u32); let mut range_start = range.start(); for line in line.split('\n') { let line_len = TextSize::from(line.len() as u32); @@ -159,8 +165,7 @@ pub(super) fn doc_comment( let next_range_start = range_start + line_len + TextSize::from(1); mem::replace(&mut range_start, next_range_start) }; - // only first line has the prefix so take it away for future iterations - let mut pos = mem::take(&mut pos); + let mut pos = TextSize::from(0); match line.find(RUSTDOC_FENCE) { Some(idx) => { @@ -196,13 +201,13 @@ pub(super) fn doc_comment( inj.add_unmapped("\n}"); - let (analysis, tmp_file_id) = Analysis::from_single_file(inj.text().to_string()); + let (analysis, tmp_file_id) = Analysis::from_single_file(inj.take_text()); - for HlRange { range, highlight, binding_hash } in - analysis.with_db(|db| super::highlight(db, tmp_file_id, None, true)).unwrap() - { - for range in inj.map_range_up(range) { - hl.add(HlRange { range, highlight: highlight | HlMod::Injected, binding_hash }); + if let Ok(ranges) = analysis.with_db(|db| super::highlight(db, tmp_file_id, None, true)) { + for HlRange { range, highlight, binding_hash } in ranges { + for range in inj.map_range_up(range) { + hl.add(HlRange { range, highlight: highlight | HlMod::Injected, binding_hash }); + } } } diff --git a/crates/ide/src/syntax_highlighting/injector.rs b/crates/ide/src/syntax_highlighting/injector.rs index 211dc54f2b..a902fd717f 100644 --- a/crates/ide/src/syntax_highlighting/injector.rs +++ b/crates/ide/src/syntax_highlighting/injector.rs @@ -29,8 +29,8 @@ impl Injector { self.buf.push_str(text); } - pub(super) fn text(&self) -> &str { - &self.buf + pub(super) fn take_text(&mut self) -> String { + std::mem::take(&mut self.buf) } pub(super) fn map_range_up(&self, range: TextRange) -> impl Iterator + '_ { diff --git a/crates/ide/src/syntax_highlighting/macro_.rs b/crates/ide/src/syntax_highlighting/macro_.rs index 7f2d61a0bd..1099d9c23b 100644 --- a/crates/ide/src/syntax_highlighting/macro_.rs +++ b/crates/ide/src/syntax_highlighting/macro_.rs @@ -1,5 +1,5 @@ //! Syntax highlighting for macro_rules!. -use syntax::{SyntaxElement, SyntaxKind, SyntaxToken, TextRange, T}; +use syntax::{SyntaxKind, SyntaxToken, TextRange, T}; use crate::{HlRange, HlTag}; @@ -19,10 +19,10 @@ impl MacroHighlighter { } } - pub(super) fn highlight(&self, element: &SyntaxElement) -> Option { + pub(super) fn highlight(&self, token: &SyntaxToken) -> Option { if let Some(state) = self.state.as_ref() { if matches!(state.rule_state, RuleState::Matcher | RuleState::Expander) { - if let Some(range) = is_metavariable(element) { + if let Some(range) = is_metavariable(token) { return Some(HlRange { range, highlight: HlTag::UnresolvedReference.into(), @@ -115,12 +115,11 @@ fn update_macro_state(state: &mut MacroMatcherParseState, tok: &SyntaxToken) { } } -fn is_metavariable(element: &SyntaxElement) -> Option { - let tok = element.as_token()?; - match tok.kind() { +fn is_metavariable(token: &SyntaxToken) -> Option { + match token.kind() { kind if kind == SyntaxKind::IDENT || kind.is_keyword() => { - if let Some(_dollar) = tok.prev_token().filter(|t| t.kind() == T![$]) { - return Some(tok.text_range()); + if let Some(_dollar) = token.prev_token().filter(|t| t.kind() == T![$]) { + return Some(token.text_range()); } } _ => (), diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html index bd8d9faae8..96044751b8 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html @@ -43,13 +43,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd 
//! This is a module to test doc injection.
 //! ```
-//! fn test() {}
+//! fn test() {}
 //! ```
 
 mod outline_module;
 
 /// ```
-/// let _ = "early doctests should not go boom";
+/// let _ = "early doctests should not go boom";
 /// ```
 struct Foo {
     bar: bool,
@@ -58,15 +58,15 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 /// This is an impl with a code block.
 ///
 /// ```
-/// fn foo() {
+/// fn foo() {
 ///
-/// }
+/// }
 /// ```
 impl Foo {
     /// ```
-    /// let _ = "Call me
+    /// let _ = "Call me
     //    KILLER WHALE
-    ///     Ishmael.";
+    ///     Ishmael.";
     /// ```
     pub const bar: bool = true;
 
@@ -75,8 +75,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     /// # Examples
     ///
     /// ```
-    /// # #![allow(unused_mut)]
-    /// let mut foo: Foo = Foo::new();
+    /// # #![allow(unused_mut)]
+    /// let mut foo: Foo = Foo::new();
     /// ```
     pub const fn new() -> Foo {
         Foo { bar: true }
@@ -87,33 +87,33 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     /// # Examples
     ///
     /// ```
-    /// use x::y;
+    /// use x::y;
     ///
-    /// let foo = Foo::new();
+    /// let foo = Foo::new();
     ///
-    /// // calls bar on foo
-    /// assert!(foo.bar());
+    /// // calls bar on foo
+    /// assert!(foo.bar());
     ///
-    /// let bar = foo.bar || Foo::bar;
+    /// let bar = foo.bar || Foo::bar;
     ///
-    /// /* multi-line
-    ///        comment */
+    /// /* multi-line
+    ///        comment */
     ///
-    /// let multi_line_string = "Foo
-    ///   bar\n
-    ///          ";
+    /// let multi_line_string = "Foo
+    ///   bar\n
+    ///          ";
     ///
     /// ```
     ///
     /// ```rust,no_run
-    /// let foobar = Foo::new().bar();
+    /// let foobar = Foo::new().bar();
     /// ```
     ///
     /// ```
-    /// // functions
-    /// fn foo<T, const X: usize>(arg: i32) {
-    ///     let x: T = X;
-    /// }
+    /// // functions
+    /// fn foo<T, const X: usize>(arg: i32) {
+    ///     let x: T = X;
+    /// }
     /// ```
     ///
     /// ```sh
@@ -138,8 +138,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 }
 
 /// ```
-/// macro_rules! noop { ($expr:expr) => { $expr }}
-/// noop!(1);
+/// macro_rules! noop { ($expr:expr) => { $expr }}
+/// noop!(1);
 /// ```
 macro_rules! noop {
     ($expr:expr) => {
@@ -148,18 +148,18 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 }
 
 /// ```rust
-/// let _ = example(&[1, 2, 3]);
+/// let _ = example(&[1, 2, 3]);
 /// ```
 ///
 /// ```
-/// loop {}
+/// loop {}
 #[cfg_attr(not(feature = "false"), doc = "loop {}")]
 #[doc = "loop {}"]
 /// ```
 ///
 #[cfg_attr(feature = "alloc", doc = "```rust")]
 #[cfg_attr(not(feature = "alloc"), doc = "```ignore")]
-/// let _ = example(&alloc::vec![1, 2, 3]);
+/// let _ = example(&alloc::vec![1, 2, 3]);
 /// ```
 pub fn mix_and_match() {}
 
diff --git a/crates/ide_db/src/defs.rs b/crates/ide_db/src/defs.rs
index 80988986ce..9d05db4a77 100644
--- a/crates/ide_db/src/defs.rs
+++ b/crates/ide_db/src/defs.rs
@@ -116,6 +116,7 @@ impl Definition {
     }
 }
 
+#[derive(Debug)]
 pub enum IdentClass {
     NameClass(NameClass),
     NameRefClass(NameRefClass),
@@ -145,6 +146,15 @@ impl IdentClass {
         Self::classify_node(sema, &parent)
     }
 
+    pub fn classify_lifetime(
+        sema: &Semantics,
+        lifetime: &ast::Lifetime,
+    ) -> Option {
+        NameClass::classify_lifetime(sema, &lifetime).map(IdentClass::NameClass).or_else(|| {
+            NameRefClass::classify_lifetime(sema, &lifetime).map(IdentClass::NameRefClass)
+        })
+    }
+
     pub fn definitions(self) -> ArrayVec {
         let mut res = ArrayVec::new();
         match self {