Merge #6231

6231: Factor macro_rules and format-string highlighting out into submodules r=Veykril a=Veykril

This moves `format`-like macro string highlighting and macro_rules highlight skipping out of the main module.

Co-authored-by: Lukas Wirth <lukastw97@gmail.com>

crates/ide/src/syntax_highlighting.rs

@@ -1,6 +1,8 @@
-mod tags;
+mod format;
 mod html;
 mod injection;
+mod macro_rules;
+mod tags;
 #[cfg(test)]
 mod tests;
 
@@ -17,9 +19,11 @@ use syntax::{
     SyntaxNode, SyntaxToken, TextRange, WalkEvent, T,
 };
 
-use crate::FileId;
+use crate::{
+    syntax_highlighting::{format::FormatStringHighlighter, macro_rules::MacroRulesHighlighter},
+    FileId,
+};
 
-use ast::FormatSpecifier;
 pub(crate) use html::highlight_as_html;
 pub use tags::{Highlight, HighlightModifier, HighlightModifiers, HighlightTag};
 
@@ -68,8 +72,9 @@ pub(crate) fn highlight(
     // When we leave a node, the we use it to flatten the highlighted ranges.
     let mut stack = HighlightedRangeStack::new();
 
-    let mut current_macro_call: Option<(ast::MacroCall, Option<MacroMatcherParseState>)> = None;
+    let mut current_macro_call: Option<ast::MacroCall> = None;
-    let mut format_string: Option<SyntaxElement> = None;
+    let mut format_string_highlighter = FormatStringHighlighter::default();
+    let mut macro_rules_highlighter = MacroRulesHighlighter::default();
 
     // Walk all nodes, keeping track of whether we are inside a macro or not.
     // If in macro, expand it first and highlight the expanded code.
@@ -99,9 +104,8 @@ pub(crate) fn highlight(
                         binding_hash: None,
                     });
                 }
-                let mut is_macro_rules = None;
                 if let Some(name) = mc.is_macro_rules() {
-                    is_macro_rules = Some(MacroMatcherParseState::new());
+                    macro_rules_highlighter.init();
                     if let Some((highlight, binding_hash)) = highlight_element(
                         &sema,
                         &mut bindings_shadow_count,
@@ -115,13 +119,14 @@ pub(crate) fn highlight(
                         });
                     }
                 }
-                current_macro_call = Some((mc.clone(), is_macro_rules));
+                current_macro_call = Some(mc.clone());
                 continue;
             }
             WalkEvent::Leave(Some(mc)) => {
-                assert!(current_macro_call.map(|it| it.0) == Some(mc));
+                assert!(current_macro_call == Some(mc));
                 current_macro_call = None;
-                format_string = None;
+                format_string_highlighter = FormatStringHighlighter::default();
+                macro_rules_highlighter = MacroRulesHighlighter::default();
             }
             _ => (),
         }
@@ -148,20 +153,6 @@ pub(crate) fn highlight(
             WalkEvent::Leave(_) => continue,
         };
 
-        // check if in matcher part of a macro_rules rule
-        if let Some((_, Some(ref mut state))) = current_macro_call {
-            if let Some(tok) = element.as_token() {
-                if matches!(
-                    update_macro_rules_state(tok, state),
-                    RuleState::Matcher | RuleState::Expander
-                ) {
-                    if skip_metavariables(element.clone()) {
-                        continue;
-                    }
-                }
-            }
-        }
-
         let range = element.text_range();
 
         let element_to_highlight = if current_macro_call.is_some() && element.kind() != COMMENT {
@@ -173,29 +164,9 @@ pub(crate) fn highlight(
             let token = sema.descend_into_macros(token.clone());
             let parent = token.parent();
 
-            // Check if macro takes a format string and remember it for highlighting later.
+            format_string_highlighter.check_for_format_string(&parent);
-            // The macros that accept a format string expand to a compiler builtin macros
+            if let Some(tok) = element.as_token() {
-            // `format_args` and `format_args_nl`.
+                macro_rules_highlighter.advance(tok);
-            if let Some(name) = parent
-                .parent()
-                .and_then(ast::MacroCall::cast)
-                .and_then(|mc| mc.path())
-                .and_then(|p| p.segment())
-                .and_then(|s| s.name_ref())
-            {
-                match name.text().as_str() {
-                    "format_args" | "format_args_nl" => {
-                        format_string = parent
-                            .children_with_tokens()
-                            .filter(|t| t.kind() != WHITESPACE)
-                            .nth(1)
-                            .filter(|e| {
-                                ast::String::can_cast(e.kind())
-                                    || ast::RawString::can_cast(e.kind())
-                            })
-                    }
-                    _ => {}
-                }
             }
 
             // We only care Name and Name_ref
@@ -214,31 +185,20 @@ pub(crate) fn highlight(
             }
         }
 
-        let is_format_string = format_string.as_ref() == Some(&element_to_highlight);
-
         if let Some((highlight, binding_hash)) = highlight_element(
             &sema,
             &mut bindings_shadow_count,
             syntactic_name_ref_highlighting,
             element_to_highlight.clone(),
         ) {
-            stack.add(HighlightedRange { range, highlight, binding_hash });
+            if macro_rules_highlighter.highlight(element_to_highlight.clone()).is_none() {
+                stack.add(HighlightedRange { range, highlight, binding_hash });
+            }
+
             if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::String::cast)
             {
-                if is_format_string {
+                format_string_highlighter.highlight_format_string(&mut stack, &string, range);
-                    stack.push();
-                    string.lex_format_specifier(|piece_range, kind| {
-                        if let Some(highlight) = highlight_format_specifier(kind) {
-                            stack.add(HighlightedRange {
-                                range: piece_range + range.start(),
-                                highlight: highlight.into(),
-                                binding_hash: None,
-                            });
-                        }
-                    });
-                    stack.pop();
-                }
                 // Highlight escape sequences
                 if let Some(char_ranges) = string.char_ranges() {
                     stack.push();
@@ -256,19 +216,7 @@ pub(crate) fn highlight(
             } else if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::RawString::cast)
             {
-                if is_format_string {
+                format_string_highlighter.highlight_format_string(&mut stack, &string, range);
-                    stack.push();
-                    string.lex_format_specifier(|piece_range, kind| {
-                        if let Some(highlight) = highlight_format_specifier(kind) {
-                            stack.add(HighlightedRange {
-                                range: piece_range + range.start(),
-                                highlight: highlight.into(),
-                                binding_hash: None,
-                            });
-                        }
-                    });
-                    stack.pop();
-                }
             }
         }
     }
@@ -436,24 +384,6 @@ impl HighlightedRangeStack {
     }
 }
 
-fn highlight_format_specifier(kind: FormatSpecifier) -> Option<HighlightTag> {
-    Some(match kind {
-        FormatSpecifier::Open
-        | FormatSpecifier::Close
-        | FormatSpecifier::Colon
-        | FormatSpecifier::Fill
-        | FormatSpecifier::Align
-        | FormatSpecifier::Sign
-        | FormatSpecifier::NumberSign
-        | FormatSpecifier::DollarSign
-        | FormatSpecifier::Dot
-        | FormatSpecifier::Asterisk
-        | FormatSpecifier::QuestionMark => HighlightTag::FormatSpecifier,
-        FormatSpecifier::Integer | FormatSpecifier::Zero => HighlightTag::NumericLiteral,
-        FormatSpecifier::Identifier => HighlightTag::Local,
-    })
-}
-
 fn macro_call_range(macro_call: &ast::MacroCall) -> Option<TextRange> {
     let path = macro_call.path()?;
     let name_ref = path.segment()?.name_ref()?;
@@ -934,99 +864,3 @@ fn highlight_name_ref_by_syntax(name: ast::NameRef, sema: &Semantics<RootDatabas
         _ => default.into(),
     }
 }
-
-struct MacroMatcherParseState {
-    /// Opening and corresponding closing bracket of the matcher or expander of the current rule
-    paren_ty: Option<(SyntaxKind, SyntaxKind)>,
-    paren_level: usize,
-    rule_state: RuleState,
-    /// Whether we are inside the outer `{` `}` macro block that holds the rules
-    in_invoc_body: bool,
-}
-
-impl MacroMatcherParseState {
-    fn new() -> Self {
-        MacroMatcherParseState {
-            paren_ty: None,
-            paren_level: 0,
-            in_invoc_body: false,
-            rule_state: RuleState::None,
-        }
-    }
-}
-
-#[derive(Copy, Clone, PartialEq)]
-enum RuleState {
-    Matcher,
-    Expander,
-    Between,
-    None,
-}
-
-impl RuleState {
-    fn transition(&mut self) {
-        *self = match self {
-            RuleState::Matcher => RuleState::Between,
-            RuleState::Expander => RuleState::None,
-            RuleState::Between => RuleState::Expander,
-            RuleState::None => RuleState::Matcher,
-        };
-    }
-}
-
-fn update_macro_rules_state(tok: &SyntaxToken, state: &mut MacroMatcherParseState) -> RuleState {
-    if !state.in_invoc_body {
-        if tok.kind() == T!['{'] {
-            state.in_invoc_body = true;
-        }
-        return state.rule_state;
-    }
-
-    match state.paren_ty {
-        Some((open, close)) => {
-            if tok.kind() == open {
-                state.paren_level += 1;
-            } else if tok.kind() == close {
-                state.paren_level -= 1;
-                if state.paren_level == 0 {
-                    let res = state.rule_state;
-                    state.rule_state.transition();
-                    state.paren_ty = None;
-                    return res;
-                }
-            }
-        }
-        None => {
-            match tok.kind() {
-                T!['('] => {
-                    state.paren_ty = Some((T!['('], T![')']));
-                }
-                T!['{'] => {
-                    state.paren_ty = Some((T!['{'], T!['}']));
-                }
-                T!['['] => {
-                    state.paren_ty = Some((T!['['], T![']']));
-                }
-                _ => (),
-            }
-            if state.paren_ty.is_some() {
-                state.paren_level = 1;
-                state.rule_state.transition();
-            }
-        }
-    }
-    state.rule_state
-}
-
-fn skip_metavariables(element: SyntaxElement) -> bool {
-    let tok = match element.as_token() {
-        Some(tok) => tok,
-        None => return false,
-    };
-    let is_fragment = || tok.prev_token().map(|tok| tok.kind()) == Some(T![$]);
-    match tok.kind() {
-        IDENT if is_fragment() => true,
-        kind if kind.is_keyword() && is_fragment() => true,
-        _ => false,
-    }
-}

crates/ide/src/syntax_highlighting/format.rs

@@ -0,0 +1,78 @@
+//! Syntax highlighting for format macro strings.
+use syntax::{
+    ast::{self, FormatSpecifier, HasFormatSpecifier},
+    AstNode, AstToken, SyntaxElement, SyntaxKind, SyntaxNode, TextRange,
+};
+
+use crate::{syntax_highlighting::HighlightedRangeStack, HighlightTag, HighlightedRange};
+
+#[derive(Default)]
+pub(super) struct FormatStringHighlighter {
+    format_string: Option<SyntaxElement>,
+}
+
+impl FormatStringHighlighter {
+    pub(super) fn check_for_format_string(&mut self, parent: &SyntaxNode) {
+        // Check if macro takes a format string and remember it for highlighting later.
+        // The macros that accept a format string expand to a compiler builtin macros
+        // `format_args` and `format_args_nl`.
+        if let Some(name) = parent
+            .parent()
+            .and_then(ast::MacroCall::cast)
+            .and_then(|mc| mc.path())
+            .and_then(|p| p.segment())
+            .and_then(|s| s.name_ref())
+        {
+            match name.text().as_str() {
+                "format_args" | "format_args_nl" => {
+                    self.format_string = parent
+                        .children_with_tokens()
+                        .filter(|t| t.kind() != SyntaxKind::WHITESPACE)
+                        .nth(1)
+                        .filter(|e| {
+                            ast::String::can_cast(e.kind()) || ast::RawString::can_cast(e.kind())
+                        })
+                }
+                _ => {}
+            }
+        }
+    }
+    pub(super) fn highlight_format_string(
+        &self,
+        range_stack: &mut HighlightedRangeStack,
+        string: &impl HasFormatSpecifier,
+        range: TextRange,
+    ) {
+        if self.format_string.as_ref() == Some(&SyntaxElement::from(string.syntax().clone())) {
+            range_stack.push();
+            string.lex_format_specifier(|piece_range, kind| {
+                if let Some(highlight) = highlight_format_specifier(kind) {
+                    range_stack.add(HighlightedRange {
+                        range: piece_range + range.start(),
+                        highlight: highlight.into(),
+                        binding_hash: None,
+                    });
+                }
+            });
+            range_stack.pop();
+        }
+    }
+}
+
+fn highlight_format_specifier(kind: FormatSpecifier) -> Option<HighlightTag> {
+    Some(match kind {
+        FormatSpecifier::Open
+        | FormatSpecifier::Close
+        | FormatSpecifier::Colon
+        | FormatSpecifier::Fill
+        | FormatSpecifier::Align
+        | FormatSpecifier::Sign
+        | FormatSpecifier::NumberSign
+        | FormatSpecifier::DollarSign
+        | FormatSpecifier::Dot
+        | FormatSpecifier::Asterisk
+        | FormatSpecifier::QuestionMark => HighlightTag::FormatSpecifier,
+        FormatSpecifier::Integer | FormatSpecifier::Zero => HighlightTag::NumericLiteral,
+        FormatSpecifier::Identifier => HighlightTag::Local,
+    })
+}

crates/ide/src/syntax_highlighting/macro_rules.rs

@@ -0,0 +1,129 @@
+//! Syntax highlighting for macro_rules!.
+use syntax::{SyntaxElement, SyntaxKind, SyntaxToken, TextRange, T};
+
+use crate::{HighlightTag, HighlightedRange};
+
+#[derive(Default)]
+pub(super) struct MacroRulesHighlighter {
+    state: Option<MacroMatcherParseState>,
+}
+
+impl MacroRulesHighlighter {
+    pub(super) fn init(&mut self) {
+        self.state = Some(MacroMatcherParseState::default());
+    }
+
+    pub(super) fn advance(&mut self, token: &SyntaxToken) {
+        if let Some(state) = self.state.as_mut() {
+            update_macro_rules_state(state, token);
+        }
+    }
+
+    pub(super) fn highlight(&self, element: SyntaxElement) -> Option<HighlightedRange> {
+        if let Some(state) = self.state.as_ref() {
+            if matches!(state.rule_state, RuleState::Matcher | RuleState::Expander) {
+                if let Some(range) = is_metavariable(element) {
+                    return Some(HighlightedRange {
+                        range,
+                        highlight: HighlightTag::UnresolvedReference.into(),
+                        binding_hash: None,
+                    });
+                }
+            }
+        }
+        None
+    }
+}
+
+struct MacroMatcherParseState {
+    /// Opening and corresponding closing bracket of the matcher or expander of the current rule
+    paren_ty: Option<(SyntaxKind, SyntaxKind)>,
+    paren_level: usize,
+    rule_state: RuleState,
+    /// Whether we are inside the outer `{` `}` macro block that holds the rules
+    in_invoc_body: bool,
+}
+
+impl Default for MacroMatcherParseState {
+    fn default() -> Self {
+        MacroMatcherParseState {
+            paren_ty: None,
+            paren_level: 0,
+            in_invoc_body: false,
+            rule_state: RuleState::None,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq)]
+enum RuleState {
+    Matcher,
+    Expander,
+    Between,
+    None,
+}
+
+impl RuleState {
+    fn transition(&mut self) {
+        *self = match self {
+            RuleState::Matcher => RuleState::Between,
+            RuleState::Expander => RuleState::None,
+            RuleState::Between => RuleState::Expander,
+            RuleState::None => RuleState::Matcher,
+        };
+    }
+}
+
+fn update_macro_rules_state(state: &mut MacroMatcherParseState, tok: &SyntaxToken) {
+    if !state.in_invoc_body {
+        if tok.kind() == T!['{'] {
+            state.in_invoc_body = true;
+        }
+        return;
+    }
+
+    match state.paren_ty {
+        Some((open, close)) => {
+            if tok.kind() == open {
+                state.paren_level += 1;
+            } else if tok.kind() == close {
+                state.paren_level -= 1;
+                if state.paren_level == 0 {
+                    state.rule_state.transition();
+                    state.paren_ty = None;
+                }
+            }
+        }
+        None => {
+            match tok.kind() {
+                T!['('] => {
+                    state.paren_ty = Some((T!['('], T![')']));
+                }
+                T!['{'] => {
+                    state.paren_ty = Some((T!['{'], T!['}']));
+                }
+                T!['['] => {
+                    state.paren_ty = Some((T!['['], T![']']));
+                }
+                _ => (),
+            }
+            if state.paren_ty.is_some() {
+                state.paren_level = 1;
+                state.rule_state.transition();
+            }
+        }
+    }
+}
+
+fn is_metavariable(element: SyntaxElement) -> Option<TextRange> {
+    let tok = element.as_token()?;
+    match tok.kind() {
+        kind if kind == SyntaxKind::IDENT || kind.is_keyword() => {
+            if let Some(_dollar) = tok.prev_token().filter(|t| t.kind() == SyntaxKind::DOLLAR) {
+                return Some(tok.text_range());
+            }
+        }
+        _ => (),
+    };
+    None
+}