From df87be88d8500c8955f882d71467e01a7d4db9ab Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Oct 2020 00:56:41 +0200 Subject: [PATCH 1/3] Factor format string highlighting out --- crates/ide/src/syntax_highlighting.rs | 83 ++------------------ crates/ide/src/syntax_highlighting/format.rs | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 75 deletions(-) create mode 100644 crates/ide/src/syntax_highlighting/format.rs diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 6aafd6fd50..f430006d7d 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -1,6 +1,7 @@ -mod tags; +mod format; mod html; mod injection; +mod tags; #[cfg(test)] mod tests; @@ -17,9 +18,8 @@ use syntax::{ SyntaxNode, SyntaxToken, TextRange, WalkEvent, T, }; -use crate::FileId; +use crate::{syntax_highlighting::format::FormatStringHighlighter, FileId}; -use ast::FormatSpecifier; pub(crate) use html::highlight_as_html; pub use tags::{Highlight, HighlightModifier, HighlightModifiers, HighlightTag}; @@ -69,7 +69,7 @@ pub(crate) fn highlight( let mut stack = HighlightedRangeStack::new(); let mut current_macro_call: Option<(ast::MacroCall, Option)> = None; - let mut format_string: Option = None; + let mut format_string_highlighter = FormatStringHighlighter::default(); // Walk all nodes, keeping track of whether we are inside a macro or not. // If in macro, expand it first and highlight the expanded code. @@ -121,7 +121,7 @@ pub(crate) fn highlight( WalkEvent::Leave(Some(mc)) => { assert!(current_macro_call.map(|it| it.0) == Some(mc)); current_macro_call = None; - format_string = None; + format_string_highlighter.reset(); } _ => (), } @@ -173,30 +173,7 @@ pub(crate) fn highlight( let token = sema.descend_into_macros(token.clone()); let parent = token.parent(); - // Check if macro takes a format string and remember it for highlighting later. - // The macros that accept a format string expand to a compiler builtin macros - // `format_args` and `format_args_nl`. - if let Some(name) = parent - .parent() - .and_then(ast::MacroCall::cast) - .and_then(|mc| mc.path()) - .and_then(|p| p.segment()) - .and_then(|s| s.name_ref()) - { - match name.text().as_str() { - "format_args" | "format_args_nl" => { - format_string = parent - .children_with_tokens() - .filter(|t| t.kind() != WHITESPACE) - .nth(1) - .filter(|e| { - ast::String::can_cast(e.kind()) - || ast::RawString::can_cast(e.kind()) - }) - } - _ => {} - } - } + format_string_highlighter.check_for_format_string(&parent); // We only care Name and Name_ref match (token.kind(), parent.kind()) { @@ -214,8 +191,6 @@ pub(crate) fn highlight( } } - let is_format_string = format_string.as_ref() == Some(&element_to_highlight); - if let Some((highlight, binding_hash)) = highlight_element( &sema, &mut bindings_shadow_count, @@ -226,19 +201,7 @@ pub(crate) fn highlight( if let Some(string) = element_to_highlight.as_token().cloned().and_then(ast::String::cast) { - if is_format_string { - stack.push(); - string.lex_format_specifier(|piece_range, kind| { - if let Some(highlight) = highlight_format_specifier(kind) { - stack.add(HighlightedRange { - range: piece_range + range.start(), - highlight: highlight.into(), - binding_hash: None, - }); - } - }); - stack.pop(); - } + format_string_highlighter.highlight_format_string(&mut stack, &string, range); // Highlight escape sequences if let Some(char_ranges) = string.char_ranges() { stack.push(); @@ -256,19 +219,7 @@ pub(crate) fn highlight( } else if let Some(string) = element_to_highlight.as_token().cloned().and_then(ast::RawString::cast) { - if is_format_string { - stack.push(); - string.lex_format_specifier(|piece_range, kind| { - if let Some(highlight) = highlight_format_specifier(kind) { - stack.add(HighlightedRange { - range: piece_range + range.start(), - highlight: highlight.into(), - binding_hash: None, - }); - } - }); - stack.pop(); - } + format_string_highlighter.highlight_format_string(&mut stack, &string, range); } } } @@ -436,24 +387,6 @@ impl HighlightedRangeStack { } } -fn highlight_format_specifier(kind: FormatSpecifier) -> Option { - Some(match kind { - FormatSpecifier::Open - | FormatSpecifier::Close - | FormatSpecifier::Colon - | FormatSpecifier::Fill - | FormatSpecifier::Align - | FormatSpecifier::Sign - | FormatSpecifier::NumberSign - | FormatSpecifier::DollarSign - | FormatSpecifier::Dot - | FormatSpecifier::Asterisk - | FormatSpecifier::QuestionMark => HighlightTag::FormatSpecifier, - FormatSpecifier::Integer | FormatSpecifier::Zero => HighlightTag::NumericLiteral, - FormatSpecifier::Identifier => HighlightTag::Local, - }) -} - fn macro_call_range(macro_call: &ast::MacroCall) -> Option { let path = macro_call.path()?; let name_ref = path.segment()?.name_ref()?; diff --git a/crates/ide/src/syntax_highlighting/format.rs b/crates/ide/src/syntax_highlighting/format.rs new file mode 100644 index 0000000000..3ab01295ab --- /dev/null +++ b/crates/ide/src/syntax_highlighting/format.rs @@ -0,0 +1,82 @@ +//! Syntax highlighting for format macro strings. +use syntax::{ + ast::{self, FormatSpecifier, HasFormatSpecifier}, + AstNode, AstToken, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, +}; + +use crate::{syntax_highlighting::HighlightedRangeStack, HighlightTag, HighlightedRange}; + +#[derive(Default)] +pub(super) struct FormatStringHighlighter { + format_string: Option, +} + +impl FormatStringHighlighter { + pub(super) fn reset(&mut self) { + self.format_string = None; + } + + pub(super) fn check_for_format_string(&mut self, parent: &SyntaxNode) { + // Check if macro takes a format string and remember it for highlighting later. + // The macros that accept a format string expand to a compiler builtin macros + // `format_args` and `format_args_nl`. + if let Some(name) = parent + .parent() + .and_then(ast::MacroCall::cast) + .and_then(|mc| mc.path()) + .and_then(|p| p.segment()) + .and_then(|s| s.name_ref()) + { + match name.text().as_str() { + "format_args" | "format_args_nl" => { + self.format_string = parent + .children_with_tokens() + .filter(|t| t.kind() != SyntaxKind::WHITESPACE) + .nth(1) + .filter(|e| { + ast::String::can_cast(e.kind()) || ast::RawString::can_cast(e.kind()) + }) + } + _ => {} + } + } + } + pub(super) fn highlight_format_string( + &self, + range_stack: &mut HighlightedRangeStack, + string: &impl HasFormatSpecifier, + range: TextRange, + ) { + if self.format_string.as_ref() == Some(&SyntaxElement::from(string.syntax().clone())) { + range_stack.push(); + string.lex_format_specifier(|piece_range, kind| { + if let Some(highlight) = highlight_format_specifier(kind) { + range_stack.add(HighlightedRange { + range: piece_range + range.start(), + highlight: highlight.into(), + binding_hash: None, + }); + } + }); + range_stack.pop(); + } + } +} + +fn highlight_format_specifier(kind: FormatSpecifier) -> Option { + Some(match kind { + FormatSpecifier::Open + | FormatSpecifier::Close + | FormatSpecifier::Colon + | FormatSpecifier::Fill + | FormatSpecifier::Align + | FormatSpecifier::Sign + | FormatSpecifier::NumberSign + | FormatSpecifier::DollarSign + | FormatSpecifier::Dot + | FormatSpecifier::Asterisk + | FormatSpecifier::QuestionMark => HighlightTag::FormatSpecifier, + FormatSpecifier::Integer | FormatSpecifier::Zero => HighlightTag::NumericLiteral, + FormatSpecifier::Identifier => HighlightTag::Local, + }) +} From 8c6dc5f28a5550acffbbb063335833304dac266d Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Oct 2020 19:23:59 +0200 Subject: [PATCH 2/3] Factor macro_rules! highlighting out --- crates/ide/src/syntax_highlighting.rs | 135 +++-------------- .../src/syntax_highlighting/macro_rules.rs | 136 ++++++++++++++++++ 2 files changed, 154 insertions(+), 117 deletions(-) create mode 100644 crates/ide/src/syntax_highlighting/macro_rules.rs diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index f430006d7d..8ecaff2047 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -1,6 +1,7 @@ mod format; mod html; mod injection; +mod macro_rules; mod tags; #[cfg(test)] mod tests; @@ -18,7 +19,10 @@ use syntax::{ SyntaxNode, SyntaxToken, TextRange, WalkEvent, T, }; -use crate::{syntax_highlighting::format::FormatStringHighlighter, FileId}; +use crate::{ + syntax_highlighting::{format::FormatStringHighlighter, macro_rules::MacroRulesHighlighter}, + FileId, +}; pub(crate) use html::highlight_as_html; pub use tags::{Highlight, HighlightModifier, HighlightModifiers, HighlightTag}; @@ -68,8 +72,9 @@ pub(crate) fn highlight( // When we leave a node, the we use it to flatten the highlighted ranges. let mut stack = HighlightedRangeStack::new(); - let mut current_macro_call: Option<(ast::MacroCall, Option)> = None; + let mut current_macro_call: Option = None; let mut format_string_highlighter = FormatStringHighlighter::default(); + let mut macro_rules_highlighter = MacroRulesHighlighter::new(); // Walk all nodes, keeping track of whether we are inside a macro or not. // If in macro, expand it first and highlight the expanded code. @@ -99,9 +104,8 @@ pub(crate) fn highlight( binding_hash: None, }); } - let mut is_macro_rules = None; if let Some(name) = mc.is_macro_rules() { - is_macro_rules = Some(MacroMatcherParseState::new()); + macro_rules_highlighter.init(); if let Some((highlight, binding_hash)) = highlight_element( &sema, &mut bindings_shadow_count, @@ -115,13 +119,14 @@ pub(crate) fn highlight( }); } } - current_macro_call = Some((mc.clone(), is_macro_rules)); + current_macro_call = Some(mc.clone()); continue; } WalkEvent::Leave(Some(mc)) => { - assert!(current_macro_call.map(|it| it.0) == Some(mc)); + assert!(current_macro_call == Some(mc)); current_macro_call = None; format_string_highlighter.reset(); + macro_rules_highlighter.reset(); } _ => (), } @@ -148,20 +153,6 @@ pub(crate) fn highlight( WalkEvent::Leave(_) => continue, }; - // check if in matcher part of a macro_rules rule - if let Some((_, Some(ref mut state))) = current_macro_call { - if let Some(tok) = element.as_token() { - if matches!( - update_macro_rules_state(tok, state), - RuleState::Matcher | RuleState::Expander - ) { - if skip_metavariables(element.clone()) { - continue; - } - } - } - } - let range = element.text_range(); let element_to_highlight = if current_macro_call.is_some() && element.kind() != COMMENT { @@ -174,6 +165,9 @@ pub(crate) fn highlight( let parent = token.parent(); format_string_highlighter.check_for_format_string(&parent); + if let Some(tok) = element.as_token() { + macro_rules_highlighter.advance(tok); + } // We only care Name and Name_ref match (token.kind(), parent.kind()) { @@ -197,7 +191,10 @@ pub(crate) fn highlight( syntactic_name_ref_highlighting, element_to_highlight.clone(), ) { - stack.add(HighlightedRange { range, highlight, binding_hash }); + if macro_rules_highlighter.highlight(element_to_highlight.clone()).is_none() { + stack.add(HighlightedRange { range, highlight, binding_hash }); + } + if let Some(string) = element_to_highlight.as_token().cloned().and_then(ast::String::cast) { @@ -867,99 +864,3 @@ fn highlight_name_ref_by_syntax(name: ast::NameRef, sema: &Semantics default.into(), } } - -struct MacroMatcherParseState { - /// Opening and corresponding closing bracket of the matcher or expander of the current rule - paren_ty: Option<(SyntaxKind, SyntaxKind)>, - paren_level: usize, - rule_state: RuleState, - /// Whether we are inside the outer `{` `}` macro block that holds the rules - in_invoc_body: bool, -} - -impl MacroMatcherParseState { - fn new() -> Self { - MacroMatcherParseState { - paren_ty: None, - paren_level: 0, - in_invoc_body: false, - rule_state: RuleState::None, - } - } -} - -#[derive(Copy, Clone, PartialEq)] -enum RuleState { - Matcher, - Expander, - Between, - None, -} - -impl RuleState { - fn transition(&mut self) { - *self = match self { - RuleState::Matcher => RuleState::Between, - RuleState::Expander => RuleState::None, - RuleState::Between => RuleState::Expander, - RuleState::None => RuleState::Matcher, - }; - } -} - -fn update_macro_rules_state(tok: &SyntaxToken, state: &mut MacroMatcherParseState) -> RuleState { - if !state.in_invoc_body { - if tok.kind() == T!['{'] { - state.in_invoc_body = true; - } - return state.rule_state; - } - - match state.paren_ty { - Some((open, close)) => { - if tok.kind() == open { - state.paren_level += 1; - } else if tok.kind() == close { - state.paren_level -= 1; - if state.paren_level == 0 { - let res = state.rule_state; - state.rule_state.transition(); - state.paren_ty = None; - return res; - } - } - } - None => { - match tok.kind() { - T!['('] => { - state.paren_ty = Some((T!['('], T![')'])); - } - T!['{'] => { - state.paren_ty = Some((T!['{'], T!['}'])); - } - T!['['] => { - state.paren_ty = Some((T!['['], T![']'])); - } - _ => (), - } - if state.paren_ty.is_some() { - state.paren_level = 1; - state.rule_state.transition(); - } - } - } - state.rule_state -} - -fn skip_metavariables(element: SyntaxElement) -> bool { - let tok = match element.as_token() { - Some(tok) => tok, - None => return false, - }; - let is_fragment = || tok.prev_token().map(|tok| tok.kind()) == Some(T![$]); - match tok.kind() { - IDENT if is_fragment() => true, - kind if kind.is_keyword() && is_fragment() => true, - _ => false, - } -} diff --git a/crates/ide/src/syntax_highlighting/macro_rules.rs b/crates/ide/src/syntax_highlighting/macro_rules.rs new file mode 100644 index 0000000000..0676e09726 --- /dev/null +++ b/crates/ide/src/syntax_highlighting/macro_rules.rs @@ -0,0 +1,136 @@ +//! Syntax highlighting for macro_rules!. +use syntax::{SyntaxElement, SyntaxKind, SyntaxToken, TextRange, T}; + +use crate::{HighlightTag, HighlightedRange}; + +pub(super) struct MacroRulesHighlighter { + state: Option, +} + +impl MacroRulesHighlighter { + pub(super) fn new() -> Self { + MacroRulesHighlighter { state: None } + } + + pub(super) fn init(&mut self) { + self.state = Some(MacroMatcherParseState::new()); + } + + pub(super) fn reset(&mut self) { + self.state = None; + } + + pub(super) fn advance(&mut self, token: &SyntaxToken) { + if let Some(state) = self.state.as_mut() { + update_macro_rules_state(state, token); + } + } + + pub(super) fn highlight(&self, element: SyntaxElement) -> Option { + if let Some(state) = self.state.as_ref() { + if matches!(state.rule_state, RuleState::Matcher | RuleState::Expander) { + if let Some(range) = is_metavariable(element) { + return Some(HighlightedRange { + range, + highlight: HighlightTag::UnresolvedReference.into(), + binding_hash: None, + }); + } + } + } + None + } +} + +struct MacroMatcherParseState { + /// Opening and corresponding closing bracket of the matcher or expander of the current rule + paren_ty: Option<(SyntaxKind, SyntaxKind)>, + paren_level: usize, + rule_state: RuleState, + /// Whether we are inside the outer `{` `}` macro block that holds the rules + in_invoc_body: bool, +} + +impl MacroMatcherParseState { + fn new() -> Self { + MacroMatcherParseState { + paren_ty: None, + paren_level: 0, + in_invoc_body: false, + rule_state: RuleState::None, + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq)] +enum RuleState { + Matcher, + Expander, + Between, + None, +} + +impl RuleState { + fn transition(&mut self) { + *self = match self { + RuleState::Matcher => RuleState::Between, + RuleState::Expander => RuleState::None, + RuleState::Between => RuleState::Expander, + RuleState::None => RuleState::Matcher, + }; + } +} + +fn update_macro_rules_state(state: &mut MacroMatcherParseState, tok: &SyntaxToken) { + if !state.in_invoc_body { + if tok.kind() == T!['{'] { + state.in_invoc_body = true; + } + return; + } + + match state.paren_ty { + Some((open, close)) => { + if tok.kind() == open { + state.paren_level += 1; + } else if tok.kind() == close { + state.paren_level -= 1; + if state.paren_level == 0 { + state.rule_state.transition(); + state.paren_ty = None; + } + } + } + None => { + match tok.kind() { + T!['('] => { + state.paren_ty = Some((T!['('], T![')'])); + } + T!['{'] => { + state.paren_ty = Some((T!['{'], T!['}'])); + } + T!['['] => { + state.paren_ty = Some((T!['['], T![']'])); + } + _ => (), + } + if state.paren_ty.is_some() { + state.paren_level = 1; + state.rule_state.transition(); + } + } + } +} + +fn is_metavariable(element: SyntaxElement) -> Option { + let tok = element.as_token()?; + match tok.kind() { + kind if kind == SyntaxKind::IDENT || kind.is_keyword() => { + if let Some(_dollar) = tok.prev_token().filter(|t| t.kind() == SyntaxKind::DOLLAR) { + return Some(tok.text_range()); + } + } + _ => (), + }; + None +} From bab29e65eb87f4765fb22d58bff723780980eeb6 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Wed, 14 Oct 2020 22:21:58 +0200 Subject: [PATCH 3/3] Default::default the highlighters --- crates/ide/src/syntax_highlighting.rs | 6 +++--- crates/ide/src/syntax_highlighting/format.rs | 4 ---- crates/ide/src/syntax_highlighting/macro_rules.rs | 15 ++++----------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 8ecaff2047..527888306b 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -74,7 +74,7 @@ pub(crate) fn highlight( let mut current_macro_call: Option = None; let mut format_string_highlighter = FormatStringHighlighter::default(); - let mut macro_rules_highlighter = MacroRulesHighlighter::new(); + let mut macro_rules_highlighter = MacroRulesHighlighter::default(); // Walk all nodes, keeping track of whether we are inside a macro or not. // If in macro, expand it first and highlight the expanded code. @@ -125,8 +125,8 @@ pub(crate) fn highlight( WalkEvent::Leave(Some(mc)) => { assert!(current_macro_call == Some(mc)); current_macro_call = None; - format_string_highlighter.reset(); - macro_rules_highlighter.reset(); + format_string_highlighter = FormatStringHighlighter::default(); + macro_rules_highlighter = MacroRulesHighlighter::default(); } _ => (), } diff --git a/crates/ide/src/syntax_highlighting/format.rs b/crates/ide/src/syntax_highlighting/format.rs index 3ab01295ab..71bde24f08 100644 --- a/crates/ide/src/syntax_highlighting/format.rs +++ b/crates/ide/src/syntax_highlighting/format.rs @@ -12,10 +12,6 @@ pub(super) struct FormatStringHighlighter { } impl FormatStringHighlighter { - pub(super) fn reset(&mut self) { - self.format_string = None; - } - pub(super) fn check_for_format_string(&mut self, parent: &SyntaxNode) { // Check if macro takes a format string and remember it for highlighting later. // The macros that accept a format string expand to a compiler builtin macros diff --git a/crates/ide/src/syntax_highlighting/macro_rules.rs b/crates/ide/src/syntax_highlighting/macro_rules.rs index 0676e09726..4462af47ed 100644 --- a/crates/ide/src/syntax_highlighting/macro_rules.rs +++ b/crates/ide/src/syntax_highlighting/macro_rules.rs @@ -3,21 +3,14 @@ use syntax::{SyntaxElement, SyntaxKind, SyntaxToken, TextRange, T}; use crate::{HighlightTag, HighlightedRange}; +#[derive(Default)] pub(super) struct MacroRulesHighlighter { state: Option, } impl MacroRulesHighlighter { - pub(super) fn new() -> Self { - MacroRulesHighlighter { state: None } - } - pub(super) fn init(&mut self) { - self.state = Some(MacroMatcherParseState::new()); - } - - pub(super) fn reset(&mut self) { - self.state = None; + self.state = Some(MacroMatcherParseState::default()); } pub(super) fn advance(&mut self, token: &SyntaxToken) { @@ -51,8 +44,8 @@ struct MacroMatcherParseState { in_invoc_body: bool, } -impl MacroMatcherParseState { - fn new() -> Self { +impl Default for MacroMatcherParseState { + fn default() -> Self { MacroMatcherParseState { paren_ty: None, paren_level: 0,