Merge #350

350: Super simple macro support r=matklad a=matklad

Super simple support for macros, mostly for figuring out how to fit them into the current architecture. Expansion is hard-coded and string based (mid-term, we should try to copy-paste macro-by-example expander from rustc). 

Ideally, we should handle

* highlighting inside the macro (done)
* extend selection inside the macro 
* completion inside the macro
* indexing structs, produced by the macro

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/ra_analysis/src/extend_selection.rs

@@ -0,0 +1,11 @@
+use ra_db::SyntaxDatabase;
+
+use crate::{
+    TextRange, FileRange,
+    db::RootDatabase,
+};
+
+pub(crate) fn extend_selection(db: &RootDatabase, frange: FileRange) -> TextRange {
+    let file = db.source_file(frange.file_id);
+    ra_editor::extend_selection(&file, frange.range).unwrap_or(frange.range)
+}

crates/ra_analysis/src/imp.rs

@@ -23,7 +23,7 @@ use crate::{
     AnalysisChange,
     Cancelable,
     completion::{CompletionItem, completions},
-    CrateId, db, Diagnostic, FileId, FilePosition, FileSystemEdit,
+    CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit,
     Query, ReferenceResolution, RootChange, SourceChange, SourceFileEdit,
     symbol_index::{LibrarySymbolsQuery, SymbolIndex, SymbolsDatabase},
 };
@@ -404,19 +404,21 @@ impl AnalysisImpl {
         Ok(res)
     }
 
-    pub fn assists(&self, file_id: FileId, range: TextRange) -> Vec<SourceChange> {
+    pub fn assists(&self, frange: FileRange) -> Vec<SourceChange> {
-        let file = self.file_syntax(file_id);
+        let file = self.file_syntax(frange.file_id);
-        let offset = range.start();
+        let offset = frange.range.start();
         let actions = vec![
             ra_editor::flip_comma(&file, offset).map(|f| f()),
             ra_editor::add_derive(&file, offset).map(|f| f()),
             ra_editor::add_impl(&file, offset).map(|f| f()),
             ra_editor::make_pub_crate(&file, offset).map(|f| f()),
-            ra_editor::introduce_variable(&file, range).map(|f| f()),
+            ra_editor::introduce_variable(&file, frange.range).map(|f| f()),
         ];
         actions
             .into_iter()
-            .filter_map(|local_edit| Some(SourceChange::from_local_edit(file_id, local_edit?)))
+            .filter_map(|local_edit| {
+                Some(SourceChange::from_local_edit(frange.file_id, local_edit?))
+            })
             .collect()
     }
 
@@ -487,13 +489,15 @@ impl AnalysisImpl {
         Ok(None)
     }
 
-    pub fn type_of(&self, file_id: FileId, range: TextRange) -> Cancelable<Option<String>> {
+    pub fn type_of(&self, frange: FileRange) -> Cancelable<Option<String>> {
-        let file = self.db.source_file(file_id);
+        let file = self.db.source_file(frange.file_id);
         let syntax = file.syntax();
-        let node = find_covering_node(syntax, range);
+        let node = find_covering_node(syntax, frange.range);
         let parent_fn = ctry!(node.ancestors().find_map(FnDef::cast));
         let function = ctry!(source_binder::function_from_source(
-            &*self.db, file_id, parent_fn
+            &*self.db,
+            frange.file_id,
+            parent_fn
         )?);
         let infer = function.infer(&*self.db)?;
         Ok(infer.type_of_node(node).map(|t| t.to_string()))

crates/ra_analysis/src/lib.rs

@@ -16,6 +16,10 @@ mod completion;
 mod symbol_index;
 pub mod mock_analysis;
 
+mod extend_selection;
+mod syntax_highlighting;
+mod macros;
+
 use std::{fmt, sync::Arc};
 
 use rustc_hash::FxHashMap;
@@ -37,7 +41,7 @@ pub use ra_editor::{
 pub use hir::FnSignatureInfo;
 
 pub use ra_db::{
-    Canceled, Cancelable, FilePosition,
+    Canceled, Cancelable, FilePosition, FileRange,
     CrateGraph, CrateId, SourceRootId, FileId
 };
 
@@ -276,8 +280,8 @@ impl Analysis {
     pub fn file_line_index(&self, file_id: FileId) -> Arc<LineIndex> {
         self.imp.file_line_index(file_id)
     }
-    pub fn extend_selection(&self, file: &SourceFileNode, range: TextRange) -> TextRange {
+    pub fn extend_selection(&self, frange: FileRange) -> TextRange {
-        ra_editor::extend_selection(file, range).unwrap_or(range)
+        extend_selection::extend_selection(&self.imp.db, frange)
     }
     pub fn matching_brace(&self, file: &SourceFileNode, offset: TextUnit) -> Option<TextUnit> {
         ra_editor::matching_brace(file, offset)
@@ -286,9 +290,9 @@ impl Analysis {
         let file = self.imp.file_syntax(file_id);
         ra_editor::syntax_tree(&file)
     }
-    pub fn join_lines(&self, file_id: FileId, range: TextRange) -> SourceChange {
+    pub fn join_lines(&self, frange: FileRange) -> SourceChange {
-        let file = self.imp.file_syntax(file_id);
+        let file = self.imp.file_syntax(frange.file_id);
-        SourceChange::from_local_edit(file_id, ra_editor::join_lines(&file, range))
+        SourceChange::from_local_edit(frange.file_id, ra_editor::join_lines(&file, frange.range))
     }
     pub fn on_enter(&self, position: FilePosition) -> Option<SourceChange> {
         let file = self.imp.file_syntax(position.file_id);
@@ -340,14 +344,13 @@ impl Analysis {
         Ok(ra_editor::runnables(&file))
     }
     pub fn highlight(&self, file_id: FileId) -> Cancelable<Vec<HighlightedRange>> {
-        let file = self.imp.file_syntax(file_id);
+        syntax_highlighting::highlight(&*self.imp.db, file_id)
-        Ok(ra_editor::highlight(&file))
     }
     pub fn completions(&self, position: FilePosition) -> Cancelable<Option<Vec<CompletionItem>>> {
         self.imp.completions(position)
     }
-    pub fn assists(&self, file_id: FileId, range: TextRange) -> Cancelable<Vec<SourceChange>> {
+    pub fn assists(&self, frange: FileRange) -> Cancelable<Vec<SourceChange>> {
-        Ok(self.imp.assists(file_id, range))
+        Ok(self.imp.assists(frange))
     }
     pub fn diagnostics(&self, file_id: FileId) -> Cancelable<Vec<Diagnostic>> {
         self.imp.diagnostics(file_id)
@@ -358,8 +361,8 @@ impl Analysis {
     ) -> Cancelable<Option<(FnSignatureInfo, Option<usize>)>> {
         self.imp.resolve_callable(position)
     }
-    pub fn type_of(&self, file_id: FileId, range: TextRange) -> Cancelable<Option<String>> {
+    pub fn type_of(&self, frange: FileRange) -> Cancelable<Option<String>> {
-        self.imp.type_of(file_id, range)
+        self.imp.type_of(frange)
     }
 }
 

crates/ra_analysis/src/macros.rs

@@ -0,0 +1,64 @@
+/// Begining of macro expansion.
+///
+/// This code should be moved out of ra_analysis into hir (?) ideally.
+use ra_syntax::{ast, AstNode, SourceFileNode, TextRange};
+
+use crate::{db::RootDatabase, FileId};
+
+pub(crate) fn expand(
+    _db: &RootDatabase,
+    _file_id: FileId,
+    macro_call: ast::MacroCall,
+) -> Option<MacroExpansion> {
+    let path = macro_call.path()?;
+    if path.qualifier().is_some() {
+        return None;
+    }
+    let name_ref = path.segment()?.name_ref()?;
+    if name_ref.text() != "ctry" {
+        return None;
+    }
+
+    let arg = macro_call.token_tree()?;
+    let text = format!(
+        r"
+        fn dummy() {{
+            match {} {{
+                None => return Ok(None),
+                Some(it) => it,
+            }}
+        }}",
+        arg.syntax().text()
+    );
+    let file = SourceFileNode::parse(&text);
+    let match_expr = file.syntax().descendants().find_map(ast::MatchExpr::cast)?;
+    let match_arg = match_expr.expr()?;
+    let ranges_map = vec![(arg.syntax().range(), match_arg.syntax().range())];
+    let res = MacroExpansion {
+        source_file: file,
+        ranges_map,
+    };
+    Some(res)
+}
+
+pub(crate) struct MacroExpansion {
+    pub(crate) source_file: SourceFileNode,
+    pub(crate) ranges_map: Vec<(TextRange, TextRange)>,
+}
+
+impl MacroExpansion {
+    pub(crate) fn source_file(&self) -> &SourceFileNode {
+        &self.source_file
+    }
+    pub(crate) fn map_range_back(&self, tgt_range: TextRange) -> Option<TextRange> {
+        for (s_range, t_range) in self.ranges_map.iter() {
+            if tgt_range.is_subrange(&t_range) {
+                let tgt_at_zero_range = tgt_range - tgt_range.start();
+                let tgt_range_offset = tgt_range.start() - t_range.start();
+                let src_range = tgt_at_zero_range + tgt_range_offset + s_range.start();
+                return Some(src_range);
+            }
+        }
+        None
+    }
+}

crates/ra_analysis/src/syntax_highlighting.rs

@@ -0,0 +1,63 @@
+use ra_syntax::{ast, AstNode,};
+use ra_editor::HighlightedRange;
+use ra_db::SyntaxDatabase;
+
+use crate::{
+    db::RootDatabase,
+    FileId, Cancelable,
+};
+
+pub(crate) fn highlight(db: &RootDatabase, file_id: FileId) -> Cancelable<Vec<HighlightedRange>> {
+    let source_file = db.source_file(file_id);
+    let mut res = ra_editor::highlight(&source_file);
+    for macro_call in source_file
+        .syntax()
+        .descendants()
+        .filter_map(ast::MacroCall::cast)
+    {
+        if let Some(exp) = crate::macros::expand(db, file_id, macro_call) {
+            let mapped_ranges = ra_editor::highlight(exp.source_file())
+                .into_iter()
+                .filter_map(|r| {
+                    let mapped_range = exp.map_range_back(r.range)?;
+                    let res = HighlightedRange {
+                        range: mapped_range,
+                        tag: r.tag,
+                    };
+                    Some(res)
+                });
+            res.extend(mapped_ranges);
+        }
+    }
+    Ok(res)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::mock_analysis::single_file;
+    use test_utils::assert_eq_dbg;
+
+    #[test]
+    fn highlights_code_inside_macros() {
+        let (analysis, file_id) = single_file(
+            "
+            fn main() {
+                ctry!({ let x = 92; x});
+            }
+        ",
+        );
+        let highlights = analysis.highlight(file_id).unwrap();
+        assert_eq_dbg(
+            r#"[HighlightedRange { range: [13; 15), tag: "keyword" },
+                HighlightedRange { range: [16; 20), tag: "function" },
+                HighlightedRange { range: [41; 46), tag: "macro" },
+                HighlightedRange { range: [49; 52), tag: "keyword" },
+                HighlightedRange { range: [57; 59), tag: "literal" },
+                HighlightedRange { range: [49; 52), tag: "keyword" },
+                HighlightedRange { range: [53; 54), tag: "function" },
+                HighlightedRange { range: [57; 59), tag: "literal" },
+                HighlightedRange { range: [61; 62), tag: "text" }]"#,
+            &highlights,
+        )
+    }
+}

crates/ra_db/src/lib.rs

@@ -8,7 +8,7 @@ pub mod mock;
 use std::sync::Arc;
 
 use ra_editor::LineIndex;
-use ra_syntax::{TextUnit, SourceFileNode};
+use ra_syntax::{TextUnit, TextRange, SourceFileNode};
 
 pub use crate::{
     cancelation::{Canceled, Cancelable},
@@ -70,3 +70,9 @@ pub struct FilePosition {
     pub file_id: FileId,
     pub offset: TextUnit,
 }
+
+#[derive(Clone, Copy, Debug)]
+pub struct FileRange {
+    pub file_id: FileId,
+    pub range: TextRange,
+}

crates/ra_editor/src/lib.rs

@@ -24,9 +24,10 @@ use ra_syntax::{
     SourceFileNode,
     Location,
     SyntaxKind::{self, *},
-    SyntaxNodeRef, TextRange, TextUnit,
+    SyntaxNodeRef, TextRange, TextUnit, Direction,
 };
 use itertools::Itertools;
+use rustc_hash::FxHashSet;
 
 #[derive(Debug)]
 pub struct HighlightedRange {
@@ -79,8 +80,13 @@ pub fn matching_brace(file: &SourceFileNode, offset: TextUnit) -> Option<TextUni
 }
 
 pub fn highlight(file: &SourceFileNode) -> Vec<HighlightedRange> {
+    // Visited nodes to handle highlighting priorities
+    let mut highlighted = FxHashSet::default();
     let mut res = Vec::new();
     for node in file.syntax().descendants() {
+        if highlighted.contains(&node) {
+            continue;
+        }
         let tag = match node.kind() {
             COMMENT => "comment",
             STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => "string",
@@ -90,7 +96,30 @@ pub fn highlight(file: &SourceFileNode) -> Vec<HighlightedRange> {
             INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE => "literal",
             LIFETIME => "parameter",
             k if k.is_keyword() => "keyword",
-            _ => continue,
+            _ => {
+                if let Some(macro_call) = ast::MacroCall::cast(node) {
+                    if let Some(path) = macro_call.path() {
+                        if let Some(segment) = path.segment() {
+                            if let Some(name_ref) = segment.name_ref() {
+                                highlighted.insert(name_ref.syntax());
+                                let range_start = name_ref.syntax().range().start();
+                                let mut range_end = name_ref.syntax().range().end();
+                                for sibling in path.syntax().siblings(Direction::Next) {
+                                    match sibling.kind() {
+                                        EXCL | IDENT => range_end = sibling.range().end(),
+                                        _ => (),
+                                    }
+                                }
+                                res.push(HighlightedRange {
+                                    range: TextRange::from_to(range_start, range_end),
+                                    tag: "macro",
+                                })
+                            }
+                        }
+                    }
+                }
+                continue;
+            }
         };
         res.push(HighlightedRange {
             range: node.range(),
@@ -235,7 +264,7 @@ fn main() {}
             r#"[HighlightedRange { range: [1; 11), tag: "comment" },
                 HighlightedRange { range: [12; 14), tag: "keyword" },
                 HighlightedRange { range: [15; 19), tag: "function" },
-                HighlightedRange { range: [29; 36), tag: "text" },
+                HighlightedRange { range: [29; 37), tag: "macro" },
                 HighlightedRange { range: [38; 50), tag: "string" },
                 HighlightedRange { range: [52; 54), tag: "literal" }]"#,
             &hls,

crates/ra_lsp_server/src/conv.rs

@@ -2,7 +2,7 @@ use languageserver_types::{
     self, Location, Position, Range, SymbolKind, TextDocumentEdit, TextDocumentIdentifier,
     TextDocumentItem, TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier, InsertTextFormat,
 };
-use ra_analysis::{FileId, FileSystemEdit, SourceChange, SourceFileEdit, FilePosition, CompletionItem, CompletionItemKind, InsertText};
+use ra_analysis::{FileId, FileSystemEdit, SourceChange, SourceFileEdit, FilePosition,FileRange,  CompletionItem, CompletionItemKind, InsertText};
 use ra_editor::{LineCol, LineIndex, translate_offset_with_edit};
 use ra_text_edit::{AtomTextEdit, TextEdit};
 use ra_syntax::{SyntaxKind, TextRange, TextUnit};
@@ -218,6 +218,17 @@ impl<'a> TryConvWith for &'a TextDocumentPositionParams {
     }
 }
 
+impl<'a> TryConvWith for (&'a TextDocumentIdentifier, Range) {
+    type Ctx = ServerWorld;
+    type Output = FileRange;
+    fn try_conv_with(self, world: &ServerWorld) -> Result<FileRange> {
+        let file_id = self.0.try_conv_with(world)?;
+        let line_index = world.analysis().file_line_index(file_id);
+        let range = self.1.conv_with(&line_index);
+        Ok(FileRange { file_id, range })
+    }
+}
+
 impl<T: TryConvWith> TryConvWith for Vec<T> {
     type Ctx = <T as TryConvWith>::Ctx;
     type Output = Vec<<T as TryConvWith>::Output>;

crates/ra_lsp_server/src/main_loop/handlers.rs

@@ -8,7 +8,7 @@ use languageserver_types::{
     PrepareRenameResponse, RenameParams, SymbolInformation, TextDocumentIdentifier, TextEdit,
     WorkspaceEdit, ParameterInformation, ParameterLabel, SignatureInformation, Hover, HoverContents,
 };
-use ra_analysis::{FileId, FoldKind, Query, RunnableKind, FilePosition, Severity};
+use ra_analysis::{FileId, FoldKind, Query, RunnableKind, FileRange, FilePosition, Severity};
 use ra_syntax::{TextUnit, text_utils::intersect};
 use ra_text_edit::text_utils::contains_offset_nonstrict;
 use rustc_hash::FxHashMap;
@@ -33,13 +33,13 @@ pub fn handle_extend_selection(
     params: req::ExtendSelectionParams,
 ) -> Result<req::ExtendSelectionResult> {
     let file_id = params.text_document.try_conv_with(&world)?;
-    let file = world.analysis().file_syntax(file_id);
     let line_index = world.analysis().file_line_index(file_id);
     let selections = params
         .selections
         .into_iter()
         .map_conv_with(&line_index)
-        .map(|r| world.analysis().extend_selection(&file, r))
+        .map(|range| FileRange { file_id, range })
+        .map(|frange| world.analysis().extend_selection(frange))
         .map_conv_with(&line_index)
         .collect();
     Ok(req::ExtendSelectionResult { selections })
@@ -71,13 +71,8 @@ pub fn handle_join_lines(
     world: ServerWorld,
     params: req::JoinLinesParams,
 ) -> Result<req::SourceChange> {
-    let file_id = params.text_document.try_conv_with(&world)?;
+    let frange = (&params.text_document, params.range).try_conv_with(&world)?;
-    let line_index = world.analysis().file_line_index(file_id);
+    world.analysis().join_lines(frange).try_conv_with(&world)
-    let range = params.range.conv_with(&line_index);
-    world
-        .analysis()
-        .join_lines(file_id, range)
-        .try_conv_with(&world)
 }
 
 pub fn handle_on_enter(
@@ -614,7 +609,10 @@ pub fn handle_code_action(
     let line_index = world.analysis().file_line_index(file_id);
     let range = params.range.conv_with(&line_index);
 
-    let assists = world.analysis().assists(file_id, range)?.into_iter();
+    let assists = world
+        .analysis()
+        .assists(FileRange { file_id, range })?
+        .into_iter();
     let fixes = world
         .analysis()
         .diagnostics(file_id)?

crates/ra_syntax/src/ast/generated.rs

@@ -1838,6 +1838,51 @@ impl<R: TreeRoot<RaTypes>> LoopExprNode<R> {
 impl<'a> ast::LoopBodyOwner<'a> for LoopExpr<'a> {}
 impl<'a> LoopExpr<'a> {}
 
+// MacroCall
+#[derive(Debug, Clone, Copy,)]
+pub struct MacroCallNode<R: TreeRoot<RaTypes> = OwnedRoot> {
+    pub(crate) syntax: SyntaxNode<R>,
+}
+pub type MacroCall<'a> = MacroCallNode<RefRoot<'a>>;
+
+impl<R1: TreeRoot<RaTypes>, R2: TreeRoot<RaTypes>> PartialEq<MacroCallNode<R1>> for MacroCallNode<R2> {
+    fn eq(&self, other: &MacroCallNode<R1>) -> bool { self.syntax == other.syntax }
+}
+impl<R: TreeRoot<RaTypes>> Eq for MacroCallNode<R> {}
+impl<R: TreeRoot<RaTypes>> Hash for MacroCallNode<R> {
+    fn hash<H: Hasher>(&self, state: &mut H) { self.syntax.hash(state) }
+}
+
+impl<'a> AstNode<'a> for MacroCall<'a> {
+    fn cast(syntax: SyntaxNodeRef<'a>) -> Option<Self> {
+        match syntax.kind() {
+            MACRO_CALL => Some(MacroCall { syntax }),
+            _ => None,
+        }
+    }
+    fn syntax(self) -> SyntaxNodeRef<'a> { self.syntax }
+}
+
+impl<R: TreeRoot<RaTypes>> MacroCallNode<R> {
+    pub fn borrowed(&self) -> MacroCall {
+        MacroCallNode { syntax: self.syntax.borrowed() }
+    }
+    pub fn owned(&self) -> MacroCallNode {
+        MacroCallNode { syntax: self.syntax.owned() }
+    }
+}
+
+
+impl<'a> MacroCall<'a> {
+    pub fn token_tree(self) -> Option<TokenTree<'a>> {
+        super::child_opt(self)
+    }
+
+    pub fn path(self) -> Option<Path<'a>> {
+        super::child_opt(self)
+    }
+}
+
 // MatchArm
 #[derive(Debug, Clone, Copy,)]
 pub struct MatchArmNode<R: TreeRoot<RaTypes> = OwnedRoot> {

crates/ra_syntax/src/grammar.ron

@@ -484,6 +484,7 @@ Grammar(
 
         "Name": (),
         "NameRef": (),
+        "MacroCall": ( options: [ "TokenTree", "Path" ] ),
         "Attr": ( options: [ ["value", "TokenTree"] ] ),
         "TokenTree": (),
         "TypeParamList": (

editors/code/src/highlighting.ts

@@ -28,7 +28,8 @@ export class Highlighter {
             ['builtin', decor('#DD6718')],
             ['text', decor('#DCDCCC')],
             ['attribute', decor('#BFEBBF')],
-            ['literal', decor('#DFAF8F')]
+            ['literal', decor('#DFAF8F')],
+            ['macro', decor('#DFAF8F')]
         ];
 
         return new Map<string, vscode.TextEditorDecorationType>(decorations);