Merge #924

924: Improve show syntax tree r=matklad a=vipentti

This implements some of the features discussed in #820. 

You can now select a range of syntax in a file and then use "Show Syntax Tree" to show its syntax. In addition you can select a range of syntax that is inside a string (typically test cases) and show its syntax as well.

Previous behavior is still available, simply use "Show Syntax Tree" without a selection, and you get the live updating syntax tree. Additionally now the live updating tree will update when the active file is changed. Previously you had to type something in the new file to get the syntax tree to update.

Co-authored-by: Ville Penttinen <villem.penttinen@gmail.com>

crates/ra_ide_api/src/lib.rs

@@ -32,13 +32,14 @@ mod references;
 mod impls;
 mod assists;
 mod diagnostics;
+mod syntax_tree;
 
 #[cfg(test)]
 mod marks;
 
 use std::sync::Arc;
 
-use ra_syntax::{SourceFile, TreeArc, TextRange, TextUnit, AstNode};
+use ra_syntax::{SourceFile, TreeArc, TextRange, TextUnit};
 use ra_text_edit::TextEdit;
 use ra_db::{
     SourceDatabase, CheckCanceled,
@@ -245,8 +246,8 @@ impl Analysis {
 
     /// Returns a syntax tree represented as `String`, for debug purposes.
     // FIXME: use a better name here.
-    pub fn syntax_tree(&self, file_id: FileId) -> String {
-        self.db.parse(file_id).syntax().debug_dump()
+    pub fn syntax_tree(&self, file_id: FileId, text_range: Option<TextRange>) -> String {
+        syntax_tree::syntax_tree(&self.db, file_id, text_range)
     }
 
     /// Returns an edit to remove all newlines in the range, cleaning up minor

crates/ra_ide_api/src/syntax_tree.rs

@@ -0,0 +1,87 @@
+use ra_db::SourceDatabase;
+use crate::db::RootDatabase;
+use ra_syntax::{
+    SourceFile, SyntaxNode, TextRange, AstNode,
+    algo::{self, visit::{visitor, Visitor}}, ast::{self, AstToken}
+};
+
+pub use ra_db::FileId;
+
+pub(crate) fn syntax_tree(
+    db: &RootDatabase,
+    file_id: FileId,
+    text_range: Option<TextRange>,
+) -> String {
+    if let Some(text_range) = text_range {
+        let file = db.parse(file_id);
+        let node = algo::find_covering_node(file.syntax(), text_range);
+
+        if let Some(tree) = syntax_tree_for_string(node, text_range) {
+            return tree;
+        }
+
+        node.debug_dump()
+    } else {
+        db.parse(file_id).syntax().debug_dump()
+    }
+}
+
+/// Attempts parsing the selected contents of a string literal
+/// as rust syntax and returns its syntax tree
+fn syntax_tree_for_string(node: &SyntaxNode, text_range: TextRange) -> Option<String> {
+    // When the range is inside a string
+    // we'll attempt parsing it as rust syntax
+    // to provide the syntax tree of the contents of the string
+    visitor()
+        .visit(|node: &ast::String| syntax_tree_for_token(node, text_range))
+        .visit(|node: &ast::RawString| syntax_tree_for_token(node, text_range))
+        .accept(node)?
+}
+
+fn syntax_tree_for_token<T: AstToken>(node: &T, text_range: TextRange) -> Option<String> {
+    // Range of the full node
+    let node_range = node.syntax().range();
+    let text = node.text().to_string();
+
+    // We start at some point inside the node
+    // Either we have selected the whole string
+    // or our selection is inside it
+    let start = text_range.start() - node_range.start();
+
+    // how many characters we have selected
+    let len = text_range.len().to_usize();
+
+    let node_len = node_range.len().to_usize();
+
+    let start = start.to_usize();
+
+    // We want to cap our length
+    let len = len.min(node_len);
+
+    // Ensure our slice is inside the actual string
+    let end = if start + len < text.len() { start + len } else { text.len() - start };
+
+    let text = &text[start..end];
+
+    // Remove possible extra string quotes from the start
+    // and the end of the string
+    let text = text
+        .trim_start_matches('r')
+        .trim_start_matches('#')
+        .trim_start_matches('"')
+        .trim_end_matches('#')
+        .trim_end_matches('"')
+        .trim()
+        // Remove custom markers
+        .replace("<|>", "");
+
+    let parsed = SourceFile::parse(&text);
+
+    // If the "file" parsed without errors,
+    // return its syntax
+    if parsed.errors().is_empty() {
+        return Some(parsed.syntax().debug_dump());
+    }
+
+    None
+}

crates/ra_ide_api/tests/test/main.rs

@@ -1,6 +1,6 @@
 use insta::assert_debug_snapshot_matches;
 use ra_ide_api::{
-    mock_analysis::{single_file, single_file_with_position, MockAnalysis},
+    mock_analysis::{single_file, single_file_with_position, single_file_with_range, MockAnalysis},
     AnalysisChange, CrateGraph, Edition::Edition2018, Query, NavigationTarget,
     ReferenceSearchResult,
 };
@@ -138,3 +138,255 @@ mod foo {
     assert_eq!(s.name(), "FooInner");
     assert_eq!(s.container_name(), Some(&SmolStr::new("foo")));
 }
+
+#[test]
+fn test_syntax_tree_without_range() {
+    // Basic syntax
+    let (analysis, file_id) = single_file(r#"fn foo() {}"#);
+    let syn = analysis.syntax_tree(file_id, None);
+
+    assert_eq!(
+        syn.trim(),
+        r#"
+SOURCE_FILE@[0; 11)
+  FN_DEF@[0; 11)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 11)
+      L_CURLY@[9; 10)
+      R_CURLY@[10; 11)
+    "#
+        .trim()
+    );
+
+    let (analysis, file_id) = single_file(
+        r#"
+fn test() {
+    assert!("
+    fn foo() {
+    }
+    ", "");
+}"#
+        .trim(),
+    );
+    let syn = analysis.syntax_tree(file_id, None);
+
+    assert_eq!(
+        syn.trim(),
+        r#"
+SOURCE_FILE@[0; 60)
+  FN_DEF@[0; 60)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 7)
+      IDENT@[3; 7) "test"
+    PARAM_LIST@[7; 9)
+      L_PAREN@[7; 8)
+      R_PAREN@[8; 9)
+    WHITESPACE@[9; 10)
+    BLOCK@[10; 60)
+      L_CURLY@[10; 11)
+      WHITESPACE@[11; 16)
+      EXPR_STMT@[16; 58)
+        MACRO_CALL@[16; 57)
+          PATH@[16; 22)
+            PATH_SEGMENT@[16; 22)
+              NAME_REF@[16; 22)
+                IDENT@[16; 22) "assert"
+          EXCL@[22; 23)
+          TOKEN_TREE@[23; 57)
+            L_PAREN@[23; 24)
+            STRING@[24; 52)
+            COMMA@[52; 53)
+            WHITESPACE@[53; 54)
+            STRING@[54; 56)
+            R_PAREN@[56; 57)
+        SEMI@[57; 58)
+      WHITESPACE@[58; 59)
+      R_CURLY@[59; 60)
+    "#
+        .trim()
+    );
+}
+
+#[test]
+fn test_syntax_tree_with_range() {
+    let (analysis, range) = single_file_with_range(r#"<|>fn foo() {}<|>"#.trim());
+    let syn = analysis.syntax_tree(range.file_id, Some(range.range));
+
+    assert_eq!(
+        syn.trim(),
+        r#"
+FN_DEF@[0; 11)
+  FN_KW@[0; 2)
+  WHITESPACE@[2; 3)
+  NAME@[3; 6)
+    IDENT@[3; 6) "foo"
+  PARAM_LIST@[6; 8)
+    L_PAREN@[6; 7)
+    R_PAREN@[7; 8)
+  WHITESPACE@[8; 9)
+  BLOCK@[9; 11)
+    L_CURLY@[9; 10)
+    R_CURLY@[10; 11)
+    "#
+        .trim()
+    );
+
+    let (analysis, range) = single_file_with_range(
+        r#"fn test() {
+    <|>assert!("
+    fn foo() {
+    }
+    ", "");<|>
+}"#
+        .trim(),
+    );
+    let syn = analysis.syntax_tree(range.file_id, Some(range.range));
+
+    assert_eq!(
+        syn.trim(),
+        r#"
+EXPR_STMT@[16; 58)
+  MACRO_CALL@[16; 57)
+    PATH@[16; 22)
+      PATH_SEGMENT@[16; 22)
+        NAME_REF@[16; 22)
+          IDENT@[16; 22) "assert"
+    EXCL@[22; 23)
+    TOKEN_TREE@[23; 57)
+      L_PAREN@[23; 24)
+      STRING@[24; 52)
+      COMMA@[52; 53)
+      WHITESPACE@[53; 54)
+      STRING@[54; 56)
+      R_PAREN@[56; 57)
+  SEMI@[57; 58)
+    "#
+        .trim()
+    );
+}
+
+#[test]
+fn test_syntax_tree_inside_string() {
+    let (analysis, range) = single_file_with_range(
+        r#"fn test() {
+    assert!("
+<|>fn foo() {
+}<|>
+fn bar() {
+}
+    ", "");
+}"#
+        .trim(),
+    );
+    let syn = analysis.syntax_tree(range.file_id, Some(range.range));
+    assert_eq!(
+        syn.trim(),
+        r#"
+SOURCE_FILE@[0; 12)
+  FN_DEF@[0; 12)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 12)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 11)
+      R_CURLY@[11; 12)
+"#
+        .trim()
+    );
+
+    // With a raw string
+    let (analysis, range) = single_file_with_range(
+        r###"fn test() {
+    assert!(r#"
+<|>fn foo() {
+}<|>
+fn bar() {
+}
+    "#, "");
+}"###
+            .trim(),
+    );
+    let syn = analysis.syntax_tree(range.file_id, Some(range.range));
+    assert_eq!(
+        syn.trim(),
+        r#"
+SOURCE_FILE@[0; 12)
+  FN_DEF@[0; 12)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 12)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 11)
+      R_CURLY@[11; 12)
+"#
+        .trim()
+    );
+
+    // With a raw string
+    let (analysis, range) = single_file_with_range(
+        r###"fn test() {
+    assert!(r<|>#"
+fn foo() {
+}
+fn bar() {
+}"<|>#, "");
+}"###
+            .trim(),
+    );
+    let syn = analysis.syntax_tree(range.file_id, Some(range.range));
+    assert_eq!(
+        syn.trim(),
+        r#"
+SOURCE_FILE@[0; 25)
+  FN_DEF@[0; 12)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 6)
+      IDENT@[3; 6) "foo"
+    PARAM_LIST@[6; 8)
+      L_PAREN@[6; 7)
+      R_PAREN@[7; 8)
+    WHITESPACE@[8; 9)
+    BLOCK@[9; 12)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 11)
+      R_CURLY@[11; 12)
+  WHITESPACE@[12; 13)
+  FN_DEF@[13; 25)
+    FN_KW@[13; 15)
+    WHITESPACE@[15; 16)
+    NAME@[16; 19)
+      IDENT@[16; 19) "bar"
+    PARAM_LIST@[19; 21)
+      L_PAREN@[19; 20)
+      R_PAREN@[20; 21)
+    WHITESPACE@[21; 22)
+    BLOCK@[22; 25)
+      L_CURLY@[22; 23)
+      WHITESPACE@[23; 24)
+      R_CURLY@[24; 25)
+
+"#
+        .trim()
+    );
+}

crates/ra_lsp_server/src/main_loop/handlers.rs

@@ -32,7 +32,9 @@ pub fn handle_analyzer_status(world: ServerWorld, _: ()) -> Result<String> {
 
 pub fn handle_syntax_tree(world: ServerWorld, params: req::SyntaxTreeParams) -> Result<String> {
     let id = params.text_document.try_conv_with(&world)?;
-    let res = world.analysis().syntax_tree(id);
+    let line_index = world.analysis().file_line_index(id);
+    let text_range = params.range.map(|p| p.conv_with(&line_index));
+    let res = world.analysis().syntax_tree(id, text_range);
     Ok(res)
 }
 

crates/ra_lsp_server/src/req.rs

@@ -39,6 +39,7 @@ impl Request for SyntaxTree {
 #[serde(rename_all = "camelCase")]
 pub struct SyntaxTreeParams {
     pub text_document: TextDocumentIdentifier,
+    pub range: Option<Range>,
 }
 
 pub enum ExtendSelection {}

editors/code/package.json

@@ -75,7 +75,7 @@
         "commands": [
             {
                 "command": "rust-analyzer.syntaxTree",
-                "title": "Show syntax tree for current file",
+                "title": "Show Syntax Tree",
                 "category": "Rust Analyzer"
             },
             {

editors/code/src/commands/syntaxTree.ts

@@ -1,11 +1,11 @@
 import * as vscode from 'vscode';
-import { TextDocumentIdentifier } from 'vscode-languageclient';
+import { Range, TextDocumentIdentifier } from 'vscode-languageclient';
 
 import { Server } from '../server';
 
 export const syntaxTreeUri = vscode.Uri.parse('rust-analyzer://syntaxtree');
 
-export class TextDocumentContentProvider
+export class SyntaxTreeContentProvider
     implements vscode.TextDocumentContentProvider {
     public eventEmitter = new vscode.EventEmitter<vscode.Uri>();
     public syntaxTree: string = 'Not available';
@@ -17,8 +17,21 @@ export class TextDocumentContentProvider
         if (editor == null) {
             return '';
         }
+
+        let range: Range | undefined;
+
+        // When the range based query is enabled we take the range of the selection
+        if (uri.query === 'range=true') {
+            range = editor.selection.isEmpty
+                ? undefined
+                : Server.client.code2ProtocolConverter.asRange(
+                      editor.selection
+                  );
+        }
+
         const request: SyntaxTreeParams = {
-            textDocument: { uri: editor.document.uri.toString() }
+            textDocument: { uri: editor.document.uri.toString() },
+            range
         };
         return Server.client.sendRequest<SyntaxTreeResult>(
             'rust-analyzer/syntaxTree',
@@ -33,6 +46,7 @@ export class TextDocumentContentProvider
 
 interface SyntaxTreeParams {
     textDocument: TextDocumentIdentifier;
+    range?: Range;
 }
 
 type SyntaxTreeResult = string;
@@ -40,11 +54,23 @@ type SyntaxTreeResult = string;
 // Opens the virtual file that will show the syntax tree
 //
 // The contents of the file come from the `TextDocumentContentProvider`
-export async function handle() {
-    const document = await vscode.workspace.openTextDocument(syntaxTreeUri);
+export function createHandle(provider: SyntaxTreeContentProvider) {
+    return async () => {
+        const editor = vscode.window.activeTextEditor;
+        const rangeEnabled = !!(editor && !editor.selection.isEmpty);
+
+        const uri = rangeEnabled
+            ? vscode.Uri.parse(`${syntaxTreeUri.toString()}?range=true`)
+            : syntaxTreeUri;
+
+        const document = await vscode.workspace.openTextDocument(uri);
+
+        provider.eventEmitter.fire(uri);
+
         return vscode.window.showTextDocument(
             document,
             vscode.ViewColumn.Two,
             true
         );
+    };
 }

editors/code/src/events/change_active_text_editor.ts

@@ -1,17 +1,25 @@
 import { TextEditor } from 'vscode';
 import { TextDocumentIdentifier } from 'vscode-languageclient';
 
+import {
+    SyntaxTreeContentProvider,
+    syntaxTreeUri
+} from '../commands/syntaxTree';
 import { Decoration } from '../highlighting';
 import { Server } from '../server';
 
-export async function handle(editor: TextEditor | undefined) {
-    if (
-        !Server.config.highlightingOn ||
-        !editor ||
-        editor.document.languageId !== 'rust'
-    ) {
+export function makeHandler(syntaxTreeProvider: SyntaxTreeContentProvider) {
+    return async function handle(editor: TextEditor | undefined) {
+        if (!editor || editor.document.languageId !== 'rust') {
             return;
         }
+
+        syntaxTreeProvider.eventEmitter.fire(syntaxTreeUri);
+
+        if (!Server.config.highlightingOn) {
+            return;
+        }
+
         const params: TextDocumentIdentifier = {
             uri: editor.document.uri.toString()
         };
@@ -20,4 +28,5 @@ export async function handle(editor: TextEditor | undefined) {
             params
         );
         Server.highlighter.setHighlights(editor, decorations);
+    };
 }

editors/code/src/events/change_text_document.ts

@@ -1,20 +1,18 @@
 import * as vscode from 'vscode';
 
 import {
-    syntaxTreeUri,
-    TextDocumentContentProvider
+    SyntaxTreeContentProvider,
+    syntaxTreeUri
 } from '../commands/syntaxTree';
 
-export function createHandler(
-    textDocumentContentProvider: TextDocumentContentProvider
-) {
+export function createHandler(syntaxTreeProvider: SyntaxTreeContentProvider) {
     return (event: vscode.TextDocumentChangeEvent) => {
         const doc = event.document;
         if (doc.languageId !== 'rust') {
             return;
         }
         afterLs(() => {
-            textDocumentContentProvider.eventEmitter.fire(syntaxTreeUri);
+            syntaxTreeProvider.eventEmitter.fire(syntaxTreeUri);
         });
     };
 }

editors/code/src/extension.ts

@@ -2,7 +2,7 @@ import * as vscode from 'vscode';
 import * as lc from 'vscode-languageclient';
 
 import * as commands from './commands';
-import { TextDocumentContentProvider } from './commands/syntaxTree';
+import { SyntaxTreeContentProvider } from './commands/syntaxTree';
 import * as events from './events';
 import * as notifications from './notifications';
 import { Server } from './server';
@@ -52,7 +52,6 @@ export function activate(context: vscode.ExtensionContext) {
     registerCommand('rust-analyzer.collectGarbage', () =>
         Server.client.sendRequest<null>('rust-analyzer/collectGarbage', null)
     );
-    registerCommand('rust-analyzer.syntaxTree', commands.syntaxTree.handle);
     registerCommand(
         'rust-analyzer.extendSelection',
         commands.extendSelection.handle
@@ -95,22 +94,27 @@ export function activate(context: vscode.ExtensionContext) {
             notifications.publishDecorations.handle
         ]
     ];
+    const syntaxTreeContentProvider = new SyntaxTreeContentProvider();
 
     // The events below are plain old javascript events, triggered and handled by vscode
     vscode.window.onDidChangeActiveTextEditor(
-        events.changeActiveTextEditor.handle
+        events.changeActiveTextEditor.makeHandler(syntaxTreeContentProvider)
     );
 
-    const textDocumentContentProvider = new TextDocumentContentProvider();
     disposeOnDeactivation(
         vscode.workspace.registerTextDocumentContentProvider(
             'rust-analyzer',
-            textDocumentContentProvider
+            syntaxTreeContentProvider
         )
     );
 
+    registerCommand(
+        'rust-analyzer.syntaxTree',
+        commands.syntaxTree.createHandle(syntaxTreeContentProvider)
+    );
+
     vscode.workspace.onDidChangeTextDocument(
-        events.changeTextDocument.createHandler(textDocumentContentProvider),
+        events.changeTextDocument.createHandler(syntaxTreeContentProvider),
         null,
         context.subscriptions
     );