Merge #3958

3958: Add proc-macro related config and tests r=matklad a=edwin0cheng

This PR do the following things:

1. Add cli argument `proc-macro` for running proc-macro server.
2. Added support for proc-macro in bench and analysis-stats
3. Added typescript config for proc-macros
4. Added an heavy test for proc-macros. 

To test it out: 

1. run `cargo xtask install --proc-macro`
2. add `"rust-analyzer.cargo.loadOutDirsFromCheck": true"` and `"rust-analyzer.procMacro.enabled": true"` in vs code config.

[Edit] Change to use `rust-analyzer proc-macro` for running proc-macro standalone process.

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

Cargo.lock

@@ -1341,6 +1341,7 @@ dependencies = [
  "ra_hir_def",
  "ra_hir_ty",
  "ra_ide",
+ "ra_proc_macro_srv",
  "ra_prof",
  "ra_project_model",
  "ra_syntax",

crates/ra_proc_macro/src/lib.rs

@@ -12,6 +12,7 @@ pub mod msg;
 use process::{ProcMacroProcessSrv, ProcMacroProcessThread};
 use ra_tt::{SmolStr, Subtree};
 use std::{
+    ffi::OsStr,
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -56,8 +57,15 @@ pub struct ProcMacroClient {
 }
 
 impl ProcMacroClient {
-    pub fn extern_process(process_path: &Path) -> Result<ProcMacroClient, std::io::Error> {
-        let (thread, process) = ProcMacroProcessSrv::run(process_path)?;
+    pub fn extern_process<I, S>(
+        process_path: &Path,
+        args: I,
+    ) -> Result<ProcMacroClient, std::io::Error>
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        let (thread, process) = ProcMacroProcessSrv::run(process_path, args)?;
         Ok(ProcMacroClient {
             kind: ProcMacroClientKind::Process { process: Arc::new(process), thread },
         })

crates/ra_proc_macro/src/process.rs

@@ -9,6 +9,7 @@ use crate::rpc::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTas
 use io::{BufRead, BufReader};
 use std::{
     convert::{TryFrom, TryInto},
+    ffi::OsStr,
     io::{self, Write},
     path::{Path, PathBuf},
     process::{Child, Command, Stdio},
@@ -44,8 +45,13 @@ impl Drop for Process {
 }
 
 impl Process {
-    fn run(process_path: &Path) -> Result<Process, io::Error> {
+    fn run<I, S>(process_path: &Path, args: I) -> Result<Process, io::Error>
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
         let child = Command::new(process_path.clone())
+            .args(args)
             .stdin(Stdio::piped())
             .stdout(Stdio::piped())
             .stderr(Stdio::null())
@@ -74,10 +80,15 @@ impl Process {
 }
 
 impl ProcMacroProcessSrv {
-    pub fn run(
+    pub fn run<I, S>(
         process_path: &Path,
-    ) -> Result<(ProcMacroProcessThread, ProcMacroProcessSrv), io::Error> {
-        let process = Process::run(process_path)?;
+        args: I,
+    ) -> Result<(ProcMacroProcessThread, ProcMacroProcessSrv), io::Error>
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        let process = Process::run(process_path, args)?;
 
         let (task_tx, task_rx) = bounded(0);
         let handle = jod_thread::spawn(move || {

crates/ra_proc_macro_srv/src/cli.rs

@@ -1,7 +1,7 @@
 //! Driver for proc macro server
 
+use crate::{expand_task, list_macros};
 use ra_proc_macro::msg::{self, Message};
-use ra_proc_macro_srv::{expand_task, list_macros};
 
 use std::io;
 
@@ -24,7 +24,8 @@ fn write_response(res: Result<msg::Response, String>) -> Result<(), io::Error> {
     let mut stdout = stdout.lock();
     msg.write(&mut stdout)
 }
-fn main() {
+
+pub fn run() {
     loop {
         let req = match read_request() {
             Err(err) => {

crates/ra_proc_macro_srv/src/lib.rs

@@ -22,7 +22,7 @@ mod dylib;
 use proc_macro::bridge::client::TokenStream;
 use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask};
 
-pub fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
+pub(crate) fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
     let expander = dylib::Expander::new(&task.lib)
         .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
 
@@ -39,7 +39,7 @@ pub fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
     }
 }
 
-pub fn list_macros(task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
+pub(crate) fn list_macros(task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
     let expander = dylib::Expander::new(&task.lib)
         .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
 
@@ -53,5 +53,7 @@ pub fn list_macros(task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
     }
 }
 
+pub mod cli;
+
 #[cfg(test)]
 mod tests;

crates/rust-analyzer/Cargo.toml

@@ -46,7 +46,7 @@ ra_db = { path = "../ra_db" }
 hir = { path = "../ra_hir", package = "ra_hir" }
 hir_def = { path = "../ra_hir_def", package = "ra_hir_def" }
 hir_ty = { path = "../ra_hir_ty", package = "ra_hir_ty" }
-
+ra_proc_macro_srv = { path = "../ra_proc_macro_srv" }
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.8"

crates/rust-analyzer/src/bin/args.rs

@@ -29,19 +29,23 @@ pub(crate) enum Command {
         with_deps: bool,
         path: PathBuf,
         load_output_dirs: bool,
+        with_proc_macro: bool,
     },
     Bench {
         path: PathBuf,
         what: BenchWhat,
         load_output_dirs: bool,
+        with_proc_macro: bool,
     },
     Diagnostics {
         path: PathBuf,
         load_output_dirs: bool,
+        with_proc_macro: bool,
         /// Include files which are not modules. In rust-analyzer
         /// this would include the parser test files.
         all: bool,
     },
+    ProcMacro,
     RunServer,
     Version,
 }
@@ -148,6 +152,7 @@ FLAGS:
     -h, --help              Prints help information
         --memory-usage
         --load-output-dirs  Load OUT_DIR values by running `cargo check` before analysis
+        --with-proc-macro    Use ra-proc-macro-srv for proc-macro expanding
     -v, --verbose
     -q, --quiet
 
@@ -165,6 +170,7 @@ ARGS:
                 let only: Option<String> = matches.opt_value_from_str(["-o", "--only"])?;
                 let with_deps: bool = matches.contains("--with-deps");
                 let load_output_dirs = matches.contains("--load-output-dirs");
+                let with_proc_macro = matches.contains("--with-proc-macro");
                 let path = {
                     let mut trailing = matches.free()?;
                     if trailing.len() != 1 {
@@ -173,7 +179,15 @@ ARGS:
                     trailing.pop().unwrap().into()
                 };
 
-                Command::Stats { randomize, memory_usage, only, with_deps, path, load_output_dirs }
+                Command::Stats {
+                    randomize,
+                    memory_usage,
+                    only,
+                    with_deps,
+                    path,
+                    load_output_dirs,
+                    with_proc_macro,
+                }
             }
             "analysis-bench" => {
                 if matches.contains(["-h", "--help"]) {
@@ -187,6 +201,7 @@ USAGE:
 FLAGS:
     -h, --help          Prints help information
     --load-output-dirs  Load OUT_DIR values by running `cargo check` before analysis
+    --with-proc-macro    Use ra-proc-macro-srv for proc-macro expanding
     -v, --verbose
 
 OPTIONS:
@@ -214,7 +229,8 @@ ARGS:
                     ),
                 };
                 let load_output_dirs = matches.contains("--load-output-dirs");
-                Command::Bench { path, what, load_output_dirs }
+                let with_proc_macro = matches.contains("--with-proc-macro");
+                Command::Bench { path, what, load_output_dirs, with_proc_macro }
             }
             "diagnostics" => {
                 if matches.contains(["-h", "--help"]) {
@@ -237,6 +253,7 @@ ARGS:
                 }
 
                 let load_output_dirs = matches.contains("--load-output-dirs");
+                let with_proc_macro = matches.contains("--with-proc-macro");
                 let all = matches.contains("--all");
                 let path = {
                     let mut trailing = matches.free()?;
@@ -246,8 +263,9 @@ ARGS:
                     trailing.pop().unwrap().into()
                 };
 
-                Command::Diagnostics { path, load_output_dirs, all }
+                Command::Diagnostics { path, load_output_dirs, with_proc_macro, all }
             }
+            "proc-macro" => Command::ProcMacro,
             _ => {
                 eprintln!(
                     "\

crates/rust-analyzer/src/bin/main.rs

@@ -25,6 +25,7 @@ fn main() -> Result<()> {
             with_deps,
             path,
             load_output_dirs,
+            with_proc_macro,
         } => cli::analysis_stats(
             args.verbosity,
             memory_usage,
@@ -33,16 +34,24 @@ fn main() -> Result<()> {
             with_deps,
             randomize,
             load_output_dirs,
+            with_proc_macro,
         )?,
 
-        args::Command::Bench { path, what, load_output_dirs } => {
-            cli::analysis_bench(args.verbosity, path.as_ref(), what, load_output_dirs)?
+        args::Command::Bench { path, what, load_output_dirs, with_proc_macro } => {
+            cli::analysis_bench(
+                args.verbosity,
+                path.as_ref(),
+                what,
+                load_output_dirs,
+                with_proc_macro,
+            )?
         }
 
-        args::Command::Diagnostics { path, load_output_dirs, all } => {
-            cli::diagnostics(path.as_ref(), load_output_dirs, all)?
+        args::Command::Diagnostics { path, load_output_dirs, with_proc_macro, all } => {
+            cli::diagnostics(path.as_ref(), load_output_dirs, with_proc_macro, all)?
         }
 
+        args::Command::ProcMacro => run_proc_macro_sv()?,
         args::Command::RunServer => run_server()?,
         args::Command::Version => println!("rust-analyzer {}", env!("REV")),
     }
@@ -56,6 +65,11 @@ fn setup_logging() -> Result<()> {
     Ok(())
 }
 
+fn run_proc_macro_sv() -> Result<()> {
+    ra_proc_macro_srv::cli::run();
+    Ok(())
+}
+
 fn run_server() -> Result<()> {
     log::info!("lifecycle: server started");
 

crates/rust-analyzer/src/cli/analysis_bench.rs

@@ -47,12 +47,13 @@ pub fn analysis_bench(
     path: &Path,
     what: BenchWhat,
     load_output_dirs: bool,
+    with_proc_macro: bool,
 ) -> Result<()> {
     ra_prof::init();
 
     let start = Instant::now();
     eprint!("loading: ");
-    let (mut host, roots) = load_cargo(path, load_output_dirs)?;
+    let (mut host, roots) = load_cargo(path, load_output_dirs, with_proc_macro)?;
     let db = host.raw_database();
     eprintln!("{:?}\n", start.elapsed());
 

crates/rust-analyzer/src/cli/analysis_stats.rs

@@ -25,9 +25,10 @@ pub fn analysis_stats(
     with_deps: bool,
     randomize: bool,
     load_output_dirs: bool,
+    with_proc_macro: bool,
 ) -> Result<()> {
     let db_load_time = Instant::now();
-    let (mut host, roots) = load_cargo(path, load_output_dirs)?;
+    let (mut host, roots) = load_cargo(path, load_output_dirs, with_proc_macro)?;
     let db = host.raw_database();
     println!("Database loaded, {} roots, {:?}", roots.len(), db_load_time.elapsed());
     let analysis_time = Instant::now();

crates/rust-analyzer/src/cli/diagnostics.rs

@@ -9,8 +9,13 @@ use std::{collections::HashSet, path::Path};
 use crate::cli::{load_cargo::load_cargo, Result};
 use hir::Semantics;
 
-pub fn diagnostics(path: &Path, load_output_dirs: bool, all: bool) -> Result<()> {
-    let (host, roots) = load_cargo(path, load_output_dirs)?;
+pub fn diagnostics(
+    path: &Path,
+    load_output_dirs: bool,
+    with_proc_macro: bool,
+    all: bool,
+) -> Result<()> {
+    let (host, roots) = load_cargo(path, load_output_dirs, with_proc_macro)?;
     let db = host.raw_database();
     let analysis = host.analysis();
     let semantics = Semantics::new(db);

crates/rust-analyzer/src/cli/load_cargo.rs

@@ -25,6 +25,7 @@ fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
 pub(crate) fn load_cargo(
     root: &Path,
     load_out_dirs_from_check: bool,
+    with_proc_macro: bool,
 ) -> Result<(AnalysisHost, FxHashMap<SourceRootId, PackageRoot>)> {
     let root = std::env::current_dir()?.join(root);
     let ws = ProjectWorkspace::discover(
@@ -69,7 +70,14 @@ pub(crate) fn load_cargo(
         })
         .collect::<FxHashMap<_, _>>();
 
-    let proc_macro_client = ProcMacroClient::dummy();
+    let proc_macro_client = if !with_proc_macro {
+        ProcMacroClient::dummy()
+    } else {
+        let mut path = std::env::current_exe()?;
+        path.pop();
+        path.push("rust-analyzer");
+        ProcMacroClient::extern_process(&path, &["proc-macro"]).unwrap()
+    };
     let host = load(&source_roots, ws, &mut vfs, receiver, extern_dirs, &proc_macro_client);
     Ok((host, source_roots))
 }
@@ -175,7 +183,7 @@ mod tests {
     #[test]
     fn test_loading_rust_analyzer() {
         let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap();
-        let (host, _roots) = load_cargo(path, false).unwrap();
+        let (host, _roots) = load_cargo(path, false, false).unwrap();
         let n_crates = Crate::all(host.raw_database()).len();
         // RA has quite a few crates, but the exact count doesn't matter
         assert!(n_crates > 20);

crates/rust-analyzer/src/config.rs

@@ -20,7 +20,7 @@ pub struct Config {
     pub with_sysroot: bool,
     pub publish_diagnostics: bool,
     pub lru_capacity: Option<usize>,
-    pub proc_macro_srv: Option<String>,
+    pub proc_macro_srv: Option<(String, Vec<String>)>,
     pub files: FilesConfig,
     pub notifications: NotificationsConfig,
 
@@ -131,6 +131,18 @@ impl Config {
         set(value, "/cargo/allFeatures", &mut self.cargo.all_features);
         set(value, "/cargo/features", &mut self.cargo.features);
         set(value, "/cargo/loadOutDirsFromCheck", &mut self.cargo.load_out_dirs_from_check);
+
+        match get::<bool>(value, "/procMacro/enabled") {
+            Some(true) => {
+                if let Ok(mut path) = std::env::current_exe() {
+                    path.pop();
+                    path.push("rust-analyzer");
+                    self.proc_macro_srv = Some((path.to_string_lossy().to_string(), vec!["proc-macro".to_string()]));
+                }
+            }
+            _ => self.proc_macro_srv = None,
+        }
+
         match get::<Vec<String>>(value, "/rustfmt/overrideCommand") {
             Some(mut args) if !args.is_empty() => {
                 let command = args.remove(0);

crates/rust-analyzer/src/world.rs

@@ -64,6 +64,7 @@ pub struct WorldState {
     pub latest_requests: Arc<RwLock<LatestRequests>>,
     pub flycheck: Option<Flycheck>,
     pub diagnostics: DiagnosticCollection,
+    pub proc_macro_client: ProcMacroClient,
 }
 
 /// An immutable snapshot of the world's state at a point in time.
@@ -147,9 +148,9 @@ impl WorldState {
 
         let proc_macro_client = match &config.proc_macro_srv {
             None => ProcMacroClient::dummy(),
-            Some(srv) => {
-                let path = Path::new(&srv);
-                match ProcMacroClient::extern_process(path) {
+            Some((path, args)) => {
+                let path = std::path::Path::new(path);
+                match ProcMacroClient::extern_process(path, args) {
                     Ok(it) => it,
                     Err(err) => {
                         log::error!(
@@ -192,6 +193,7 @@ impl WorldState {
             latest_requests: Default::default(),
             flycheck,
             diagnostics: Default::default(),
+            proc_macro_client,
         }
     }
 

crates/rust-analyzer/tests/heavy_tests/main.rs

@@ -9,7 +9,7 @@ use lsp_types::{
 };
 use rust_analyzer::req::{
     CodeActionParams, CodeActionRequest, Completion, CompletionParams, DidOpenTextDocument,
-    Formatting, GotoDefinition, OnEnter, Runnables, RunnablesParams,
+    Formatting, GotoDefinition, HoverRequest, OnEnter, Runnables, RunnablesParams,
 };
 use serde_json::json;
 use tempfile::TempDir;
@@ -625,3 +625,92 @@ fn main() { message(); }
     ));
     assert!(format!("{}", res).contains("hello.rs"));
 }
+
+#[test]
+fn resolve_proc_macro() {
+    if skip_slow_tests() {
+        return;
+    }
+    let server = Project::with_fixture(
+        r###"
+//- foo/Cargo.toml
+[package]
+name = "foo"
+version = "0.0.0"
+edition = "2018"
+[dependencies]
+bar = {path = "../bar"}
+
+//- foo/src/main.rs
+use bar::Bar;
+trait Bar {
+  fn bar();
+}
+#[derive(Bar)]
+struct Foo {}
+fn main() {
+  Foo::bar();
+}
+
+//- bar/Cargo.toml
+[package]
+name = "bar"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+//- bar/src/lib.rs
+extern crate proc_macro;
+use proc_macro::{Delimiter, Group, Ident, Span, TokenStream, TokenTree};
+macro_rules! t {
+    ($n:literal) => {
+        TokenTree::from(Ident::new($n, Span::call_site()))
+    };
+    ({}) => {
+        TokenTree::from(Group::new(Delimiter::Brace, TokenStream::new()))
+    };
+    (()) => {
+        TokenTree::from(Group::new(Delimiter::Parenthesis, TokenStream::new()))
+    };
+}
+#[proc_macro_derive(Bar)]
+pub fn foo(_input: TokenStream) -> TokenStream {
+    // We hard code the output here for preventing to use any deps
+    let mut res = TokenStream::new();
+
+    // impl Bar for Foo { fn bar() {} }
+    let mut tokens = vec![t!("impl"), t!("Bar"), t!("for"), t!("Foo")];
+    let mut fn_stream = TokenStream::new();
+    fn_stream.extend(vec![t!("fn"), t!("bar"), t!(()), t!({})]);
+    tokens.push(Group::new(Delimiter::Brace, fn_stream).into());
+    res.extend(tokens);
+    res
+}
+
+"###,
+    )
+    .with_config(|config| {
+        // FIXME: Use env!("CARGO_BIN_EXE_ra-analyzer") instead after
+        // https://github.com/rust-lang/cargo/pull/7697 landed
+        let macro_srv_path = std::path::Path::new(std::env!("CARGO_MANIFEST_DIR"))
+            .join("../../target/debug/rust-analyzer")
+            .to_string_lossy()
+            .to_string();
+
+        config.cargo.load_out_dirs_from_check = true;
+        config.proc_macro_srv = Some((macro_srv_path, vec!["proc-macro".to_string()]));
+    })
+    .root("foo")
+    .root("bar")
+    .server();
+    server.wait_until_workspace_is_loaded();
+    let res = server.send_request::<HoverRequest>(TextDocumentPositionParams::new(
+        server.doc_id("foo/src/main.rs"),
+        Position::new(7, 9),
+    ));
+
+    let value = res.get("contents").unwrap().get("value").unwrap().to_string();
+    assert_eq!(value, r#""```rust\nfoo::Bar\nfn bar()\n```""#)
+}

editors/code/package.json

@@ -388,6 +388,11 @@
                     "description": "Enable logging of VS Code extensions itself",
                     "type": "boolean",
                     "default": false
+                },
+                "rust-analyzer.procMacro.enabled": {
+                    "description": "Enable Proc macro support, cargo.loadOutDirsFromCheck must be enabled.",
+                    "type": "boolean",
+                    "default": false
                 }
             }
         },

editors/code/src/config.ts

@@ -12,6 +12,7 @@ export class Config {
     private readonly requiresReloadOpts = [
         "serverPath",
         "cargo",
+        "procMacro",
         "files",
         "highlighting",
         "updates.channel",