diff --git a/Cargo.lock b/Cargo.lock index 34c5351623..975c1aef86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -424,6 +424,17 @@ dependencies = [ "regex", ] +[[package]] +name = "goblin" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd5e3132801a1ac34ac53b97acde50c4685414dd2f291b9ea52afa6f07468c8" +dependencies = [ + "log", + "plain", + "scroll", +] + [[package]] name = "heck" version = "0.3.1" @@ -586,6 +597,15 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" +[[package]] +name = "libloading" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c979a19ffb457f0273965c333053f3d586bf759bf7b683fbebc37f9a9ebedc4" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "linked-hash-map" version = "0.5.2" @@ -825,6 +845,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad1f1b834a05d42dae330066e9699a173b28185b3bdc3dbf14ca239585de8cc" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "ppv-lite86" version = "0.2.6" @@ -1082,10 +1108,13 @@ version = "0.1.0" dependencies = [ "cargo_metadata", "difference", + "goblin", + "libloading", "ra_mbe", "ra_proc_macro", "ra_tt", "serde_derive", + "test_utils", ] [[package]] @@ -1403,6 +1432,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scroll" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb2332cb595d33f7edd5700f4cbf94892e680c7f0ae56adab58a35190b66cb1" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "semver" version = "0.9.0" diff --git a/crates/ra_proc_macro_srv/Cargo.toml b/crates/ra_proc_macro_srv/Cargo.toml index f08de5fc77..1e0f503392 100644 --- a/crates/ra_proc_macro_srv/Cargo.toml +++ b/crates/ra_proc_macro_srv/Cargo.toml @@ -12,9 +12,12 @@ doctest = false ra_tt = { path = "../ra_tt" } ra_mbe = { path = "../ra_mbe" } ra_proc_macro = { path = "../ra_proc_macro" } +goblin = "0.2.1" +libloading = "0.6.0" +test_utils = { path = "../test_utils" } [dev-dependencies] cargo_metadata = "0.9.1" difference = "2.0.0" # used as proc macro test target -serde_derive = "=1.0.104" \ No newline at end of file +serde_derive = "=1.0.104" diff --git a/crates/ra_proc_macro_srv/src/dylib.rs b/crates/ra_proc_macro_srv/src/dylib.rs new file mode 100644 index 0000000000..ec63d587bb --- /dev/null +++ b/crates/ra_proc_macro_srv/src/dylib.rs @@ -0,0 +1,211 @@ +//! Handles dynamic library loading for proc macro + +use crate::{proc_macro::bridge, rustc_server::TokenStream}; +use std::path::Path; + +use goblin::{mach::Mach, Object}; +use libloading::Library; +use ra_proc_macro::ProcMacroKind; + +use std::io::Error as IoError; +use std::io::ErrorKind as IoErrorKind; + +const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_"; + +fn invalid_data_err(e: impl Into>) -> IoError { + IoError::new(IoErrorKind::InvalidData, e) +} + +fn get_symbols_from_lib(file: &Path) -> Result, IoError> { + let buffer = std::fs::read(file)?; + let object = Object::parse(&buffer).map_err(invalid_data_err)?; + + match object { + Object::Elf(elf) => { + let symbols = elf.dynstrtab.to_vec().map_err(invalid_data_err)?; + let names = symbols.iter().map(|s| s.to_string()).collect(); + Ok(names) + } + Object::PE(pe) => { + let symbol_names = + pe.exports.iter().flat_map(|s| s.name).map(|n| n.to_string()).collect(); + Ok(symbol_names) + } + Object::Mach(mach) => match mach { + Mach::Binary(binary) => { + let exports = binary.exports().map_err(invalid_data_err)?; + let names = exports + .into_iter() + .map(|s| { + // In macos doc: + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dlsym.3.html + // Unlike other dyld API's, the symbol name passed to dlsym() must NOT be + // prepended with an underscore. + if s.name.starts_with("_") { + s.name[1..].to_string() + } else { + s.name + } + }) + .collect(); + Ok(names) + } + Mach::Fat(_) => Ok(vec![]), + }, + Object::Archive(_) | Object::Unknown(_) => Ok(vec![]), + } +} + +fn is_derive_registrar_symbol(symbol: &str) -> bool { + symbol.contains(NEW_REGISTRAR_SYMBOL) +} + +fn find_registrar_symbol(file: &Path) -> Result, IoError> { + let symbols = get_symbols_from_lib(file)?; + Ok(symbols.into_iter().find(|s| is_derive_registrar_symbol(s))) +} + +/// Loads dynamic library in platform dependent manner. +/// +/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described +/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample) +/// and [here](https://github.com/rust-lang/rust/issues/60593). +/// +/// Usage of RTLD_DEEPBIND +/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1) +/// +/// It seems that on Windows that behaviour is default, so we do nothing in that case. +#[cfg(windows)] +fn load_library(file: &Path) -> Result { + Library::new(file) +} + +#[cfg(unix)] +fn load_library(file: &Path) -> Result { + use libloading::os::unix::Library as UnixLibrary; + use std::os::raw::c_int; + + const RTLD_NOW: c_int = 0x00002; + const RTLD_DEEPBIND: c_int = 0x00008; + + UnixLibrary::open(Some(file), RTLD_NOW | RTLD_DEEPBIND).map(|lib| lib.into()) +} + +struct ProcMacroLibraryLibloading { + // Hold the dylib to prevent it for unloadeding + _lib: Library, + exported_macros: Vec, +} + +impl ProcMacroLibraryLibloading { + fn open(file: &Path) -> Result { + let symbol_name = find_registrar_symbol(file)? + .ok_or(invalid_data_err(format!("Cannot find registrar symbol in file {:?}", file)))?; + + let lib = load_library(file).map_err(invalid_data_err)?; + let exported_macros = { + let macros: libloading::Symbol<&&[bridge::client::ProcMacro]> = + unsafe { lib.get(symbol_name.as_bytes()) }.map_err(invalid_data_err)?; + macros.to_vec() + }; + + Ok(ProcMacroLibraryLibloading { _lib: lib, exported_macros }) + } +} + +type ProcMacroLibraryImpl = ProcMacroLibraryLibloading; + +pub struct Expander { + libs: Vec, +} + +impl Expander { + pub fn new>(lib: &P) -> Result { + let mut libs = vec![]; + /* Some libraries for dynamic loading require canonicalized path (even when it is + already absolute + */ + let lib = + lib.as_ref().canonicalize().expect(&format!("Cannot canonicalize {:?}", lib.as_ref())); + + let library = ProcMacroLibraryImpl::open(&lib).map_err(|e| e.to_string())?; + libs.push(library); + + Ok(Expander { libs }) + } + + pub fn expand( + &self, + macro_name: &str, + macro_body: &ra_tt::Subtree, + attributes: Option<&ra_tt::Subtree>, + ) -> Result { + let parsed_body = TokenStream::with_subtree(macro_body.clone()); + + let parsed_attributes = attributes + .map_or(crate::rustc_server::TokenStream::new(), |attr| { + TokenStream::with_subtree(attr.clone()) + }); + + for lib in &self.libs { + for proc_macro in &lib.exported_macros { + match proc_macro { + bridge::client::ProcMacro::CustomDerive { trait_name, client, .. } + if *trait_name == macro_name => + { + let res = client.run( + &crate::proc_macro::bridge::server::SameThread, + crate::rustc_server::Rustc::default(), + parsed_body, + ); + return res.map(|it| it.subtree); + } + bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { + let res = client.run( + &crate::proc_macro::bridge::server::SameThread, + crate::rustc_server::Rustc::default(), + parsed_body, + ); + return res.map(|it| it.subtree); + } + bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { + let res = client.run( + &crate::proc_macro::bridge::server::SameThread, + crate::rustc_server::Rustc::default(), + parsed_attributes, + parsed_body, + ); + + return res.map(|it| it.subtree); + } + _ => continue, + } + } + } + + Err(bridge::PanicMessage::String("Nothing to expand".to_string())) + } + + pub fn list_macros(&self) -> Result, bridge::PanicMessage> { + let mut result = vec![]; + + for lib in &self.libs { + for proc_macro in &lib.exported_macros { + let res = match proc_macro { + bridge::client::ProcMacro::CustomDerive { trait_name, .. } => { + (trait_name.to_string(), ProcMacroKind::CustomDerive) + } + bridge::client::ProcMacro::Bang { name, .. } => { + (name.to_string(), ProcMacroKind::FuncLike) + } + bridge::client::ProcMacro::Attr { name, .. } => { + (name.to_string(), ProcMacroKind::Attr) + } + }; + result.push(res); + } + } + + Ok(result) + } +} diff --git a/crates/ra_proc_macro_srv/src/lib.rs b/crates/ra_proc_macro_srv/src/lib.rs index f376df2367..59716cbb3b 100644 --- a/crates/ra_proc_macro_srv/src/lib.rs +++ b/crates/ra_proc_macro_srv/src/lib.rs @@ -17,13 +17,41 @@ mod proc_macro; #[doc(hidden)] mod rustc_server; +mod dylib; + use proc_macro::bridge::client::TokenStream; use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask}; -pub fn expand_task(_task: &ExpansionTask) -> Result { - unimplemented!() +pub fn expand_task(task: &ExpansionTask) -> Result { + let expander = dylib::Expander::new(&task.lib) + .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib)); + + match expander.expand(&task.macro_name, &task.macro_body, task.attributes.as_ref()) { + Ok(expansion) => Ok(ExpansionResult { expansion }), + Err(msg) => { + let reason = format!( + "Cannot perform expansion for {}: error {:?}!", + &task.macro_name, + msg.as_str() + ); + Err(reason) + } + } } -pub fn list_macros(_task: &ListMacrosTask) -> Result { - unimplemented!() +pub fn list_macros(task: &ListMacrosTask) -> Result { + let expander = dylib::Expander::new(&task.lib) + .expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib)); + + match expander.list_macros() { + Ok(macros) => Ok(ListMacrosResult { macros }), + Err(msg) => { + let reason = + format!("Cannot perform expansion for {:?}: error {:?}!", &task.lib, msg.as_str()); + Err(reason) + } + } } + +#[cfg(test)] +mod tests; diff --git a/crates/ra_proc_macro_srv/src/rustc_server.rs b/crates/ra_proc_macro_srv/src/rustc_server.rs index 92d1fd989d..ec0d356922 100644 --- a/crates/ra_proc_macro_srv/src/rustc_server.rs +++ b/crates/ra_proc_macro_srv/src/rustc_server.rs @@ -34,6 +34,10 @@ impl TokenStream { TokenStream { subtree: Default::default() } } + pub fn with_subtree(subtree: tt::Subtree) -> Self { + TokenStream { subtree } + } + pub fn is_empty(&self) -> bool { self.subtree.token_trees.is_empty() } diff --git a/crates/ra_proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt b/crates/ra_proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt new file mode 100644 index 0000000000..24507d98d7 --- /dev/null +++ b/crates/ra_proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt @@ -0,0 +1,188 @@ +SUBTREE $ + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT allow 4294967295 + SUBTREE () 4294967295 + IDENT non_upper_case_globals 4294967295 + PUNCH , [alone] 4294967295 + IDENT unused_attributes 4294967295 + PUNCH , [alone] 4294967295 + IDENT unused_qualifications 4294967295 + IDENT const 4294967295 + IDENT _IMPL_SERIALIZE_FOR_Foo 4294967295 + PUNCH : [alone] 4294967295 + SUBTREE () 4294967295 + PUNCH = [alone] 4294967295 + SUBTREE {} 4294967295 + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT allow 4294967295 + SUBTREE () 4294967295 + IDENT unknown_lints 4294967295 + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT cfg_attr 4294967295 + SUBTREE () 4294967295 + IDENT feature 4294967295 + PUNCH = [alone] 4294967295 + SUBTREE $ + LITERAL "cargo-clippy" 0 + PUNCH , [alone] 4294967295 + IDENT allow 4294967295 + SUBTREE () 4294967295 + IDENT useless_attribute 4294967295 + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT allow 4294967295 + SUBTREE () 4294967295 + IDENT rust_2018_idioms 4294967295 + IDENT extern 4294967295 + IDENT crate 4294967295 + IDENT serde 4294967295 + IDENT as 4294967295 + IDENT _serde 4294967295 + PUNCH ; [alone] 4294967295 + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT allow 4294967295 + SUBTREE () 4294967295 + IDENT unused_macros 4294967295 + IDENT macro_rules 4294967295 + PUNCH ! [alone] 4294967295 + IDENT try 4294967295 + SUBTREE {} 4294967295 + SUBTREE () 4294967295 + PUNCH $ [alone] 4294967295 + IDENT __expr 4294967295 + PUNCH : [alone] 4294967295 + IDENT expr 4294967295 + PUNCH = [joint] 4294967295 + PUNCH > [alone] 4294967295 + SUBTREE {} 4294967295 + IDENT match 4294967295 + PUNCH $ [alone] 4294967295 + IDENT __expr 4294967295 + SUBTREE {} 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT export 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Ok 4294967295 + SUBTREE () 4294967295 + IDENT __val 4294967295 + PUNCH = [joint] 4294967295 + PUNCH > [alone] 4294967295 + IDENT __val 4294967295 + PUNCH , [alone] 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT export 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Err 4294967295 + SUBTREE () 4294967295 + IDENT __err 4294967295 + PUNCH = [joint] 4294967295 + PUNCH > [alone] 4294967295 + SUBTREE {} 4294967295 + IDENT return 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT export 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Err 4294967295 + SUBTREE () 4294967295 + IDENT __err 4294967295 + PUNCH ; [alone] 4294967295 + PUNCH # [alone] 4294967295 + SUBTREE [] 4294967295 + IDENT automatically_derived 4294967295 + IDENT impl 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Serialize 4294967295 + IDENT for 4294967295 + IDENT Foo 1 + SUBTREE {} 4294967295 + IDENT fn 4294967295 + IDENT serialize 4294967295 + PUNCH < [alone] 4294967295 + IDENT __S 4294967295 + PUNCH > [alone] 4294967295 + SUBTREE () 4294967295 + PUNCH & [alone] 4294967295 + IDENT self 4294967295 + PUNCH , [alone] 4294967295 + IDENT __serializer 4294967295 + PUNCH : [alone] 4294967295 + IDENT __S 4294967295 + PUNCH - [joint] 4294967295 + PUNCH > [alone] 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT export 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Result 4294967295 + PUNCH < [alone] 4294967295 + IDENT __S 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Ok 4294967295 + PUNCH , [alone] 4294967295 + IDENT __S 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Error 4294967295 + PUNCH > [alone] 4294967295 + IDENT where 4294967295 + IDENT __S 4294967295 + PUNCH : [alone] 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Serializer 4294967295 + PUNCH , [alone] 4294967295 + SUBTREE {} 4294967295 + IDENT let 4294967295 + IDENT __serde_state 4294967295 + PUNCH = [alone] 4294967295 + IDENT try 4294967295 + PUNCH ! [alone] 4294967295 + SUBTREE () 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT Serializer 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT serialize_struct 4294967295 + SUBTREE () 4294967295 + IDENT __serializer 4294967295 + PUNCH , [alone] 4294967295 + LITERAL "Foo" 4294967295 + PUNCH , [alone] 4294967295 + IDENT false 4294967295 + IDENT as 4294967295 + IDENT usize 4294967295 + PUNCH ; [alone] 4294967295 + IDENT _serde 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT ser 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT SerializeStruct 4294967295 + PUNCH : [joint] 4294967295 + PUNCH : [alone] 4294967295 + IDENT end 4294967295 + SUBTREE () 4294967295 + IDENT __serde_state 4294967295 + PUNCH ; [alone] 4294967295 \ No newline at end of file diff --git a/crates/ra_proc_macro_srv/src/tests/mod.rs b/crates/ra_proc_macro_srv/src/tests/mod.rs new file mode 100644 index 0000000000..03f79bc5d6 --- /dev/null +++ b/crates/ra_proc_macro_srv/src/tests/mod.rs @@ -0,0 +1,47 @@ +//! proc-macro tests + +#[macro_use] +mod utils; +use test_utils::assert_eq_text; +use utils::*; + +#[test] +fn test_derive_serialize_proc_macro() { + assert_expand( + "serde_derive", + "Serialize", + "1.0.104", + r##"struct Foo {}"##, + include_str!("fixtures/test_serialize_proc_macro.txt"), + ); +} + +#[test] +fn test_derive_serialize_proc_macro_failed() { + assert_expand( + "serde_derive", + "Serialize", + "1.0.104", + r##" + struct {} +"##, + r##" +SUBTREE $ + IDENT compile_error 4294967295 + PUNCH ! [alone] 4294967295 + SUBTREE {} 4294967295 + LITERAL "expected identifier" 4294967295 +"##, + ); +} + +#[test] +fn test_derive_proc_macro_list() { + let res = list("serde_derive", "1.0.104").join("\n"); + + assert_eq_text!( + &res, + r#"Serialize [CustomDerive] +Deserialize [CustomDerive]"# + ); +} diff --git a/crates/ra_proc_macro_srv/src/tests/utils.rs b/crates/ra_proc_macro_srv/src/tests/utils.rs new file mode 100644 index 0000000000..1ee4094492 --- /dev/null +++ b/crates/ra_proc_macro_srv/src/tests/utils.rs @@ -0,0 +1,65 @@ +//! utils used in proc-macro tests + +use crate::dylib; +use crate::list_macros; +pub use difference::Changeset as __Changeset; +use ra_proc_macro::ListMacrosTask; +use std::str::FromStr; +use test_utils::assert_eq_text; + +mod fixtures { + use cargo_metadata::{parse_messages, Message}; + use std::process::Command; + + // Use current project metadata to get the proc-macro dylib path + pub fn dylib_path(crate_name: &str, version: &str) -> std::path::PathBuf { + let command = Command::new("cargo") + .args(&["check", "--message-format", "json"]) + .output() + .unwrap() + .stdout; + + for message in parse_messages(command.as_slice()) { + match message.unwrap() { + Message::CompilerArtifact(artifact) => { + if artifact.target.kind.contains(&"proc-macro".to_string()) { + let repr = format!("{} {}", crate_name, version); + if artifact.package_id.repr.starts_with(&repr) { + return artifact.filenames[0].clone(); + } + } + } + _ => (), // Unknown message + } + } + + panic!("No proc-macro dylib for {} found!", crate_name); + } +} + +fn parse_string(code: &str) -> Option { + Some(crate::rustc_server::TokenStream::from_str(code).unwrap()) +} + +pub fn assert_expand( + crate_name: &str, + macro_name: &str, + version: &str, + fixture: &str, + expect: &str, +) { + let path = fixtures::dylib_path(crate_name, version); + let expander = dylib::Expander::new(&path).unwrap(); + let fixture = parse_string(fixture).unwrap(); + + let res = expander.expand(macro_name, &fixture.subtree, None).unwrap(); + assert_eq_text!(&format!("{:?}", res), &expect.trim()); +} + +pub fn list(crate_name: &str, version: &str) -> Vec { + let path = fixtures::dylib_path(crate_name, version); + let task = ListMacrosTask { lib: path }; + + let res = list_macros(&task).unwrap(); + res.macros.into_iter().map(|(name, kind)| format!("{} [{:?}]", name, kind)).collect() +}