3920: Implement expand_task and list_macros in proc_macro_srv r=matklad a=edwin0cheng

This PR finish up the remain `proc_macro_srv` implementation :

1. Added dylib loading code for proc-macro crate dylib. Note that we have to add some special flags for unix loading because of a bug in old version of glibc, see https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1 and https://github.com/rust-lang/rust/issues/60593 for details.

2. Added tests for proc-macro expansion: We use a trick here by adding `serde_derive` to dev-dependencies and calling `cargo-metadata` for searching its dylib path, and expand it in our tests. 

[EDIT]
Note that this PR **DO NOT** implement the final glue code with rust-analzyer and proc-macro-srv yet.
 

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>
This commit is contained in:
bors[bot] 2020-04-11 11:49:07 +00:00 committed by GitHub
commit 0ecdba20df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 600 additions and 5 deletions

49
Cargo.lock generated
View file

@ -424,6 +424,17 @@ dependencies = [
"regex",
]
[[package]]
name = "goblin"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd5e3132801a1ac34ac53b97acde50c4685414dd2f291b9ea52afa6f07468c8"
dependencies = [
"log",
"plain",
"scroll",
]
[[package]]
name = "heck"
version = "0.3.1"
@ -586,6 +597,15 @@ version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0"
[[package]]
name = "libloading"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c979a19ffb457f0273965c333053f3d586bf759bf7b683fbebc37f9a9ebedc4"
dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "linked-hash-map"
version = "0.5.2"
@ -825,6 +845,12 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ad1f1b834a05d42dae330066e9699a173b28185b3bdc3dbf14ca239585de8cc"
[[package]]
name = "plain"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "ppv-lite86"
version = "0.2.6"
@ -1082,10 +1108,13 @@ version = "0.1.0"
dependencies = [
"cargo_metadata",
"difference",
"goblin",
"libloading",
"ra_mbe",
"ra_proc_macro",
"ra_tt",
"serde_derive",
"test_utils",
]
[[package]]
@ -1403,6 +1432,26 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scroll"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb2332cb595d33f7edd5700f4cbf94892e680c7f0ae56adab58a35190b66cb1"
dependencies = [
"scroll_derive",
]
[[package]]
name = "scroll_derive"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "semver"
version = "0.9.0"

View file

@ -12,9 +12,12 @@ doctest = false
ra_tt = { path = "../ra_tt" }
ra_mbe = { path = "../ra_mbe" }
ra_proc_macro = { path = "../ra_proc_macro" }
goblin = "0.2.1"
libloading = "0.6.0"
test_utils = { path = "../test_utils" }
[dev-dependencies]
cargo_metadata = "0.9.1"
difference = "2.0.0"
# used as proc macro test target
serde_derive = "=1.0.104"
serde_derive = "=1.0.104"

View file

@ -0,0 +1,211 @@
//! Handles dynamic library loading for proc macro
use crate::{proc_macro::bridge, rustc_server::TokenStream};
use std::path::Path;
use goblin::{mach::Mach, Object};
use libloading::Library;
use ra_proc_macro::ProcMacroKind;
use std::io::Error as IoError;
use std::io::ErrorKind as IoErrorKind;
const NEW_REGISTRAR_SYMBOL: &str = "_rustc_proc_macro_decls_";
fn invalid_data_err(e: impl Into<Box<dyn std::error::Error + Send + Sync>>) -> IoError {
IoError::new(IoErrorKind::InvalidData, e)
}
fn get_symbols_from_lib(file: &Path) -> Result<Vec<String>, IoError> {
let buffer = std::fs::read(file)?;
let object = Object::parse(&buffer).map_err(invalid_data_err)?;
match object {
Object::Elf(elf) => {
let symbols = elf.dynstrtab.to_vec().map_err(invalid_data_err)?;
let names = symbols.iter().map(|s| s.to_string()).collect();
Ok(names)
}
Object::PE(pe) => {
let symbol_names =
pe.exports.iter().flat_map(|s| s.name).map(|n| n.to_string()).collect();
Ok(symbol_names)
}
Object::Mach(mach) => match mach {
Mach::Binary(binary) => {
let exports = binary.exports().map_err(invalid_data_err)?;
let names = exports
.into_iter()
.map(|s| {
// In macos doc:
// https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dlsym.3.html
// Unlike other dyld API's, the symbol name passed to dlsym() must NOT be
// prepended with an underscore.
if s.name.starts_with("_") {
s.name[1..].to_string()
} else {
s.name
}
})
.collect();
Ok(names)
}
Mach::Fat(_) => Ok(vec![]),
},
Object::Archive(_) | Object::Unknown(_) => Ok(vec![]),
}
}
fn is_derive_registrar_symbol(symbol: &str) -> bool {
symbol.contains(NEW_REGISTRAR_SYMBOL)
}
fn find_registrar_symbol(file: &Path) -> Result<Option<String>, IoError> {
let symbols = get_symbols_from_lib(file)?;
Ok(symbols.into_iter().find(|s| is_derive_registrar_symbol(s)))
}
/// Loads dynamic library in platform dependent manner.
///
/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
/// and [here](https://github.com/rust-lang/rust/issues/60593).
///
/// Usage of RTLD_DEEPBIND
/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
///
/// It seems that on Windows that behaviour is default, so we do nothing in that case.
#[cfg(windows)]
fn load_library(file: &Path) -> Result<Library, libloading::Error> {
Library::new(file)
}
#[cfg(unix)]
fn load_library(file: &Path) -> Result<Library, libloading::Error> {
use libloading::os::unix::Library as UnixLibrary;
use std::os::raw::c_int;
const RTLD_NOW: c_int = 0x00002;
const RTLD_DEEPBIND: c_int = 0x00008;
UnixLibrary::open(Some(file), RTLD_NOW | RTLD_DEEPBIND).map(|lib| lib.into())
}
struct ProcMacroLibraryLibloading {
// Hold the dylib to prevent it for unloadeding
_lib: Library,
exported_macros: Vec<bridge::client::ProcMacro>,
}
impl ProcMacroLibraryLibloading {
fn open(file: &Path) -> Result<Self, IoError> {
let symbol_name = find_registrar_symbol(file)?
.ok_or(invalid_data_err(format!("Cannot find registrar symbol in file {:?}", file)))?;
let lib = load_library(file).map_err(invalid_data_err)?;
let exported_macros = {
let macros: libloading::Symbol<&&[bridge::client::ProcMacro]> =
unsafe { lib.get(symbol_name.as_bytes()) }.map_err(invalid_data_err)?;
macros.to_vec()
};
Ok(ProcMacroLibraryLibloading { _lib: lib, exported_macros })
}
}
type ProcMacroLibraryImpl = ProcMacroLibraryLibloading;
pub struct Expander {
libs: Vec<ProcMacroLibraryImpl>,
}
impl Expander {
pub fn new<P: AsRef<Path>>(lib: &P) -> Result<Expander, String> {
let mut libs = vec![];
/* Some libraries for dynamic loading require canonicalized path (even when it is
already absolute
*/
let lib =
lib.as_ref().canonicalize().expect(&format!("Cannot canonicalize {:?}", lib.as_ref()));
let library = ProcMacroLibraryImpl::open(&lib).map_err(|e| e.to_string())?;
libs.push(library);
Ok(Expander { libs })
}
pub fn expand(
&self,
macro_name: &str,
macro_body: &ra_tt::Subtree,
attributes: Option<&ra_tt::Subtree>,
) -> Result<ra_tt::Subtree, bridge::PanicMessage> {
let parsed_body = TokenStream::with_subtree(macro_body.clone());
let parsed_attributes = attributes
.map_or(crate::rustc_server::TokenStream::new(), |attr| {
TokenStream::with_subtree(attr.clone())
});
for lib in &self.libs {
for proc_macro in &lib.exported_macros {
match proc_macro {
bridge::client::ProcMacro::CustomDerive { trait_name, client, .. }
if *trait_name == macro_name =>
{
let res = client.run(
&crate::proc_macro::bridge::server::SameThread,
crate::rustc_server::Rustc::default(),
parsed_body,
);
return res.map(|it| it.subtree);
}
bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => {
let res = client.run(
&crate::proc_macro::bridge::server::SameThread,
crate::rustc_server::Rustc::default(),
parsed_body,
);
return res.map(|it| it.subtree);
}
bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => {
let res = client.run(
&crate::proc_macro::bridge::server::SameThread,
crate::rustc_server::Rustc::default(),
parsed_attributes,
parsed_body,
);
return res.map(|it| it.subtree);
}
_ => continue,
}
}
}
Err(bridge::PanicMessage::String("Nothing to expand".to_string()))
}
pub fn list_macros(&self) -> Result<Vec<(String, ProcMacroKind)>, bridge::PanicMessage> {
let mut result = vec![];
for lib in &self.libs {
for proc_macro in &lib.exported_macros {
let res = match proc_macro {
bridge::client::ProcMacro::CustomDerive { trait_name, .. } => {
(trait_name.to_string(), ProcMacroKind::CustomDerive)
}
bridge::client::ProcMacro::Bang { name, .. } => {
(name.to_string(), ProcMacroKind::FuncLike)
}
bridge::client::ProcMacro::Attr { name, .. } => {
(name.to_string(), ProcMacroKind::Attr)
}
};
result.push(res);
}
}
Ok(result)
}
}

View file

@ -17,13 +17,41 @@ mod proc_macro;
#[doc(hidden)]
mod rustc_server;
mod dylib;
use proc_macro::bridge::client::TokenStream;
use ra_proc_macro::{ExpansionResult, ExpansionTask, ListMacrosResult, ListMacrosTask};
pub fn expand_task(_task: &ExpansionTask) -> Result<ExpansionResult, String> {
unimplemented!()
pub fn expand_task(task: &ExpansionTask) -> Result<ExpansionResult, String> {
let expander = dylib::Expander::new(&task.lib)
.expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
match expander.expand(&task.macro_name, &task.macro_body, task.attributes.as_ref()) {
Ok(expansion) => Ok(ExpansionResult { expansion }),
Err(msg) => {
let reason = format!(
"Cannot perform expansion for {}: error {:?}!",
&task.macro_name,
msg.as_str()
);
Err(reason)
}
}
}
pub fn list_macros(_task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
unimplemented!()
pub fn list_macros(task: &ListMacrosTask) -> Result<ListMacrosResult, String> {
let expander = dylib::Expander::new(&task.lib)
.expect(&format!("Cannot expand with provided libraries: ${:?}", &task.lib));
match expander.list_macros() {
Ok(macros) => Ok(ListMacrosResult { macros }),
Err(msg) => {
let reason =
format!("Cannot perform expansion for {:?}: error {:?}!", &task.lib, msg.as_str());
Err(reason)
}
}
}
#[cfg(test)]
mod tests;

View file

@ -34,6 +34,10 @@ impl TokenStream {
TokenStream { subtree: Default::default() }
}
pub fn with_subtree(subtree: tt::Subtree) -> Self {
TokenStream { subtree }
}
pub fn is_empty(&self) -> bool {
self.subtree.token_trees.is_empty()
}

View file

@ -0,0 +1,188 @@
SUBTREE $
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT allow 4294967295
SUBTREE () 4294967295
IDENT non_upper_case_globals 4294967295
PUNCH , [alone] 4294967295
IDENT unused_attributes 4294967295
PUNCH , [alone] 4294967295
IDENT unused_qualifications 4294967295
IDENT const 4294967295
IDENT _IMPL_SERIALIZE_FOR_Foo 4294967295
PUNCH : [alone] 4294967295
SUBTREE () 4294967295
PUNCH = [alone] 4294967295
SUBTREE {} 4294967295
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT allow 4294967295
SUBTREE () 4294967295
IDENT unknown_lints 4294967295
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT cfg_attr 4294967295
SUBTREE () 4294967295
IDENT feature 4294967295
PUNCH = [alone] 4294967295
SUBTREE $
LITERAL "cargo-clippy" 0
PUNCH , [alone] 4294967295
IDENT allow 4294967295
SUBTREE () 4294967295
IDENT useless_attribute 4294967295
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT allow 4294967295
SUBTREE () 4294967295
IDENT rust_2018_idioms 4294967295
IDENT extern 4294967295
IDENT crate 4294967295
IDENT serde 4294967295
IDENT as 4294967295
IDENT _serde 4294967295
PUNCH ; [alone] 4294967295
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT allow 4294967295
SUBTREE () 4294967295
IDENT unused_macros 4294967295
IDENT macro_rules 4294967295
PUNCH ! [alone] 4294967295
IDENT try 4294967295
SUBTREE {} 4294967295
SUBTREE () 4294967295
PUNCH $ [alone] 4294967295
IDENT __expr 4294967295
PUNCH : [alone] 4294967295
IDENT expr 4294967295
PUNCH = [joint] 4294967295
PUNCH > [alone] 4294967295
SUBTREE {} 4294967295
IDENT match 4294967295
PUNCH $ [alone] 4294967295
IDENT __expr 4294967295
SUBTREE {} 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT export 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Ok 4294967295
SUBTREE () 4294967295
IDENT __val 4294967295
PUNCH = [joint] 4294967295
PUNCH > [alone] 4294967295
IDENT __val 4294967295
PUNCH , [alone] 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT export 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Err 4294967295
SUBTREE () 4294967295
IDENT __err 4294967295
PUNCH = [joint] 4294967295
PUNCH > [alone] 4294967295
SUBTREE {} 4294967295
IDENT return 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT export 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Err 4294967295
SUBTREE () 4294967295
IDENT __err 4294967295
PUNCH ; [alone] 4294967295
PUNCH # [alone] 4294967295
SUBTREE [] 4294967295
IDENT automatically_derived 4294967295
IDENT impl 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Serialize 4294967295
IDENT for 4294967295
IDENT Foo 1
SUBTREE {} 4294967295
IDENT fn 4294967295
IDENT serialize 4294967295
PUNCH < [alone] 4294967295
IDENT __S 4294967295
PUNCH > [alone] 4294967295
SUBTREE () 4294967295
PUNCH & [alone] 4294967295
IDENT self 4294967295
PUNCH , [alone] 4294967295
IDENT __serializer 4294967295
PUNCH : [alone] 4294967295
IDENT __S 4294967295
PUNCH - [joint] 4294967295
PUNCH > [alone] 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT export 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Result 4294967295
PUNCH < [alone] 4294967295
IDENT __S 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Ok 4294967295
PUNCH , [alone] 4294967295
IDENT __S 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Error 4294967295
PUNCH > [alone] 4294967295
IDENT where 4294967295
IDENT __S 4294967295
PUNCH : [alone] 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Serializer 4294967295
PUNCH , [alone] 4294967295
SUBTREE {} 4294967295
IDENT let 4294967295
IDENT __serde_state 4294967295
PUNCH = [alone] 4294967295
IDENT try 4294967295
PUNCH ! [alone] 4294967295
SUBTREE () 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT Serializer 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT serialize_struct 4294967295
SUBTREE () 4294967295
IDENT __serializer 4294967295
PUNCH , [alone] 4294967295
LITERAL "Foo" 4294967295
PUNCH , [alone] 4294967295
IDENT false 4294967295
IDENT as 4294967295
IDENT usize 4294967295
PUNCH ; [alone] 4294967295
IDENT _serde 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT ser 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT SerializeStruct 4294967295
PUNCH : [joint] 4294967295
PUNCH : [alone] 4294967295
IDENT end 4294967295
SUBTREE () 4294967295
IDENT __serde_state 4294967295
PUNCH ; [alone] 4294967295

View file

@ -0,0 +1,47 @@
//! proc-macro tests
#[macro_use]
mod utils;
use test_utils::assert_eq_text;
use utils::*;
#[test]
fn test_derive_serialize_proc_macro() {
assert_expand(
"serde_derive",
"Serialize",
"1.0.104",
r##"struct Foo {}"##,
include_str!("fixtures/test_serialize_proc_macro.txt"),
);
}
#[test]
fn test_derive_serialize_proc_macro_failed() {
assert_expand(
"serde_derive",
"Serialize",
"1.0.104",
r##"
struct {}
"##,
r##"
SUBTREE $
IDENT compile_error 4294967295
PUNCH ! [alone] 4294967295
SUBTREE {} 4294967295
LITERAL "expected identifier" 4294967295
"##,
);
}
#[test]
fn test_derive_proc_macro_list() {
let res = list("serde_derive", "1.0.104").join("\n");
assert_eq_text!(
&res,
r#"Serialize [CustomDerive]
Deserialize [CustomDerive]"#
);
}

View file

@ -0,0 +1,65 @@
//! utils used in proc-macro tests
use crate::dylib;
use crate::list_macros;
pub use difference::Changeset as __Changeset;
use ra_proc_macro::ListMacrosTask;
use std::str::FromStr;
use test_utils::assert_eq_text;
mod fixtures {
use cargo_metadata::{parse_messages, Message};
use std::process::Command;
// Use current project metadata to get the proc-macro dylib path
pub fn dylib_path(crate_name: &str, version: &str) -> std::path::PathBuf {
let command = Command::new("cargo")
.args(&["check", "--message-format", "json"])
.output()
.unwrap()
.stdout;
for message in parse_messages(command.as_slice()) {
match message.unwrap() {
Message::CompilerArtifact(artifact) => {
if artifact.target.kind.contains(&"proc-macro".to_string()) {
let repr = format!("{} {}", crate_name, version);
if artifact.package_id.repr.starts_with(&repr) {
return artifact.filenames[0].clone();
}
}
}
_ => (), // Unknown message
}
}
panic!("No proc-macro dylib for {} found!", crate_name);
}
}
fn parse_string(code: &str) -> Option<crate::rustc_server::TokenStream> {
Some(crate::rustc_server::TokenStream::from_str(code).unwrap())
}
pub fn assert_expand(
crate_name: &str,
macro_name: &str,
version: &str,
fixture: &str,
expect: &str,
) {
let path = fixtures::dylib_path(crate_name, version);
let expander = dylib::Expander::new(&path).unwrap();
let fixture = parse_string(fixture).unwrap();
let res = expander.expand(macro_name, &fixture.subtree, None).unwrap();
assert_eq_text!(&format!("{:?}", res), &expect.trim());
}
pub fn list(crate_name: &str, version: &str) -> Vec<String> {
let path = fixtures::dylib_path(crate_name, version);
let task = ListMacrosTask { lib: path };
let res = list_macros(&task).unwrap();
res.macros.into_iter().map(|(name, kind)| format!("{} [{:?}]", name, kind)).collect()
}