Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-11-14 17:07:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

Load proc-macros asynchronously

Browse source
This commit is contained in:
Lukas Wirth 2023-03-25 18:06:06 +01:00
parent e9fb2ffe45
commit 607375dc20
10 changed files with 154 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
crates/base-db/src/input.rs
View file

@ -6,15 +6,16 @@
//! actual IO. See `vfs` and `project_model` in the `rust-analyzer` crate for how
//! actual IO is done and lowered to input.
use std::{fmt, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
use std::{fmt, mem, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
use cfg::CfgOptions;
use rustc_hash::FxHashMap;
use stdx::hash::{NoHashHashMap, NoHashHashSet};
use syntax::SmolStr;
use tt::token_id::Subtree;
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath};
pub type ProcMacroPaths = FxHashMap<CrateId, Result<(Option<String>, AbsPathBuf), String>>;
pub type ProcMacros = FxHashMap<CrateId, ProcMacroLoadResult>;
/// Files are grouped into source roots. A source root is a directory on the
@ -455,16 +456,11 @@ impl CrateGraph {
}
/// Extends this crate graph by adding a complete disjoint second crate
/// graph.
/// graph and adjust the ids in the [`ProcMacroPaths`] accordingly.
///
/// The ids of the crates in the `other` graph are shifted by the return
/// amount.
pub fn extend(
&mut self,
other: CrateGraph,
proc_macros: &mut ProcMacros,
other_proc_macros: ProcMacros,
) -> u32 {
pub fn extend(&mut self, other: CrateGraph, proc_macros: &mut ProcMacroPaths) -> u32 {
let start = self.arena.len() as u32;
self.arena.extend(other.arena.into_iter().map(|(id, mut data)| {
let new_id = id.shift(start);
@ -473,8 +469,11 @@ impl CrateGraph {
}
(new_id, data)
}));
proc_macros
.extend(other_proc_macros.into_iter().map(|(id, macros)| (id.shift(start), macros)));
*proc_macros = mem::take(proc_macros)
.into_iter()
.map(|(id, macros)| (id.shift(start), macros))
.collect();
start
}

4
crates/base-db/src/lib.rs
View file

@ -16,8 +16,8 @@ pub use crate::{
input::{
CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency,
Edition, Env, LangCrateOrigin, ProcMacro, ProcMacroExpander, ProcMacroExpansionError,
ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacros, SourceRoot, SourceRootId,
TargetLayoutLoadResult,
ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot,
SourceRootId, TargetLayoutLoadResult,
},
};
pub use salsa::{self, Cancelled};

14
crates/project-model/src/tests.rs
View file

@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use base_db::{CrateGraph, FileId, ProcMacros};
use base_db::{CrateGraph, FileId, ProcMacroPaths};
use cfg::{CfgAtom, CfgDiff};
use expect_test::{expect, Expect};
use paths::{AbsPath, AbsPathBuf};
@ -14,11 +14,14 @@ use crate::{
WorkspaceBuildScripts,
};
fn load_cargo(file: &str) -> (CrateGraph, ProcMacros) {
fn load_cargo(file: &str) -> (CrateGraph, ProcMacroPaths) {
load_cargo_with_overrides(file, CfgOverrides::default())
}
fn load_cargo_with_overrides(file: &str, cfg_overrides: CfgOverrides) -> (CrateGraph, ProcMacros) {
fn load_cargo_with_overrides(
file: &str,
cfg_overrides: CfgOverrides,
) -> (CrateGraph, ProcMacroPaths) {
let meta = get_test_json_file(file);
let cargo_workspace = CargoWorkspace::new(meta);
let project_workspace = ProjectWorkspace::Cargo {
@ -34,7 +37,7 @@ fn load_cargo_with_overrides(file: &str, cfg_overrides: CfgOverrides) -> (CrateG
to_crate_graph(project_workspace)
}
fn load_rust_project(file: &str) -> (CrateGraph, ProcMacros) {
fn load_rust_project(file: &str) -> (CrateGraph, ProcMacroPaths) {
let data = get_test_json_file(file);
let project = rooted_project_json(data);
let sysroot = Ok(get_fake_sysroot());
@ -92,9 +95,8 @@ fn rooted_project_json(data: ProjectJsonData) -> ProjectJson {
ProjectJson::new(base, data)
}
fn to_crate_graph(project_workspace: ProjectWorkspace) -> (CrateGraph, ProcMacros) {
fn to_crate_graph(project_workspace: ProjectWorkspace) -> (CrateGraph, ProcMacroPaths) {
project_workspace.to_crate_graph(
&mut |_, _| Ok(Vec::new()),
&mut {
let mut counter = 0;
move |_path| {

41
crates/project-model/src/workspace.rs
View file

@ -7,7 +7,7 @@ use std::{collections::VecDeque, fmt, fs, process::Command, sync::Arc};
use anyhow::{bail, format_err, Context, Result};
use base_db::{
CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Edition, Env,
FileId, LangCrateOrigin, ProcMacroLoadResult, ProcMacros, TargetLayoutLoadResult,
FileId, LangCrateOrigin, ProcMacroPaths, TargetLayoutLoadResult,
};
use cfg::{CfgDiff, CfgOptions};
use paths::{AbsPath, AbsPathBuf};
@ -576,16 +576,14 @@ impl ProjectWorkspace {
pub fn to_crate_graph(
&self,
load_proc_macro: &mut dyn FnMut(&str, &AbsPath) -> ProcMacroLoadResult,
load: &mut dyn FnMut(&AbsPath) -> Option<FileId>,
extra_env: &FxHashMap<String, String>,
) -> (CrateGraph, ProcMacros) {
) -> (CrateGraph, ProcMacroPaths) {
let _p = profile::span("ProjectWorkspace::to_crate_graph");
let (mut crate_graph, proc_macros) = match self {
ProjectWorkspace::Json { project, sysroot, rustc_cfg } => project_json_to_crate_graph(
rustc_cfg.clone(),
load_proc_macro,
load,
project,
sysroot.as_ref().ok(),
@ -602,7 +600,6 @@ impl ProjectWorkspace {
toolchain: _,
target_layout,
} => cargo_to_crate_graph(
load_proc_macro,
load,
rustc.as_ref().ok(),
cargo,
@ -679,15 +676,14 @@ impl ProjectWorkspace {
fn project_json_to_crate_graph(
rustc_cfg: Vec<CfgFlag>,
load_proc_macro: &mut dyn FnMut(&str, &AbsPath) -> ProcMacroLoadResult,
load: &mut dyn FnMut(&AbsPath) -> Option<FileId>,
project: &ProjectJson,
sysroot: Option<&Sysroot>,
extra_env: &FxHashMap<String, String>,
target_layout: TargetLayoutLoadResult,
) -> (CrateGraph, ProcMacros) {
) -> (CrateGraph, ProcMacroPaths) {
let mut crate_graph = CrateGraph::default();
let mut proc_macros = FxHashMap::<_, _>::default();
let mut proc_macros = FxHashMap::default();
let sysroot_deps = sysroot.as_ref().map(|sysroot| {
sysroot_to_crate_graph(
&mut crate_graph,
@ -708,16 +704,15 @@ fn project_json_to_crate_graph(
})
.map(|(crate_id, krate, file_id)| {
let env = krate.env.clone().into_iter().collect();
if let Some(it) = krate.proc_macro_dylib_path.clone() {
if let Some(path) = krate.proc_macro_dylib_path.clone() {
proc_macros.insert(
crate_id,
load_proc_macro(
krate.display_name.as_ref().map(|it| it.canonical_name()).unwrap_or(""),
&it,
),
Ok((
krate.display_name.as_ref().map(|it| it.canonical_name().to_owned()),
path,
)),
);
}
let target_cfgs = match krate.target.as_deref() {
Some(target) => cfg_cache
.entry(target)
@ -782,7 +777,6 @@ fn project_json_to_crate_graph(
}
fn cargo_to_crate_graph(
load_proc_macro: &mut dyn FnMut(&str, &AbsPath) -> ProcMacroLoadResult,
load: &mut dyn FnMut(&AbsPath) -> Option<FileId>,
rustc: Option<&(CargoWorkspace, WorkspaceBuildScripts)>,
cargo: &CargoWorkspace,
@ -791,7 +785,7 @@ fn cargo_to_crate_graph(
override_cfg: &CfgOverrides,
build_scripts: &WorkspaceBuildScripts,
target_layout: TargetLayoutLoadResult,
) -> (CrateGraph, ProcMacros) {
) -> (CrateGraph, ProcMacroPaths) {
let _p = profile::span("cargo_to_crate_graph");
let mut crate_graph = CrateGraph::default();
let mut proc_macros = FxHashMap::default();
@ -862,7 +856,6 @@ fn cargo_to_crate_graph(
&cargo[pkg],
build_scripts.get_output(pkg),
cfg_options.clone(),
&mut |path| load_proc_macro(&cargo[tgt].name, path),
file_id,
&cargo[tgt].name,
cargo[tgt].is_proc_macro,
@ -938,7 +931,6 @@ fn cargo_to_crate_graph(
&mut proc_macros,
&mut pkg_to_lib_crate,
load,
load_proc_macro,
rustc_workspace,
cargo,
&public_deps,
@ -966,7 +958,7 @@ fn detached_files_to_crate_graph(
detached_files: &[AbsPathBuf],
sysroot: Option<&Sysroot>,
target_layout: TargetLayoutLoadResult,
) -> (CrateGraph, ProcMacros) {
) -> (CrateGraph, ProcMacroPaths) {
let _p = profile::span("detached_files_to_crate_graph");
let mut crate_graph = CrateGraph::default();
let (public_deps, _libproc_macro) = match sysroot {
@ -1018,10 +1010,9 @@ fn detached_files_to_crate_graph(
fn handle_rustc_crates(
crate_graph: &mut CrateGraph,
proc_macros: &mut ProcMacros,
proc_macros: &mut ProcMacroPaths,
pkg_to_lib_crate: &mut FxHashMap<Package, CrateId>,
load: &mut dyn FnMut(&AbsPath) -> Option<FileId>,
load_proc_macro: &mut dyn FnMut(&str, &AbsPath) -> ProcMacroLoadResult,
rustc_workspace: &CargoWorkspace,
cargo: &CargoWorkspace,
public_deps: &SysrootPublicDeps,
@ -1084,7 +1075,6 @@ fn handle_rustc_crates(
&rustc_workspace[pkg],
build_scripts.get_output(pkg),
cfg_options.clone(),
&mut |path| load_proc_macro(&rustc_workspace[tgt].name, path),
file_id,
&rustc_workspace[tgt].name,
rustc_workspace[tgt].is_proc_macro,
@ -1146,11 +1136,10 @@ fn handle_rustc_crates(
fn add_target_crate_root(
crate_graph: &mut CrateGraph,
proc_macros: &mut ProcMacros,
proc_macros: &mut ProcMacroPaths,
pkg: &PackageData,
build_data: Option<&BuildScriptOutput>,
cfg_options: CfgOptions,
load_proc_macro: &mut dyn FnMut(&AbsPath) -> ProcMacroLoadResult,
file_id: FileId,
cargo_name: &str,
is_proc_macro: bool,
@ -1197,11 +1186,11 @@ fn add_target_crate_root(
target_layout,
);
let proc_macro = match build_data.as_ref().map(|it| &it.proc_macro_dylib_path) {
Some(it) => it.as_deref().map(load_proc_macro),
Some(it) => it.clone().map(Ok),
None => Some(Err("crate has not (yet) been built".into())),
};
if let Some(proc_macro) = proc_macro {
proc_macros.insert(crate_id, proc_macro);
proc_macros.insert(crate_id, proc_macro.map(|path| (Some(cargo_name.to_owned()), path)));
}
crate_id

22
crates/rust-analyzer/src/cli/load_cargo.rs
View file

@ -69,7 +69,7 @@ pub fn load_workspace(
Box::new(loader)
};
let proc_macro_client = match &load_config.with_proc_macro_server {
let proc_macro_server = match &load_config.with_proc_macro_server {
ProcMacroServerChoice::Sysroot => ws
.find_sysroot_proc_macro_srv()
.ok_or_else(|| "failed to find sysroot proc-macro server".to_owned())
@ -83,9 +83,6 @@ pub fn load_workspace(
};
let (crate_graph, proc_macros) = ws.to_crate_graph(
&mut |_, path: &AbsPath| {
load_proc_macro(proc_macro_client.as_ref().map_err(|e| &**e), path, &[])
},
&mut |path: &AbsPath| {
let contents = loader.load_sync(path);
let path = vfs::VfsPath::from(path.to_path_buf());
@ -94,6 +91,21 @@ pub fn load_workspace(
},
extra_env,
);
let proc_macros = {
let proc_macro_server = match &proc_macro_server {
Ok(it) => Ok(it),
Err(e) => Err(e.as_str()),
};
proc_macros
.into_iter()
.map(|(crate_id, path)| {
(
crate_id,
path.and_then(|(_, path)| load_proc_macro(proc_macro_server, &path, &[])),
)
})
.collect()
};
let project_folders = ProjectFolders::new(&[ws], &[]);
loader.set_config(vfs::loader::Config {
@ -114,7 +126,7 @@ pub fn load_workspace(
if load_config.prefill_caches {
host.analysis().parallel_prime_caches(1, |_| {})?;
}
Ok((host, vfs, proc_macro_client.ok()))
Ok((host, vfs, proc_macro_server.ok()))
}
fn load_crate_graph(

10
crates/rust-analyzer/src/global_state.rs
View file

@ -59,10 +59,11 @@ pub(crate) struct GlobalState {
pub(crate) mem_docs: MemDocs,
pub(crate) semantic_tokens_cache: Arc<Mutex<FxHashMap<Url, SemanticTokens>>>,
pub(crate) shutdown_requested: bool,
pub(crate) proc_macro_changed: bool,
pub(crate) last_reported_status: Option<lsp_ext::ServerStatusParams>,
pub(crate) source_root_config: SourceRootConfig,
pub(crate) proc_macro_clients: Vec<Result<ProcMacroServer, String>>,
pub(crate) proc_macro_changed: bool,
pub(crate) proc_macro_clients: Arc<[Result<ProcMacroServer, String>]>,
pub(crate) flycheck: Arc<[FlycheckHandle]>,
pub(crate) flycheck_sender: Sender<flycheck::Message>,
@ -151,10 +152,11 @@ impl GlobalState {
mem_docs: MemDocs::default(),
semantic_tokens_cache: Arc::new(Default::default()),
shutdown_requested: false,
proc_macro_changed: false,
last_reported_status: None,
source_root_config: SourceRootConfig::default(),
proc_macro_clients: vec![],
proc_macro_changed: false,
proc_macro_clients: Arc::new([]),
flycheck: Arc::new([]),
flycheck_sender,

5
crates/rust-analyzer/src/handlers.rs
View file

@ -5,6 +5,7 @@
use std::{
io::Write as _,
process::{self, Stdio},
sync::Arc,
};
use anyhow::Context;
@ -44,7 +45,7 @@ use crate::{
};
pub(crate) fn handle_workspace_reload(state: &mut GlobalState, _: ()) -> Result<()> {
state.proc_macro_clients.clear();
state.proc_macro_clients = Arc::new([]);
state.proc_macro_changed = false;
state.fetch_workspaces_queue.request_op("reload workspace request".to_string());
@ -53,7 +54,7 @@ pub(crate) fn handle_workspace_reload(state: &mut GlobalState, _: ()) -> Result<
}
pub(crate) fn handle_proc_macros_reload(state: &mut GlobalState, _: ()) -> Result<()> {
state.proc_macro_clients.clear();
state.proc_macro_clients = Arc::new([]);
state.proc_macro_changed = false;
state.fetch_build_data_queue.request_op("reload proc macros request".to_string());

18
crates/rust-analyzer/src/main_loop.rs
View file

@ -24,7 +24,7 @@ use crate::{
handlers, lsp_ext,
lsp_utils::{apply_document_changes, notification_is, Progress},
mem_docs::DocumentData,
reload::{self, BuildDataProgress, ProjectWorkspaceProgress},
reload::{self, BuildDataProgress, ProcMacroProgress, ProjectWorkspaceProgress},
Result,
};
@ -68,6 +68,7 @@ pub(crate) enum Task {
PrimeCaches(PrimeCachesProgress),
FetchWorkspace(ProjectWorkspaceProgress),
FetchBuildData(BuildDataProgress),
LoadProcMacros(ProcMacroProgress),
}
#[derive(Debug)]
@ -487,6 +488,21 @@ impl GlobalState {
}
};
if let Some(state) = state {
self.report_progress("Building", state, msg, None, None);
}
}
Task::LoadProcMacros(progress) => {
let (state, msg) = match progress {
ProcMacroProgress::Begin => (Some(Progress::Begin), None),
ProcMacroProgress::Report(msg) => (Some(Progress::Report), Some(msg)),
ProcMacroProgress::End(proc_macro_load_result) => {
self.set_proc_macros(proc_macro_load_result);
(Some(Progress::End), None)
}
};
if let Some(state) = state {
self.report_progress("Loading", state, msg, None, None);
}

105
crates/rust-analyzer/src/reload.rs
View file

@ -12,7 +12,7 @@
//! correct. Instead, we try to provide a best-effort service. Even if the
//! project is currently loading and we don't have a full project model, we
//! still want to respond to various requests.
use std::{collections::hash_map::Entry, mem, sync::Arc};
use std::{collections::hash_map::Entry, iter, mem, sync::Arc};
use flycheck::{FlycheckConfig, FlycheckHandle};
use hir::db::DefDatabase;
@ -20,7 +20,7 @@ use ide::Change;
use ide_db::{
base_db::{
CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacroKind,
ProcMacroLoadResult, ProcMacros, SourceRoot, VfsPath,
ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot, VfsPath,
},
FxHashMap,
};
@ -54,6 +54,13 @@ pub(crate) enum BuildDataProgress {
End((Arc<Vec<ProjectWorkspace>>, Vec<anyhow::Result<WorkspaceBuildScripts>>)),
}
#[derive(Debug)]
pub(crate) enum ProcMacroProgress {
Begin,
Report(String),
End(ProcMacros),
}
impl GlobalState {
pub(crate) fn is_quiescent(&self) -> bool {
!(self.last_reported_status.is_none()
@ -216,6 +223,59 @@ impl GlobalState {
});
}
pub(crate) fn load_proc_macros(&mut self, paths: Vec<ProcMacroPaths>) {
tracing::info!("will load proc macros");
let dummy_replacements = self.config.dummy_replacements().clone();
let proc_macro_clients = self.proc_macro_clients.clone();
self.task_pool.handle.spawn_with_sender(move |sender| {
sender.send(Task::LoadProcMacros(ProcMacroProgress::Begin)).unwrap();
let dummy_replacements = &dummy_replacements;
let progress = {
let sender = sender.clone();
&move |msg| {
sender.send(Task::LoadProcMacros(ProcMacroProgress::Report(msg))).unwrap()
}
};
let mut res = FxHashMap::default();
for (client, paths) in proc_macro_clients
.iter()
.map(|res| res.as_ref().map_err(|e| &**e))
.chain(iter::repeat_with(|| Err("Proc macros are disabled")))
.zip(paths)
{
res.extend(paths.into_iter().map(move |(crate_id, res)| {
(
crate_id,
res.and_then(|(crate_name, path)| {
progress(path.display().to_string());
load_proc_macro(
client,
&path,
crate_name
.as_deref()
.and_then(|crate_name| {
dummy_replacements.get(crate_name).map(|v| &**v)
})
.unwrap_or_default(),
)
}),
)
}));
}
sender.send(Task::LoadProcMacros(ProcMacroProgress::End(res))).unwrap();
});
}
pub(crate) fn set_proc_macros(&mut self, proc_macros: ProcMacros) {
let mut change = Change::new();
change.set_proc_macros(proc_macros);
self.analysis_host.apply_change(change);
}
pub(crate) fn switch_workspaces(&mut self, cause: Cause) {
let _p = profile::span("GlobalState::switch_workspaces");
tracing::info!(%cause, "will switch workspaces");
@ -303,8 +363,6 @@ impl GlobalState {
);
}
let mut change = Change::new();
let files_config = self.config.files();
let project_folders = ProjectFolders::new(&self.workspaces, &files_config.exclude);
@ -353,11 +411,10 @@ impl GlobalState {
watch,
version: self.vfs_config_version,
});
self.source_root_config = project_folders.source_root_config;
// Create crate graph from all the workspaces
let (crate_graph, proc_macros) = {
let dummy_replacements = self.config.dummy_replacements();
let (crate_graph, proc_macro_paths) = {
let vfs = &mut self.vfs.write().0;
let loader = &mut self.loader;
let mem_docs = &self.mem_docs;
@ -376,34 +433,22 @@ impl GlobalState {
};
let mut crate_graph = CrateGraph::default();
let mut proc_macros = ProcMacros::default();
for (idx, ws) in self.workspaces.iter().enumerate() {
let proc_macro_client = match self.proc_macro_clients.get(idx) {
Some(res) => res.as_ref().map_err(|e| &**e),
None => Err("Proc macros are disabled"),
};
let mut load_proc_macro = move |crate_name: &str, path: &AbsPath| {
load_proc_macro(
proc_macro_client,
path,
dummy_replacements.get(crate_name).map(|v| &**v).unwrap_or_default(),
)
};
let (other, other_proc_macros) = ws.to_crate_graph(
&mut load_proc_macro,
&mut load,
&self.config.cargo().extra_env,
);
crate_graph.extend(other, &mut proc_macros, other_proc_macros);
let mut proc_macros = Vec::default();
for ws in &**self.workspaces {
let (other, mut crate_proc_macros) =
ws.to_crate_graph(&mut load, &self.config.cargo().extra_env);
crate_graph.extend(other, &mut crate_proc_macros);
proc_macros.push(crate_proc_macros);
}
(crate_graph, proc_macros)
};
let mut change = Change::new();
change.set_crate_graph(crate_graph);
change.set_proc_macros(proc_macros);
self.source_root_config = project_folders.source_root_config;
self.analysis_host.apply_change(change);
if same_workspaces {
self.load_proc_macros(proc_macro_paths);
}
self.process_changes();
self.reload_flycheck();
tracing::info!("did switch workspaces");

2
docs/dev/lsp-extensions.md
View file

@ -1,5 +1,5 @@
<!---
lsp_ext.rs hash: 37f31ae648632897
lsp_ext.rs hash: 92fe1037312754df
If you need to change the above hash to make the test pass, please check if you
need to adjust this doc as well and ping this issue: