Split out project loading capabilities from rust-analyzer crate

Cargo.lock

@@ -962,6 +962,23 @@ dependencies = [
  "text-size",
 ]
 
+[[package]]
+name = "load-cargo"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "crossbeam-channel",
+ "ide",
+ "ide-db",
+ "itertools",
+ "proc-macro-api",
+ "project-model",
+ "tracing",
+ "tt",
+ "vfs",
+ "vfs-notify",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.4.10"
@@ -1535,6 +1552,7 @@ dependencies = [
  "ide-db",
  "ide-ssr",
  "itertools",
+ "load-cargo",
  "lsp-server 0.7.1",
  "lsp-types",
  "mbe",
@@ -1564,7 +1582,6 @@ dependencies = [
  "tracing-subscriber",
  "tracing-tree",
  "triomphe",
- "tt",
  "vfs",
  "vfs-notify",
  "winapi",

Cargo.toml

@@ -61,6 +61,7 @@ ide-diagnostics = { path = "./crates/ide-diagnostics", version = "0.0.0" }
 ide-ssr = { path = "./crates/ide-ssr", version = "0.0.0" }
 intern = { path = "./crates/intern", version = "0.0.0" }
 limit = { path = "./crates/limit", version = "0.0.0" }
+load-cargo = { path = "./crates/load-cargo", version = "0.0.0" }
 mbe = { path = "./crates/mbe", version = "0.0.0" }
 parser = { path = "./crates/parser", version = "0.0.0" }
 paths = { path = "./crates/paths", version = "0.0.0" }

crates/load-cargo/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "load-cargo"
+version = "0.0.0"
+rust-version.workspace = true
+edition.workspace = true
+license.workspace = true
+authors.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.62"
+crossbeam-channel = "0.5.5"
+itertools = "0.10.5"
+tracing = "0.1.35"
+
+ide.workspace = true
+ide-db.workspace =true
+proc-macro-api.workspace = true
+project-model.workspace = true
+tt.workspace = true
+vfs.workspace = true
+vfs-notify.workspace = true

crates/load-cargo/src/lib.rs

@@ -0,0 +1,441 @@
+//! Loads a Cargo project into a static instance of analysis, without support
+//! for incorporating changes.
+// Note, don't remove any public api from this. This API is consumed by external tools
+// to run rust-analyzer as a library.
+use std::{collections::hash_map::Entry, mem, path::Path, sync};
+
+use ::tt::token_id as tt;
+use crossbeam_channel::{unbounded, Receiver};
+use ide::{AnalysisHost, Change, SourceRoot};
+use ide_db::{
+    base_db::{
+        CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacroKind,
+        ProcMacroLoadResult, ProcMacros,
+    },
+    FxHashMap,
+};
+use itertools::Itertools;
+use proc_macro_api::{MacroDylib, ProcMacroServer};
+use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace};
+use vfs::{file_set::FileSetConfig, loader::Handle, AbsPath, AbsPathBuf, VfsPath};
+
+pub struct LoadCargoConfig {
+    pub load_out_dirs_from_check: bool,
+    pub with_proc_macro_server: ProcMacroServerChoice,
+    pub prefill_caches: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ProcMacroServerChoice {
+    Sysroot,
+    Explicit(AbsPathBuf),
+    None,
+}
+
+pub fn load_workspace_at(
+    root: &Path,
+    cargo_config: &CargoConfig,
+    load_config: &LoadCargoConfig,
+    progress: &dyn Fn(String),
+) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> {
+    let root = AbsPathBuf::assert(std::env::current_dir()?.join(root));
+    let root = ProjectManifest::discover_single(&root)?;
+    let mut workspace = ProjectWorkspace::load(root, cargo_config, progress)?;
+
+    if load_config.load_out_dirs_from_check {
+        let build_scripts = workspace.run_build_scripts(cargo_config, progress)?;
+        workspace.set_build_scripts(build_scripts)
+    }
+
+    load_workspace(workspace, &cargo_config.extra_env, load_config)
+}
+
+pub fn load_workspace(
+    ws: ProjectWorkspace,
+    extra_env: &FxHashMap<String, String>,
+    load_config: &LoadCargoConfig,
+) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> {
+    let (sender, receiver) = unbounded();
+    let mut vfs = vfs::Vfs::default();
+    let mut loader = {
+        let loader =
+            vfs_notify::NotifyHandle::spawn(Box::new(move |msg| sender.send(msg).unwrap()));
+        Box::new(loader)
+    };
+
+    let proc_macro_server = match &load_config.with_proc_macro_server {
+        ProcMacroServerChoice::Sysroot => ws
+            .find_sysroot_proc_macro_srv()
+            .and_then(|it| ProcMacroServer::spawn(it).map_err(Into::into)),
+        ProcMacroServerChoice::Explicit(path) => {
+            ProcMacroServer::spawn(path.clone()).map_err(Into::into)
+        }
+        ProcMacroServerChoice::None => Err(anyhow::format_err!("proc macro server disabled")),
+    };
+
+    let (crate_graph, proc_macros) = ws.to_crate_graph(
+        &mut |path: &AbsPath| {
+            let contents = loader.load_sync(path);
+            let path = vfs::VfsPath::from(path.to_path_buf());
+            vfs.set_file_contents(path.clone(), contents);
+            vfs.file_id(&path)
+        },
+        extra_env,
+    );
+    let proc_macros = {
+        let proc_macro_server = match &proc_macro_server {
+            Ok(it) => Ok(it),
+            Err(e) => Err(e.to_string()),
+        };
+        proc_macros
+            .into_iter()
+            .map(|(crate_id, path)| {
+                (
+                    crate_id,
+                    path.map_or_else(
+                        |_| Err("proc macro crate is missing dylib".to_owned()),
+                        |(_, path)| {
+                            proc_macro_server.as_ref().map_err(Clone::clone).and_then(
+                                |proc_macro_server| load_proc_macro(proc_macro_server, &path, &[]),
+                            )
+                        },
+                    ),
+                )
+            })
+            .collect()
+    };
+
+    let project_folders = ProjectFolders::new(&[ws], &[]);
+    loader.set_config(vfs::loader::Config {
+        load: project_folders.load,
+        watch: vec![],
+        version: 0,
+    });
+
+    let host = load_crate_graph(
+        crate_graph,
+        proc_macros,
+        project_folders.source_root_config,
+        &mut vfs,
+        &receiver,
+    );
+
+    if load_config.prefill_caches {
+        host.analysis().parallel_prime_caches(1, |_| {})?;
+    }
+    Ok((host, vfs, proc_macro_server.ok()))
+}
+
+#[derive(Default)]
+pub struct ProjectFolders {
+    pub load: Vec<vfs::loader::Entry>,
+    pub watch: Vec<usize>,
+    pub source_root_config: SourceRootConfig,
+}
+
+impl ProjectFolders {
+    pub fn new(workspaces: &[ProjectWorkspace], global_excludes: &[AbsPathBuf]) -> ProjectFolders {
+        let mut res = ProjectFolders::default();
+        let mut fsc = FileSetConfig::builder();
+        let mut local_filesets = vec![];
+
+        // Dedup source roots
+        // Depending on the project setup, we can have duplicated source roots, or for example in
+        // the case of the rustc workspace, we can end up with two source roots that are almost the
+        // same but not quite, like:
+        // PackageRoot { is_local: false, include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri")], exclude: [] }
+        // PackageRoot {
+        //     is_local: true,
+        //     include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri"), AbsPathBuf(".../rust/build/x86_64-pc-windows-msvc/stage0-tools/x86_64-pc-windows-msvc/release/build/cargo-miri-85801cd3d2d1dae4/out")],
+        //     exclude: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri/.git"), AbsPathBuf(".../rust/src/tools/miri/cargo-miri/target")]
+        // }
+        //
+        // The first one comes from the explicit rustc workspace which points to the rustc workspace itself
+        // The second comes from the rustc workspace that we load as the actual project workspace
+        // These `is_local` differing in this kind of way gives us problems, especially when trying to filter diagnostics as we don't report diagnostics for external libraries.
+        // So we need to deduplicate these, usually it would be enough to deduplicate by `include`, but as the rustc example shows here that doesn't work,
+        // so we need to also coalesce the includes if they overlap.
+
+        let mut roots: Vec<_> = workspaces
+            .iter()
+            .flat_map(|ws| ws.to_roots())
+            .update(|root| root.include.sort())
+            .sorted_by(|a, b| a.include.cmp(&b.include))
+            .collect();
+
+        // map that tracks indices of overlapping roots
+        let mut overlap_map = FxHashMap::<_, Vec<_>>::default();
+        let mut done = false;
+
+        while !mem::replace(&mut done, true) {
+            // maps include paths to indices of the corresponding root
+            let mut include_to_idx = FxHashMap::default();
+            // Find and note down the indices of overlapping roots
+            for (idx, root) in roots.iter().enumerate().filter(|(_, it)| !it.include.is_empty()) {
+                for include in &root.include {
+                    match include_to_idx.entry(include) {
+                        Entry::Occupied(e) => {
+                            overlap_map.entry(*e.get()).or_default().push(idx);
+                        }
+                        Entry::Vacant(e) => {
+                            e.insert(idx);
+                        }
+                    }
+                }
+            }
+            for (k, v) in overlap_map.drain() {
+                done = false;
+                for v in v {
+                    let r = mem::replace(
+                        &mut roots[v],
+                        PackageRoot { is_local: false, include: vec![], exclude: vec![] },
+                    );
+                    roots[k].is_local |= r.is_local;
+                    roots[k].include.extend(r.include);
+                    roots[k].exclude.extend(r.exclude);
+                }
+                roots[k].include.sort();
+                roots[k].exclude.sort();
+                roots[k].include.dedup();
+                roots[k].exclude.dedup();
+            }
+        }
+
+        for root in roots.into_iter().filter(|it| !it.include.is_empty()) {
+            let file_set_roots: Vec<VfsPath> =
+                root.include.iter().cloned().map(VfsPath::from).collect();
+
+            let entry = {
+                let mut dirs = vfs::loader::Directories::default();
+                dirs.extensions.push("rs".into());
+                dirs.include.extend(root.include);
+                dirs.exclude.extend(root.exclude);
+                for excl in global_excludes {
+                    if dirs
+                        .include
+                        .iter()
+                        .any(|incl| incl.starts_with(excl) || excl.starts_with(incl))
+                    {
+                        dirs.exclude.push(excl.clone());
+                    }
+                }
+
+                vfs::loader::Entry::Directories(dirs)
+            };
+
+            if root.is_local {
+                res.watch.push(res.load.len());
+            }
+            res.load.push(entry);
+
+            if root.is_local {
+                local_filesets.push(fsc.len());
+            }
+            fsc.add_file_set(file_set_roots)
+        }
+
+        let fsc = fsc.build();
+        res.source_root_config = SourceRootConfig { fsc, local_filesets };
+
+        res
+    }
+}
+
+#[derive(Default, Debug)]
+pub struct SourceRootConfig {
+    pub fsc: FileSetConfig,
+    pub local_filesets: Vec<usize>,
+}
+
+impl SourceRootConfig {
+    pub fn partition(&self, vfs: &vfs::Vfs) -> Vec<SourceRoot> {
+        self.fsc
+            .partition(vfs)
+            .into_iter()
+            .enumerate()
+            .map(|(idx, file_set)| {
+                let is_local = self.local_filesets.contains(&idx);
+                if is_local {
+                    SourceRoot::new_local(file_set)
+                } else {
+                    SourceRoot::new_library(file_set)
+                }
+            })
+            .collect()
+    }
+}
+
+/// Load the proc-macros for the given lib path, replacing all expanders whose names are in `dummy_replace`
+/// with an identity dummy expander.
+pub fn load_proc_macro(
+    server: &ProcMacroServer,
+    path: &AbsPath,
+    dummy_replace: &[Box<str>],
+) -> ProcMacroLoadResult {
+    let res: Result<Vec<_>, String> = (|| {
+        let dylib = MacroDylib::new(path.to_path_buf());
+        let vec = server.load_dylib(dylib).map_err(|e| format!("{e}"))?;
+        if vec.is_empty() {
+            return Err("proc macro library returned no proc macros".to_string());
+        }
+        Ok(vec
+            .into_iter()
+            .map(|expander| expander_to_proc_macro(expander, dummy_replace))
+            .collect())
+    })();
+    match res {
+        Ok(proc_macros) => {
+            tracing::info!(
+                "Loaded proc-macros for {path}: {:?}",
+                proc_macros.iter().map(|it| it.name.clone()).collect::<Vec<_>>()
+            );
+            Ok(proc_macros)
+        }
+        Err(e) => {
+            tracing::warn!("proc-macro loading for {path} failed: {e}");
+            Err(e)
+        }
+    }
+}
+
+fn load_crate_graph(
+    crate_graph: CrateGraph,
+    proc_macros: ProcMacros,
+    source_root_config: SourceRootConfig,
+    vfs: &mut vfs::Vfs,
+    receiver: &Receiver<vfs::loader::Message>,
+) -> AnalysisHost {
+    let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok());
+    let mut host = AnalysisHost::new(lru_cap);
+    let mut analysis_change = Change::new();
+
+    host.raw_database_mut().enable_proc_attr_macros();
+
+    // wait until Vfs has loaded all roots
+    for task in receiver {
+        match task {
+            vfs::loader::Message::Progress { n_done, n_total, config_version: _ } => {
+                if n_done == n_total {
+                    break;
+                }
+            }
+            vfs::loader::Message::Loaded { files } => {
+                for (path, contents) in files {
+                    vfs.set_file_contents(path.into(), contents);
+                }
+            }
+        }
+    }
+    let changes = vfs.take_changes();
+    for file in changes {
+        if file.exists() {
+            let contents = vfs.file_contents(file.file_id);
+            if let Ok(text) = std::str::from_utf8(contents) {
+                analysis_change.change_file(file.file_id, Some(text.into()))
+            }
+        }
+    }
+    let source_roots = source_root_config.partition(vfs);
+    analysis_change.set_roots(source_roots);
+
+    analysis_change.set_crate_graph(crate_graph);
+    analysis_change.set_proc_macros(proc_macros);
+
+    host.apply_change(analysis_change);
+    host
+}
+
+fn expander_to_proc_macro(
+    expander: proc_macro_api::ProcMacro,
+    dummy_replace: &[Box<str>],
+) -> ProcMacro {
+    let name = From::from(expander.name());
+    let kind = match expander.kind() {
+        proc_macro_api::ProcMacroKind::CustomDerive => ProcMacroKind::CustomDerive,
+        proc_macro_api::ProcMacroKind::FuncLike => ProcMacroKind::FuncLike,
+        proc_macro_api::ProcMacroKind::Attr => ProcMacroKind::Attr,
+    };
+    let expander: sync::Arc<dyn ProcMacroExpander> =
+        if dummy_replace.iter().any(|replace| &**replace == name) {
+            match kind {
+                ProcMacroKind::Attr => sync::Arc::new(IdentityExpander),
+                _ => sync::Arc::new(EmptyExpander),
+            }
+        } else {
+            sync::Arc::new(Expander(expander))
+        };
+    ProcMacro { name, kind, expander }
+}
+
+#[derive(Debug)]
+struct Expander(proc_macro_api::ProcMacro);
+
+impl ProcMacroExpander for Expander {
+    fn expand(
+        &self,
+        subtree: &tt::Subtree,
+        attrs: Option<&tt::Subtree>,
+        env: &Env,
+    ) -> Result<tt::Subtree, ProcMacroExpansionError> {
+        let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
+        match self.0.expand(subtree, attrs, env) {
+            Ok(Ok(subtree)) => Ok(subtree),
+            Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
+            Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
+        }
+    }
+}
+
+/// Dummy identity expander, used for attribute proc-macros that are deliberately ignored by the user.
+#[derive(Debug)]
+struct IdentityExpander;
+
+impl ProcMacroExpander for IdentityExpander {
+    fn expand(
+        &self,
+        subtree: &tt::Subtree,
+        _: Option<&tt::Subtree>,
+        _: &Env,
+    ) -> Result<tt::Subtree, ProcMacroExpansionError> {
+        Ok(subtree.clone())
+    }
+}
+
+/// Empty expander, used for proc-macros that are deliberately ignored by the user.
+#[derive(Debug)]
+struct EmptyExpander;
+
+impl ProcMacroExpander for EmptyExpander {
+    fn expand(
+        &self,
+        _: &tt::Subtree,
+        _: Option<&tt::Subtree>,
+        _: &Env,
+    ) -> Result<tt::Subtree, ProcMacroExpansionError> {
+        Ok(tt::Subtree::empty())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use ide_db::base_db::SourceDatabase;
+
+    use super::*;
+
+    #[test]
+    fn test_loading_rust_analyzer() {
+        let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap();
+        let cargo_config = CargoConfig::default();
+        let load_cargo_config = LoadCargoConfig {
+            load_out_dirs_from_check: false,
+            with_proc_macro_server: ProcMacroServerChoice::None,
+            prefill_caches: false,
+        };
+        let (host, _vfs, _proc_macro) =
+            load_workspace_at(path, &cargo_config, &load_cargo_config, &|_| {}).unwrap();
+
+        let n_crates = host.raw_database().crate_graph().iter().count();
+        // RA has quite a few crates, but the exact count doesn't matter
+        assert!(n_crates > 20);
+    }
+}

crates/rust-analyzer/Cargo.toml

@@ -62,13 +62,13 @@ ide-db.workspace = true
 # This should only be used in CLI
 ide-ssr.workspace = true
 ide.workspace = true
+load-cargo.workspace = true
 proc-macro-api.workspace = true
 profile.workspace = true
 project-model.workspace = true
 stdx.workspace = true
 syntax.workspace = true
 toolchain.workspace = true
-tt.workspace = true
 vfs-notify.workspace = true
 vfs.workspace = true
 

crates/rust-analyzer/src/cli.rs

@@ -1,7 +1,6 @@
 //! Various batch processing tasks, intended primarily for debugging.
 
 pub mod flags;
-pub mod load_cargo;
 mod parse;
 mod symbols;
 mod highlight;

crates/rust-analyzer/src/cli/analysis_stats.rs

@@ -24,6 +24,7 @@ use ide_db::{
     LineIndexDatabase,
 };
 use itertools::Itertools;
+use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
 use oorandom::Rand32;
 use profile::{Bytes, StopWatch};
 use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource};
@@ -34,9 +35,7 @@ use vfs::{AbsPathBuf, Vfs, VfsPath};
 
 use crate::cli::{
     flags::{self, OutputFormat},
-    full_name_of_item,
+    full_name_of_item, print_memory_usage,
-    load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice},
-    print_memory_usage,
     progress_report::ProgressReport,
     report_metric, Verbosity,
 };

crates/rust-analyzer/src/cli/diagnostics.rs

@@ -7,11 +7,9 @@ use rustc_hash::FxHashSet;
 use hir::{db::HirDatabase, Crate, Module};
 use ide::{AssistResolveStrategy, DiagnosticsConfig, Severity};
 use ide_db::base_db::SourceDatabaseExt;
+use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
 
-use crate::cli::{
+use crate::cli::flags;
-    flags,
-    load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice},
-};
 
 impl flags::Diagnostics {
     pub fn run(self) -> anyhow::Result<()> {

crates/rust-analyzer/src/cli/load_cargo.rs

@@ -1,204 +0,0 @@
-//! Loads a Cargo project into a static instance of analysis, without support
-//! for incorporating changes.
-use std::path::Path;
-
-use crossbeam_channel::{unbounded, Receiver};
-use ide::{AnalysisHost, Change};
-use ide_db::{
-    base_db::{CrateGraph, ProcMacros},
-    FxHashMap,
-};
-use proc_macro_api::ProcMacroServer;
-use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace};
-use triomphe::Arc;
-use vfs::{loader::Handle, AbsPath, AbsPathBuf};
-
-use crate::reload::{load_proc_macro, ProjectFolders, SourceRootConfig};
-
-// Note: Since this type is used by external tools that use rust-analyzer as a library
-// what otherwise would be `pub(crate)` has to be `pub` here instead.
-pub struct LoadCargoConfig {
-    pub load_out_dirs_from_check: bool,
-    pub with_proc_macro_server: ProcMacroServerChoice,
-    pub prefill_caches: bool,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ProcMacroServerChoice {
-    Sysroot,
-    Explicit(AbsPathBuf),
-    None,
-}
-
-// Note: Since this function is used by external tools that use rust-analyzer as a library
-// what otherwise would be `pub(crate)` has to be `pub` here instead.
-pub fn load_workspace_at(
-    root: &Path,
-    cargo_config: &CargoConfig,
-    load_config: &LoadCargoConfig,
-    progress: &dyn Fn(String),
-) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> {
-    let root = AbsPathBuf::assert(std::env::current_dir()?.join(root));
-    let root = ProjectManifest::discover_single(&root)?;
-    let mut workspace = ProjectWorkspace::load(root, cargo_config, progress)?;
-
-    if load_config.load_out_dirs_from_check {
-        let build_scripts = workspace.run_build_scripts(cargo_config, progress)?;
-        workspace.set_build_scripts(build_scripts)
-    }
-
-    load_workspace(workspace, &cargo_config.extra_env, load_config)
-}
-
-// Note: Since this function is used by external tools that use rust-analyzer as a library
-// what otherwise would be `pub(crate)` has to be `pub` here instead.
-//
-// The reason both, `load_workspace_at` and `load_workspace` are `pub` is that some of
-// these tools need access to `ProjectWorkspace`, too, which `load_workspace_at` hides.
-pub fn load_workspace(
-    ws: ProjectWorkspace,
-    extra_env: &FxHashMap<String, String>,
-    load_config: &LoadCargoConfig,
-) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> {
-    let (sender, receiver) = unbounded();
-    let mut vfs = vfs::Vfs::default();
-    let mut loader = {
-        let loader =
-            vfs_notify::NotifyHandle::spawn(Box::new(move |msg| sender.send(msg).unwrap()));
-        Box::new(loader)
-    };
-
-    let proc_macro_server = match &load_config.with_proc_macro_server {
-        ProcMacroServerChoice::Sysroot => ws
-            .find_sysroot_proc_macro_srv()
-            .and_then(|it| ProcMacroServer::spawn(it).map_err(Into::into)),
-        ProcMacroServerChoice::Explicit(path) => {
-            ProcMacroServer::spawn(path.clone()).map_err(Into::into)
-        }
-        ProcMacroServerChoice::None => Err(anyhow::format_err!("proc macro server disabled")),
-    };
-
-    let (crate_graph, proc_macros) = ws.to_crate_graph(
-        &mut |path: &AbsPath| {
-            let contents = loader.load_sync(path);
-            let path = vfs::VfsPath::from(path.to_path_buf());
-            vfs.set_file_contents(path.clone(), contents);
-            vfs.file_id(&path)
-        },
-        extra_env,
-    );
-    let proc_macros = {
-        let proc_macro_server = match &proc_macro_server {
-            Ok(it) => Ok(it),
-            Err(e) => Err(e.to_string()),
-        };
-        proc_macros
-            .into_iter()
-            .map(|(crate_id, path)| {
-                (
-                    crate_id,
-                    path.map_or_else(
-                        |_| Err("proc macro crate is missing dylib".to_owned()),
-                        |(_, path)| {
-                            proc_macro_server.as_ref().map_err(Clone::clone).and_then(
-                                |proc_macro_server| load_proc_macro(proc_macro_server, &path, &[]),
-                            )
-                        },
-                    ),
-                )
-            })
-            .collect()
-    };
-
-    let project_folders = ProjectFolders::new(&[ws], &[]);
-    loader.set_config(vfs::loader::Config {
-        load: project_folders.load,
-        watch: vec![],
-        version: 0,
-    });
-
-    tracing::debug!("crate graph: {:?}", crate_graph);
-    let host = load_crate_graph(
-        crate_graph,
-        proc_macros,
-        project_folders.source_root_config,
-        &mut vfs,
-        &receiver,
-    );
-
-    if load_config.prefill_caches {
-        host.analysis().parallel_prime_caches(1, |_| {})?;
-    }
-    Ok((host, vfs, proc_macro_server.ok()))
-}
-
-fn load_crate_graph(
-    crate_graph: CrateGraph,
-    proc_macros: ProcMacros,
-    source_root_config: SourceRootConfig,
-    vfs: &mut vfs::Vfs,
-    receiver: &Receiver<vfs::loader::Message>,
-) -> AnalysisHost {
-    let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok());
-    let mut host = AnalysisHost::new(lru_cap);
-    let mut analysis_change = Change::new();
-
-    host.raw_database_mut().enable_proc_attr_macros();
-
-    // wait until Vfs has loaded all roots
-    for task in receiver {
-        match task {
-            vfs::loader::Message::Progress { n_done, n_total, config_version: _ } => {
-                if n_done == n_total {
-                    break;
-                }
-            }
-            vfs::loader::Message::Loaded { files } => {
-                for (path, contents) in files {
-                    vfs.set_file_contents(path.into(), contents);
-                }
-            }
-        }
-    }
-    let changes = vfs.take_changes();
-    for file in changes {
-        if file.exists() {
-            let contents = vfs.file_contents(file.file_id);
-            if let Ok(text) = std::str::from_utf8(contents) {
-                analysis_change.change_file(file.file_id, Some(Arc::from(text)))
-            }
-        }
-    }
-    let source_roots = source_root_config.partition(vfs);
-    analysis_change.set_roots(source_roots);
-
-    analysis_change.set_crate_graph(crate_graph);
-    analysis_change.set_proc_macros(proc_macros);
-
-    host.apply_change(analysis_change);
-    host
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use hir::Crate;
-
-    #[test]
-    fn test_loading_rust_analyzer() {
-        let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap();
-        let cargo_config = CargoConfig::default();
-        let load_cargo_config = LoadCargoConfig {
-            load_out_dirs_from_check: false,
-            with_proc_macro_server: ProcMacroServerChoice::None,
-            prefill_caches: false,
-        };
-        let (host, _vfs, _proc_macro) =
-            load_workspace_at(path, &cargo_config, &load_cargo_config, &|_| {}).unwrap();
-
-        let n_crates = Crate::all(host.raw_database()).len();
-        // RA has quite a few crates, but the exact count doesn't matter
-        assert!(n_crates > 20);
-    }
-}

crates/rust-analyzer/src/cli/lsif.rs

@@ -8,22 +8,22 @@ use ide::{
     Analysis, FileId, FileRange, MonikerKind, PackageInformation, RootDatabase, StaticIndex,
     StaticIndexedFile, TokenId, TokenStaticData,
 };
-use ide_db::LineIndexDatabase;
+use ide_db::{
-
+    base_db::salsa::{self, ParallelDatabase},
-use ide_db::base_db::salsa::{self, ParallelDatabase};
+    line_index::WideEncoding,
-use ide_db::line_index::WideEncoding;
+    LineIndexDatabase,
+};
+use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
 use lsp_types::{self, lsif};
 use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource};
 use vfs::{AbsPathBuf, Vfs};
 
-use crate::cli::load_cargo::ProcMacroServerChoice;
+use crate::{
-use crate::cli::{
+    cli::flags,
-    flags,
+    line_index::{LineEndings, LineIndex, PositionEncoding},
-    load_cargo::{load_workspace, LoadCargoConfig},
+    to_proto,
+    version::version,
 };
-use crate::line_index::{LineEndings, LineIndex, PositionEncoding};
-use crate::to_proto;
-use crate::version::version;
 
 /// Need to wrap Snapshot to provide `Clone` impl for `map_with`
 struct Snap<DB>(DB);

crates/rust-analyzer/src/cli/run_tests.rs

@@ -7,12 +7,9 @@ use profile::StopWatch;
 use project_model::{CargoConfig, RustLibSource};
 use syntax::TextRange;
 
-use crate::cli::{
+use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
-    flags, full_name_of_item,
+
-    load_cargo::load_workspace_at,
+use crate::cli::{flags, full_name_of_item, Result};
-    load_cargo::{LoadCargoConfig, ProcMacroServerChoice},
-    Result,
-};
 
 impl flags::RunTests {
     pub fn run(self) -> Result<()> {

crates/rust-analyzer/src/cli/scip.rs

@@ -6,22 +6,19 @@ use std::{
     time::Instant,
 };
 
-use crate::{
-    cli::load_cargo::ProcMacroServerChoice,
-    line_index::{LineEndings, LineIndex, PositionEncoding},
-};
 use ide::{
     LineCol, MonikerDescriptorKind, StaticIndex, StaticIndexedFile, TextRange, TokenId,
     TokenStaticData,
 };
 use ide_db::LineIndexDatabase;
+use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
 use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource};
 use scip::types as scip_types;
 use std::env;
 
-use crate::cli::{
+use crate::{
-    flags,
+    cli::flags,
-    load_cargo::{load_workspace, LoadCargoConfig},
+    line_index::{LineEndings, LineIndex, PositionEncoding},
 };
 
 impl flags::Scip {
@@ -275,7 +272,7 @@ mod test {
         let change_fixture = ChangeFixture::parse(ra_fixture);
         host.raw_database_mut().apply_change(change_fixture.change);
         let (file_id, range_or_offset) =
-            change_fixture.file_position.expect("expected a marker ($0)");
+            change_fixture.file_position.expect("expected a marker ()");
         let offset = range_or_offset.expect_offset();
         (host, FilePosition { file_id, offset })
     }

crates/rust-analyzer/src/cli/ssr.rs

@@ -2,12 +2,10 @@
 
 use anyhow::Context;
 use ide_ssr::MatchFinder;
+use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
 use project_model::{CargoConfig, RustLibSource};
 
-use crate::cli::{
+use crate::cli::flags;
-    flags,
-    load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice},
-};
 
 impl flags::Ssr {
     pub fn run(self) -> anyhow::Result<()> {

crates/rust-analyzer/src/global_state.rs

@@ -9,6 +9,7 @@ use crossbeam_channel::{unbounded, Receiver, Sender};
 use flycheck::FlycheckHandle;
 use ide::{Analysis, AnalysisHost, Cancellable, Change, FileId};
 use ide_db::base_db::{CrateId, FileLoader, ProcMacroPaths, SourceDatabase};
+use load_cargo::SourceRootConfig;
 use lsp_types::{SemanticTokens, Url};
 use nohash_hasher::IntMap;
 use parking_lot::{Mutex, RwLock};
@@ -27,7 +28,7 @@ use crate::{
     main_loop::Task,
     mem_docs::MemDocs,
     op_queue::OpQueue,
-    reload::{self, SourceRootConfig},
+    reload,
     task_pool::TaskPool,
     to_proto::url_from_abs_path,
 };

crates/rust-analyzer/src/integrated_benchmarks.rs

@@ -20,7 +20,7 @@ use test_utils::project_root;
 use triomphe::Arc;
 use vfs::{AbsPathBuf, VfsPath};
 
-use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
+use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
 
 #[test]
 fn integrated_highlighting_benchmark() {

crates/rust-analyzer/src/reload.rs

@@ -12,26 +12,22 @@
 //! correct. Instead, we try to provide a best-effort service. Even if the
 //! project is currently loading and we don't have a full project model, we
 //! still want to respond to various  requests.
-use std::{collections::hash_map::Entry, iter, mem, sync};
+use std::{iter, mem};
 
 use flycheck::{FlycheckConfig, FlycheckHandle};
 use hir::db::DefDatabase;
 use ide::Change;
 use ide_db::{
-    base_db::{
+    base_db::{salsa::Durability, CrateGraph, ProcMacroPaths, ProcMacros},
-        salsa::Durability, CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError,
-        ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot, VfsPath,
-    },
     FxHashMap,
 };
-use itertools::Itertools;
+use load_cargo::{load_proc_macro, ProjectFolders};
-use proc_macro_api::{MacroDylib, ProcMacroServer};
+use proc_macro_api::ProcMacroServer;
-use project_model::{PackageRoot, ProjectWorkspace, WorkspaceBuildScripts};
+use project_model::{ProjectWorkspace, WorkspaceBuildScripts};
 use rustc_hash::FxHashSet;
 use stdx::{format_to, thread::ThreadIntent};
-use syntax::SmolStr;
 use triomphe::Arc;
-use vfs::{file_set::FileSetConfig, AbsPath, AbsPathBuf, ChangeKind};
+use vfs::{AbsPath, ChangeKind};
 
 use crate::{
     config::{Config, FilesWatcher, LinkedProject},
@@ -41,8 +37,6 @@ use crate::{
     op_queue::Cause,
 };
 
-use ::tt::token_id as tt;
-
 #[derive(Debug)]
 pub(crate) enum ProjectWorkspaceProgress {
     Begin,
@@ -619,253 +613,6 @@ impl GlobalState {
     }
 }
 
-#[derive(Default)]
-pub(crate) struct ProjectFolders {
-    pub(crate) load: Vec<vfs::loader::Entry>,
-    pub(crate) watch: Vec<usize>,
-    pub(crate) source_root_config: SourceRootConfig,
-}
-
-impl ProjectFolders {
-    pub(crate) fn new(
-        workspaces: &[ProjectWorkspace],
-        global_excludes: &[AbsPathBuf],
-    ) -> ProjectFolders {
-        let mut res = ProjectFolders::default();
-        let mut fsc = FileSetConfig::builder();
-        let mut local_filesets = vec![];
-
-        // Dedup source roots
-        // Depending on the project setup, we can have duplicated source roots, or for example in
-        // the case of the rustc workspace, we can end up with two source roots that are almost the
-        // same but not quite, like:
-        // PackageRoot { is_local: false, include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri")], exclude: [] }
-        // PackageRoot {
-        //     is_local: true,
-        //     include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri"), AbsPathBuf(".../rust/build/x86_64-pc-windows-msvc/stage0-tools/x86_64-pc-windows-msvc/release/build/cargo-miri-85801cd3d2d1dae4/out")],
-        //     exclude: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri/.git"), AbsPathBuf(".../rust/src/tools/miri/cargo-miri/target")]
-        // }
-        //
-        // The first one comes from the explicit rustc workspace which points to the rustc workspace itself
-        // The second comes from the rustc workspace that we load as the actual project workspace
-        // These `is_local` differing in this kind of way gives us problems, especially when trying to filter diagnostics as we don't report diagnostics for external libraries.
-        // So we need to deduplicate these, usually it would be enough to deduplicate by `include`, but as the rustc example shows here that doesn't work,
-        // so we need to also coalesce the includes if they overlap.
-
-        let mut roots: Vec<_> = workspaces
-            .iter()
-            .flat_map(|ws| ws.to_roots())
-            .update(|root| root.include.sort())
-            .sorted_by(|a, b| a.include.cmp(&b.include))
-            .collect();
-
-        // map that tracks indices of overlapping roots
-        let mut overlap_map = FxHashMap::<_, Vec<_>>::default();
-        let mut done = false;
-
-        while !mem::replace(&mut done, true) {
-            // maps include paths to indices of the corresponding root
-            let mut include_to_idx = FxHashMap::default();
-            // Find and note down the indices of overlapping roots
-            for (idx, root) in roots.iter().enumerate().filter(|(_, it)| !it.include.is_empty()) {
-                for include in &root.include {
-                    match include_to_idx.entry(include) {
-                        Entry::Occupied(e) => {
-                            overlap_map.entry(*e.get()).or_default().push(idx);
-                        }
-                        Entry::Vacant(e) => {
-                            e.insert(idx);
-                        }
-                    }
-                }
-            }
-            for (k, v) in overlap_map.drain() {
-                done = false;
-                for v in v {
-                    let r = mem::replace(
-                        &mut roots[v],
-                        PackageRoot { is_local: false, include: vec![], exclude: vec![] },
-                    );
-                    roots[k].is_local |= r.is_local;
-                    roots[k].include.extend(r.include);
-                    roots[k].exclude.extend(r.exclude);
-                }
-                roots[k].include.sort();
-                roots[k].exclude.sort();
-                roots[k].include.dedup();
-                roots[k].exclude.dedup();
-            }
-        }
-
-        for root in roots.into_iter().filter(|it| !it.include.is_empty()) {
-            let file_set_roots: Vec<VfsPath> =
-                root.include.iter().cloned().map(VfsPath::from).collect();
-
-            let entry = {
-                let mut dirs = vfs::loader::Directories::default();
-                dirs.extensions.push("rs".into());
-                dirs.include.extend(root.include);
-                dirs.exclude.extend(root.exclude);
-                for excl in global_excludes {
-                    if dirs
-                        .include
-                        .iter()
-                        .any(|incl| incl.starts_with(excl) || excl.starts_with(incl))
-                    {
-                        dirs.exclude.push(excl.clone());
-                    }
-                }
-
-                vfs::loader::Entry::Directories(dirs)
-            };
-
-            if root.is_local {
-                res.watch.push(res.load.len());
-            }
-            res.load.push(entry);
-
-            if root.is_local {
-                local_filesets.push(fsc.len());
-            }
-            fsc.add_file_set(file_set_roots)
-        }
-
-        let fsc = fsc.build();
-        res.source_root_config = SourceRootConfig { fsc, local_filesets };
-
-        res
-    }
-}
-
-#[derive(Default, Debug)]
-pub(crate) struct SourceRootConfig {
-    pub(crate) fsc: FileSetConfig,
-    pub(crate) local_filesets: Vec<usize>,
-}
-
-impl SourceRootConfig {
-    pub(crate) fn partition(&self, vfs: &vfs::Vfs) -> Vec<SourceRoot> {
-        let _p = profile::span("SourceRootConfig::partition");
-        self.fsc
-            .partition(vfs)
-            .into_iter()
-            .enumerate()
-            .map(|(idx, file_set)| {
-                let is_local = self.local_filesets.contains(&idx);
-                if is_local {
-                    SourceRoot::new_local(file_set)
-                } else {
-                    SourceRoot::new_library(file_set)
-                }
-            })
-            .collect()
-    }
-}
-
-/// Load the proc-macros for the given lib path, replacing all expanders whose names are in `dummy_replace`
-/// with an identity dummy expander.
-pub(crate) fn load_proc_macro(
-    server: &ProcMacroServer,
-    path: &AbsPath,
-    dummy_replace: &[Box<str>],
-) -> ProcMacroLoadResult {
-    let res: Result<Vec<_>, String> = (|| {
-        let dylib = MacroDylib::new(path.to_path_buf());
-        let vec = server.load_dylib(dylib).map_err(|e| format!("{e}"))?;
-        if vec.is_empty() {
-            return Err("proc macro library returned no proc macros".to_string());
-        }
-        Ok(vec
-            .into_iter()
-            .map(|expander| expander_to_proc_macro(expander, dummy_replace))
-            .collect())
-    })();
-    return match res {
-        Ok(proc_macros) => {
-            tracing::info!(
-                "Loaded proc-macros for {path}: {:?}",
-                proc_macros.iter().map(|it| it.name.clone()).collect::<Vec<_>>()
-            );
-            Ok(proc_macros)
-        }
-        Err(e) => {
-            tracing::warn!("proc-macro loading for {path} failed: {e}");
-            Err(e)
-        }
-    };
-
-    fn expander_to_proc_macro(
-        expander: proc_macro_api::ProcMacro,
-        dummy_replace: &[Box<str>],
-    ) -> ProcMacro {
-        let name = SmolStr::from(expander.name());
-        let kind = match expander.kind() {
-            proc_macro_api::ProcMacroKind::CustomDerive => ProcMacroKind::CustomDerive,
-            proc_macro_api::ProcMacroKind::FuncLike => ProcMacroKind::FuncLike,
-            proc_macro_api::ProcMacroKind::Attr => ProcMacroKind::Attr,
-        };
-        let expander: sync::Arc<dyn ProcMacroExpander> =
-            if dummy_replace.iter().any(|replace| &**replace == name) {
-                match kind {
-                    ProcMacroKind::Attr => sync::Arc::new(IdentityExpander),
-                    _ => sync::Arc::new(EmptyExpander),
-                }
-            } else {
-                sync::Arc::new(Expander(expander))
-            };
-        ProcMacro { name, kind, expander }
-    }
-
-    #[derive(Debug)]
-    struct Expander(proc_macro_api::ProcMacro);
-
-    impl ProcMacroExpander for Expander {
-        fn expand(
-            &self,
-            subtree: &tt::Subtree,
-            attrs: Option<&tt::Subtree>,
-            env: &Env,
-        ) -> Result<tt::Subtree, ProcMacroExpansionError> {
-            let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect();
-            match self.0.expand(subtree, attrs, env) {
-                Ok(Ok(subtree)) => Ok(subtree),
-                Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
-                Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
-            }
-        }
-    }
-
-    /// Dummy identity expander, used for attribute proc-macros that are deliberately ignored by the user.
-    #[derive(Debug)]
-    struct IdentityExpander;
-
-    impl ProcMacroExpander for IdentityExpander {
-        fn expand(
-            &self,
-            subtree: &tt::Subtree,
-            _: Option<&tt::Subtree>,
-            _: &Env,
-        ) -> Result<tt::Subtree, ProcMacroExpansionError> {
-            Ok(subtree.clone())
-        }
-    }
-
-    /// Empty expander, used for proc-macros that are deliberately ignored by the user.
-    #[derive(Debug)]
-    struct EmptyExpander;
-
-    impl ProcMacroExpander for EmptyExpander {
-        fn expand(
-            &self,
-            _: &tt::Subtree,
-            _: Option<&tt::Subtree>,
-            _: &Env,
-        ) -> Result<tt::Subtree, ProcMacroExpansionError> {
-            Ok(tt::Subtree::empty())
-        }
-    }
-}
-
 pub(crate) fn should_refresh_for_change(path: &AbsPath, change_kind: ChangeKind) -> bool {
     const IMPLICIT_TARGET_FILES: &[&str] = &["build.rs", "src/main.rs", "src/lib.rs"];
     const IMPLICIT_TARGET_DIRS: &[&str] = &["src/bin", "examples", "tests", "benches"];