Auto merge of #17900 - darichey:exclude-vendored-libraries, r=davidbarsky

Add scip/lsif flag to exclude vendored libaries

#17809 changed StaticIndex to include vendored libraries. This PR adds a flag to disable that behavior.

At work, our monorepo has too many rust targets to index all at once, so we split them up into several shards. Since all of our libraries are vendored, if rust-analyzer includes them, sharding no longer has much benefit, because every shard will have to index the entire transitive dependency graphs of all of its targets. We get around the issue presented in #17809 because some other shard will index the libraries directly.

crates/ide/src/lib.rs

@@ -104,7 +104,9 @@ pub use crate::{
     rename::RenameError,
     runnables::{Runnable, RunnableKind, TestId},
     signature_help::SignatureHelp,
-    static_index::{StaticIndex, StaticIndexedFile, TokenId, TokenStaticData},
+    static_index::{
+        StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
+    },
     syntax_highlighting::{
         tags::{Highlight, HlMod, HlMods, HlOperator, HlPunct, HlTag},
         HighlightConfig, HlRange,

crates/ide/src/static_index.rs

@@ -124,6 +124,11 @@ fn documentation_for_definition(
     )
 }
 
+pub enum VendoredLibrariesConfig<'a> {
+    Included { workspace_root: &'a VfsPath },
+    Excluded,
+}
+
 impl StaticIndex<'_> {
     fn add_file(&mut self, file_id: FileId) {
         let current_crate = crates_for(self.db, file_id).pop().map(Into::into);
@@ -240,15 +245,22 @@ impl StaticIndex<'_> {
         self.files.push(result);
     }
 
-    pub fn compute<'a>(analysis: &'a Analysis, workspace_root: &VfsPath) -> StaticIndex<'a> {
+    pub fn compute<'a>(
+        analysis: &'a Analysis,
+        vendored_libs_config: VendoredLibrariesConfig<'_>,
+    ) -> StaticIndex<'a> {
         let db = &*analysis.db;
         let work = all_modules(db).into_iter().filter(|module| {
             let file_id = module.definition_source_file_id(db).original_file(db);
             let source_root = db.file_source_root(file_id.into());
             let source_root = db.source_root(source_root);
-            let is_vendored = source_root
-                .path_for_file(&file_id.into())
-                .is_some_and(|module_path| module_path.starts_with(workspace_root));
+            let is_vendored = match vendored_libs_config {
+                VendoredLibrariesConfig::Included { workspace_root } => source_root
+                    .path_for_file(&file_id.into())
+                    .is_some_and(|module_path| module_path.starts_with(workspace_root)),
+                VendoredLibrariesConfig::Excluded => false,
+            };
+
             !source_root.is_library || is_vendored
         });
         let mut this = StaticIndex {
@@ -278,10 +290,11 @@ mod tests {
     use ide_db::{base_db::VfsPath, FileRange, FxHashSet};
     use syntax::TextSize;
 
-    fn check_all_ranges(ra_fixture: &str) {
+    use super::VendoredLibrariesConfig;
+
+    fn check_all_ranges(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
         let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
-        let s =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let s = StaticIndex::compute(&analysis, vendored_libs_config);
         let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
         for f in s.files {
             for (range, _) in f.tokens {
@@ -298,10 +311,9 @@ mod tests {
     }
 
     #[track_caller]
-    fn check_definitions(ra_fixture: &str) {
+    fn check_definitions(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
         let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
-        let s =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let s = StaticIndex::compute(&analysis, vendored_libs_config);
         let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
         for (_, t) in s.tokens.iter() {
             if let Some(t) = t.definition {
@@ -329,6 +341,9 @@ struct Foo;
 enum E { X(Foo) }
    //^   ^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
         check_definitions(
             r#"
@@ -337,6 +352,9 @@ struct Foo;
 enum E { X(Foo) }
    //^   ^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
@@ -359,6 +377,9 @@ pub func() {
 
 }
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
@@ -377,9 +398,30 @@ struct ExternalLibrary(i32);
 struct VendoredLibrary(i32);
      //^^^^^^^^^^^^^^^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
+    #[test]
+    fn vendored_crate_excluded() {
+        check_all_ranges(
+            r#"
+//- /workspace/main.rs crate:main deps:external,vendored
+struct Main(i32);
+     //^^^^ ^^^
+
+//- /external/lib.rs new_source_root:library crate:external@0.1.0,https://a.b/foo.git library
+struct ExternalLibrary(i32);
+
+//- /workspace/vendored/lib.rs new_source_root:library crate:vendored@0.1.0,https://a.b/bar.git library
+struct VendoredLibrary(i32);
+"#,
+            VendoredLibrariesConfig::Excluded,
+        )
+    }
+
     #[test]
     fn derives() {
         check_all_ranges(
@@ -394,6 +436,9 @@ pub macro Copy {}
 struct Hello(i32);
      //^^^^^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 }

crates/rust-analyzer/src/cli/flags.rs

@@ -138,6 +138,9 @@ xflags::xflags! {
 
         cmd lsif {
             required path: PathBuf
+
+            /// Exclude code from vendored libraries from the resulting index.
+            optional --exclude-vendored-libraries
         }
 
         cmd scip {
@@ -148,6 +151,9 @@ xflags::xflags! {
 
             /// A path to an json configuration file that can be used to customize cargo behavior.
             optional --config-path config_path: PathBuf
+
+            /// Exclude code from vendored libraries from the resulting index.
+            optional --exclude-vendored-libraries
         }
     }
 }
@@ -259,6 +265,8 @@ pub struct Search {
 #[derive(Debug)]
 pub struct Lsif {
     pub path: PathBuf,
+
+    pub exclude_vendored_libraries: bool,
 }
 
 #[derive(Debug)]
@@ -267,6 +275,7 @@ pub struct Scip {
 
     pub output: Option<PathBuf>,
     pub config_path: Option<PathBuf>,
+    pub exclude_vendored_libraries: bool,
 }
 
 impl RustAnalyzer {

crates/rust-analyzer/src/cli/lsif.rs

@@ -5,7 +5,7 @@ use std::time::Instant;
 
 use ide::{
     Analysis, AnalysisHost, FileId, FileRange, MonikerKind, PackageInformation, RootDatabase,
-    StaticIndex, StaticIndexedFile, TokenId, TokenStaticData,
+    StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
 };
 use ide_db::{line_index::WideEncoding, LineIndexDatabase};
 use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
@@ -296,7 +296,13 @@ impl flags::Lsif {
         let db = host.raw_database();
         let analysis = host.analysis();
 
-        let si = StaticIndex::compute(&analysis, &path.clone().into());
+        let vendored_libs_config = if self.exclude_vendored_libraries {
+            VendoredLibrariesConfig::Excluded
+        } else {
+            VendoredLibrariesConfig::Included { workspace_root: &path.clone().into() }
+        };
+
+        let si = StaticIndex::compute(&analysis, vendored_libs_config);
 
         let mut lsif = LsifManager::new(&analysis, db, &vfs);
         lsif.add_vertex(lsif::Vertex::MetaData(lsif::MetaData {

crates/rust-analyzer/src/cli/scip.rs

@@ -4,7 +4,7 @@ use std::{path::PathBuf, time::Instant};
 
 use ide::{
     AnalysisHost, LineCol, MonikerDescriptorKind, MonikerResult, StaticIndex, StaticIndexedFile,
-    SymbolInformationKind, TextRange, TokenId,
+    SymbolInformationKind, TextRange, TokenId, VendoredLibrariesConfig,
 };
 use ide_db::LineIndexDatabase;
 use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
@@ -63,7 +63,13 @@ impl flags::Scip {
         let db = host.raw_database();
         let analysis = host.analysis();
 
-        let si = StaticIndex::compute(&analysis, &root.clone().into());
+        let vendored_libs_config = if self.exclude_vendored_libraries {
+            VendoredLibrariesConfig::Excluded
+        } else {
+            VendoredLibrariesConfig::Included { workspace_root: &root.clone().into() }
+        };
+
+        let si = StaticIndex::compute(&analysis, vendored_libs_config);
 
         let metadata = scip_types::Metadata {
             version: scip_types::ProtocolVersion::UnspecifiedProtocolVersion.into(),
@@ -352,8 +358,12 @@ mod test {
         let (host, position) = position(ra_fixture);
 
         let analysis = host.analysis();
-        let si =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let si = StaticIndex::compute(
+            &analysis,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
+        );
 
         let FilePosition { file_id, offset } = position;
 
@@ -617,8 +627,12 @@ pub mod example_mod {
         host.raw_database_mut().apply_change(change_fixture.change);
 
         let analysis = host.analysis();
-        let si =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let si = StaticIndex::compute(
+            &analysis,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
+        );
 
         let file = si.files.first().unwrap();
         let (_, token_id) = file.tokens.first().unwrap();