Merge #2008

2008: Prepare SourceDatabase API for lazy file loading r=matklad a=matklad



Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/ra_cli/src/analysis_bench.rs

@@ -8,7 +8,7 @@ use std::{
 
 use ra_db::{
     salsa::{Database, Durability},
-    FileId, SourceDatabase,
+    FileId, SourceDatabaseExt,
 };
 use ra_ide_api::{Analysis, AnalysisChange, AnalysisHost, FilePosition, LineCol};
 

crates/ra_cli/src/analysis_stats.rs

@@ -2,7 +2,7 @@
 
 use std::{collections::HashSet, fmt::Write, path::Path, time::Instant};
 
-use ra_db::SourceDatabase;
+use ra_db::SourceDatabaseExt;
 use ra_hir::{AssocItem, Crate, HasBodySource, HasSource, HirDisplay, ModuleDef, Ty, TypeWalk};
 use ra_syntax::AstNode;
 

crates/ra_db/src/input.rs

@@ -57,7 +57,7 @@ impl SourceRoot {
     pub fn walk(&self) -> impl Iterator<Item = FileId> + '_ {
         self.files.values().copied()
     }
-    pub(crate) fn file_by_relative_path(&self, path: &RelativePath) -> Option<FileId> {
+    pub fn file_by_relative_path(&self, path: &RelativePath) -> Option<FileId> {
         self.files.get(path).copied()
     }
 }

crates/ra_db/src/lib.rs

@@ -64,21 +64,39 @@ pub struct FileRange {
 
 pub const DEFAULT_LRU_CAP: usize = 128;
 
+pub trait FileLoader {
+    /// Text of the file.
+    fn file_text(&self, file_id: FileId) -> Arc<String>;
+    fn resolve_relative_path(&self, anchor: FileId, relative_path: &RelativePath)
+        -> Option<FileId>;
+    fn relevant_crates(&self, file_id: FileId) -> Arc<Vec<CrateId>>;
+}
+
 /// Database which stores all significant input facts: source code and project
 /// model. Everything else in rust-analyzer is derived from these queries.
 #[salsa::query_group(SourceDatabaseStorage)]
-pub trait SourceDatabase: CheckCanceled + std::fmt::Debug {
+pub trait SourceDatabase: CheckCanceled + FileLoader + std::fmt::Debug {
-    /// Text of the file.
-    #[salsa::input]
-    fn file_text(&self, file_id: FileId) -> Arc<String>;
-
-    #[salsa::transparent]
-    fn resolve_relative_path(&self, anchor: FileId, relative_path: &RelativePath)
-        -> Option<FileId>;
-
     // Parses the file into the syntax tree.
     #[salsa::invoke(parse_query)]
     fn parse(&self, file_id: FileId) -> Parse<ast::SourceFile>;
+
+    /// The crate graph.
+    #[salsa::input]
+    fn crate_graph(&self) -> Arc<CrateGraph>;
+}
+
+fn parse_query(db: &impl SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> {
+    let _p = profile("parse_query");
+    let text = db.file_text(file_id);
+    SourceFile::parse(&*text)
+}
+
+/// We don't want to give HIR knowledge of source roots, hence we extract these
+/// methods into a separate DB.
+#[salsa::query_group(SourceDatabaseExtStorage)]
+pub trait SourceDatabaseExt: SourceDatabase {
+    #[salsa::input]
+    fn file_text(&self, file_id: FileId) -> Arc<String>;
     /// Path to a file, relative to the root of its source root.
     #[salsa::input]
     fn file_relative_path(&self, file_id: FileId) -> RelativePathBuf;
@@ -88,40 +106,48 @@ pub trait SourceDatabase: CheckCanceled + std::fmt::Debug {
     /// Contents of the source root.
     #[salsa::input]
     fn source_root(&self, id: SourceRootId) -> Arc<SourceRoot>;
+
     fn source_root_crates(&self, id: SourceRootId) -> Arc<Vec<CrateId>>;
-    /// The crate graph.
-    #[salsa::input]
-    fn crate_graph(&self) -> Arc<CrateGraph>;
 }
 
-fn resolve_relative_path(
+fn source_root_crates(
-    db: &impl SourceDatabase,
+    db: &(impl SourceDatabaseExt + SourceDatabase),
-    anchor: FileId,
+    id: SourceRootId,
-    relative_path: &RelativePath,
+) -> Arc<Vec<CrateId>> {
-) -> Option<FileId> {
-    let path = {
-        let mut path = db.file_relative_path(anchor);
-        // Workaround for relative path API: turn `lib.rs` into ``.
-        if !path.pop() {
-            path = RelativePathBuf::default();
-        }
-        path.push(relative_path);
-        path.normalize()
-    };
-    let source_root = db.file_source_root(anchor);
-    let source_root = db.source_root(source_root);
-    source_root.file_by_relative_path(&path)
-}
-
-fn source_root_crates(db: &impl SourceDatabase, id: SourceRootId) -> Arc<Vec<CrateId>> {
     let root = db.source_root(id);
     let graph = db.crate_graph();
     let res = root.walk().filter_map(|it| graph.crate_id_for_crate_root(it)).collect::<Vec<_>>();
     Arc::new(res)
 }
 
-fn parse_query(db: &impl SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> {
+/// Silly workaround for cyclic deps between the traits
-    let _p = profile("parse_query");
+pub struct FileLoaderDelegate<T>(pub T);
-    let text = db.file_text(file_id);
+
-    SourceFile::parse(&*text)
+impl<T: SourceDatabaseExt> FileLoader for FileLoaderDelegate<&'_ T> {
+    fn file_text(&self, file_id: FileId) -> Arc<String> {
+        SourceDatabaseExt::file_text(self.0, file_id)
+    }
+    fn resolve_relative_path(
+        &self,
+        anchor: FileId,
+        relative_path: &RelativePath,
+    ) -> Option<FileId> {
+        let path = {
+            let mut path = self.0.file_relative_path(anchor);
+            // Workaround for relative path API: turn `lib.rs` into ``.
+            if !path.pop() {
+                path = RelativePathBuf::default();
+            }
+            path.push(relative_path);
+            path.normalize()
+        };
+        let source_root = self.0.file_source_root(anchor);
+        let source_root = self.0.source_root(source_root);
+        source_root.file_by_relative_path(&path)
+    }
+
+    fn relevant_crates(&self, file_id: FileId) -> Arc<Vec<CrateId>> {
+        let source_root = self.0.file_source_root(file_id);
+        self.0.source_root_crates(source_root)
+    }
 }

crates/ra_hir/src/from_source.rs

@@ -189,14 +189,14 @@ impl Module {
             ModuleSource::SourceFile(_) => None,
         };
 
-        let source_root_id = db.file_source_root(src.file_id.original_file(db));
+        db.relevant_crates(src.file_id.original_file(db))
-        db.source_root_crates(source_root_id).iter().map(|&crate_id| Crate { crate_id }).find_map(
+            .iter()
-            |krate| {
+            .map(|&crate_id| Crate { crate_id })
+            .find_map(|krate| {
                 let def_map = db.crate_def_map(krate);
                 let module_id = def_map.find_module_by_source(src.file_id, decl_id)?;
                 Some(Module { krate, module_id })
-            },
+            })
-        )
     }
 }
 

crates/ra_hir/src/ids.rs

@@ -85,11 +85,7 @@ impl HirFileId {
                 // Note:
                 // The final goal we would like to make all parse_macro success,
                 // such that the following log will not call anyway.
-                log::warn!(
+                log::warn!("fail on macro_parse: (reason: {})", err,);
-                    "fail on macro_parse: (reason: {}) {}",
-                    err,
-                    macro_call_id.debug_dump(db)
-                );
             })
             .ok()?;
         match macro_file.macro_file_kind {
@@ -367,35 +363,6 @@ impl AstItemDef<ast::TypeAliasDef> for TypeAliasId {
     }
 }
 
-impl MacroCallId {
-    pub fn debug_dump(self, db: &impl AstDatabase) -> String {
-        let loc = self.loc(db);
-        let node = loc.ast_id.to_node(db);
-        let syntax_str = {
-            let mut res = String::new();
-            node.syntax().text().for_each_chunk(|chunk| {
-                if !res.is_empty() {
-                    res.push(' ')
-                }
-                res.push_str(chunk)
-            });
-            res
-        };
-
-        // dump the file name
-        let file_id: HirFileId = self.loc(db).ast_id.file_id();
-        let original = file_id.original_file(db);
-        let macro_rules = db.macro_def(loc.def);
-
-        format!(
-            "macro call [file: {:?}] : {}\nhas rules: {}",
-            db.file_relative_path(original),
-            syntax_str,
-            macro_rules.is_some()
-        )
-    }
-}
-
 /// This exists just for Chalk, because Chalk just has a single `StructId` where
 /// we have different kinds of ADTs, primitive types and special type
 /// constructors like tuples and function pointers.

crates/ra_hir/src/mock.rs

@@ -5,10 +5,10 @@ use std::{panic, sync::Arc};
 use parking_lot::Mutex;
 use ra_cfg::CfgOptions;
 use ra_db::{
-    salsa, CrateGraph, CrateId, Edition, FileId, FilePosition, SourceDatabase, SourceRoot,
+    salsa, CrateGraph, CrateId, Edition, FileId, FileLoader, FileLoaderDelegate, FilePosition,
-    SourceRootId,
+    SourceDatabase, SourceDatabaseExt, SourceRoot, SourceRootId,
 };
-use relative_path::RelativePathBuf;
+use relative_path::{RelativePath, RelativePathBuf};
 use rustc_hash::FxHashMap;
 use test_utils::{extract_offset, parse_fixture, CURSOR_MARKER};
 
@@ -17,6 +17,7 @@ use crate::{db, debug::HirDebugHelper, diagnostics::DiagnosticSink};
 pub const WORKSPACE: SourceRootId = SourceRootId(0);
 
 #[salsa::database(
+    ra_db::SourceDatabaseExtStorage,
     ra_db::SourceDatabaseStorage,
     db::InternDatabaseStorage,
     db::AstDatabaseStorage,
@@ -34,6 +35,22 @@ pub struct MockDatabase {
 
 impl panic::RefUnwindSafe for MockDatabase {}
 
+impl FileLoader for MockDatabase {
+    fn file_text(&self, file_id: FileId) -> Arc<String> {
+        FileLoaderDelegate(self).file_text(file_id)
+    }
+    fn resolve_relative_path(
+        &self,
+        anchor: FileId,
+        relative_path: &RelativePath,
+    ) -> Option<FileId> {
+        FileLoaderDelegate(self).resolve_relative_path(anchor, relative_path)
+    }
+    fn relevant_crates(&self, file_id: FileId) -> Arc<Vec<CrateId>> {
+        FileLoaderDelegate(self).relevant_crates(file_id)
+    }
+}
+
 impl HirDebugHelper for MockDatabase {
     fn crate_name(&self, krate: CrateId) -> Option<String> {
         self.crate_names.get(&krate).cloned()

crates/ra_hir/src/nameres/tests/incremental.rs

@@ -2,7 +2,7 @@ use super::*;
 
 use std::sync::Arc;
 
-use ra_db::SourceDatabase;
+use ra_db::{SourceDatabase, SourceDatabaseExt};
 
 fn check_def_map_is_not_recomputed(initial: &str, file_change: &str) {
     let (mut db, pos) = MockDatabase::with_position(initial);

crates/ra_ide_api/src/change.rs

@@ -4,7 +4,7 @@ use std::{fmt, sync::Arc, time};
 
 use ra_db::{
     salsa::{Database, Durability, SweepStrategy},
-    CrateGraph, CrateId, FileId, SourceDatabase, SourceRoot, SourceRootId,
+    CrateGraph, CrateId, FileId, SourceDatabase, SourceDatabaseExt, SourceRoot, SourceRootId,
 };
 use ra_prof::{memory_usage, profile, Bytes};
 use ra_syntax::SourceFile;

crates/ra_ide_api/src/db.rs

@@ -4,8 +4,10 @@ use std::sync::Arc;
 
 use ra_db::{
     salsa::{self, Database, Durability},
-    Canceled, CheckCanceled, CrateId, FileId, SourceDatabase, SourceRootId,
+    Canceled, CheckCanceled, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase,
+    SourceDatabaseExt, SourceRootId,
 };
+use relative_path::RelativePath;
 use rustc_hash::FxHashMap;
 
 use crate::{
@@ -15,6 +17,7 @@ use crate::{
 
 #[salsa::database(
     ra_db::SourceDatabaseStorage,
+    ra_db::SourceDatabaseExtStorage,
     LineIndexDatabaseStorage,
     symbol_index::SymbolsDatabaseStorage,
     hir::db::InternDatabaseStorage,
@@ -31,6 +34,22 @@ pub(crate) struct RootDatabase {
     pub(crate) last_gc_check: crate::wasm_shims::Instant,
 }
 
+impl FileLoader for RootDatabase {
+    fn file_text(&self, file_id: FileId) -> Arc<String> {
+        FileLoaderDelegate(self).file_text(file_id)
+    }
+    fn resolve_relative_path(
+        &self,
+        anchor: FileId,
+        relative_path: &RelativePath,
+    ) -> Option<FileId> {
+        FileLoaderDelegate(self).resolve_relative_path(anchor, relative_path)
+    }
+    fn relevant_crates(&self, file_id: FileId) -> Arc<Vec<CrateId>> {
+        FileLoaderDelegate(self).relevant_crates(file_id)
+    }
+}
+
 impl hir::debug::HirDebugHelper for RootDatabase {
     fn crate_name(&self, krate: CrateId) -> Option<String> {
         self.debug_data.crate_names.get(&krate).cloned()

crates/ra_ide_api/src/diagnostics.rs

@@ -4,7 +4,7 @@ use std::cell::RefCell;
 
 use hir::diagnostics::{AstDiagnostic, Diagnostic as _, DiagnosticSink};
 use itertools::Itertools;
-use ra_db::SourceDatabase;
+use ra_db::{SourceDatabase, SourceDatabaseExt};
 use ra_prof::profile;
 use ra_syntax::{
     algo,

crates/ra_ide_api/src/lib.rs

@@ -52,7 +52,7 @@ use std::sync::Arc;
 use ra_cfg::CfgOptions;
 use ra_db::{
     salsa::{self, ParallelDatabase},
-    CheckCanceled, SourceDatabase,
+    CheckCanceled, FileLoader, SourceDatabase,
 };
 use ra_syntax::{SourceFile, TextRange, TextUnit};
 use ra_text_edit::TextEdit;
@@ -289,10 +289,14 @@ impl AnalysisHost {
     pub fn per_query_memory_usage(&mut self) -> Vec<(String, ra_prof::Bytes)> {
         self.db.per_query_memory_usage()
     }
-    pub fn raw_database(&self) -> &(impl hir::db::HirDatabase + salsa::Database) {
+    pub fn raw_database(
+        &self,
+    ) -> &(impl hir::db::HirDatabase + salsa::Database + ra_db::SourceDatabaseExt) {
         &self.db
     }
-    pub fn raw_database_mut(&mut self) -> &mut (impl hir::db::HirDatabase + salsa::Database) {
+    pub fn raw_database_mut(
+        &mut self,
+    ) -> &mut (impl hir::db::HirDatabase + salsa::Database + ra_db::SourceDatabaseExt) {
         &mut self.db
     }
 }

crates/ra_ide_api/src/references.rs

@@ -1,7 +1,7 @@
 //! FIXME: write short doc here
 
 use hir::{Either, ModuleSource};
-use ra_db::SourceDatabase;
+use ra_db::{SourceDatabase, SourceDatabaseExt};
 use ra_syntax::{algo::find_node_at_offset, ast, AstNode, SourceFile, SyntaxNode};
 use relative_path::{RelativePath, RelativePathBuf};
 

crates/ra_ide_api/src/symbol_index.rs

@@ -29,7 +29,7 @@ use std::{
 use fst::{self, Streamer};
 use ra_db::{
     salsa::{self, ParallelDatabase},
-    SourceDatabase, SourceRootId,
+    SourceDatabaseExt, SourceRootId,
 };
 use ra_syntax::{
     ast::{self, NameOwner},