Merge #678

678: WIP: automatically collect garbage r=matklad a=matklad



Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

Cargo.lock

@@ -380,6 +380,11 @@ name = "fnv"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "fs_extra"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "fsevent"
 version = "0.2.17"
@@ -551,6 +556,34 @@ name = "itoa"
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "jemalloc-ctl"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "jemalloc-sys"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "jemallocator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "join_to_string"
 version = "0.1.3"
@@ -890,7 +923,7 @@ dependencies = [
  "ra_syntax 0.1.0",
  "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "salsa 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "test_utils 0.1.0",
 ]
 
@@ -920,6 +953,8 @@ dependencies = [
  "fst 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "insta 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "jemalloc-ctl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1061,7 +1096,7 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1090,7 +1125,7 @@ name = "rand_hc"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1128,7 +1163,7 @@ name = "rand_xorshift"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1266,7 +1301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "salsa"
-version = "0.10.0-alpha5"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1275,13 +1310,13 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "salsa-macros 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "salsa-macros"
-version = "0.10.0-alpha5"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1868,6 +1903,7 @@ dependencies = [
 "checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646"
 "checksum flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd731387787f54fa333fa426e173fe42ea3d1123636b2b27ad802025fc5d182"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674"
 "checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05"
 "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
 "checksum fst 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "db72126ca7dff566cdbbdd54af44668c544897d9d3862b198141f176f1238bdf"
@@ -1887,6 +1923,9 @@ dependencies = [
 "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
 "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358"
 "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b"
+"checksum jemalloc-ctl 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e93b0f37e7d735c6b610176d5b1bde8e1621ff3f6f7ac23cdfa4e7f7d0111b5"
+"checksum jemalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "bfc62c8e50e381768ce8ee0428ee53741929f7ebd73e4d83f669bcf7693e00ae"
+"checksum jemallocator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9f0cd42ac65f758063fea55126b0148b1ce0a6354ff78e07a4d6806bc65c4ab3"
 "checksum join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc7a5290e8c2606ce2be49f456d50f69173cb96d1541e4f66e34ac8b331a98f"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
 "checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1"
@@ -1950,8 +1989,8 @@ dependencies = [
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 "checksum rusty-fork 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9591f190d2852720b679c21f66ad929f9f1d7bb09d1193c26167586029d8489c"
 "checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7"
-"checksum salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "8b5e2535d707dc5ced81106d3b71d806cfeef8a6e8a567472fde7ffd56b770dd"
+"checksum salsa 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cad0e2348e2f80725b2980914a08a00267136c3ecf720896d3f7f08eef51e08f"
-"checksum salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c5da4c649f6d4fc1864fcd9a379b1f7c6d570b278559c84a6e15981c949cc6"
+"checksum salsa-macros 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c79744109df21b80aef1367669b0a9e4985bc966e76bf0e9321b222ec0b9fbb"
 "checksum same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20c4be53a8a1ff4c1f1b2bd14570d2f634628709752f0702ecdd2b3f9a5267"
 "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"

crates/ra_db/Cargo.toml

@@ -5,8 +5,8 @@ version = "0.1.0"
 authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
 
 [dependencies]
+salsa = "0.10.0"
 relative-path = "0.4.0"
-salsa = "0.10.0-alpha5"
 rustc-hash = "1.0"
 parking_lot = "0.7.0"
 

crates/ra_ide_api/Cargo.toml

@@ -14,6 +14,8 @@ fst = "0.3.1"
 rustc-hash = "1.0"
 parking_lot = "0.7.0"
 unicase = "2.2.0"
+jemallocator = "0.1.9"
+jemalloc-ctl = "0.2.0"
 
 ra_syntax = { path = "../ra_syntax" }
 ra_ide_api_light = { path = "../ra_ide_api_light" }

crates/ra_ide_api/src/db.rs

@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use ra_db::{
     CheckCanceled, FileId, Canceled, SourceDatabase,
@@ -17,6 +20,8 @@ use crate::{LineIndex, symbol_index::{self, SymbolsDatabase}};
 pub(crate) struct RootDatabase {
     runtime: salsa::Runtime<RootDatabase>,
     interner: Arc<hir::HirInterner>,
+    pub(crate) last_gc: time::Instant,
+    pub(crate) last_gc_check: time::Instant,
 }
 
 impl salsa::Database for RootDatabase {
@@ -33,6 +38,8 @@ impl Default for RootDatabase {
         let mut db = RootDatabase {
             runtime: salsa::Runtime::default(),
             interner: Default::default(),
+            last_gc: time::Instant::now(),
+            last_gc_check: time::Instant::now(),
         };
         db.set_crate_graph(Default::default());
         db.set_local_roots(Default::default());
@@ -46,6 +53,8 @@ impl salsa::ParallelDatabase for RootDatabase {
         salsa::Snapshot::new(RootDatabase {
             runtime: self.runtime.snapshot(self),
             interner: Arc::clone(&self.interner),
+            last_gc: self.last_gc.clone(),
+            last_gc_check: self.last_gc_check.clone(),
         })
     }
 }

crates/ra_ide_api/src/imp.rs

@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use hir::{
     self, Problem, source_binder
@@ -19,12 +22,14 @@ use crate::{
     CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit,
     Query, RootChange, SourceChange, SourceFileEdit,
     symbol_index::{FileSymbol, SymbolsDatabase},
+    status::syntax_tree_stats
 };
 
+const GC_COOLDOWN: time::Duration = time::Duration::from_millis(100);
+
 impl db::RootDatabase {
     pub(crate) fn apply_change(&mut self, change: AnalysisChange) {
         log::info!("apply_change {:?}", change);
-        // self.gc_syntax_trees();
         if !change.new_roots.is_empty() {
             let mut local_roots = Vec::clone(&self.local_roots());
             for (root_id, is_local) in change.new_roots {
@@ -72,18 +77,36 @@ impl db::RootDatabase {
         self.set_source_root(root_id, Arc::new(source_root));
     }
 
-    /// Ideally, we should call this function from time to time to collect heavy
+    pub(crate) fn maybe_collect_garbage(&mut self) {
-    /// syntax trees. However, if we actually do that, everything is recomputed
+        if self.last_gc_check.elapsed() > GC_COOLDOWN {
-    /// for some reason. Needs investigation.
+            self.last_gc_check = time::Instant::now();
+            let retained_trees = syntax_tree_stats(self).retained;
+            if retained_trees > 100 {
+                log::info!(
+                    "automatic garbadge collection, {} retained trees",
+                    retained_trees
+                );
+                self.collect_garbage();
+            }
+        }
+    }
+
     pub(crate) fn collect_garbage(&mut self) {
-        self.query(ra_db::ParseQuery)
+        self.last_gc = time::Instant::now();
-            .sweep(SweepStrategy::default().discard_values());
+
-        self.query(hir::db::HirParseQuery)
+        let sweep = SweepStrategy::default()
-            .sweep(SweepStrategy::default().discard_values());
+            .discard_values()
-        self.query(hir::db::FileItemsQuery)
+            .sweep_all_revisions();
-            .sweep(SweepStrategy::default().discard_values());
+
-        self.query(hir::db::FileItemQuery)
+        self.query(ra_db::ParseQuery).sweep(sweep);
-            .sweep(SweepStrategy::default().discard_values());
+
+        self.query(hir::db::HirParseQuery).sweep(sweep);
+        self.query(hir::db::FileItemsQuery).sweep(sweep);
+        self.query(hir::db::FileItemQuery).sweep(sweep);
+
+        self.query(hir::db::LowerModuleQuery).sweep(sweep);
+        self.query(hir::db::LowerModuleSourceMapQuery).sweep(sweep);
+        self.query(hir::db::BodySyntaxMappingQuery).sweep(sweep);
     }
 }
 

crates/ra_ide_api/src/lib.rs

@@ -59,6 +59,11 @@ pub use ra_db::{
     Canceled, CrateGraph, CrateId, FileId, FilePosition, FileRange, SourceRootId
 };
 
+// We use jemalloc mainly to get heap usage statistics, actual performance
+// differnece is not measures.
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
 pub type Cancelable<T> = Result<T, Canceled>;
 
 #[derive(Default)]
@@ -286,6 +291,10 @@ impl AnalysisHost {
         self.db.apply_change(change)
     }
 
+    pub fn maybe_collect_garbage(&mut self) {
+        self.db.maybe_collect_garbage();
+    }
+
     pub fn collect_garbage(&mut self) {
         self.db.collect_garbage();
     }

crates/ra_ide_api/src/status.rs

@@ -15,9 +15,13 @@ use crate::{
     symbol_index::{SymbolIndex, LibrarySymbolsQuery},
 };
 
+pub(crate) fn syntax_tree_stats(db: &RootDatabase) -> SyntaxTreeStats {
+    db.query(ParseQuery).entries::<SyntaxTreeStats>()
+}
+
 pub(crate) fn status(db: &RootDatabase) -> String {
     let files_stats = db.query(FileTextQuery).entries::<FilesStats>();
-    let syntax_tree_stats = db.query(ParseQuery).entries::<SyntaxTreeStats>();
+    let syntax_tree_stats = syntax_tree_stats(db);
     let symbols_stats = db
         .query(LibrarySymbolsQuery)
         .entries::<LibrarySymbolsStats>();
@@ -26,8 +30,13 @@ pub(crate) fn status(db: &RootDatabase) -> String {
         interner.len()
     };
     format!(
-        "{}\n{}\n{}\nn_defs {}\n",
+        "{}\n{}\n{}\n{} defs\n\nmemory:\n{}\ngc {:?} seconds ago",
-        files_stats, symbols_stats, syntax_tree_stats, n_defs
+        files_stats,
+        symbols_stats,
+        syntax_tree_stats,
+        n_defs,
+        MemoryStats::current(),
+        db.last_gc.elapsed().as_secs(),
     )
 }
 
@@ -58,9 +67,9 @@ impl FromIterator<TableEntry<FileId, Arc<String>>> for FilesStats {
 }
 
 #[derive(Default)]
-struct SyntaxTreeStats {
+pub(crate) struct SyntaxTreeStats {
     total: usize,
-    retained: usize,
+    pub(crate) retained: usize,
     retained_size: Bytes,
 }
 
@@ -118,6 +127,31 @@ impl FromIterator<TableEntry<SourceRootId, Arc<SymbolIndex>>> for LibrarySymbols
     }
 }
 
+struct MemoryStats {
+    allocated: Bytes,
+    resident: Bytes,
+}
+
+impl MemoryStats {
+    fn current() -> MemoryStats {
+        jemalloc_ctl::epoch().unwrap();
+        MemoryStats {
+            allocated: Bytes(jemalloc_ctl::stats::allocated().unwrap()),
+            resident: Bytes(jemalloc_ctl::stats::resident().unwrap()),
+        }
+    }
+}
+
+impl fmt::Display for MemoryStats {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            fmt,
+            "{} allocated {} resident",
+            self.allocated, self.resident,
+        )
+    }
+}
+
 #[derive(Default)]
 struct Bytes(usize);
 

crates/ra_lsp_server/src/main_loop.rs

@@ -172,6 +172,7 @@ fn main_loop_inner(
 
     let (libdata_sender, libdata_receiver) = unbounded();
     loop {
+        state.maybe_collect_garbage();
         log::trace!("selecting");
         let event = select! {
             recv(msg_receiver) -> msg => match msg {
@@ -207,7 +208,7 @@ fn main_loop_inner(
                     };
                     match req.cast::<req::CollectGarbage>() {
                         Ok((id, ())) => {
-                            state.collect_garbadge();
+                            state.collect_garbage();
                             let resp = RawResponse::ok::<req::CollectGarbage>(id, &());
                             msg_sender.send(RawMessage::Response(resp)).unwrap()
                         }

crates/ra_lsp_server/src/server_world.rs

@@ -232,7 +232,11 @@ impl ServerWorldState {
         }
     }
 
-    pub fn collect_garbadge(&mut self) {
+    pub fn maybe_collect_garbage(&mut self) {
+        self.analysis_host.maybe_collect_garbage()
+    }
+
+    pub fn collect_garbage(&mut self) {
         self.analysis_host.collect_garbage()
     }
 }