From 1d5eaefe8a8e4f8b267d51ee8ece866741586ada Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sun, 6 Jan 2019 15:05:12 +0100 Subject: [PATCH 01/24] initial Watcher impl --- Cargo.lock | 266 ++++++++++++++++++++++++-- crates/ra_lsp_server/src/main_loop.rs | 19 +- crates/ra_vfs/Cargo.toml | 2 + crates/ra_vfs/src/lib.rs | 59 ++++-- crates/ra_vfs/src/watcher.rs | 96 ++++++++++ 5 files changed, 414 insertions(+), 28 deletions(-) create mode 100644 crates/ra_vfs/src/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index 3b8df0c8a0..fccf8a65e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,11 @@ name = "bit-vec" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bitflags" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bitflags" version = "1.0.4" @@ -109,6 +114,15 @@ name = "byteorder" version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bytes" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cargo_metadata" version = "0.7.1" @@ -335,6 +349,16 @@ name = "fake-simd" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "filetime" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flexi_logger" version = "0.10.5" @@ -351,6 +375,24 @@ name = "fnv" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "fsevent" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fsevent-sys" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fst" version = "0.3.3" @@ -374,6 +416,11 @@ name = "fuchsia-zircon-sys" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "futures" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "gen_lsp_server" version = "0.1.0" @@ -436,6 +483,28 @@ name = "indexmap" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "inotify" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "inotify-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "insta" version = "0.5.2" @@ -455,6 +524,15 @@ dependencies = [ "serde_yaml 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "iovec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itertools" version = "0.8.0" @@ -473,11 +551,25 @@ name = "join_to_string" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lazy_static" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lazycell" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "libc" version = "0.2.48" @@ -511,7 +603,7 @@ version = "0.55.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "num-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.85 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.85 (registry+https://github.com/rust-lang/crates.io-index)", @@ -553,17 +645,84 @@ name = "memoffset" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "mio" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio-extras" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nodrop" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "num-derive" -version = "0.2.3" +name = "notify" +version = "4.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", + "fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-extras 2.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-derive" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.15.26 (registry+https://github.com/rust-lang/crates.io-index)", @@ -726,7 +885,7 @@ dependencies = [ "ra_syntax 0.1.0", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "salsa 0.10.0-alpha4 (registry+https://github.com/rust-lang/crates.io-index)", + "salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)", "test_utils 0.1.0", ] @@ -849,7 +1008,9 @@ name = "ra_vfs" version = "0.1.0" dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "ra_arena 0.1.0", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1097,7 +1258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "salsa" -version = "0.10.0-alpha4" +version = "0.10.0-alpha5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "derive-new 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1106,13 +1267,13 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "salsa-macros 0.10.0-alpha4 (registry+https://github.com/rust-lang/crates.io-index)", + "salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "salsa-macros" -version = "0.10.0-alpha4" +version = "0.10.0-alpha5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1198,6 +1359,11 @@ dependencies = [ "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "slug" version = "0.1.4" @@ -1381,6 +1547,42 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-executor" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-io" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tools" version = "0.1.0" @@ -1555,6 +1757,11 @@ dependencies = [ "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi" version = "0.3.6" @@ -1564,6 +1771,11 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -1582,6 +1794,15 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "yaml-rust" version = "0.4.2" @@ -1602,10 +1823,12 @@ dependencies = [ "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum bit-set 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6f1efcc46c18245a69c38fcc5cc650f16d3a59d034f3106e9ed63748f695730a" "checksum bit-vec 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4440d5cb623bb7390ae27fec0bb6c61111969860f8e3ae198bfa0663645e67cf" +"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" +"checksum bytes 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "40ade3d27603c2cb345eb0912aec461a6dec7e06a4ae48589904e808335c7afa" "checksum cargo_metadata 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "585784cac9b05c93a53b17a0b24a5cdd1dfdda5256f030e089b549d2390cc720" "checksum cc 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4a8b715cb4597106ea87c7c84b2f1d452c7492033765df7f32651e66fcf749" "checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4" @@ -1633,11 +1856,15 @@ dependencies = [ "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" "checksum flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd731387787f54fa333fa426e173fe42ea3d1123636b2b27ad802025fc5d182" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05" +"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874" "checksum fst 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "db72126ca7dff566cdbbdd54af44668c544897d9d3862b198141f176f1238bdf" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" @@ -1645,11 +1872,16 @@ dependencies = [ "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum im 12.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0627d417829c1d763d602687634869f254fc79f7e22dea6c824dab993db857e4" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" +"checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718" +"checksum inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0" "checksum insta 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6ff57d9cbc4664b54a972c321155c7703794bc0f5c9944f29c36f40d10d626f3" +"checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc7a5290e8c2606ce2be49f456d50f69173cb96d1541e4f66e34ac8b331a98f" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" +"checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" "checksum libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)" = "e962c7641008ac010fa60a7dfdc1712449f29c44ef2d4702394aea943ee75047" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" @@ -1660,8 +1892,13 @@ dependencies = [ "checksum memchr 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e1dd4eaac298c32ce07eb6ed9242eda7d82955b9170b7d6db59b2e02cc63fcb8" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" +"checksum mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)" = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" +"checksum mio-extras 2.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "46e73a04c2fa6250b8d802134d56d554a9ec2922bf977777c805ea5def61ce40" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" -"checksum num-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8af1847c907c2f04d7bfd572fb25bbb4385c637fe5be163cf2f8c5d778fe1e7d" +"checksum notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c968cf37cf949114b00d51b0b23536d1c3a4a3963767cf4c969c65a6af78dc7d" +"checksum num-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d9fe8fcafd1b86a37ce8a1cfa15ae504817e0c8c2e7ad42767371461ac1d316d" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" "checksum num_cpus 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5a69d464bdc213aaaff628444e99578ede64e9c854025aa43b9796530afa9238" @@ -1704,8 +1941,8 @@ dependencies = [ "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rusty-fork 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9591f190d2852720b679c21f66ad929f9f1d7bb09d1193c26167586029d8489c" "checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" -"checksum salsa 0.10.0-alpha4 (registry+https://github.com/rust-lang/crates.io-index)" = "3f442595eae948da8fbb2aa1e13940d9d2d70031753a27a5d1434f91b706ff12" -"checksum salsa-macros 0.10.0-alpha4 (registry+https://github.com/rust-lang/crates.io-index)" = "2e6c1a1bee4eb44881438e80c1a26db1c3b957b6cc51765615d429019babdec2" +"checksum salsa 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "8b5e2535d707dc5ced81106d3b71d806cfeef8a6e8a567472fde7ffd56b770dd" +"checksum salsa-macros 0.10.0-alpha5 (registry+https://github.com/rust-lang/crates.io-index)" = "e7c5da4c649f6d4fc1864fcd9a379b1f7c6d570b278559c84a6e15981c949cc6" "checksum same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20c4be53a8a1ff4c1f1b2bd14570d2f634628709752f0702ecdd2b3f9a5267" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" @@ -1715,6 +1952,7 @@ dependencies = [ "checksum serde_json 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "4b90a9fbe1211e57d3e1c15670f1cb00802988fb23a1a4aad7a2b63544f1920e" "checksum serde_yaml 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0887a8e097a69559b56aa2526bf7aff7c3048cf627dff781f0b56a6001534593" "checksum sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "51b9d1f3b5de8a167ab06834a7c883bd197f2191e1dda1a22d9ccfeedbf9aded" +"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum slug 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373" "checksum smallvec 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "88aea073965ab29f6edb5493faf96ad662fb18aa9eeb186a3b7057951605ed15" "checksum smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9af1035bc5d742ab6b7ab16713e41cc2ffe78cb474f6f43cd696b2d16052007e" @@ -1733,6 +1971,9 @@ dependencies = [ "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +"checksum tokio-executor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30c6dbf2d1ad1de300b393910e8a3aa272b724a400b6531da03eed99e329fbf0" +"checksum tokio-io 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b53aeb9d3f5ccf2ebb29e19788f96987fa1355f8fe45ea193928eaaaf3ae820f" +"checksum tokio-reactor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "afbcdb0f0d2a1e4c440af82d7bbf0bf91a8a8c0575bcd20c05d15be7e9d3a02f" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-trie 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71a9c5b1fe77426cf144cc30e49e955270f5086e31a6441dfa8b32efc09b9d77" "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" @@ -1757,8 +1998,11 @@ dependencies = [ "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b9f3bf741a801531993db6478b95682117471f76916f5e690dd8d45395b09349" "checksum walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d9d7ed3431229a144296213105a390676cc49c9b6a72bd19f3176c98e129fa1" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum yaml-rust 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95acf0db5515d07da9965ec0e0ba6cc2d825e2caeb7303b66ca441729801254e" diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs index ddd20a41fc..e5a0603d1e 100644 --- a/crates/ra_lsp_server/src/main_loop.rs +++ b/crates/ra_lsp_server/src/main_loop.rs @@ -11,7 +11,7 @@ use gen_lsp_server::{ }; use lsp_types::NumberOrString; use ra_ide_api::{Canceled, FileId, LibraryData}; -use ra_vfs::VfsTask; +use ra_vfs::{VfsTask, WatcherChange}; use rustc_hash::FxHashSet; use serde::{de::DeserializeOwned, Serialize}; use threadpool::ThreadPool; @@ -113,6 +113,7 @@ enum Event { Msg(RawMessage), Task(Task), Vfs(VfsTask), + Watcher(WatcherChange), Lib(LibraryData), } @@ -149,6 +150,7 @@ impl fmt::Debug for Event { Event::Task(it) => fmt::Debug::fmt(it, f), Event::Vfs(it) => fmt::Debug::fmt(it, f), Event::Lib(it) => fmt::Debug::fmt(it, f), + Event::Watcher(it) => fmt::Debug::fmt(it, f), } } } @@ -183,6 +185,10 @@ fn main_loop_inner( Ok(task) => Event::Vfs(task), Err(RecvError) => bail!("vfs died"), }, + recv(state.vfs.read().change_receiver()) -> change => match change { + Ok(change) => Event::Watcher(change), + Err(RecvError) => bail!("vfs watcher died"), + }, recv(libdata_receiver) -> data => Event::Lib(data.unwrap()) }; log::info!("loop_turn = {:?}", event); @@ -194,6 +200,10 @@ fn main_loop_inner( state.vfs.write().handle_task(task); state_changed = true; } + Event::Watcher(change) => { + state.vfs.write().handle_change(change); + state_changed = true; + } Event::Lib(lib) => { feedback(internal_mode, "library loaded", msg_sender); state.add_lib(lib); @@ -365,7 +375,7 @@ fn on_notification( if let Some(file_id) = state .vfs .write() - .add_file_overlay(&path, params.text_document.text) + .add_file_overlay(&path, Some(params.text_document.text)) { subs.add_sub(FileId(file_id.0.into())); } @@ -384,7 +394,10 @@ fn on_notification( .pop() .ok_or_else(|| format_err!("empty changes"))? .text; - state.vfs.write().change_file_overlay(path.as_path(), text); + state + .vfs + .write() + .change_file_overlay(path.as_path(), Some(text)); return Ok(()); } Err(not) => not, diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index e637063c97..f7a972e91b 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -10,6 +10,8 @@ relative-path = "0.4.0" rustc-hash = "1.0" crossbeam-channel = "0.3.5" log = "0.4.6" +notify = "4" +drop_bomb = "0.1.0" thread_worker = { path = "../thread_worker" } ra_arena = { path = "../ra_arena" } diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index cdea18d734..5336822b33 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -14,26 +14,26 @@ //! which are watched for changes. Typically, there will be a root for each //! Cargo package. mod io; +mod watcher; use std::{ - fmt, - mem, - thread, cmp::Reverse, - path::{Path, PathBuf}, ffi::OsStr, + fmt, fs, mem, + path::{Path, PathBuf}, sync::Arc, - fs, + thread, }; -use rustc_hash::{FxHashMap, FxHashSet}; -use relative_path::RelativePathBuf; use crossbeam_channel::Receiver; -use walkdir::DirEntry; +use ra_arena::{impl_arena_id, Arena, RawId}; +use relative_path::RelativePathBuf; +use rustc_hash::{FxHashMap, FxHashSet}; use thread_worker::WorkerHandle; -use ra_arena::{Arena, RawId, impl_arena_id}; +use walkdir::DirEntry; pub use crate::io::TaskResult as VfsTask; +pub use crate::watcher::{Watcher, WatcherChange}; /// `RootFilter` is a predicate that checks if a file can belong to a root. If /// several filters match a file (nested dirs), the most nested one wins. @@ -85,6 +85,7 @@ pub struct Vfs { pending_changes: Vec, worker: io::Worker, worker_handle: WorkerHandle, + watcher: Watcher, } impl fmt::Debug for Vfs { @@ -97,12 +98,15 @@ impl Vfs { pub fn new(mut roots: Vec) -> (Vfs, Vec) { let (worker, worker_handle) = io::start(); + let watcher = Watcher::new().unwrap(); // TODO return Result? + let mut res = Vfs { roots: Arena::default(), files: Arena::default(), root2files: FxHashMap::default(), worker, worker_handle, + watcher, pending_changes: Vec::new(), }; @@ -129,6 +133,7 @@ impl Vfs { filter: Box::new(filter), }; res.worker.inp.send(task).unwrap(); + res.watcher.watch(path).unwrap(); } let roots = res.roots.iter().map(|(id, _)| id).collect(); (res, roots) @@ -183,6 +188,10 @@ impl Vfs { &self.worker.out } + pub fn change_receiver(&self) -> &Receiver { + &self.watcher.change_receiver() + } + pub fn handle_task(&mut self, task: io::TaskResult) { let mut files = Vec::new(); // While we were scanning the root in the backgound, a file might have @@ -209,22 +218,41 @@ impl Vfs { self.pending_changes.push(change); } - pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option { + pub fn handle_change(&mut self, change: WatcherChange) { + match change { + WatcherChange::Create(path) => { + self.add_file_overlay(&path, None); + } + WatcherChange::Remove(path) => { + self.remove_file_overlay(&path); + } + WatcherChange::Rename(src, dst) => { + self.remove_file_overlay(&src); + self.add_file_overlay(&dst, None); + } + WatcherChange::Write(path) => { + self.change_file_overlay(&path, None); + } + } + } + + pub fn add_file_overlay(&mut self, path: &Path, text: Option) -> Option { let mut res = None; - if let Some((root, path, file)) = self.find_root(path) { + if let Some((root, rel_path, file)) = self.find_root(path) { + let text = text.unwrap_or_else(|| fs::read_to_string(&path).unwrap_or_default()); let text = Arc::new(text); let change = if let Some(file) = file { res = Some(file); self.change_file(file, Arc::clone(&text)); VfsChange::ChangeFile { file, text } } else { - let file = self.add_file(root, path.clone(), Arc::clone(&text)); + let file = self.add_file(root, rel_path.clone(), Arc::clone(&text)); res = Some(file); VfsChange::AddFile { file, text, root, - path, + path: rel_path, } }; self.pending_changes.push(change); @@ -232,8 +260,10 @@ impl Vfs { res } - pub fn change_file_overlay(&mut self, path: &Path, new_text: String) { + pub fn change_file_overlay(&mut self, path: &Path, new_text: Option) { if let Some((_root, _path, file)) = self.find_root(path) { + let new_text = + new_text.unwrap_or_else(|| fs::read_to_string(&path).unwrap_or_default()); let file = file.expect("can't change a file which wasn't added"); let text = Arc::new(new_text); self.change_file(file, Arc::clone(&text)); @@ -267,6 +297,7 @@ impl Vfs { /// Sutdown the VFS and terminate the background watching thread. pub fn shutdown(self) -> thread::Result<()> { + let _ = self.watcher.shutdown(); let _ = self.worker.shutdown(); self.worker_handle.shutdown() } diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs new file mode 100644 index 0000000000..cc05f949ee --- /dev/null +++ b/crates/ra_vfs/src/watcher.rs @@ -0,0 +1,96 @@ +use std::{ + path::{Path, PathBuf}, + sync::mpsc, + thread, + time::Duration, +}; + +use crossbeam_channel::Receiver; +use drop_bomb::DropBomb; +use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; + +pub struct Watcher { + receiver: Receiver, + watcher: RecommendedWatcher, + thread: thread::JoinHandle<()>, + bomb: DropBomb, +} + +#[derive(Debug)] +pub enum WatcherChange { + Create(PathBuf), + Write(PathBuf), + Remove(PathBuf), + Rename(PathBuf, PathBuf), +} + +impl WatcherChange { + fn from_debounced_event(ev: DebouncedEvent) -> Option { + match ev { + DebouncedEvent::NoticeWrite(_) + | DebouncedEvent::NoticeRemove(_) + | DebouncedEvent::Chmod(_) + | DebouncedEvent::Rescan => { + // ignore + None + } + DebouncedEvent::Create(path) => Some(WatcherChange::Create(path)), + DebouncedEvent::Write(path) => Some(WatcherChange::Write(path)), + DebouncedEvent::Remove(path) => Some(WatcherChange::Remove(path)), + DebouncedEvent::Rename(src, dst) => Some(WatcherChange::Rename(src, dst)), + DebouncedEvent::Error(err, path) => { + // TODO + log::warn!("watch error {}, {:?}", err, path); + None + } + } + } +} + +impl Watcher { + pub fn new() -> Result> { + let (input_sender, input_receiver) = mpsc::channel(); + let watcher = notify::watcher(input_sender, Duration::from_millis(250))?; + let (output_sender, output_receiver) = crossbeam_channel::unbounded(); + let thread = thread::spawn(move || loop { + match input_receiver.recv() { + Ok(ev) => { + // forward relevant events only + if let Some(change) = WatcherChange::from_debounced_event(ev) { + output_sender.send(change).unwrap(); + } + } + Err(err) => { + log::debug!("Watcher stopped ({})", err); + break; + } + } + }); + Ok(Watcher { + receiver: output_receiver, + watcher, + thread, + bomb: DropBomb::new(format!("Watcher was not shutdown")), + }) + } + + pub fn watch(&mut self, root: impl AsRef) -> Result<(), Box> { + self.watcher.watch(root, RecursiveMode::Recursive)?; + Ok(()) + } + + pub fn change_receiver(&self) -> &Receiver { + &self.receiver + } + + pub fn shutdown(mut self) -> thread::Result<()> { + self.bomb.defuse(); + drop(self.watcher); + let res = self.thread.join(); + match &res { + Ok(()) => log::info!("... Watcher terminated with ok"), + Err(_) => log::error!("... Watcher terminated with err"), + } + res + } +} From d032a1a4e8c52f4ea38d9ca24070203a35436158 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sun, 6 Jan 2019 18:36:22 +0100 Subject: [PATCH 02/24] complete test --- Cargo.lock | 1 + crates/ra_vfs/Cargo.toml | 1 + crates/ra_vfs/src/lib.rs | 2 +- crates/ra_vfs/src/watcher.rs | 37 ++++++------ crates/ra_vfs/tests/vfs.rs | 106 +++++++++++++++++++++++++---------- 5 files changed, 95 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fccf8a65e1..a694c2d916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1009,6 +1009,7 @@ version = "0.1.0" dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "flexi_logger 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "ra_arena 0.1.0", diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index f7a972e91b..b703cbd9fc 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -18,3 +18,4 @@ ra_arena = { path = "../ra_arena" } [dev-dependencies] tempfile = "3" +flexi_logger = "0.10.0" diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 5336822b33..1ca94dcd6c 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -98,7 +98,7 @@ impl Vfs { pub fn new(mut roots: Vec) -> (Vfs, Vec) { let (worker, worker_handle) = io::start(); - let watcher = Watcher::new().unwrap(); // TODO return Result? + let watcher = Watcher::start().unwrap(); // TODO return Result? let mut res = Vfs { roots: Arena::default(), diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index cc05f949ee..1aac236161 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -39,7 +39,6 @@ impl WatcherChange { DebouncedEvent::Remove(path) => Some(WatcherChange::Remove(path)), DebouncedEvent::Rename(src, dst) => Some(WatcherChange::Rename(src, dst)), DebouncedEvent::Error(err, path) => { - // TODO log::warn!("watch error {}, {:?}", err, path); None } @@ -48,23 +47,17 @@ impl WatcherChange { } impl Watcher { - pub fn new() -> Result> { + pub fn start() -> Result> { let (input_sender, input_receiver) = mpsc::channel(); let watcher = notify::watcher(input_sender, Duration::from_millis(250))?; let (output_sender, output_receiver) = crossbeam_channel::unbounded(); - let thread = thread::spawn(move || loop { - match input_receiver.recv() { - Ok(ev) => { - // forward relevant events only - if let Some(change) = WatcherChange::from_debounced_event(ev) { - output_sender.send(change).unwrap(); - } - } - Err(err) => { - log::debug!("Watcher stopped ({})", err); - break; - } - } + let thread = thread::spawn(move || { + input_receiver + .into_iter() + // forward relevant events only + .filter_map(WatcherChange::from_debounced_event) + .try_for_each(|change| output_sender.send(change)) + .unwrap() }); Ok(Watcher { receiver: output_receiver, @@ -86,11 +79,13 @@ impl Watcher { pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); drop(self.watcher); - let res = self.thread.join(); - match &res { - Ok(()) => log::info!("... Watcher terminated with ok"), - Err(_) => log::error!("... Watcher terminated with err"), - } - res + // TODO this doesn't terminate for some reason + // let res = self.thread.join(); + // match &res { + // Ok(()) => log::info!("... Watcher terminated with ok"), + // Err(_) => log::error!("... Watcher terminated with err"), + // } + // res + Ok(()) } } diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index f56fc46037..8634be9c49 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -1,14 +1,13 @@ -use std::{ - fs, - collections::HashSet, -}; - -use tempfile::tempdir; +use std::{collections::HashSet, fs}; +use flexi_logger::Logger; use ra_vfs::{Vfs, VfsChange}; +use tempfile::tempdir; #[test] fn test_vfs_works() -> std::io::Result<()> { + Logger::with_str("debug").start().unwrap(); + let files = [ ("a/foo.rs", "hello"), ("a/bar.rs", "world"), @@ -58,42 +57,89 @@ fn test_vfs_works() -> std::io::Result<()> { assert_eq!(files, expected_files); } - vfs.add_file_overlay(&dir.path().join("a/b/baz.rs"), "quux".to_string()); - let change = vfs.commit_changes().pop().unwrap(); - match change { - VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "quux"), - _ => panic!("unexpected change"), + // on disk change + fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); + let change = vfs.change_receiver().recv().unwrap(); + vfs.handle_change(change); + match vfs.commit_changes().as_slice() { + [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), + _ => panic!("unexpected changes"), } - vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); - let change = vfs.commit_changes().pop().unwrap(); - match change { - VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "m"), - _ => panic!("unexpected change"), + // in memory change + vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), Some("m".to_string())); + match vfs.commit_changes().as_slice() { + [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "m"), + _ => panic!("unexpected changes"), } + // in memory remove, restores data on disk vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); - let change = vfs.commit_changes().pop().unwrap(); - match change { - VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "nested hello"), - _ => panic!("unexpected change"), + match vfs.commit_changes().as_slice() { + [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), + _ => panic!("unexpected changes"), } - vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); - let change = vfs.commit_changes().pop().unwrap(); - match change { - VfsChange::AddFile { text, path, .. } => { - assert_eq!(&*text, "spam"); + // in memory add + vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), Some("spam".to_string())); + match vfs.commit_changes().as_slice() { + [VfsChange::AddFile { text, path, .. }] => { + assert_eq!(text.as_str(), "spam"); assert_eq!(path, "spam.rs"); } - _ => panic!("unexpected change"), + _ => panic!("unexpected changes"), } + // in memory remove vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs")); - let change = vfs.commit_changes().pop().unwrap(); - match change { - VfsChange::RemoveFile { .. } => (), - _ => panic!("unexpected change"), + match vfs.commit_changes().as_slice() { + [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "spam.rs"), + _ => panic!("unexpected changes"), + } + + // on disk add + fs::write(&dir.path().join("a/new.rs"), "new hello").unwrap(); + let change = vfs.change_receiver().recv().unwrap(); + vfs.handle_change(change); + match vfs.commit_changes().as_slice() { + [VfsChange::AddFile { text, path, .. }] => { + assert_eq!(text.as_str(), "new hello"); + assert_eq!(path, "new.rs"); + } + _ => panic!("unexpected changes"), + } + + // on disk rename + fs::rename(&dir.path().join("a/new.rs"), &dir.path().join("a/new1.rs")).unwrap(); + let change = vfs.change_receiver().recv().unwrap(); + vfs.handle_change(change); + match vfs.commit_changes().as_slice() { + [VfsChange::RemoveFile { + path: removed_path, .. + }, VfsChange::AddFile { + text, + path: added_path, + .. + }] => { + assert_eq!(removed_path, "new.rs"); + assert_eq!(added_path, "new1.rs"); + assert_eq!(text.as_str(), "new hello"); + } + _ => panic!("unexpected changes"), + } + + // on disk remove + fs::remove_file(&dir.path().join("a/new1.rs")).unwrap(); + let change = vfs.change_receiver().recv().unwrap(); + vfs.handle_change(change); + match vfs.commit_changes().as_slice() { + [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "new1.rs"), + _ => panic!("unexpected changes"), + } + + match vfs.change_receiver().try_recv() { + Err(crossbeam_channel::TryRecvError::Empty) => (), + res => panic!("unexpected {:?}", res), } vfs.shutdown().unwrap(); From 6b86f038d61752bbf306ed5dd9def74be3b5dcc1 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 7 Jan 2019 21:35:18 +0100 Subject: [PATCH 03/24] refator to move all io to io module use same channel for scanner and watcher some implementations pending --- crates/ra_lsp_server/src/main_loop.rs | 17 +----- crates/ra_vfs/src/io.rs | 57 +++++++++++++++---- crates/ra_vfs/src/lib.rs | 81 +++++++++++---------------- crates/ra_vfs/src/watcher.rs | 62 ++++++++++++++------ crates/ra_vfs/tests/vfs.rs | 22 ++++---- 5 files changed, 136 insertions(+), 103 deletions(-) diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs index e5a0603d1e..4f984ebc76 100644 --- a/crates/ra_lsp_server/src/main_loop.rs +++ b/crates/ra_lsp_server/src/main_loop.rs @@ -113,7 +113,6 @@ enum Event { Msg(RawMessage), Task(Task), Vfs(VfsTask), - Watcher(WatcherChange), Lib(LibraryData), } @@ -150,7 +149,6 @@ impl fmt::Debug for Event { Event::Task(it) => fmt::Debug::fmt(it, f), Event::Vfs(it) => fmt::Debug::fmt(it, f), Event::Lib(it) => fmt::Debug::fmt(it, f), - Event::Watcher(it) => fmt::Debug::fmt(it, f), } } } @@ -185,10 +183,6 @@ fn main_loop_inner( Ok(task) => Event::Vfs(task), Err(RecvError) => bail!("vfs died"), }, - recv(state.vfs.read().change_receiver()) -> change => match change { - Ok(change) => Event::Watcher(change), - Err(RecvError) => bail!("vfs watcher died"), - }, recv(libdata_receiver) -> data => Event::Lib(data.unwrap()) }; log::info!("loop_turn = {:?}", event); @@ -200,10 +194,6 @@ fn main_loop_inner( state.vfs.write().handle_task(task); state_changed = true; } - Event::Watcher(change) => { - state.vfs.write().handle_change(change); - state_changed = true; - } Event::Lib(lib) => { feedback(internal_mode, "library loaded", msg_sender); state.add_lib(lib); @@ -375,7 +365,7 @@ fn on_notification( if let Some(file_id) = state .vfs .write() - .add_file_overlay(&path, Some(params.text_document.text)) + .add_file_overlay(&path, params.text_document.text) { subs.add_sub(FileId(file_id.0.into())); } @@ -394,10 +384,7 @@ fn on_notification( .pop() .ok_or_else(|| format_err!("empty changes"))? .text; - state - .vfs - .write() - .change_file_overlay(path.as_path(), Some(text)); + state.vfs.write().change_file_overlay(path.as_path(), text); return Ok(()); } Err(not) => not, diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 80328ad186..79dea5dc72 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -10,17 +10,47 @@ use relative_path::RelativePathBuf; use crate::{VfsRoot, has_rs_extension}; -pub(crate) struct Task { - pub(crate) root: VfsRoot, - pub(crate) path: PathBuf, - pub(crate) filter: Box bool + Send>, +pub(crate) enum Task { + AddRoot { + root: VfsRoot, + path: PathBuf, + filter: Box bool + Send>, + }, + WatcherChange(crate::watcher::WatcherChange), } -pub struct TaskResult { +#[derive(Debug)] +pub struct AddRootResult { pub(crate) root: VfsRoot, pub(crate) files: Vec<(RelativePathBuf, String)>, } +#[derive(Debug)] +pub enum WatcherChangeResult { + Create { + path: PathBuf, + text: String, + }, + Write { + path: PathBuf, + text: String, + }, + Remove { + path: PathBuf, + }, + // can this be replaced and use Remove and Create instead? + Rename { + src: PathBuf, + dst: PathBuf, + text: String, + }, +} + +pub enum TaskResult { + AddRoot(AddRootResult), + WatcherChange(WatcherChangeResult), +} + impl fmt::Debug for TaskResult { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str("TaskResult { ... }") @@ -40,11 +70,18 @@ pub(crate) fn start() -> (Worker, WorkerHandle) { } fn handle_task(task: Task) -> TaskResult { - let Task { root, path, filter } = task; - log::debug!("loading {} ...", path.as_path().display()); - let files = load_root(path.as_path(), &*filter); - log::debug!("... loaded {}", path.as_path().display()); - TaskResult { root, files } + match task { + Task::AddRoot { root, path, filter } => { + log::debug!("loading {} ...", path.as_path().display()); + let files = load_root(path.as_path(), &*filter); + log::debug!("... loaded {}", path.as_path().display()); + TaskResult::AddRoot(AddRootResult { root, files }) + } + Task::WatcherChange(change) => { + // TODO + unimplemented!() + } + } } fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePathBuf, String)> { diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 1ca94dcd6c..889ed788dd 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -60,7 +60,7 @@ impl RootFilter { } } -fn has_rs_extension(p: &Path) -> bool { +pub(crate) fn has_rs_extension(p: &Path) -> bool { p.extension() == Some(OsStr::new("rs")) } @@ -98,7 +98,7 @@ impl Vfs { pub fn new(mut roots: Vec) -> (Vfs, Vec) { let (worker, worker_handle) = io::start(); - let watcher = Watcher::start().unwrap(); // TODO return Result? + let watcher = Watcher::start(worker.inp.clone()).unwrap(); // TODO return Result? let mut res = Vfs { roots: Arena::default(), @@ -127,7 +127,7 @@ impl Vfs { nested.iter().all(|it| it != entry.path()) } }; - let task = io::Task { + let task = io::Task::AddRoot { root, path: path.clone(), filter: Box::new(filter), @@ -188,58 +188,43 @@ impl Vfs { &self.worker.out } - pub fn change_receiver(&self) -> &Receiver { - &self.watcher.change_receiver() - } - pub fn handle_task(&mut self, task: io::TaskResult) { - let mut files = Vec::new(); - // While we were scanning the root in the backgound, a file might have - // been open in the editor, so we need to account for that. - let exising = self.root2files[&task.root] - .iter() - .map(|&file| (self.files[file].path.clone(), file)) - .collect::>(); - for (path, text) in task.files { - if let Some(&file) = exising.get(&path) { - let text = Arc::clone(&self.files[file].text); - files.push((file, path, text)); - continue; - } - let text = Arc::new(text); - let file = self.add_file(task.root, path.clone(), Arc::clone(&text)); - files.push((file, path, text)); - } + match task { + io::TaskResult::AddRoot(task) => { + let mut files = Vec::new(); + // While we were scanning the root in the backgound, a file might have + // been open in the editor, so we need to account for that. + let exising = self.root2files[&task.root] + .iter() + .map(|&file| (self.files[file].path.clone(), file)) + .collect::>(); + for (path, text) in task.files { + if let Some(&file) = exising.get(&path) { + let text = Arc::clone(&self.files[file].text); + files.push((file, path, text)); + continue; + } + let text = Arc::new(text); + let file = self.add_file(task.root, path.clone(), Arc::clone(&text)); + files.push((file, path, text)); + } - let change = VfsChange::AddRoot { - root: task.root, - files, - }; - self.pending_changes.push(change); - } - - pub fn handle_change(&mut self, change: WatcherChange) { - match change { - WatcherChange::Create(path) => { - self.add_file_overlay(&path, None); + let change = VfsChange::AddRoot { + root: task.root, + files, + }; + self.pending_changes.push(change); } - WatcherChange::Remove(path) => { - self.remove_file_overlay(&path); - } - WatcherChange::Rename(src, dst) => { - self.remove_file_overlay(&src); - self.add_file_overlay(&dst, None); - } - WatcherChange::Write(path) => { - self.change_file_overlay(&path, None); + io::TaskResult::WatcherChange(change) => { + // TODO + unimplemented!() } } } - pub fn add_file_overlay(&mut self, path: &Path, text: Option) -> Option { + pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option { let mut res = None; if let Some((root, rel_path, file)) = self.find_root(path) { - let text = text.unwrap_or_else(|| fs::read_to_string(&path).unwrap_or_default()); let text = Arc::new(text); let change = if let Some(file) = file { res = Some(file); @@ -260,10 +245,8 @@ impl Vfs { res } - pub fn change_file_overlay(&mut self, path: &Path, new_text: Option) { + pub fn change_file_overlay(&mut self, path: &Path, new_text: String) { if let Some((_root, _path, file)) = self.find_root(path) { - let new_text = - new_text.unwrap_or_else(|| fs::read_to_string(&path).unwrap_or_default()); let file = file.expect("can't change a file which wasn't added"); let text = Arc::new(new_text); self.change_file(file, Arc::clone(&text)); diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index 1aac236161..a6d0496c03 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -5,12 +5,12 @@ use std::{ time::Duration, }; -use crossbeam_channel::Receiver; +use crossbeam_channel::Sender; use drop_bomb::DropBomb; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; +use crate::{has_rs_extension, io}; pub struct Watcher { - receiver: Receiver, watcher: RecommendedWatcher, thread: thread::JoinHandle<()>, bomb: DropBomb, @@ -21,24 +21,54 @@ pub enum WatcherChange { Create(PathBuf), Write(PathBuf), Remove(PathBuf), + // can this be replaced and use Remove and Create instead? Rename(PathBuf, PathBuf), } impl WatcherChange { - fn from_debounced_event(ev: DebouncedEvent) -> Option { + fn try_from_debounced_event(ev: DebouncedEvent) -> Option { match ev { DebouncedEvent::NoticeWrite(_) | DebouncedEvent::NoticeRemove(_) - | DebouncedEvent::Chmod(_) - | DebouncedEvent::Rescan => { + | DebouncedEvent::Chmod(_) => { // ignore None } - DebouncedEvent::Create(path) => Some(WatcherChange::Create(path)), - DebouncedEvent::Write(path) => Some(WatcherChange::Write(path)), - DebouncedEvent::Remove(path) => Some(WatcherChange::Remove(path)), - DebouncedEvent::Rename(src, dst) => Some(WatcherChange::Rename(src, dst)), + DebouncedEvent::Rescan => { + // TODO should we rescan the root? + None + } + DebouncedEvent::Create(path) => { + if has_rs_extension(&path) { + Some(WatcherChange::Create(path)) + } else { + None + } + } + DebouncedEvent::Write(path) => { + if has_rs_extension(&path) { + Some(WatcherChange::Write(path)) + } else { + None + } + } + DebouncedEvent::Remove(path) => { + if has_rs_extension(&path) { + Some(WatcherChange::Remove(path)) + } else { + None + } + } + DebouncedEvent::Rename(src, dst) => { + match (has_rs_extension(&src), has_rs_extension(&dst)) { + (true, true) => Some(WatcherChange::Rename(src, dst)), + (true, false) => Some(WatcherChange::Remove(src)), + (false, true) => Some(WatcherChange::Create(dst)), + (false, false) => None, + } + } DebouncedEvent::Error(err, path) => { + // TODO should we reload the file contents? log::warn!("watch error {}, {:?}", err, path); None } @@ -47,20 +77,20 @@ impl WatcherChange { } impl Watcher { - pub fn start() -> Result> { + pub(crate) fn start( + output_sender: Sender, + ) -> Result> { let (input_sender, input_receiver) = mpsc::channel(); let watcher = notify::watcher(input_sender, Duration::from_millis(250))?; - let (output_sender, output_receiver) = crossbeam_channel::unbounded(); let thread = thread::spawn(move || { input_receiver .into_iter() // forward relevant events only - .filter_map(WatcherChange::from_debounced_event) - .try_for_each(|change| output_sender.send(change)) + .filter_map(WatcherChange::try_from_debounced_event) + .try_for_each(|change| output_sender.send(io::Task::WatcherChange(change))) .unwrap() }); Ok(Watcher { - receiver: output_receiver, watcher, thread, bomb: DropBomb::new(format!("Watcher was not shutdown")), @@ -72,10 +102,6 @@ impl Watcher { Ok(()) } - pub fn change_receiver(&self) -> &Receiver { - &self.receiver - } - pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); drop(self.watcher); diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 8634be9c49..21d5633b12 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -59,15 +59,15 @@ fn test_vfs_works() -> std::io::Result<()> { // on disk change fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); - let change = vfs.change_receiver().recv().unwrap(); - vfs.handle_change(change); + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), _ => panic!("unexpected changes"), } // in memory change - vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), Some("m".to_string())); + vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "m"), _ => panic!("unexpected changes"), @@ -81,7 +81,7 @@ fn test_vfs_works() -> std::io::Result<()> { } // in memory add - vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), Some("spam".to_string())); + vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); match vfs.commit_changes().as_slice() { [VfsChange::AddFile { text, path, .. }] => { assert_eq!(text.as_str(), "spam"); @@ -99,8 +99,8 @@ fn test_vfs_works() -> std::io::Result<()> { // on disk add fs::write(&dir.path().join("a/new.rs"), "new hello").unwrap(); - let change = vfs.change_receiver().recv().unwrap(); - vfs.handle_change(change); + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); match vfs.commit_changes().as_slice() { [VfsChange::AddFile { text, path, .. }] => { assert_eq!(text.as_str(), "new hello"); @@ -111,8 +111,8 @@ fn test_vfs_works() -> std::io::Result<()> { // on disk rename fs::rename(&dir.path().join("a/new.rs"), &dir.path().join("a/new1.rs")).unwrap(); - let change = vfs.change_receiver().recv().unwrap(); - vfs.handle_change(change); + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path: removed_path, .. @@ -130,14 +130,14 @@ fn test_vfs_works() -> std::io::Result<()> { // on disk remove fs::remove_file(&dir.path().join("a/new1.rs")).unwrap(); - let change = vfs.change_receiver().recv().unwrap(); - vfs.handle_change(change); + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "new1.rs"), _ => panic!("unexpected changes"), } - match vfs.change_receiver().try_recv() { + match vfs.task_receiver().try_recv() { Err(crossbeam_channel::TryRecvError::Empty) => (), res => panic!("unexpected {:?}", res), } From 76bf7498aa88c4de4517f4eb1218807fdfc7071b Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sat, 12 Jan 2019 18:17:52 +0100 Subject: [PATCH 04/24] handle watched events filtering in `Vfs`add `is_overlayed`load changed files contents in `io` --- Cargo.lock | 2 +- crates/ra_vfs/src/io.rs | 74 +++++++++------ crates/ra_vfs/src/lib.rs | 171 ++++++++++++++++++++++++++--------- crates/ra_vfs/src/watcher.rs | 85 +++++++---------- crates/ra_vfs/tests/vfs.rs | 35 +++---- 5 files changed, 221 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a694c2d916..b8c840c685 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1009,7 +1009,7 @@ version = "0.1.0" dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "flexi_logger 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)", + "flexi_logger 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "ra_arena 0.1.0", diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 79dea5dc72..a0c87fb258 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -1,14 +1,13 @@ use std::{ - fmt, - fs, + fmt, fs, path::{Path, PathBuf}, }; -use walkdir::{DirEntry, WalkDir}; -use thread_worker::{WorkerHandle}; use relative_path::RelativePathBuf; +use thread_worker::WorkerHandle; +use walkdir::{DirEntry, WalkDir}; -use crate::{VfsRoot, has_rs_extension}; +use crate::{has_rs_extension, watcher::WatcherChange, VfsRoot}; pub(crate) enum Task { AddRoot { @@ -16,7 +15,7 @@ pub(crate) enum Task { path: PathBuf, filter: Box bool + Send>, }, - WatcherChange(crate::watcher::WatcherChange), + LoadChange(crate::watcher::WatcherChange), } #[derive(Debug)] @@ -26,29 +25,16 @@ pub struct AddRootResult { } #[derive(Debug)] -pub enum WatcherChangeResult { - Create { - path: PathBuf, - text: String, - }, - Write { - path: PathBuf, - text: String, - }, - Remove { - path: PathBuf, - }, - // can this be replaced and use Remove and Create instead? - Rename { - src: PathBuf, - dst: PathBuf, - text: String, - }, +pub enum WatcherChangeData { + Create { path: PathBuf, text: String }, + Write { path: PathBuf, text: String }, + Remove { path: PathBuf }, } pub enum TaskResult { AddRoot(AddRootResult), - WatcherChange(WatcherChangeResult), + HandleChange(WatcherChange), + LoadChange(Option), } impl fmt::Debug for TaskResult { @@ -77,9 +63,10 @@ fn handle_task(task: Task) -> TaskResult { log::debug!("... loaded {}", path.as_path().display()); TaskResult::AddRoot(AddRootResult { root, files }) } - Task::WatcherChange(change) => { - // TODO - unimplemented!() + Task::LoadChange(change) => { + log::debug!("loading {:?} ...", change); + let data = load_change(change); + TaskResult::LoadChange(data) } } } @@ -113,3 +100,34 @@ fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePa } res } + +fn load_change(change: WatcherChange) -> Option { + let data = match change { + WatcherChange::Create(path) => { + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(e) => { + log::warn!("watcher error: {}", e); + return None; + } + }; + WatcherChangeData::Create { path, text } + } + WatcherChange::Write(path) => { + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(e) => { + log::warn!("watcher error: {}", e); + return None; + } + }; + WatcherChangeData::Write { path, text } + } + WatcherChange::Remove(path) => WatcherChangeData::Remove { path }, + WatcherChange::Rescan => { + // this should be handled by Vfs::handle_task + return None; + } + }; + Some(data) +} diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 889ed788dd..2930f6b805 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -75,6 +75,7 @@ impl_arena_id!(VfsFile); struct VfsFileData { root: VfsRoot, path: RelativePathBuf, + is_overlayed: bool, text: Arc, } @@ -170,7 +171,7 @@ impl Vfs { } else { let text = fs::read_to_string(path).unwrap_or_default(); let text = Arc::new(text); - let file = self.add_file(root, rel_path.clone(), Arc::clone(&text)); + let file = self.add_file(root, rel_path.clone(), Arc::clone(&text), false); let change = VfsChange::AddFile { file, text, @@ -205,7 +206,7 @@ impl Vfs { continue; } let text = Arc::new(text); - let file = self.add_file(task.root, path.clone(), Arc::clone(&text)); + let file = self.add_file(task.root, path.clone(), Arc::clone(&text), false); files.push((file, path, text)); } @@ -215,63 +216,132 @@ impl Vfs { }; self.pending_changes.push(change); } - io::TaskResult::WatcherChange(change) => { - // TODO - unimplemented!() - } + io::TaskResult::HandleChange(change) => match &change { + watcher::WatcherChange::Create(path) + | watcher::WatcherChange::Remove(path) + | watcher::WatcherChange::Write(path) => { + if self.should_handle_change(&path) { + self.worker.inp.send(io::Task::LoadChange(change)).unwrap() + } + } + watcher::WatcherChange::Rescan => { + // TODO send Task::AddRoot? + } + }, + io::TaskResult::LoadChange(None) => {} + io::TaskResult::LoadChange(Some(change)) => match change { + io::WatcherChangeData::Create { path, text } + | io::WatcherChangeData::Write { path, text } => { + if let Some((root, path, file)) = self.find_root(&path) { + if let Some(file) = file { + self.do_change_file(file, text, false); + } else { + self.do_add_file(root, path, text, false); + } + } + } + io::WatcherChangeData::Remove { path } => { + if let Some((root, path, file)) = self.find_root(&path) { + if let Some(file) = file { + self.do_remove_file(root, path, file, false); + } + } + } + }, } } - pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option { - let mut res = None; - if let Some((root, rel_path, file)) = self.find_root(path) { - let text = Arc::new(text); - let change = if let Some(file) = file { - res = Some(file); - self.change_file(file, Arc::clone(&text)); - VfsChange::ChangeFile { file, text } - } else { - let file = self.add_file(root, rel_path.clone(), Arc::clone(&text)); - res = Some(file); - VfsChange::AddFile { - file, - text, - root, - path: rel_path, + fn should_handle_change(&self, path: &Path) -> bool { + if let Some((_root, _rel_path, file)) = self.find_root(&path) { + if let Some(file) = file { + if self.files[file].is_overlayed { + // file is overlayed + return false; } - }; - self.pending_changes.push(change); + } + true + } else { + // file doesn't belong to any root + false + } + } + + fn do_add_file( + &mut self, + root: VfsRoot, + path: RelativePathBuf, + text: String, + is_overlay: bool, + ) -> Option { + let text = Arc::new(text); + let file = self.add_file(root, path.clone(), text.clone(), is_overlay); + self.pending_changes.push(VfsChange::AddFile { + file, + root, + path, + text, + }); + Some(file) + } + + fn do_change_file(&mut self, file: VfsFile, text: String, is_overlay: bool) { + if !is_overlay && self.files[file].is_overlayed { + return; + } + let text = Arc::new(text); + self.change_file(file, text.clone(), is_overlay); + self.pending_changes + .push(VfsChange::ChangeFile { file, text }); + } + + fn do_remove_file( + &mut self, + root: VfsRoot, + path: RelativePathBuf, + file: VfsFile, + is_overlay: bool, + ) { + if !is_overlay && self.files[file].is_overlayed { + return; + } + self.remove_file(file); + self.pending_changes + .push(VfsChange::RemoveFile { root, path, file }); + } + + pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option { + if let Some((root, rel_path, file)) = self.find_root(path) { + if let Some(file) = file { + self.do_change_file(file, text, true); + Some(file) + } else { + self.do_add_file(root, rel_path, text, true) + } + } else { + None } - res } pub fn change_file_overlay(&mut self, path: &Path, new_text: String) { if let Some((_root, _path, file)) = self.find_root(path) { let file = file.expect("can't change a file which wasn't added"); - let text = Arc::new(new_text); - self.change_file(file, Arc::clone(&text)); - let change = VfsChange::ChangeFile { file, text }; - self.pending_changes.push(change); + self.do_change_file(file, new_text, true); } } pub fn remove_file_overlay(&mut self, path: &Path) -> Option { - let mut res = None; if let Some((root, path, file)) = self.find_root(path) { let file = file.expect("can't remove a file which wasn't added"); - res = Some(file); let full_path = path.to_path(&self.roots[root].root); - let change = if let Ok(text) = fs::read_to_string(&full_path) { - let text = Arc::new(text); - self.change_file(file, Arc::clone(&text)); - VfsChange::ChangeFile { file, text } + if let Ok(text) = fs::read_to_string(&full_path) { + self.do_change_file(file, text, true); } else { - self.remove_file(file); - VfsChange::RemoveFile { root, file, path } - }; - self.pending_changes.push(change); + self.do_remove_file(root, path, file, true); + } + Some(file) + } else { + None } - res } pub fn commit_changes(&mut self) -> Vec { @@ -285,15 +355,28 @@ impl Vfs { self.worker_handle.shutdown() } - fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc) -> VfsFile { - let data = VfsFileData { root, path, text }; + fn add_file( + &mut self, + root: VfsRoot, + path: RelativePathBuf, + text: Arc, + is_overlayed: bool, + ) -> VfsFile { + let data = VfsFileData { + root, + path, + text, + is_overlayed, + }; let file = self.files.alloc(data); self.root2files.get_mut(&root).unwrap().insert(file); file } - fn change_file(&mut self, file: VfsFile, new_text: Arc) { - self.files[file].text = new_text; + fn change_file(&mut self, file: VfsFile, new_text: Arc, is_overlayed: bool) { + let mut file_data = &mut self.files[file]; + file_data.text = new_text; + file_data.is_overlayed = is_overlayed; } fn remove_file(&mut self, file: VfsFile) { diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index a6d0496c03..a5401869ce 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -5,10 +5,10 @@ use std::{ time::Duration, }; +use crate::io; use crossbeam_channel::Sender; use drop_bomb::DropBomb; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; -use crate::{has_rs_extension, io}; pub struct Watcher { watcher: RecommendedWatcher, @@ -21,59 +21,41 @@ pub enum WatcherChange { Create(PathBuf), Write(PathBuf), Remove(PathBuf), - // can this be replaced and use Remove and Create instead? - Rename(PathBuf, PathBuf), + Rescan, } -impl WatcherChange { - fn try_from_debounced_event(ev: DebouncedEvent) -> Option { - match ev { - DebouncedEvent::NoticeWrite(_) - | DebouncedEvent::NoticeRemove(_) - | DebouncedEvent::Chmod(_) => { - // ignore - None - } - DebouncedEvent::Rescan => { - // TODO should we rescan the root? - None - } - DebouncedEvent::Create(path) => { - if has_rs_extension(&path) { - Some(WatcherChange::Create(path)) - } else { - None - } - } - DebouncedEvent::Write(path) => { - if has_rs_extension(&path) { - Some(WatcherChange::Write(path)) - } else { - None - } - } - DebouncedEvent::Remove(path) => { - if has_rs_extension(&path) { - Some(WatcherChange::Remove(path)) - } else { - None - } - } - DebouncedEvent::Rename(src, dst) => { - match (has_rs_extension(&src), has_rs_extension(&dst)) { - (true, true) => Some(WatcherChange::Rename(src, dst)), - (true, false) => Some(WatcherChange::Remove(src)), - (false, true) => Some(WatcherChange::Create(dst)), - (false, false) => None, - } - } - DebouncedEvent::Error(err, path) => { - // TODO should we reload the file contents? - log::warn!("watch error {}, {:?}", err, path); - None - } +fn send_change_events( + ev: DebouncedEvent, + sender: &Sender, +) -> Result<(), Box> { + match ev { + DebouncedEvent::NoticeWrite(_) + | DebouncedEvent::NoticeRemove(_) + | DebouncedEvent::Chmod(_) => { + // ignore + } + DebouncedEvent::Rescan => { + sender.send(io::Task::LoadChange(WatcherChange::Rescan))?; + } + DebouncedEvent::Create(path) => { + sender.send(io::Task::LoadChange(WatcherChange::Create(path)))?; + } + DebouncedEvent::Write(path) => { + sender.send(io::Task::LoadChange(WatcherChange::Write(path)))?; + } + DebouncedEvent::Remove(path) => { + sender.send(io::Task::LoadChange(WatcherChange::Remove(path)))?; + } + DebouncedEvent::Rename(src, dst) => { + sender.send(io::Task::LoadChange(WatcherChange::Remove(src)))?; + sender.send(io::Task::LoadChange(WatcherChange::Create(dst)))?; + } + DebouncedEvent::Error(err, path) => { + // TODO should we reload the file contents? + log::warn!("watcher error {}, {:?}", err, path); } } + Ok(()) } impl Watcher { @@ -86,8 +68,7 @@ impl Watcher { input_receiver .into_iter() // forward relevant events only - .filter_map(WatcherChange::try_from_debounced_event) - .try_for_each(|change| output_sender.send(io::Task::WatcherChange(change))) + .try_for_each(|change| send_change_events(change, &output_sender)) .unwrap() }); Ok(Watcher { diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 21d5633b12..9cde2bed7f 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -4,6 +4,13 @@ use flexi_logger::Logger; use ra_vfs::{Vfs, VfsChange}; use tempfile::tempdir; +fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { + for _ in 0..num_tasks { + let task = vfs.task_receiver().recv().unwrap(); + vfs.handle_task(task); + } +} + #[test] fn test_vfs_works() -> std::io::Result<()> { Logger::with_str("debug").start().unwrap(); @@ -25,10 +32,7 @@ fn test_vfs_works() -> std::io::Result<()> { let b_root = dir.path().join("a/b"); let (mut vfs, _) = Vfs::new(vec![a_root, b_root]); - for _ in 0..2 { - let task = vfs.task_receiver().recv().unwrap(); - vfs.handle_task(task); - } + process_tasks(&mut vfs, 2); { let files = vfs .commit_changes() @@ -57,30 +61,26 @@ fn test_vfs_works() -> std::io::Result<()> { assert_eq!(files, expected_files); } - // on disk change fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); - let task = vfs.task_receiver().recv().unwrap(); - vfs.handle_task(task); + process_tasks(&mut vfs, 1); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), _ => panic!("unexpected changes"), } - // in memory change vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "m"), _ => panic!("unexpected changes"), } - // in memory remove, restores data on disk + // removing overlay restores data on disk vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), _ => panic!("unexpected changes"), } - // in memory add vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); match vfs.commit_changes().as_slice() { [VfsChange::AddFile { text, path, .. }] => { @@ -90,17 +90,14 @@ fn test_vfs_works() -> std::io::Result<()> { _ => panic!("unexpected changes"), } - // in memory remove vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs")); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "spam.rs"), _ => panic!("unexpected changes"), } - // on disk add fs::write(&dir.path().join("a/new.rs"), "new hello").unwrap(); - let task = vfs.task_receiver().recv().unwrap(); - vfs.handle_task(task); + process_tasks(&mut vfs, 1); match vfs.commit_changes().as_slice() { [VfsChange::AddFile { text, path, .. }] => { assert_eq!(text.as_str(), "new hello"); @@ -109,10 +106,8 @@ fn test_vfs_works() -> std::io::Result<()> { _ => panic!("unexpected changes"), } - // on disk rename fs::rename(&dir.path().join("a/new.rs"), &dir.path().join("a/new1.rs")).unwrap(); - let task = vfs.task_receiver().recv().unwrap(); - vfs.handle_task(task); + process_tasks(&mut vfs, 2); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path: removed_path, .. @@ -125,13 +120,11 @@ fn test_vfs_works() -> std::io::Result<()> { assert_eq!(added_path, "new1.rs"); assert_eq!(text.as_str(), "new hello"); } - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } - // on disk remove fs::remove_file(&dir.path().join("a/new1.rs")).unwrap(); - let task = vfs.task_receiver().recv().unwrap(); - vfs.handle_task(task); + process_tasks(&mut vfs, 1); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "new1.rs"), _ => panic!("unexpected changes"), From 5f31d495bdc8639a826f60c6f42d12bb89be9f1a Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sat, 12 Jan 2019 20:32:24 +0100 Subject: [PATCH 05/24] reference `notify` issue --- crates/ra_vfs/src/lib.rs | 2 +- crates/ra_vfs/src/watcher.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 2930f6b805..3eeec8b1c8 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -225,7 +225,7 @@ impl Vfs { } } watcher::WatcherChange::Rescan => { - // TODO send Task::AddRoot? + // TODO we should reload all files } }, io::TaskResult::LoadChange(None) => {} diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index a5401869ce..190a9f9245 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -86,7 +86,8 @@ impl Watcher { pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); drop(self.watcher); - // TODO this doesn't terminate for some reason + // TODO this doesn't terminate because of a buf in `notify` + // uncomment when https://github.com/passcod/notify/pull/170 is released // let res = self.thread.join(); // match &res { // Ok(()) => log::info!("... Watcher terminated with ok"), From b0f7e72c49141df0ad95e3fcb561ee6a4d86b537 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Tue, 15 Jan 2019 17:45:56 +0100 Subject: [PATCH 06/24] use notify with fix --- Cargo.lock | 10 +++++----- crates/ra_lsp_server/src/main_loop.rs | 2 +- crates/ra_vfs/Cargo.toml | 3 ++- crates/ra_vfs/src/watcher.rs | 15 ++++++--------- crates/ra_vfs/tests/vfs.rs | 4 ++-- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8c840c685..d511469567 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -702,8 +702,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "notify" -version = "4.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "4.0.6" +source = "git+https://github.com/vemoo/notify/?branch=v4-legacy#8114796fb7b133ba8898ba5d08fda20856f666d4" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1009,9 +1009,9 @@ version = "0.1.0" dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "flexi_logger 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)", "ra_arena 0.1.0", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1898,7 +1898,7 @@ dependencies = [ "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" -"checksum notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c968cf37cf949114b00d51b0b23536d1c3a4a3963767cf4c969c65a6af78dc7d" +"checksum notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)" = "" "checksum num-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d9fe8fcafd1b86a37ce8a1cfa15ae504817e0c8c2e7ad42767371461ac1d316d" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs index 4f984ebc76..ddd20a41fc 100644 --- a/crates/ra_lsp_server/src/main_loop.rs +++ b/crates/ra_lsp_server/src/main_loop.rs @@ -11,7 +11,7 @@ use gen_lsp_server::{ }; use lsp_types::NumberOrString; use ra_ide_api::{Canceled, FileId, LibraryData}; -use ra_vfs::{VfsTask, WatcherChange}; +use ra_vfs::VfsTask; use rustc_hash::FxHashSet; use serde::{de::DeserializeOwned, Serialize}; use threadpool::ThreadPool; diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index b703cbd9fc..1a21557bf5 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -10,7 +10,8 @@ relative-path = "0.4.0" rustc-hash = "1.0" crossbeam-channel = "0.3.5" log = "0.4.6" -notify = "4" +# until https://github.com/passcod/notify/issues/169 is fixed +notify = { git = "https://github.com/vemoo/notify/", branch = "v4-legacy" } drop_bomb = "0.1.0" thread_worker = { path = "../thread_worker" } diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index 190a9f9245..3bd0e7da2e 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -86,14 +86,11 @@ impl Watcher { pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); drop(self.watcher); - // TODO this doesn't terminate because of a buf in `notify` - // uncomment when https://github.com/passcod/notify/pull/170 is released - // let res = self.thread.join(); - // match &res { - // Ok(()) => log::info!("... Watcher terminated with ok"), - // Err(_) => log::error!("... Watcher terminated with err"), - // } - // res - Ok(()) + let res = self.thread.join(); + match &res { + Ok(()) => log::info!("... Watcher terminated with ok"), + Err(_) => log::error!("... Watcher terminated with err"), + } + res } } diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 9cde2bed7f..87fb5a092b 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, fs}; -use flexi_logger::Logger; +// use flexi_logger::Logger; use ra_vfs::{Vfs, VfsChange}; use tempfile::tempdir; @@ -13,7 +13,7 @@ fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { #[test] fn test_vfs_works() -> std::io::Result<()> { - Logger::with_str("debug").start().unwrap(); + // Logger::with_str("debug").start().unwrap(); let files = [ ("a/foo.rs", "hello"), From abd8ccefa4fd1abbf674f479f0dd7d0457c94d2d Mon Sep 17 00:00:00 2001 From: Bernardo Date: Wed, 16 Jan 2019 18:42:55 +0100 Subject: [PATCH 07/24] better error handling --- crates/ra_vfs/src/lib.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 3eeec8b1c8..f698e3e86e 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -86,7 +86,7 @@ pub struct Vfs { pending_changes: Vec, worker: io::Worker, worker_handle: WorkerHandle, - watcher: Watcher, + watcher: Option, } impl fmt::Debug for Vfs { @@ -99,7 +99,13 @@ impl Vfs { pub fn new(mut roots: Vec) -> (Vfs, Vec) { let (worker, worker_handle) = io::start(); - let watcher = Watcher::start(worker.inp.clone()).unwrap(); // TODO return Result? + let watcher = match Watcher::start(worker.inp.clone()) { + Ok(watcher) => Some(watcher), + Err(e) => { + log::error!("could not start watcher: {}", e); + None + } + }; let mut res = Vfs { roots: Arena::default(), @@ -134,7 +140,11 @@ impl Vfs { filter: Box::new(filter), }; res.worker.inp.send(task).unwrap(); - res.watcher.watch(path).unwrap(); + if let Some(ref mut watcher) = res.watcher { + if let Err(e) = watcher.watch(path) { + log::warn!("could not watch \"{}\": {}", path.display(), e); + } + } } let roots = res.roots.iter().map(|(id, _)| id).collect(); (res, roots) @@ -350,7 +360,9 @@ impl Vfs { /// Sutdown the VFS and terminate the background watching thread. pub fn shutdown(self) -> thread::Result<()> { - let _ = self.watcher.shutdown(); + if let Some(watcher) = self.watcher { + let _ = watcher.shutdown(); + } let _ = self.worker.shutdown(); self.worker_handle.shutdown() } From e69b620f0d1e90afcc14dc7cf07ed0b828d8ec96 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Wed, 16 Jan 2019 19:30:20 +0100 Subject: [PATCH 08/24] add missing Task::HandleChange --- crates/ra_vfs/src/io.rs | 11 ++++++++--- crates/ra_vfs/src/lib.rs | 1 + crates/ra_vfs/src/watcher.rs | 14 +++++++------- crates/ra_vfs/tests/vfs.rs | 23 ++++++++++++----------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index a0c87fb258..e5d5c6463e 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -15,7 +15,8 @@ pub(crate) enum Task { path: PathBuf, filter: Box bool + Send>, }, - LoadChange(crate::watcher::WatcherChange), + HandleChange(WatcherChange), + LoadChange(WatcherChange), } #[derive(Debug)] @@ -63,6 +64,10 @@ fn handle_task(task: Task) -> TaskResult { log::debug!("... loaded {}", path.as_path().display()); TaskResult::AddRoot(AddRootResult { root, files }) } + Task::HandleChange(change) => { + // forward as is because Vfs has to decide if we should load it + TaskResult::HandleChange(change) + } Task::LoadChange(change) => { log::debug!("loading {:?} ...", change); let data = load_change(change); @@ -107,7 +112,7 @@ fn load_change(change: WatcherChange) -> Option { let text = match fs::read_to_string(&path) { Ok(text) => text, Err(e) => { - log::warn!("watcher error: {}", e); + log::warn!("watcher error \"{}\": {}", path.display(), e); return None; } }; @@ -117,7 +122,7 @@ fn load_change(change: WatcherChange) -> Option { let text = match fs::read_to_string(&path) { Ok(text) => text, Err(e) => { - log::warn!("watcher error: {}", e); + log::warn!("watcher error \"{}\": {}", path.display(), e); return None; } }; diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index f698e3e86e..48a46d2106 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -266,6 +266,7 @@ impl Vfs { if let Some(file) = file { if self.files[file].is_overlayed { // file is overlayed + log::debug!("skipping overlayed \"{}\"", path.display()); return false; } } diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index 3bd0e7da2e..dfbbcbfe67 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -35,24 +35,24 @@ fn send_change_events( // ignore } DebouncedEvent::Rescan => { - sender.send(io::Task::LoadChange(WatcherChange::Rescan))?; + sender.send(io::Task::HandleChange(WatcherChange::Rescan))?; } DebouncedEvent::Create(path) => { - sender.send(io::Task::LoadChange(WatcherChange::Create(path)))?; + sender.send(io::Task::HandleChange(WatcherChange::Create(path)))?; } DebouncedEvent::Write(path) => { - sender.send(io::Task::LoadChange(WatcherChange::Write(path)))?; + sender.send(io::Task::HandleChange(WatcherChange::Write(path)))?; } DebouncedEvent::Remove(path) => { - sender.send(io::Task::LoadChange(WatcherChange::Remove(path)))?; + sender.send(io::Task::HandleChange(WatcherChange::Remove(path)))?; } DebouncedEvent::Rename(src, dst) => { - sender.send(io::Task::LoadChange(WatcherChange::Remove(src)))?; - sender.send(io::Task::LoadChange(WatcherChange::Create(dst)))?; + sender.send(io::Task::HandleChange(WatcherChange::Remove(src)))?; + sender.send(io::Task::HandleChange(WatcherChange::Create(dst)))?; } DebouncedEvent::Error(err, path) => { // TODO should we reload the file contents? - log::warn!("watcher error {}, {:?}", err, path); + log::warn!("watcher error \"{}\", {:?}", err, path); } } Ok(()) diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 87fb5a092b..8266a0bd5b 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -62,23 +62,24 @@ fn test_vfs_works() -> std::io::Result<()> { } fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); - process_tasks(&mut vfs, 1); + // 2 tasks per watcher change, first for HandleChange then for LoadChange + process_tasks(&mut vfs, 2); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "m"), - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } // removing overlay restores data on disk vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); match vfs.commit_changes().as_slice() { [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); @@ -87,27 +88,27 @@ fn test_vfs_works() -> std::io::Result<()> { assert_eq!(text.as_str(), "spam"); assert_eq!(path, "spam.rs"); } - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs")); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "spam.rs"), - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } fs::write(&dir.path().join("a/new.rs"), "new hello").unwrap(); - process_tasks(&mut vfs, 1); + process_tasks(&mut vfs, 2); match vfs.commit_changes().as_slice() { [VfsChange::AddFile { text, path, .. }] => { assert_eq!(text.as_str(), "new hello"); assert_eq!(path, "new.rs"); } - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } fs::rename(&dir.path().join("a/new.rs"), &dir.path().join("a/new1.rs")).unwrap(); - process_tasks(&mut vfs, 2); + process_tasks(&mut vfs, 4); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path: removed_path, .. @@ -124,10 +125,10 @@ fn test_vfs_works() -> std::io::Result<()> { } fs::remove_file(&dir.path().join("a/new1.rs")).unwrap(); - process_tasks(&mut vfs, 1); + process_tasks(&mut vfs, 2); match vfs.commit_changes().as_slice() { [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "new1.rs"), - _ => panic!("unexpected changes"), + xs => panic!("unexpected changes {:?}", xs), } match vfs.task_receiver().try_recv() { From f181e36a44d1998d3239f09365b16bfea50288a4 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sat, 19 Jan 2019 00:53:06 +0100 Subject: [PATCH 09/24] handle recursive watching ourselves --- Cargo.lock | 32 ++++++++++++++++++++ crates/ra_vfs/Cargo.toml | 1 + crates/ra_vfs/src/lib.rs | 4 +-- crates/ra_vfs/src/watcher.rs | 57 ++++++++++++++++++++++++++++-------- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d511469567..2d18122d88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -446,6 +446,18 @@ name = "glob" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "globset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "heck" version = "0.3.1" @@ -469,6 +481,23 @@ dependencies = [ "unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ignore" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "im" version = "12.3.0" @@ -1010,6 +1039,7 @@ dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)", + "ignore 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)", "ra_arena 0.1.0", @@ -1868,9 +1898,11 @@ dependencies = [ "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" +"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +"checksum ignore 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ad03ca67dc12474ecd91fdb94d758cbd20cb4e7a78ebe831df26a9b7511e1162" "checksum im 12.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0627d417829c1d763d602687634869f254fc79f7e22dea6c824dab993db857e4" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718" diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index 1a21557bf5..58196555c4 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -12,6 +12,7 @@ crossbeam-channel = "0.3.5" log = "0.4.6" # until https://github.com/passcod/notify/issues/169 is fixed notify = { git = "https://github.com/vemoo/notify/", branch = "v4-legacy" } +ignore = "0.4" drop_bomb = "0.1.0" thread_worker = { path = "../thread_worker" } diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 48a46d2106..ad40db340d 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -141,9 +141,7 @@ impl Vfs { }; res.worker.inp.send(task).unwrap(); if let Some(ref mut watcher) = res.watcher { - if let Err(e) = watcher.watch(path) { - log::warn!("could not watch \"{}\": {}", path.display(), e); - } + watcher.watch(path); } } let roots = res.roots.iter().map(|(id, _)| id).collect(); diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index dfbbcbfe67..013611e1ab 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -1,17 +1,17 @@ +use crate::io; +use crossbeam_channel::Sender; +use drop_bomb::DropBomb; +use ignore; +use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; use std::{ path::{Path, PathBuf}, - sync::mpsc, + sync::{mpsc, Arc, Mutex}, thread, time::Duration, }; -use crate::io; -use crossbeam_channel::Sender; -use drop_bomb::DropBomb; -use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; - pub struct Watcher { - watcher: RecommendedWatcher, + watcher: Arc>, thread: thread::JoinHandle<()>, bomb: DropBomb, } @@ -24,9 +24,10 @@ pub enum WatcherChange { Rescan, } -fn send_change_events( +fn handle_change_event( ev: DebouncedEvent, sender: &Sender, + watcher: &Arc>, ) -> Result<(), Box> { match ev { DebouncedEvent::NoticeWrite(_) @@ -38,6 +39,9 @@ fn send_change_events( sender.send(io::Task::HandleChange(WatcherChange::Rescan))?; } DebouncedEvent::Create(path) => { + if path.is_dir() { + watch_recursive(watcher, &path); + } sender.send(io::Task::HandleChange(WatcherChange::Create(path)))?; } DebouncedEvent::Write(path) => { @@ -58,17 +62,45 @@ fn send_change_events( Ok(()) } +fn watch_one(watcher: &mut RecommendedWatcher, path: &Path) { + match watcher.watch(path, RecursiveMode::NonRecursive) { + Ok(()) => log::debug!("watching \"{}\"", path.display()), + Err(e) => log::warn!("could not watch \"{}\": {}", path.display(), e), + } +} + +fn watch_recursive(watcher: &Arc>, path: &Path) { + log::debug!("watch_recursive \"{}\"", path.display()); + let mut w = watcher.lock().unwrap(); + // TODO it seems path itself isn't checked against ignores + // check if path should be ignored before walking it + for res in ignore::Walk::new(path) { + match res { + Ok(entry) => { + if entry.path().is_dir() { + watch_one(&mut w, entry.path()); + } + } + Err(e) => log::warn!("watcher error: {}", e), + } + } +} + impl Watcher { pub(crate) fn start( output_sender: Sender, ) -> Result> { let (input_sender, input_receiver) = mpsc::channel(); - let watcher = notify::watcher(input_sender, Duration::from_millis(250))?; + let watcher = Arc::new(Mutex::new(notify::watcher( + input_sender, + Duration::from_millis(250), + )?)); + let w = watcher.clone(); let thread = thread::spawn(move || { input_receiver .into_iter() // forward relevant events only - .try_for_each(|change| send_change_events(change, &output_sender)) + .try_for_each(|change| handle_change_event(change, &output_sender, &w)) .unwrap() }); Ok(Watcher { @@ -78,9 +110,8 @@ impl Watcher { }) } - pub fn watch(&mut self, root: impl AsRef) -> Result<(), Box> { - self.watcher.watch(root, RecursiveMode::Recursive)?; - Ok(()) + pub fn watch(&mut self, root: impl AsRef) { + watch_recursive(&self.watcher, root.as_ref()); } pub fn shutdown(mut self) -> thread::Result<()> { From fb1d748a2c49597934337432a78be2a5a098ca0e Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sat, 19 Jan 2019 01:15:22 +0100 Subject: [PATCH 10/24] actually drop watcher, use parking_lot::Mutex --- Cargo.lock | 1 + crates/ra_vfs/Cargo.toml | 1 + crates/ra_vfs/src/watcher.rs | 26 +++++++++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d18122d88..9cd0b3d9f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1042,6 +1042,7 @@ dependencies = [ "ignore 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)", + "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "ra_arena 0.1.0", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index 58196555c4..b77b7139fd 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -14,6 +14,7 @@ log = "0.4.6" notify = { git = "https://github.com/vemoo/notify/", branch = "v4-legacy" } ignore = "0.4" drop_bomb = "0.1.0" +parking_lot = "0.7.0" thread_worker = { path = "../thread_worker" } ra_arena = { path = "../ra_arena" } diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index 013611e1ab..9d552f8860 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -3,15 +3,16 @@ use crossbeam_channel::Sender; use drop_bomb::DropBomb; use ignore; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; +use parking_lot::Mutex; use std::{ path::{Path, PathBuf}, - sync::{mpsc, Arc, Mutex}, + sync::{mpsc, Arc}, thread, time::Duration, }; pub struct Watcher { - watcher: Arc>, + watcher: Arc>>, thread: thread::JoinHandle<()>, bomb: DropBomb, } @@ -27,7 +28,7 @@ pub enum WatcherChange { fn handle_change_event( ev: DebouncedEvent, sender: &Sender, - watcher: &Arc>, + watcher: &Arc>>, ) -> Result<(), Box> { match ev { DebouncedEvent::NoticeWrite(_) @@ -69,16 +70,23 @@ fn watch_one(watcher: &mut RecommendedWatcher, path: &Path) { } } -fn watch_recursive(watcher: &Arc>, path: &Path) { +fn watch_recursive(watcher: &Arc>>, path: &Path) { log::debug!("watch_recursive \"{}\"", path.display()); - let mut w = watcher.lock().unwrap(); + let mut watcher = watcher.lock(); + let mut watcher = match *watcher { + Some(ref mut watcher) => watcher, + None => { + // watcher has been dropped + return; + } + }; // TODO it seems path itself isn't checked against ignores // check if path should be ignored before walking it for res in ignore::Walk::new(path) { match res { Ok(entry) => { if entry.path().is_dir() { - watch_one(&mut w, entry.path()); + watch_one(&mut watcher, entry.path()); } } Err(e) => log::warn!("watcher error: {}", e), @@ -91,10 +99,10 @@ impl Watcher { output_sender: Sender, ) -> Result> { let (input_sender, input_receiver) = mpsc::channel(); - let watcher = Arc::new(Mutex::new(notify::watcher( + let watcher = Arc::new(Mutex::new(Some(notify::watcher( input_sender, Duration::from_millis(250), - )?)); + )?))); let w = watcher.clone(); let thread = thread::spawn(move || { input_receiver @@ -116,7 +124,7 @@ impl Watcher { pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); - drop(self.watcher); + drop(self.watcher.lock().take()); let res = self.thread.join(); match &res { Ok(()) => log::info!("... Watcher terminated with ok"), From eacf7aeb42d7ba54c305664773e77eb592b51b99 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sat, 19 Jan 2019 22:28:51 +0100 Subject: [PATCH 11/24] ignore check event dir for ignore, cleanup tests --- crates/ra_vfs/src/watcher.rs | 61 ++++++++++++---- crates/ra_vfs/tests/vfs.rs | 133 ++++++++++++++++++++++------------- 2 files changed, 130 insertions(+), 64 deletions(-) diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index 9d552f8860..f0ef9bc0ea 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -1,7 +1,7 @@ use crate::io; use crossbeam_channel::Sender; use drop_bomb::DropBomb; -use ignore; +use ignore::{gitignore::Gitignore, Walk}; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; use parking_lot::Mutex; use std::{ @@ -40,8 +40,11 @@ fn handle_change_event( sender.send(io::Task::HandleChange(WatcherChange::Rescan))?; } DebouncedEvent::Create(path) => { - if path.is_dir() { - watch_recursive(watcher, &path); + // we have to check if `path` is ignored because Walk iterator doesn't check it + // also childs are only ignored if they match a pattern + // (see `matched` vs `matched_path_or_any_parents` in `Gitignore`) + if path.is_dir() && !should_ignore_dir(&path) { + watch_recursive(watcher, &path, Some(sender)); } sender.send(io::Task::HandleChange(WatcherChange::Create(path)))?; } @@ -63,15 +66,18 @@ fn handle_change_event( Ok(()) } -fn watch_one(watcher: &mut RecommendedWatcher, path: &Path) { - match watcher.watch(path, RecursiveMode::NonRecursive) { - Ok(()) => log::debug!("watching \"{}\"", path.display()), - Err(e) => log::warn!("could not watch \"{}\": {}", path.display(), e), +fn watch_one(watcher: &mut RecommendedWatcher, dir: &Path) { + match watcher.watch(dir, RecursiveMode::NonRecursive) { + Ok(()) => log::debug!("watching \"{}\"", dir.display()), + Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), } } -fn watch_recursive(watcher: &Arc>>, path: &Path) { - log::debug!("watch_recursive \"{}\"", path.display()); +fn watch_recursive( + watcher: &Arc>>, + dir: &Path, + sender: Option<&Sender>, +) { let mut watcher = watcher.lock(); let mut watcher = match *watcher { Some(ref mut watcher) => watcher, @@ -80,20 +86,47 @@ fn watch_recursive(watcher: &Arc>>, path: &Path return; } }; - // TODO it seems path itself isn't checked against ignores - // check if path should be ignored before walking it - for res in ignore::Walk::new(path) { + for res in Walk::new(dir) { match res { Ok(entry) => { if entry.path().is_dir() { watch_one(&mut watcher, entry.path()); } + if let Some(sender) = sender { + // emit as create because we haven't seen it yet + if let Err(e) = sender.send(io::Task::HandleChange(WatcherChange::Create( + entry.path().to_path_buf(), + ))) { + log::warn!("watcher error: {}", e) + } + } } Err(e) => log::warn!("watcher error: {}", e), } } } +fn should_ignore_dir(dir: &Path) -> bool { + let mut parent = dir; + loop { + parent = match parent.parent() { + Some(p) => p, + None => break, + }; + let gitignore = parent.join(".gitignore"); + if gitignore.exists() { + let gitignore = Gitignore::new(gitignore).0; + if gitignore.matched_path_or_any_parents(dir, true).is_ignore() { + log::debug!("ignored {}", dir.display()); + return true; + } + } + } + false +} + +const WATCHER_DELAY: Duration = Duration::from_millis(250); + impl Watcher { pub(crate) fn start( output_sender: Sender, @@ -101,7 +134,7 @@ impl Watcher { let (input_sender, input_receiver) = mpsc::channel(); let watcher = Arc::new(Mutex::new(Some(notify::watcher( input_sender, - Duration::from_millis(250), + WATCHER_DELAY, )?))); let w = watcher.clone(); let thread = thread::spawn(move || { @@ -119,7 +152,7 @@ impl Watcher { } pub fn watch(&mut self, root: impl AsRef) { - watch_recursive(&self.watcher, root.as_ref()); + watch_recursive(&self.watcher, root.as_ref(), None); } pub fn shutdown(mut self) -> thread::Result<()> { diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 8266a0bd5b..71b25a5c96 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -11,9 +11,21 @@ fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { } } +macro_rules! assert_match { + ($x:expr, $pat:pat) => { + assert_match!($x, $pat, assert!(true)) + }; + ($x:expr, $pat:pat, $assert:expr) => { + match $x { + $pat => $assert, + x => assert!(false, "Expected {}, got {:?}", stringify!($pat), x), + }; + }; +} + #[test] fn test_vfs_works() -> std::io::Result<()> { - // Logger::with_str("debug").start().unwrap(); + // Logger::with_str("vfs=debug,ra_vfs=debug").start().unwrap(); let files = [ ("a/foo.rs", "hello"), @@ -21,13 +33,16 @@ fn test_vfs_works() -> std::io::Result<()> { ("a/b/baz.rs", "nested hello"), ]; - let dir = tempdir()?; + let dir = tempdir().unwrap(); for (path, text) in files.iter() { let file_path = dir.path().join(path); - fs::create_dir_all(file_path.parent().unwrap())?; + fs::create_dir_all(file_path.parent().unwrap()).unwrap(); fs::write(file_path, text)? } + let gitignore = dir.path().join("a/.gitignore"); + fs::write(gitignore, "/target").unwrap(); + let a_root = dir.path().join("a"); let b_root = dir.path().join("a/b"); @@ -62,79 +77,97 @@ fn test_vfs_works() -> std::io::Result<()> { } fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); - // 2 tasks per watcher change, first for HandleChange then for LoadChange + // 2 tasks per change, HandleChange and then LoadChange process_tasks(&mut vfs, 2); - match vfs.commit_changes().as_slice() { - [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), - xs => panic!("unexpected changes {:?}", xs), - } + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::ChangeFile { text, .. }], + assert_eq!(text.as_str(), "quux") + ); vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); - match vfs.commit_changes().as_slice() { - [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "m"), - xs => panic!("unexpected changes {:?}", xs), - } + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::ChangeFile { text, .. }], + assert_eq!(text.as_str(), "m") + ); // removing overlay restores data on disk vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); - match vfs.commit_changes().as_slice() { - [VfsChange::ChangeFile { text, .. }] => assert_eq!(text.as_str(), "quux"), - xs => panic!("unexpected changes {:?}", xs), - } + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::ChangeFile { text, .. }], + assert_eq!(text.as_str(), "quux") + ); vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); - match vfs.commit_changes().as_slice() { - [VfsChange::AddFile { text, path, .. }] => { + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::AddFile { text, path, .. }], + { assert_eq!(text.as_str(), "spam"); assert_eq!(path, "spam.rs"); } - xs => panic!("unexpected changes {:?}", xs), - } + ); vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs")); - match vfs.commit_changes().as_slice() { - [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "spam.rs"), - xs => panic!("unexpected changes {:?}", xs), - } + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::RemoveFile { path, .. }], + assert_eq!(path, "spam.rs") + ); - fs::write(&dir.path().join("a/new.rs"), "new hello").unwrap(); - process_tasks(&mut vfs, 2); - match vfs.commit_changes().as_slice() { - [VfsChange::AddFile { text, path, .. }] => { - assert_eq!(text.as_str(), "new hello"); - assert_eq!(path, "new.rs"); - } - xs => panic!("unexpected changes {:?}", xs), - } - - fs::rename(&dir.path().join("a/new.rs"), &dir.path().join("a/new1.rs")).unwrap(); + fs::create_dir_all(dir.path().join("a/c")).unwrap(); + fs::write(dir.path().join("a/c/new.rs"), "new hello").unwrap(); process_tasks(&mut vfs, 4); - match vfs.commit_changes().as_slice() { + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::AddFile { text, path, .. }], + { + assert_eq!(text.as_str(), "new hello"); + assert_eq!(path, "c/new.rs"); + } + ); + + fs::rename( + &dir.path().join("a/c/new.rs"), + &dir.path().join("a/c/new1.rs"), + ) + .unwrap(); + process_tasks(&mut vfs, 4); + assert_match!( + vfs.commit_changes().as_slice(), [VfsChange::RemoveFile { path: removed_path, .. }, VfsChange::AddFile { text, path: added_path, .. - }] => { - assert_eq!(removed_path, "new.rs"); - assert_eq!(added_path, "new1.rs"); + }], + { + assert_eq!(removed_path, "c/new.rs"); + assert_eq!(added_path, "c/new1.rs"); assert_eq!(text.as_str(), "new hello"); } - xs => panic!("unexpected changes {:?}", xs), - } + ); - fs::remove_file(&dir.path().join("a/new1.rs")).unwrap(); + fs::remove_file(&dir.path().join("a/c/new1.rs")).unwrap(); process_tasks(&mut vfs, 2); - match vfs.commit_changes().as_slice() { - [VfsChange::RemoveFile { path, .. }] => assert_eq!(path, "new1.rs"), - xs => panic!("unexpected changes {:?}", xs), - } + assert_match!( + vfs.commit_changes().as_slice(), + [VfsChange::RemoveFile { path, .. }], + assert_eq!(path, "c/new1.rs") + ); - match vfs.task_receiver().try_recv() { - Err(crossbeam_channel::TryRecvError::Empty) => (), - res => panic!("unexpected {:?}", res), - } + fs::create_dir_all(dir.path().join("a/target")).unwrap(); + // should be ignored + fs::write(&dir.path().join("a/target/new.rs"), "ignore me").unwrap(); + process_tasks(&mut vfs, 1); // 1 task because no LoadChange will happen, just HandleChange for dir creation + + assert_match!( + vfs.task_receiver().try_recv(), + Err(crossbeam_channel::TryRecvError::Empty) + ); vfs.shutdown().unwrap(); Ok(()) From f88355ccb5e8ea2381e13eabcdb64880e757aff1 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Sun, 20 Jan 2019 22:13:21 +0100 Subject: [PATCH 12/24] refactor, put watcher with `io::Worker` use `RootFilter` to filter recursive watches untested --- Cargo.lock | 32 ---------- crates/ra_vfs/Cargo.toml | 1 - crates/ra_vfs/src/io.rs | 107 +++++++++++++++++++++++++------ crates/ra_vfs/src/lib.rs | 100 +++++++++++++++++------------ crates/ra_vfs/src/watcher.rs | 120 ++++++++++++----------------------- 5 files changed, 188 insertions(+), 172 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cd0b3d9f3..41e84b7e94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -446,18 +446,6 @@ name = "glob" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "globset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "heck" version = "0.3.1" @@ -481,23 +469,6 @@ dependencies = [ "unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ignore" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "im" version = "12.3.0" @@ -1039,7 +1010,6 @@ dependencies = [ "crossbeam-channel 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)", - "ignore 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1899,11 +1869,9 @@ dependencies = [ "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" "checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" -"checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -"checksum ignore 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ad03ca67dc12474ecd91fdb94d758cbd20cb4e7a78ebe831df26a9b7511e1162" "checksum im 12.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0627d417829c1d763d602687634869f254fc79f7e22dea6c824dab993db857e4" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718" diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index b77b7139fd..c7c4769b3d 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -12,7 +12,6 @@ crossbeam-channel = "0.3.5" log = "0.4.6" # until https://github.com/passcod/notify/issues/169 is fixed notify = { git = "https://github.com/vemoo/notify/", branch = "v4-legacy" } -ignore = "0.4" drop_bomb = "0.1.0" parking_lot = "0.7.0" diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index e5d5c6463e..25acec9b1d 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -1,13 +1,20 @@ use std::{ fmt, fs, path::{Path, PathBuf}, + sync::Arc, + thread, }; +use crossbeam_channel::{Receiver, Sender}; +use parking_lot::Mutex; use relative_path::RelativePathBuf; use thread_worker::WorkerHandle; use walkdir::{DirEntry, WalkDir}; -use crate::{has_rs_extension, watcher::WatcherChange, VfsRoot}; +use crate::{ + watcher::{Watcher, WatcherChange}, + VfsRoot, +}; pub(crate) enum Task { AddRoot { @@ -17,6 +24,10 @@ pub(crate) enum Task { }, HandleChange(WatcherChange), LoadChange(WatcherChange), + Watch { + dir: PathBuf, + filter: Box bool + Send>, + }, } #[derive(Debug)] @@ -35,7 +46,8 @@ pub enum WatcherChangeData { pub enum TaskResult { AddRoot(AddRootResult), HandleChange(WatcherChange), - LoadChange(Option), + LoadChange(WatcherChangeData), + NoOp, } impl fmt::Debug for TaskResult { @@ -44,21 +56,74 @@ impl fmt::Debug for TaskResult { } } -pub(crate) type Worker = thread_worker::Worker; - -pub(crate) fn start() -> (Worker, WorkerHandle) { - thread_worker::spawn("vfs", 128, |input_receiver, output_sender| { - input_receiver - .into_iter() - .map(handle_task) - .try_for_each(|it| output_sender.send(it)) - .unwrap() - }) +pub(crate) struct Worker { + worker: thread_worker::Worker, + worker_handle: WorkerHandle, + watcher: Arc>>, } -fn handle_task(task: Task) -> TaskResult { +impl Worker { + pub(crate) fn start() -> Worker { + let watcher = Arc::new(Mutex::new(None)); + let watcher_clone = watcher.clone(); + let (worker, worker_handle) = + thread_worker::spawn("vfs", 128, move |input_receiver, output_sender| { + let res = input_receiver + .into_iter() + .map(|t| handle_task(t, &watcher_clone)) + .try_for_each(|it| output_sender.send(it)); + res.unwrap() + }); + match Watcher::start(worker.inp.clone()) { + Ok(w) => { + watcher.lock().replace(w); + } + Err(e) => log::error!("could not start watcher: {}", e), + }; + Worker { + worker, + worker_handle, + watcher, + } + } + + pub(crate) fn sender(&self) -> &Sender { + &self.worker.inp + } + + pub(crate) fn receiver(&self) -> &Receiver { + &self.worker.out + } + + pub(crate) fn shutdown(self) -> thread::Result<()> { + if let Some(watcher) = self.watcher.lock().take() { + let _ = watcher.shutdown(); + } + self.worker_handle.shutdown() + } +} + +fn watch( + watcher: &Arc>>, + dir: &Path, + filter_entry: impl Fn(&DirEntry) -> bool, + emit_for_existing: bool, +) { + let mut watcher = watcher.lock(); + let watcher = match *watcher { + Some(ref mut w) => w, + None => { + // watcher dropped or couldn't start + return; + } + }; + watcher.watch_recursive(dir, filter_entry, emit_for_existing) +} + +fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult { match task { Task::AddRoot { root, path, filter } => { + watch(watcher, &path, &*filter, false); log::debug!("loading {} ...", path.as_path().display()); let files = load_root(path.as_path(), &*filter); log::debug!("... loaded {}", path.as_path().display()); @@ -70,8 +135,14 @@ fn handle_task(task: Task) -> TaskResult { } Task::LoadChange(change) => { log::debug!("loading {:?} ...", change); - let data = load_change(change); - TaskResult::LoadChange(data) + match load_change(change) { + Some(data) => TaskResult::LoadChange(data), + None => TaskResult::NoOp, + } + } + Task::Watch { dir, filter } => { + watch(watcher, &dir, &*filter, true); + TaskResult::NoOp } } } @@ -90,9 +161,6 @@ fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePa continue; } let path = entry.path(); - if !has_rs_extension(path) { - continue; - } let text = match fs::read_to_string(path) { Ok(text) => text, Err(e) => { @@ -109,6 +177,9 @@ fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePa fn load_change(change: WatcherChange) -> Option { let data = match change { WatcherChange::Create(path) => { + if path.is_dir() { + return None; + } let text = match fs::read_to_string(&path) { Ok(text) => text, Err(e) => { diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index ad40db340d..1961808904 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -20,7 +20,7 @@ use std::{ cmp::Reverse, ffi::OsStr, fmt, fs, mem, - path::{Path, PathBuf}, + path::{Component, Path, PathBuf}, sync::Arc, thread, }; @@ -29,30 +29,37 @@ use crossbeam_channel::Receiver; use ra_arena::{impl_arena_id, Arena, RawId}; use relative_path::RelativePathBuf; use rustc_hash::{FxHashMap, FxHashSet}; -use thread_worker::WorkerHandle; use walkdir::DirEntry; pub use crate::io::TaskResult as VfsTask; -pub use crate::watcher::{Watcher, WatcherChange}; +pub use crate::watcher::WatcherChange; /// `RootFilter` is a predicate that checks if a file can belong to a root. If /// several filters match a file (nested dirs), the most nested one wins. -struct RootFilter { +pub(crate) struct RootFilter { root: PathBuf, - file_filter: fn(&Path) -> bool, + filter: fn(RootEntry) -> bool, +} + +pub(crate) struct RootEntry<'a, 'b> { + root: &'a Path, + path: &'b Path, } impl RootFilter { fn new(root: PathBuf) -> RootFilter { RootFilter { root, - file_filter: has_rs_extension, + filter: default_filter, } } /// Check if this root can contain `path`. NB: even if this returns /// true, the `path` might actually be conained in some nested root. - fn can_contain(&self, path: &Path) -> Option { - if !(self.file_filter)(path) { + pub(crate) fn can_contain(&self, path: &Path) -> Option { + if !(self.filter)(RootEntry { + root: &self.root, + path, + }) { return None; } let path = path.strip_prefix(&self.root).ok()?; @@ -60,8 +67,17 @@ impl RootFilter { } } -pub(crate) fn has_rs_extension(p: &Path) -> bool { - p.extension() == Some(OsStr::new("rs")) +pub(crate) fn default_filter(entry: RootEntry) -> bool { + if entry.path.is_dir() { + // first component relative to root is "target" + entry + .path + .strip_prefix(entry.root) + .map(|p| p.components().next() != Some(Component::Normal(OsStr::new("target")))) + .unwrap_or(false) + } else { + entry.path.extension() == Some(OsStr::new("rs")) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -80,13 +96,11 @@ struct VfsFileData { } pub struct Vfs { - roots: Arena, + roots: Arena>, files: Arena, root2files: FxHashMap>, pending_changes: Vec, worker: io::Worker, - worker_handle: WorkerHandle, - watcher: Option, } impl fmt::Debug for Vfs { @@ -97,41 +111,35 @@ impl fmt::Debug for Vfs { impl Vfs { pub fn new(mut roots: Vec) -> (Vfs, Vec) { - let (worker, worker_handle) = io::start(); - - let watcher = match Watcher::start(worker.inp.clone()) { - Ok(watcher) => Some(watcher), - Err(e) => { - log::error!("could not start watcher: {}", e); - None - } - }; + let worker = io::Worker::start(); let mut res = Vfs { roots: Arena::default(), files: Arena::default(), root2files: FxHashMap::default(), worker, - worker_handle, - watcher, pending_changes: Vec::new(), }; // A hack to make nesting work. roots.sort_by_key(|it| Reverse(it.as_os_str().len())); for (i, path) in roots.iter().enumerate() { - let root = res.roots.alloc(RootFilter::new(path.clone())); + let root_filter = Arc::new(RootFilter::new(path.clone())); + + let root = res.roots.alloc(root_filter.clone()); res.root2files.insert(root, Default::default()); + let nested = roots[..i] .iter() .filter(|it| it.starts_with(path)) .map(|it| it.clone()) .collect::>(); + let filter = move |entry: &DirEntry| { - if entry.file_type().is_file() { - has_rs_extension(entry.path()) + if entry.file_type().is_dir() && nested.iter().any(|it| it == entry.path()) { + false } else { - nested.iter().all(|it| it != entry.path()) + root_filter.can_contain(entry.path()).is_some() } }; let task = io::Task::AddRoot { @@ -139,10 +147,7 @@ impl Vfs { path: path.clone(), filter: Box::new(filter), }; - res.worker.inp.send(task).unwrap(); - if let Some(ref mut watcher) = res.watcher { - watcher.watch(path); - } + res.worker.sender().send(task).unwrap(); } let roots = res.roots.iter().map(|(id, _)| id).collect(); (res, roots) @@ -194,7 +199,7 @@ impl Vfs { } pub fn task_receiver(&self) -> &Receiver { - &self.worker.out + self.worker.receiver() } pub fn handle_task(&mut self, task: io::TaskResult) { @@ -225,19 +230,35 @@ impl Vfs { self.pending_changes.push(change); } io::TaskResult::HandleChange(change) => match &change { + watcher::WatcherChange::Create(path) if path.is_dir() => { + if let Some((root, _path, _file)) = self.find_root(&path) { + let root_filter = self.roots[root].clone(); + let filter = + move |entry: &DirEntry| root_filter.can_contain(entry.path()).is_some(); + self.worker + .sender() + .send(io::Task::Watch { + dir: path.to_path_buf(), + filter: Box::new(filter), + }) + .unwrap() + } + } watcher::WatcherChange::Create(path) | watcher::WatcherChange::Remove(path) | watcher::WatcherChange::Write(path) => { if self.should_handle_change(&path) { - self.worker.inp.send(io::Task::LoadChange(change)).unwrap() + self.worker + .sender() + .send(io::Task::LoadChange(change)) + .unwrap() } } watcher::WatcherChange::Rescan => { // TODO we should reload all files } }, - io::TaskResult::LoadChange(None) => {} - io::TaskResult::LoadChange(Some(change)) => match change { + io::TaskResult::LoadChange(change) => match change { io::WatcherChangeData::Create { path, text } | io::WatcherChangeData::Write { path, text } => { if let Some((root, path, file)) = self.find_root(&path) { @@ -256,6 +277,7 @@ impl Vfs { } } }, + io::TaskResult::NoOp => {} } } @@ -359,11 +381,7 @@ impl Vfs { /// Sutdown the VFS and terminate the background watching thread. pub fn shutdown(self) -> thread::Result<()> { - if let Some(watcher) = self.watcher { - let _ = watcher.shutdown(); - } - let _ = self.worker.shutdown(); - self.worker_handle.shutdown() + self.worker.shutdown() } fn add_file( diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index f0ef9bc0ea..d8c35f2a30 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -1,20 +1,20 @@ use crate::io; use crossbeam_channel::Sender; use drop_bomb::DropBomb; -use ignore::{gitignore::Gitignore, Walk}; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; -use parking_lot::Mutex; use std::{ path::{Path, PathBuf}, - sync::{mpsc, Arc}, + sync::mpsc, thread, time::Duration, }; +use walkdir::{DirEntry, WalkDir}; -pub struct Watcher { - watcher: Arc>>, +pub(crate) struct Watcher { + watcher: RecommendedWatcher, thread: thread::JoinHandle<()>, bomb: DropBomb, + sender: Sender, } #[derive(Debug)] @@ -28,7 +28,6 @@ pub enum WatcherChange { fn handle_change_event( ev: DebouncedEvent, sender: &Sender, - watcher: &Arc>>, ) -> Result<(), Box> { match ev { DebouncedEvent::NoticeWrite(_) @@ -40,12 +39,6 @@ fn handle_change_event( sender.send(io::Task::HandleChange(WatcherChange::Rescan))?; } DebouncedEvent::Create(path) => { - // we have to check if `path` is ignored because Walk iterator doesn't check it - // also childs are only ignored if they match a pattern - // (see `matched` vs `matched_path_or_any_parents` in `Gitignore`) - if path.is_dir() && !should_ignore_dir(&path) { - watch_recursive(watcher, &path, Some(sender)); - } sender.send(io::Task::HandleChange(WatcherChange::Create(path)))?; } DebouncedEvent::Write(path) => { @@ -66,65 +59,6 @@ fn handle_change_event( Ok(()) } -fn watch_one(watcher: &mut RecommendedWatcher, dir: &Path) { - match watcher.watch(dir, RecursiveMode::NonRecursive) { - Ok(()) => log::debug!("watching \"{}\"", dir.display()), - Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), - } -} - -fn watch_recursive( - watcher: &Arc>>, - dir: &Path, - sender: Option<&Sender>, -) { - let mut watcher = watcher.lock(); - let mut watcher = match *watcher { - Some(ref mut watcher) => watcher, - None => { - // watcher has been dropped - return; - } - }; - for res in Walk::new(dir) { - match res { - Ok(entry) => { - if entry.path().is_dir() { - watch_one(&mut watcher, entry.path()); - } - if let Some(sender) = sender { - // emit as create because we haven't seen it yet - if let Err(e) = sender.send(io::Task::HandleChange(WatcherChange::Create( - entry.path().to_path_buf(), - ))) { - log::warn!("watcher error: {}", e) - } - } - } - Err(e) => log::warn!("watcher error: {}", e), - } - } -} - -fn should_ignore_dir(dir: &Path) -> bool { - let mut parent = dir; - loop { - parent = match parent.parent() { - Some(p) => p, - None => break, - }; - let gitignore = parent.join(".gitignore"); - if gitignore.exists() { - let gitignore = Gitignore::new(gitignore).0; - if gitignore.matched_path_or_any_parents(dir, true).is_ignore() { - log::debug!("ignored {}", dir.display()); - return true; - } - } - } - false -} - const WATCHER_DELAY: Duration = Duration::from_millis(250); impl Watcher { @@ -132,32 +66,58 @@ impl Watcher { output_sender: Sender, ) -> Result> { let (input_sender, input_receiver) = mpsc::channel(); - let watcher = Arc::new(Mutex::new(Some(notify::watcher( - input_sender, - WATCHER_DELAY, - )?))); - let w = watcher.clone(); + let watcher = notify::watcher(input_sender, WATCHER_DELAY)?; + let sender = output_sender.clone(); let thread = thread::spawn(move || { input_receiver .into_iter() // forward relevant events only - .try_for_each(|change| handle_change_event(change, &output_sender, &w)) + .try_for_each(|change| handle_change_event(change, &output_sender)) .unwrap() }); Ok(Watcher { watcher, thread, + sender, bomb: DropBomb::new(format!("Watcher was not shutdown")), }) } - pub fn watch(&mut self, root: impl AsRef) { - watch_recursive(&self.watcher, root.as_ref(), None); + pub fn watch_recursive( + &mut self, + dir: &Path, + filter_entry: impl Fn(&DirEntry) -> bool, + emit_for_existing: bool, + ) { + for res in WalkDir::new(dir).into_iter().filter_entry(filter_entry) { + match res { + Ok(entry) => { + if entry.path().is_dir() { + match self.watcher.watch(dir, RecursiveMode::NonRecursive) { + Ok(()) => log::debug!("watching \"{}\"", dir.display()), + Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), + } + } + if emit_for_existing { + // emit as create because we haven't seen it yet + if let Err(e) = + self.sender + .send(io::Task::HandleChange(WatcherChange::Create( + entry.path().to_path_buf(), + ))) + { + log::warn!("watcher error: {}", e) + } + } + } + Err(e) => log::warn!("watcher error: {}", e), + } + } } pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); - drop(self.watcher.lock().take()); + drop(self.watcher); let res = self.thread.join(); match &res { Ok(()) => log::info!("... Watcher terminated with ok"), From 7f7c4e7465f58cdbfdaaf232d571960f1b754b7c Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 21 Jan 2019 18:37:46 +0100 Subject: [PATCH 13/24] do not emit create for directory again --- crates/ra_vfs/src/io.rs | 7 ++++--- crates/ra_vfs/src/watcher.rs | 4 ++-- crates/ra_vfs/tests/vfs.rs | 3 --- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 25acec9b1d..a74222c02e 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -68,11 +68,11 @@ impl Worker { let watcher_clone = watcher.clone(); let (worker, worker_handle) = thread_worker::spawn("vfs", 128, move |input_receiver, output_sender| { - let res = input_receiver + input_receiver .into_iter() .map(|t| handle_task(t, &watcher_clone)) - .try_for_each(|it| output_sender.send(it)); - res.unwrap() + .try_for_each(|it| output_sender.send(it)) + .unwrap() }); match Watcher::start(worker.inp.clone()) { Ok(w) => { @@ -99,6 +99,7 @@ impl Worker { if let Some(watcher) = self.watcher.lock().take() { let _ = watcher.shutdown(); } + let _ = self.worker.shutdown(); self.worker_handle.shutdown() } } diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/watcher.rs index d8c35f2a30..6069358913 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/watcher.rs @@ -87,7 +87,7 @@ impl Watcher { &mut self, dir: &Path, filter_entry: impl Fn(&DirEntry) -> bool, - emit_for_existing: bool, + emit_for_contents: bool, ) { for res in WalkDir::new(dir).into_iter().filter_entry(filter_entry) { match res { @@ -98,7 +98,7 @@ impl Watcher { Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), } } - if emit_for_existing { + if emit_for_contents && entry.depth() > 0 { // emit as create because we haven't seen it yet if let Err(e) = self.sender diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 71b25a5c96..b18ea74a32 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -40,9 +40,6 @@ fn test_vfs_works() -> std::io::Result<()> { fs::write(file_path, text)? } - let gitignore = dir.path().join("a/.gitignore"); - fs::write(gitignore, "/target").unwrap(); - let a_root = dir.path().join("a"); let b_root = dir.path().join("a/b"); From 277e0f1baa21b8f3e5b040b78ce2bd6beca6cd7c Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 21 Jan 2019 18:59:54 +0100 Subject: [PATCH 14/24] move watcher to io module --- crates/ra_vfs/src/{io.rs => io/mod.rs} | 10 ++++--- crates/ra_vfs/src/{ => io}/watcher.rs | 14 +++++----- crates/ra_vfs/src/lib.rs | 36 ++++++++++++-------------- 3 files changed, 29 insertions(+), 31 deletions(-) rename crates/ra_vfs/src/{io.rs => io/mod.rs} (97%) rename crates/ra_vfs/src/{ => io}/watcher.rs (100%) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io/mod.rs similarity index 97% rename from crates/ra_vfs/src/io.rs rename to crates/ra_vfs/src/io/mod.rs index a74222c02e..6d5af76904 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io/mod.rs @@ -11,10 +11,11 @@ use relative_path::RelativePathBuf; use thread_worker::WorkerHandle; use walkdir::{DirEntry, WalkDir}; -use crate::{ - watcher::{Watcher, WatcherChange}, - VfsRoot, -}; +mod watcher; +use watcher::Watcher; +pub use watcher::WatcherChange; + +use crate::VfsRoot; pub(crate) enum Task { AddRoot { @@ -22,6 +23,7 @@ pub(crate) enum Task { path: PathBuf, filter: Box bool + Send>, }, + /// this variant should only be created by the watcher HandleChange(WatcherChange), LoadChange(WatcherChange), Watch { diff --git a/crates/ra_vfs/src/watcher.rs b/crates/ra_vfs/src/io/watcher.rs similarity index 100% rename from crates/ra_vfs/src/watcher.rs rename to crates/ra_vfs/src/io/watcher.rs index 6069358913..e332984772 100644 --- a/crates/ra_vfs/src/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -10,13 +10,6 @@ use std::{ }; use walkdir::{DirEntry, WalkDir}; -pub(crate) struct Watcher { - watcher: RecommendedWatcher, - thread: thread::JoinHandle<()>, - bomb: DropBomb, - sender: Sender, -} - #[derive(Debug)] pub enum WatcherChange { Create(PathBuf), @@ -61,6 +54,13 @@ fn handle_change_event( const WATCHER_DELAY: Duration = Duration::from_millis(250); +pub(crate) struct Watcher { + watcher: RecommendedWatcher, + thread: thread::JoinHandle<()>, + bomb: DropBomb, + sender: Sender, +} + impl Watcher { pub(crate) fn start( output_sender: Sender, diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 1961808904..5db0d86460 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -14,7 +14,6 @@ //! which are watched for changes. Typically, there will be a root for each //! Cargo package. mod io; -mod watcher; use std::{ cmp::Reverse, @@ -32,7 +31,7 @@ use rustc_hash::{FxHashMap, FxHashSet}; use walkdir::DirEntry; pub use crate::io::TaskResult as VfsTask; -pub use crate::watcher::WatcherChange; +use io::{Task, TaskResult, WatcherChange, WatcherChangeData, Worker}; /// `RootFilter` is a predicate that checks if a file can belong to a root. If /// several filters match a file (nested dirs), the most nested one wins. @@ -100,7 +99,7 @@ pub struct Vfs { files: Arena, root2files: FxHashMap>, pending_changes: Vec, - worker: io::Worker, + worker: Worker, } impl fmt::Debug for Vfs { @@ -204,7 +203,7 @@ impl Vfs { pub fn handle_task(&mut self, task: io::TaskResult) { match task { - io::TaskResult::AddRoot(task) => { + TaskResult::AddRoot(task) => { let mut files = Vec::new(); // While we were scanning the root in the backgound, a file might have // been open in the editor, so we need to account for that. @@ -229,38 +228,35 @@ impl Vfs { }; self.pending_changes.push(change); } - io::TaskResult::HandleChange(change) => match &change { - watcher::WatcherChange::Create(path) if path.is_dir() => { + TaskResult::HandleChange(change) => match &change { + WatcherChange::Create(path) if path.is_dir() => { if let Some((root, _path, _file)) = self.find_root(&path) { let root_filter = self.roots[root].clone(); let filter = move |entry: &DirEntry| root_filter.can_contain(entry.path()).is_some(); self.worker .sender() - .send(io::Task::Watch { + .send(Task::Watch { dir: path.to_path_buf(), filter: Box::new(filter), }) .unwrap() } } - watcher::WatcherChange::Create(path) - | watcher::WatcherChange::Remove(path) - | watcher::WatcherChange::Write(path) => { + WatcherChange::Create(path) + | WatcherChange::Remove(path) + | WatcherChange::Write(path) => { if self.should_handle_change(&path) { - self.worker - .sender() - .send(io::Task::LoadChange(change)) - .unwrap() + self.worker.sender().send(Task::LoadChange(change)).unwrap() } } - watcher::WatcherChange::Rescan => { + WatcherChange::Rescan => { // TODO we should reload all files } }, - io::TaskResult::LoadChange(change) => match change { - io::WatcherChangeData::Create { path, text } - | io::WatcherChangeData::Write { path, text } => { + TaskResult::LoadChange(change) => match change { + WatcherChangeData::Create { path, text } + | WatcherChangeData::Write { path, text } => { if let Some((root, path, file)) = self.find_root(&path) { if let Some(file) = file { self.do_change_file(file, text, false); @@ -269,7 +265,7 @@ impl Vfs { } } } - io::WatcherChangeData::Remove { path } => { + WatcherChangeData::Remove { path } => { if let Some((root, path, file)) = self.find_root(&path) { if let Some(file) = file { self.do_remove_file(root, path, file, false); @@ -277,7 +273,7 @@ impl Vfs { } } }, - io::TaskResult::NoOp => {} + TaskResult::NoOp => {} } } From 2a1afad3eda7d8c5635de6e7f524ed943cecc22b Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 21 Jan 2019 19:11:39 +0100 Subject: [PATCH 15/24] avoid boxing --- crates/ra_vfs/src/io/mod.rs | 45 ++++++++++++++++++++++++--------- crates/ra_vfs/src/io/watcher.rs | 16 +++++------- crates/ra_vfs/src/lib.rs | 17 +++---------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/crates/ra_vfs/src/io/mod.rs b/crates/ra_vfs/src/io/mod.rs index 6d5af76904..daac6c6f28 100644 --- a/crates/ra_vfs/src/io/mod.rs +++ b/crates/ra_vfs/src/io/mod.rs @@ -9,26 +9,27 @@ use crossbeam_channel::{Receiver, Sender}; use parking_lot::Mutex; use relative_path::RelativePathBuf; use thread_worker::WorkerHandle; -use walkdir::{DirEntry, WalkDir}; +use walkdir::WalkDir; mod watcher; use watcher::Watcher; pub use watcher::WatcherChange; -use crate::VfsRoot; +use crate::{RootFilter, VfsRoot}; pub(crate) enum Task { AddRoot { root: VfsRoot, path: PathBuf, - filter: Box bool + Send>, + root_filter: Arc, + nested_roots: Vec, }, /// this variant should only be created by the watcher HandleChange(WatcherChange), LoadChange(WatcherChange), Watch { dir: PathBuf, - filter: Box bool + Send>, + root_filter: Arc, }, } @@ -109,7 +110,7 @@ impl Worker { fn watch( watcher: &Arc>>, dir: &Path, - filter_entry: impl Fn(&DirEntry) -> bool, + filter_entry: &RootFilter, emit_for_existing: bool, ) { let mut watcher = watcher.lock(); @@ -125,10 +126,19 @@ fn watch( fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult { match task { - Task::AddRoot { root, path, filter } => { - watch(watcher, &path, &*filter, false); + Task::AddRoot { + root, + path, + root_filter, + nested_roots, + } => { + watch(watcher, &path, &*root_filter, false); log::debug!("loading {} ...", path.as_path().display()); - let files = load_root(path.as_path(), &*filter); + let files = load_root( + path.as_path(), + root_filter.as_ref(), + nested_roots.as_slice(), + ); log::debug!("... loaded {}", path.as_path().display()); TaskResult::AddRoot(AddRootResult { root, files }) } @@ -143,16 +153,27 @@ fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult None => TaskResult::NoOp, } } - Task::Watch { dir, filter } => { - watch(watcher, &dir, &*filter, true); + Task::Watch { dir, root_filter } => { + watch(watcher, &dir, root_filter.as_ref(), true); TaskResult::NoOp } } } -fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePathBuf, String)> { +fn load_root( + root: &Path, + root_filter: &RootFilter, + nested_roots: &[PathBuf], +) -> Vec<(RelativePathBuf, String)> { let mut res = Vec::new(); - for entry in WalkDir::new(root).into_iter().filter_entry(filter) { + for entry in WalkDir::new(root).into_iter().filter_entry(|entry| { + if entry.file_type().is_dir() && nested_roots.iter().any(|it| it == entry.path()) { + // do not load files of a nested root + false + } else { + root_filter.can_contain(entry.path()).is_some() + } + }) { let entry = match entry { Ok(entry) => entry, Err(e) => { diff --git a/crates/ra_vfs/src/io/watcher.rs b/crates/ra_vfs/src/io/watcher.rs index e332984772..5e9bc8ff3f 100644 --- a/crates/ra_vfs/src/io/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -1,4 +1,4 @@ -use crate::io; +use crate::{io, RootFilter}; use crossbeam_channel::Sender; use drop_bomb::DropBomb; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; @@ -8,7 +8,7 @@ use std::{ thread, time::Duration, }; -use walkdir::{DirEntry, WalkDir}; +use walkdir::WalkDir; #[derive(Debug)] pub enum WatcherChange { @@ -83,13 +83,11 @@ impl Watcher { }) } - pub fn watch_recursive( - &mut self, - dir: &Path, - filter_entry: impl Fn(&DirEntry) -> bool, - emit_for_contents: bool, - ) { - for res in WalkDir::new(dir).into_iter().filter_entry(filter_entry) { + pub fn watch_recursive(&mut self, dir: &Path, filter: &RootFilter, emit_for_contents: bool) { + for res in WalkDir::new(dir) + .into_iter() + .filter_entry(|entry| filter.can_contain(entry.path()).is_some()) + { match res { Ok(entry) => { if entry.path().is_dir() { diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 5db0d86460..1d0af6a098 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -28,7 +28,6 @@ use crossbeam_channel::Receiver; use ra_arena::{impl_arena_id, Arena, RawId}; use relative_path::RelativePathBuf; use rustc_hash::{FxHashMap, FxHashSet}; -use walkdir::DirEntry; pub use crate::io::TaskResult as VfsTask; use io::{Task, TaskResult, WatcherChange, WatcherChangeData, Worker}; @@ -128,23 +127,17 @@ impl Vfs { let root = res.roots.alloc(root_filter.clone()); res.root2files.insert(root, Default::default()); - let nested = roots[..i] + let nested_roots = roots[..i] .iter() .filter(|it| it.starts_with(path)) .map(|it| it.clone()) .collect::>(); - let filter = move |entry: &DirEntry| { - if entry.file_type().is_dir() && nested.iter().any(|it| it == entry.path()) { - false - } else { - root_filter.can_contain(entry.path()).is_some() - } - }; let task = io::Task::AddRoot { root, path: path.clone(), - filter: Box::new(filter), + root_filter, + nested_roots, }; res.worker.sender().send(task).unwrap(); } @@ -232,13 +225,11 @@ impl Vfs { WatcherChange::Create(path) if path.is_dir() => { if let Some((root, _path, _file)) = self.find_root(&path) { let root_filter = self.roots[root].clone(); - let filter = - move |entry: &DirEntry| root_filter.can_contain(entry.path()).is_some(); self.worker .sender() .send(Task::Watch { dir: path.to_path_buf(), - filter: Box::new(filter), + root_filter, }) .unwrap() } From eeed6cf53b9f6112329cc8a274dcf63bce887c50 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 21 Jan 2019 21:48:07 +0100 Subject: [PATCH 16/24] fix recursive watch --- crates/ra_vfs/src/io/watcher.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/ra_vfs/src/io/watcher.rs b/crates/ra_vfs/src/io/watcher.rs index 5e9bc8ff3f..b370c5fbcd 100644 --- a/crates/ra_vfs/src/io/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -91,9 +91,14 @@ impl Watcher { match res { Ok(entry) => { if entry.path().is_dir() { - match self.watcher.watch(dir, RecursiveMode::NonRecursive) { - Ok(()) => log::debug!("watching \"{}\"", dir.display()), - Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), + match self + .watcher + .watch(entry.path(), RecursiveMode::NonRecursive) + { + Ok(()) => log::debug!("watching \"{}\"", entry.path().display()), + Err(e) => { + log::warn!("could not watch \"{}\": {}", entry.path().display(), e) + } } } if emit_for_contents && entry.depth() > 0 { From 10a24cf649b4e136bb4f25cd295c2fb15125d71a Mon Sep 17 00:00:00 2001 From: Bernardo Date: Mon, 21 Jan 2019 22:12:34 +0100 Subject: [PATCH 17/24] simplify and optimize `RootFilter`by determining if is contained first --- crates/ra_vfs/src/lib.rs | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 1d0af6a098..f4447be43d 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -17,16 +17,15 @@ mod io; use std::{ cmp::Reverse, - ffi::OsStr, fmt, fs, mem, - path::{Component, Path, PathBuf}, + path::{Path, PathBuf}, sync::Arc, thread, }; use crossbeam_channel::Receiver; use ra_arena::{impl_arena_id, Arena, RawId}; -use relative_path::RelativePathBuf; +use relative_path::{Component, RelativePath, RelativePathBuf}; use rustc_hash::{FxHashMap, FxHashSet}; pub use crate::io::TaskResult as VfsTask; @@ -36,12 +35,7 @@ use io::{Task, TaskResult, WatcherChange, WatcherChangeData, Worker}; /// several filters match a file (nested dirs), the most nested one wins. pub(crate) struct RootFilter { root: PathBuf, - filter: fn(RootEntry) -> bool, -} - -pub(crate) struct RootEntry<'a, 'b> { - root: &'a Path, - path: &'b Path, + filter: fn(&Path, &RelativePath) -> bool, } impl RootFilter { @@ -54,27 +48,20 @@ impl RootFilter { /// Check if this root can contain `path`. NB: even if this returns /// true, the `path` might actually be conained in some nested root. pub(crate) fn can_contain(&self, path: &Path) -> Option { - if !(self.filter)(RootEntry { - root: &self.root, - path, - }) { + let rel_path = path.strip_prefix(&self.root).ok()?; + let rel_path = RelativePathBuf::from_path(rel_path).ok()?; + if !(self.filter)(path, rel_path.as_relative_path()) { return None; } - let path = path.strip_prefix(&self.root).ok()?; - RelativePathBuf::from_path(path).ok() + Some(rel_path) } } -pub(crate) fn default_filter(entry: RootEntry) -> bool { - if entry.path.is_dir() { - // first component relative to root is "target" - entry - .path - .strip_prefix(entry.root) - .map(|p| p.components().next() != Some(Component::Normal(OsStr::new("target")))) - .unwrap_or(false) +pub(crate) fn default_filter(path: &Path, rel_path: &RelativePath) -> bool { + if path.is_dir() { + rel_path.components().next() != Some(Component::Normal("target")) } else { - entry.path.extension() == Some(OsStr::new("rs")) + rel_path.extension() == Some("rs") } } From 0a086508524bed87bb15113437e9c2b1e1be4c42 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Tue, 22 Jan 2019 18:14:31 +0100 Subject: [PATCH 18/24] hardcode ".git" and "node_modules" also --- crates/ra_vfs/src/{io/mod.rs => io.rs} | 2 +- crates/ra_vfs/src/lib.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) rename crates/ra_vfs/src/{io/mod.rs => io.rs} (99%) diff --git a/crates/ra_vfs/src/io/mod.rs b/crates/ra_vfs/src/io.rs similarity index 99% rename from crates/ra_vfs/src/io/mod.rs rename to crates/ra_vfs/src/io.rs index daac6c6f28..3ab52ac3d4 100644 --- a/crates/ra_vfs/src/io/mod.rs +++ b/crates/ra_vfs/src/io.rs @@ -132,7 +132,7 @@ fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult root_filter, nested_roots, } => { - watch(watcher, &path, &*root_filter, false); + watch(watcher, &path, root_filter.as_ref(), false); log::debug!("loading {} ...", path.as_path().display()); let files = load_root( path.as_path(), diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index f4447be43d..4f3896a82a 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -59,7 +59,15 @@ impl RootFilter { pub(crate) fn default_filter(path: &Path, rel_path: &RelativePath) -> bool { if path.is_dir() { - rel_path.components().next() != Some(Component::Normal("target")) + for (i, c) in rel_path.components().enumerate() { + if let Component::Normal(c) = c { + // hardcoded for now + if (i == 0 && c == "target") || c == ".git" || c == "node_modules" { + return false; + } + } + } + true } else { rel_path.extension() == Some("rs") } From be14ab217ce29542a8b2c84282e822adcc69646c Mon Sep 17 00:00:00 2001 From: Bernardo Date: Tue, 22 Jan 2019 18:38:34 +0100 Subject: [PATCH 19/24] better test, avoid duplicated events --- crates/ra_vfs/src/io.rs | 7 ++++++- crates/ra_vfs/src/io/watcher.rs | 22 ++++++++++++---------- crates/ra_vfs/tests/vfs.rs | 23 ++++++++++++----------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 3ab52ac3d4..335f4f2e1f 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -55,7 +55,12 @@ pub enum TaskResult { impl fmt::Debug for TaskResult { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("TaskResult { ... }") + match self { + TaskResult::AddRoot(..) => f.write_str("TaskResult::AddRoot(..)"), + TaskResult::HandleChange(c) => write!(f, "TaskResult::HandleChange({:?})", c), + TaskResult::LoadChange(c) => write!(f, "TaskResult::LoadChange({:?})", c), + TaskResult::NoOp => f.write_str("TaskResult::NoOp"), + } } } diff --git a/crates/ra_vfs/src/io/watcher.rs b/crates/ra_vfs/src/io/watcher.rs index b370c5fbcd..68bb6b6925 100644 --- a/crates/ra_vfs/src/io/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -100,16 +100,18 @@ impl Watcher { log::warn!("could not watch \"{}\": {}", entry.path().display(), e) } } - } - if emit_for_contents && entry.depth() > 0 { - // emit as create because we haven't seen it yet - if let Err(e) = - self.sender - .send(io::Task::HandleChange(WatcherChange::Create( - entry.path().to_path_buf(), - ))) - { - log::warn!("watcher error: {}", e) + } else { + if emit_for_contents && entry.depth() > 0 { + // emit only for files otherwise we will cause watch_recursive to be called again with a dir that we are already watching + // emit as create because we haven't seen it yet + if let Err(e) = + self.sender + .send(io::Task::HandleChange(WatcherChange::Create( + entry.path().to_path_buf(), + ))) + { + log::warn!("watcher error: {}", e) + } } } } diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index b18ea74a32..d3271570a9 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -1,12 +1,13 @@ use std::{collections::HashSet, fs}; -// use flexi_logger::Logger; +use flexi_logger::Logger; use ra_vfs::{Vfs, VfsChange}; use tempfile::tempdir; fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { for _ in 0..num_tasks { let task = vfs.task_receiver().recv().unwrap(); + log::debug!("{:?}", task); vfs.handle_task(task); } } @@ -25,7 +26,7 @@ macro_rules! assert_match { #[test] fn test_vfs_works() -> std::io::Result<()> { - // Logger::with_str("vfs=debug,ra_vfs=debug").start().unwrap(); + Logger::with_str("vfs=debug,ra_vfs=debug").start().unwrap(); let files = [ ("a/foo.rs", "hello"), @@ -114,21 +115,21 @@ fn test_vfs_works() -> std::io::Result<()> { assert_eq!(path, "spam.rs") ); - fs::create_dir_all(dir.path().join("a/c")).unwrap(); - fs::write(dir.path().join("a/c/new.rs"), "new hello").unwrap(); + fs::create_dir_all(dir.path().join("a/sub1/sub2")).unwrap(); + fs::write(dir.path().join("a/sub1/sub2/new.rs"), "new hello").unwrap(); process_tasks(&mut vfs, 4); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::AddFile { text, path, .. }], { assert_eq!(text.as_str(), "new hello"); - assert_eq!(path, "c/new.rs"); + assert_eq!(path, "sub1/sub2/new.rs"); } ); fs::rename( - &dir.path().join("a/c/new.rs"), - &dir.path().join("a/c/new1.rs"), + &dir.path().join("a/sub1/sub2/new.rs"), + &dir.path().join("a/sub1/sub2/new1.rs"), ) .unwrap(); process_tasks(&mut vfs, 4); @@ -142,18 +143,18 @@ fn test_vfs_works() -> std::io::Result<()> { .. }], { - assert_eq!(removed_path, "c/new.rs"); - assert_eq!(added_path, "c/new1.rs"); + assert_eq!(removed_path, "sub1/sub2/new.rs"); + assert_eq!(added_path, "sub1/sub2/new1.rs"); assert_eq!(text.as_str(), "new hello"); } ); - fs::remove_file(&dir.path().join("a/c/new1.rs")).unwrap(); + fs::remove_file(&dir.path().join("a/sub1/sub2/new1.rs")).unwrap(); process_tasks(&mut vfs, 2); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::RemoveFile { path, .. }], - assert_eq!(path, "c/new1.rs") + assert_eq!(path, "sub1/sub2/new1.rs") ); fs::create_dir_all(dir.path().join("a/target")).unwrap(); From 34a34f9399015bbd351113675928295f42f74369 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Wed, 23 Jan 2019 18:42:26 +0100 Subject: [PATCH 20/24] use released `notifiy` version, add TODO comment --- Cargo.lock | 8 ++++---- crates/ra_vfs/Cargo.toml | 3 +-- crates/ra_vfs/src/lib.rs | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41e84b7e94..db09967ba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -702,8 +702,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "notify" -version = "4.0.6" -source = "git+https://github.com/vemoo/notify/?branch=v4-legacy#8114796fb7b133ba8898ba5d08fda20856f666d4" +version = "4.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1011,7 +1011,7 @@ dependencies = [ "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "flexi_logger 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)", + "notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "ra_arena 0.1.0", "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1899,7 +1899,7 @@ dependencies = [ "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" -"checksum notify 4.0.6 (git+https://github.com/vemoo/notify/?branch=v4-legacy)" = "" +"checksum notify 4.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c968cf37cf949114b00d51b0b23536d1c3a4a3963767cf4c969c65a6af78dc7d" "checksum num-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d9fe8fcafd1b86a37ce8a1cfa15ae504817e0c8c2e7ad42767371461ac1d316d" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml index c7c4769b3d..383381d2a9 100644 --- a/crates/ra_vfs/Cargo.toml +++ b/crates/ra_vfs/Cargo.toml @@ -10,8 +10,7 @@ relative-path = "0.4.0" rustc-hash = "1.0" crossbeam-channel = "0.3.5" log = "0.4.6" -# until https://github.com/passcod/notify/issues/169 is fixed -notify = { git = "https://github.com/vemoo/notify/", branch = "v4-legacy" } +notify = "4.0.7" drop_bomb = "0.1.0" parking_lot = "0.7.0" diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 4f3896a82a..bcff7928b7 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -61,7 +61,7 @@ pub(crate) fn default_filter(path: &Path, rel_path: &RelativePath) -> bool { if path.is_dir() { for (i, c) in rel_path.components().enumerate() { if let Component::Normal(c) = c { - // hardcoded for now + // TODO hardcoded for now if (i == 0 && c == "target") || c == ".git" || c == "node_modules" { return false; } From cfbf47b0023585a30d825b5c5da8e2fb9d6fc337 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Wed, 23 Jan 2019 20:43:35 +0100 Subject: [PATCH 21/24] review fixes --- crates/ra_vfs/src/io.rs | 29 +++++++++-------------------- crates/ra_vfs/src/lib.rs | 1 - crates/ra_vfs/tests/vfs.rs | 2 +- 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 335f4f2e1f..83a021c2fa 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -50,7 +50,6 @@ pub enum TaskResult { AddRoot(AddRootResult), HandleChange(WatcherChange), LoadChange(WatcherChangeData), - NoOp, } impl fmt::Debug for TaskResult { @@ -59,7 +58,6 @@ impl fmt::Debug for TaskResult { TaskResult::AddRoot(..) => f.write_str("TaskResult::AddRoot(..)"), TaskResult::HandleChange(c) => write!(f, "TaskResult::HandleChange({:?})", c), TaskResult::LoadChange(c) => write!(f, "TaskResult::LoadChange({:?})", c), - TaskResult::NoOp => f.write_str("TaskResult::NoOp"), } } } @@ -78,7 +76,7 @@ impl Worker { thread_worker::spawn("vfs", 128, move |input_receiver, output_sender| { input_receiver .into_iter() - .map(|t| handle_task(t, &watcher_clone)) + .filter_map(|t| handle_task(t, &watcher_clone)) .try_for_each(|it| output_sender.send(it)) .unwrap() }); @@ -118,18 +116,12 @@ fn watch( filter_entry: &RootFilter, emit_for_existing: bool, ) { - let mut watcher = watcher.lock(); - let watcher = match *watcher { - Some(ref mut w) => w, - None => { - // watcher dropped or couldn't start - return; - } - }; - watcher.watch_recursive(dir, filter_entry, emit_for_existing) + if let Some(watcher) = watcher.lock().as_mut() { + watcher.watch_recursive(dir, filter_entry, emit_for_existing) + } } -fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult { +fn handle_task(task: Task, watcher: &Arc>>) -> Option { match task { Task::AddRoot { root, @@ -145,22 +137,19 @@ fn handle_task(task: Task, watcher: &Arc>>) -> TaskResult nested_roots.as_slice(), ); log::debug!("... loaded {}", path.as_path().display()); - TaskResult::AddRoot(AddRootResult { root, files }) + Some(TaskResult::AddRoot(AddRootResult { root, files })) } Task::HandleChange(change) => { // forward as is because Vfs has to decide if we should load it - TaskResult::HandleChange(change) + Some(TaskResult::HandleChange(change)) } Task::LoadChange(change) => { log::debug!("loading {:?} ...", change); - match load_change(change) { - Some(data) => TaskResult::LoadChange(data), - None => TaskResult::NoOp, - } + load_change(change).map(TaskResult::LoadChange) } Task::Watch { dir, root_filter } => { watch(watcher, &dir, root_filter.as_ref(), true); - TaskResult::NoOp + None } } } diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index bcff7928b7..f6b45c18a7 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -259,7 +259,6 @@ impl Vfs { } } }, - TaskResult::NoOp => {} } } diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index d3271570a9..bf44e97c57 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -117,7 +117,7 @@ fn test_vfs_works() -> std::io::Result<()> { fs::create_dir_all(dir.path().join("a/sub1/sub2")).unwrap(); fs::write(dir.path().join("a/sub1/sub2/new.rs"), "new hello").unwrap(); - process_tasks(&mut vfs, 4); + process_tasks(&mut vfs, 3); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::AddFile { text, path, .. }], From 86fadbd4e59f12535edcc280ec227d7ee8a0848d Mon Sep 17 00:00:00 2001 From: Bernardo Date: Thu, 24 Jan 2019 19:04:56 +0100 Subject: [PATCH 22/24] extract `Roots` struct --- crates/ra_vfs/src/lib.rs | 73 ++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index f6b45c18a7..cba3a463ae 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -18,6 +18,7 @@ mod io; use std::{ cmp::Reverse, fmt, fs, mem, + ops::{Deref, DerefMut}, path::{Path, PathBuf}, sync::Arc, thread, @@ -88,8 +89,38 @@ struct VfsFileData { text: Arc, } -pub struct Vfs { +pub(crate) struct Roots { roots: Arena>, +} + +impl Roots { + pub(crate) fn new() -> Roots { + Roots { + roots: Arena::default(), + } + } + pub(crate) fn find(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf)> { + self.roots + .iter() + .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it))) + } +} + +impl Deref for Roots { + type Target = Arena>; + fn deref(&self) -> &Self::Target { + &self.roots + } +} + +impl DerefMut for Roots { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.roots + } +} + +pub struct Vfs { + roots: Arc, files: Arena, root2files: FxHashMap>, pending_changes: Vec, @@ -103,26 +134,22 @@ impl fmt::Debug for Vfs { } impl Vfs { - pub fn new(mut roots: Vec) -> (Vfs, Vec) { + pub fn new(roots: Vec) -> (Vfs, Vec) { + let mut root_paths = roots; let worker = io::Worker::start(); - let mut res = Vfs { - roots: Arena::default(), - files: Arena::default(), - root2files: FxHashMap::default(), - worker, - pending_changes: Vec::new(), - }; + let mut roots = Roots::new(); + let mut root2files = FxHashMap::default(); // A hack to make nesting work. - roots.sort_by_key(|it| Reverse(it.as_os_str().len())); - for (i, path) in roots.iter().enumerate() { + root_paths.sort_by_key(|it| Reverse(it.as_os_str().len())); + for (i, path) in root_paths.iter().enumerate() { let root_filter = Arc::new(RootFilter::new(path.clone())); - let root = res.roots.alloc(root_filter.clone()); - res.root2files.insert(root, Default::default()); + let root = roots.alloc(root_filter.clone()); + root2files.insert(root, Default::default()); - let nested_roots = roots[..i] + let nested_roots = root_paths[..i] .iter() .filter(|it| it.starts_with(path)) .map(|it| it.clone()) @@ -134,10 +161,17 @@ impl Vfs { root_filter, nested_roots, }; - res.worker.sender().send(task).unwrap(); + worker.sender().send(task).unwrap(); } - let roots = res.roots.iter().map(|(id, _)| id).collect(); - (res, roots) + let res = Vfs { + roots: Arc::new(roots), + files: Arena::default(), + root2files, + worker, + pending_changes: Vec::new(), + }; + let vfs_roots = res.roots.iter().map(|(id, _)| id).collect(); + (res, vfs_roots) } pub fn root2path(&self, root: VfsRoot) -> PathBuf { @@ -399,10 +433,7 @@ impl Vfs { } fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option)> { - let (root, path) = self - .roots - .iter() - .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?; + let (root, path) = self.roots.find(&path)?; let file = self.root2files[&root] .iter() .map(|&it| it) From d63e1cebff771621b90bdce25ac013eecb415e1e Mon Sep 17 00:00:00 2001 From: Bernardo Date: Fri, 25 Jan 2019 18:39:35 +0100 Subject: [PATCH 23/24] use `Roots` in watcher --- crates/ra_vfs/src/io.rs | 200 ++++++++--------------------- crates/ra_vfs/src/io/watcher.rs | 215 +++++++++++++++++++++----------- crates/ra_vfs/src/lib.rs | 170 +++++++++++-------------- crates/ra_vfs/tests/vfs.rs | 21 ++-- 4 files changed, 275 insertions(+), 331 deletions(-) diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs index 83a021c2fa..7ca1e98358 100644 --- a/crates/ra_vfs/src/io.rs +++ b/crates/ra_vfs/src/io.rs @@ -1,95 +1,72 @@ -use std::{ - fmt, fs, - path::{Path, PathBuf}, - sync::Arc, - thread, -}; +use std::{fs, sync::Arc, thread}; use crossbeam_channel::{Receiver, Sender}; -use parking_lot::Mutex; use relative_path::RelativePathBuf; use thread_worker::WorkerHandle; use walkdir::WalkDir; mod watcher; use watcher::Watcher; -pub use watcher::WatcherChange; -use crate::{RootFilter, VfsRoot}; +use crate::{RootFilter, Roots, VfsRoot}; pub(crate) enum Task { AddRoot { root: VfsRoot, - path: PathBuf, - root_filter: Arc, - nested_roots: Vec, - }, - /// this variant should only be created by the watcher - HandleChange(WatcherChange), - LoadChange(WatcherChange), - Watch { - dir: PathBuf, - root_filter: Arc, + filter: Arc, }, } #[derive(Debug)] -pub struct AddRootResult { - pub(crate) root: VfsRoot, - pub(crate) files: Vec<(RelativePathBuf, String)>, -} - -#[derive(Debug)] -pub enum WatcherChangeData { - Create { path: PathBuf, text: String }, - Write { path: PathBuf, text: String }, - Remove { path: PathBuf }, -} - pub enum TaskResult { - AddRoot(AddRootResult), - HandleChange(WatcherChange), - LoadChange(WatcherChangeData), -} - -impl fmt::Debug for TaskResult { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - TaskResult::AddRoot(..) => f.write_str("TaskResult::AddRoot(..)"), - TaskResult::HandleChange(c) => write!(f, "TaskResult::HandleChange({:?})", c), - TaskResult::LoadChange(c) => write!(f, "TaskResult::LoadChange({:?})", c), - } - } + BulkLoadRoot { + root: VfsRoot, + files: Vec<(RelativePathBuf, String)>, + }, + AddSingleFile { + root: VfsRoot, + path: RelativePathBuf, + text: String, + }, + ChangeSingleFile { + root: VfsRoot, + path: RelativePathBuf, + text: String, + }, + RemoveSingleFile { + root: VfsRoot, + path: RelativePathBuf, + }, } pub(crate) struct Worker { worker: thread_worker::Worker, worker_handle: WorkerHandle, - watcher: Arc>>, } impl Worker { - pub(crate) fn start() -> Worker { - let watcher = Arc::new(Mutex::new(None)); - let watcher_clone = watcher.clone(); + pub(crate) fn start(roots: Arc) -> Worker { let (worker, worker_handle) = thread_worker::spawn("vfs", 128, move |input_receiver, output_sender| { - input_receiver + let mut watcher = match Watcher::start(roots, output_sender.clone()) { + Ok(w) => Some(w), + Err(e) => { + log::error!("could not start watcher: {}", e); + None + } + }; + let res = input_receiver .into_iter() - .filter_map(|t| handle_task(t, &watcher_clone)) - .try_for_each(|it| output_sender.send(it)) - .unwrap() + .filter_map(|t| handle_task(t, &mut watcher)) + .try_for_each(|it| output_sender.send(it)); + if let Some(watcher) = watcher { + let _ = watcher.shutdown(); + } + res.unwrap() }); - match Watcher::start(worker.inp.clone()) { - Ok(w) => { - watcher.lock().replace(w); - } - Err(e) => log::error!("could not start watcher: {}", e), - }; Worker { worker, worker_handle, - watcher, } } @@ -102,72 +79,31 @@ impl Worker { } pub(crate) fn shutdown(self) -> thread::Result<()> { - if let Some(watcher) = self.watcher.lock().take() { - let _ = watcher.shutdown(); - } let _ = self.worker.shutdown(); self.worker_handle.shutdown() } } -fn watch( - watcher: &Arc>>, - dir: &Path, - filter_entry: &RootFilter, - emit_for_existing: bool, -) { - if let Some(watcher) = watcher.lock().as_mut() { - watcher.watch_recursive(dir, filter_entry, emit_for_existing) - } -} - -fn handle_task(task: Task, watcher: &Arc>>) -> Option { +fn handle_task(task: Task, watcher: &mut Option) -> Option { match task { - Task::AddRoot { - root, - path, - root_filter, - nested_roots, - } => { - watch(watcher, &path, root_filter.as_ref(), false); - log::debug!("loading {} ...", path.as_path().display()); - let files = load_root( - path.as_path(), - root_filter.as_ref(), - nested_roots.as_slice(), - ); - log::debug!("... loaded {}", path.as_path().display()); - Some(TaskResult::AddRoot(AddRootResult { root, files })) - } - Task::HandleChange(change) => { - // forward as is because Vfs has to decide if we should load it - Some(TaskResult::HandleChange(change)) - } - Task::LoadChange(change) => { - log::debug!("loading {:?} ...", change); - load_change(change).map(TaskResult::LoadChange) - } - Task::Watch { dir, root_filter } => { - watch(watcher, &dir, root_filter.as_ref(), true); - None + Task::AddRoot { root, filter } => { + if let Some(watcher) = watcher { + watcher.watch_root(&filter) + } + log::debug!("loading {} ...", filter.root.as_path().display()); + let files = load_root(filter.as_ref()); + log::debug!("... loaded {}", filter.root.as_path().display()); + Some(TaskResult::BulkLoadRoot { root, files }) } } } -fn load_root( - root: &Path, - root_filter: &RootFilter, - nested_roots: &[PathBuf], -) -> Vec<(RelativePathBuf, String)> { +fn load_root(filter: &RootFilter) -> Vec<(RelativePathBuf, String)> { let mut res = Vec::new(); - for entry in WalkDir::new(root).into_iter().filter_entry(|entry| { - if entry.file_type().is_dir() && nested_roots.iter().any(|it| it == entry.path()) { - // do not load files of a nested root - false - } else { - root_filter.can_contain(entry.path()).is_some() - } - }) { + for entry in WalkDir::new(&filter.root) + .into_iter() + .filter_entry(filter.entry_filter()) + { let entry = match entry { Ok(entry) => entry, Err(e) => { @@ -186,42 +122,8 @@ fn load_root( continue; } }; - let path = RelativePathBuf::from_path(path.strip_prefix(root).unwrap()).unwrap(); + let path = RelativePathBuf::from_path(path.strip_prefix(&filter.root).unwrap()).unwrap(); res.push((path.to_owned(), text)) } res } - -fn load_change(change: WatcherChange) -> Option { - let data = match change { - WatcherChange::Create(path) => { - if path.is_dir() { - return None; - } - let text = match fs::read_to_string(&path) { - Ok(text) => text, - Err(e) => { - log::warn!("watcher error \"{}\": {}", path.display(), e); - return None; - } - }; - WatcherChangeData::Create { path, text } - } - WatcherChange::Write(path) => { - let text = match fs::read_to_string(&path) { - Ok(text) => text, - Err(e) => { - log::warn!("watcher error \"{}\": {}", path.display(), e); - return None; - } - }; - WatcherChangeData::Write { path, text } - } - WatcherChange::Remove(path) => WatcherChangeData::Remove { path }, - WatcherChange::Rescan => { - // this should be handled by Vfs::handle_task - return None; - } - }; - Some(data) -} diff --git a/crates/ra_vfs/src/io/watcher.rs b/crates/ra_vfs/src/io/watcher.rs index 68bb6b6925..1d7ce21367 100644 --- a/crates/ra_vfs/src/io/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -1,118 +1,72 @@ -use crate::{io, RootFilter}; +use crate::{io, RootFilter, Roots, VfsRoot}; use crossbeam_channel::Sender; use drop_bomb::DropBomb; use notify::{DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher}; +use parking_lot::Mutex; use std::{ + fs, path::{Path, PathBuf}, - sync::mpsc, + sync::{mpsc, Arc}, thread, time::Duration, }; use walkdir::WalkDir; #[derive(Debug)] -pub enum WatcherChange { - Create(PathBuf), - Write(PathBuf), - Remove(PathBuf), - Rescan, -} - -fn handle_change_event( - ev: DebouncedEvent, - sender: &Sender, -) -> Result<(), Box> { - match ev { - DebouncedEvent::NoticeWrite(_) - | DebouncedEvent::NoticeRemove(_) - | DebouncedEvent::Chmod(_) => { - // ignore - } - DebouncedEvent::Rescan => { - sender.send(io::Task::HandleChange(WatcherChange::Rescan))?; - } - DebouncedEvent::Create(path) => { - sender.send(io::Task::HandleChange(WatcherChange::Create(path)))?; - } - DebouncedEvent::Write(path) => { - sender.send(io::Task::HandleChange(WatcherChange::Write(path)))?; - } - DebouncedEvent::Remove(path) => { - sender.send(io::Task::HandleChange(WatcherChange::Remove(path)))?; - } - DebouncedEvent::Rename(src, dst) => { - sender.send(io::Task::HandleChange(WatcherChange::Remove(src)))?; - sender.send(io::Task::HandleChange(WatcherChange::Create(dst)))?; - } - DebouncedEvent::Error(err, path) => { - // TODO should we reload the file contents? - log::warn!("watcher error \"{}\", {:?}", err, path); - } - } - Ok(()) +enum ChangeKind { + Create, + Write, + Remove, } const WATCHER_DELAY: Duration = Duration::from_millis(250); pub(crate) struct Watcher { - watcher: RecommendedWatcher, thread: thread::JoinHandle<()>, bomb: DropBomb, - sender: Sender, + watcher: Arc>>, } impl Watcher { pub(crate) fn start( - output_sender: Sender, + roots: Arc, + output_sender: Sender, ) -> Result> { let (input_sender, input_receiver) = mpsc::channel(); - let watcher = notify::watcher(input_sender, WATCHER_DELAY)?; + let watcher = Arc::new(Mutex::new(Some(notify::watcher( + input_sender, + WATCHER_DELAY, + )?))); let sender = output_sender.clone(); + let watcher_clone = watcher.clone(); let thread = thread::spawn(move || { + let worker = WatcherWorker { + roots, + watcher: watcher_clone, + sender, + }; input_receiver .into_iter() // forward relevant events only - .try_for_each(|change| handle_change_event(change, &output_sender)) + .try_for_each(|change| worker.handle_debounced_event(change)) .unwrap() }); Ok(Watcher { - watcher, thread, - sender, + watcher, bomb: DropBomb::new(format!("Watcher was not shutdown")), }) } - pub fn watch_recursive(&mut self, dir: &Path, filter: &RootFilter, emit_for_contents: bool) { - for res in WalkDir::new(dir) + pub fn watch_root(&mut self, filter: &RootFilter) { + for res in WalkDir::new(&filter.root) .into_iter() - .filter_entry(|entry| filter.can_contain(entry.path()).is_some()) + .filter_entry(filter.entry_filter()) { match res { Ok(entry) => { if entry.path().is_dir() { - match self - .watcher - .watch(entry.path(), RecursiveMode::NonRecursive) - { - Ok(()) => log::debug!("watching \"{}\"", entry.path().display()), - Err(e) => { - log::warn!("could not watch \"{}\": {}", entry.path().display(), e) - } - } - } else { - if emit_for_contents && entry.depth() > 0 { - // emit only for files otherwise we will cause watch_recursive to be called again with a dir that we are already watching - // emit as create because we haven't seen it yet - if let Err(e) = - self.sender - .send(io::Task::HandleChange(WatcherChange::Create( - entry.path().to_path_buf(), - ))) - { - log::warn!("watcher error: {}", e) - } - } + watch_one(self.watcher.as_ref(), entry.path()); } } Err(e) => log::warn!("watcher error: {}", e), @@ -122,7 +76,7 @@ impl Watcher { pub fn shutdown(mut self) -> thread::Result<()> { self.bomb.defuse(); - drop(self.watcher); + drop(self.watcher.lock().take()); let res = self.thread.join(); match &res { Ok(()) => log::info!("... Watcher terminated with ok"), @@ -131,3 +85,116 @@ impl Watcher { res } } + +struct WatcherWorker { + watcher: Arc>>, + roots: Arc, + sender: Sender, +} + +impl WatcherWorker { + fn handle_debounced_event(&self, ev: DebouncedEvent) -> Result<(), Box> { + match ev { + DebouncedEvent::NoticeWrite(_) + | DebouncedEvent::NoticeRemove(_) + | DebouncedEvent::Chmod(_) => { + // ignore + } + DebouncedEvent::Rescan => { + // TODO rescan all roots + } + DebouncedEvent::Create(path) => { + self.handle_change(path, ChangeKind::Create); + } + DebouncedEvent::Write(path) => { + self.handle_change(path, ChangeKind::Write); + } + DebouncedEvent::Remove(path) => { + self.handle_change(path, ChangeKind::Remove); + } + DebouncedEvent::Rename(src, dst) => { + self.handle_change(src, ChangeKind::Remove); + self.handle_change(dst, ChangeKind::Create); + } + DebouncedEvent::Error(err, path) => { + // TODO should we reload the file contents? + log::warn!("watcher error \"{}\", {:?}", err, path); + } + } + Ok(()) + } + + fn handle_change(&self, path: PathBuf, kind: ChangeKind) { + if let Err(e) = self.try_handle_change(path, kind) { + log::warn!("watcher error: {}", e) + } + } + + fn try_handle_change( + &self, + path: PathBuf, + kind: ChangeKind, + ) -> Result<(), Box> { + let (root, rel_path) = match self.roots.find(&path) { + Some(x) => x, + None => return Ok(()), + }; + match kind { + ChangeKind::Create => { + if path.is_dir() { + self.watch_recursive(&path, root); + } else { + let text = fs::read_to_string(&path)?; + self.sender.send(io::TaskResult::AddSingleFile { + root, + path: rel_path, + text, + })? + } + } + ChangeKind::Write => { + let text = fs::read_to_string(&path)?; + self.sender.send(io::TaskResult::ChangeSingleFile { + root, + path: rel_path, + text, + })? + } + ChangeKind::Remove => self.sender.send(io::TaskResult::RemoveSingleFile { + root, + path: rel_path, + })?, + } + Ok(()) + } + + fn watch_recursive(&self, dir: &Path, root: VfsRoot) { + let filter = &self.roots[root]; + for res in WalkDir::new(dir) + .into_iter() + .filter_entry(|entry| filter.can_contain(entry.path()).is_some()) + { + match res { + Ok(entry) => { + if entry.path().is_dir() { + watch_one(self.watcher.as_ref(), entry.path()); + } else { + // emit only for files otherwise we will cause watch_recursive to be called again with a dir that we are already watching + // emit as create because we haven't seen it yet + self.handle_change(entry.path().to_path_buf(), ChangeKind::Create); + } + } + Err(e) => log::warn!("watcher error: {}", e), + } + } + } +} + +fn watch_one(watcher: &Mutex>, dir: &Path) { + if let Some(watcher) = watcher.lock().as_mut() { + match watcher.watch(dir, RecursiveMode::NonRecursive) { + Ok(()) => log::debug!("watching \"{}\"", dir.display()), + Err(e) => log::warn!("could not watch \"{}\": {}", dir.display(), e), + } + } +} diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index cba3a463ae..661892f8a2 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -28,22 +28,25 @@ use crossbeam_channel::Receiver; use ra_arena::{impl_arena_id, Arena, RawId}; use relative_path::{Component, RelativePath, RelativePathBuf}; use rustc_hash::{FxHashMap, FxHashSet}; +use walkdir::DirEntry; pub use crate::io::TaskResult as VfsTask; -use io::{Task, TaskResult, WatcherChange, WatcherChangeData, Worker}; +use io::{TaskResult, Worker}; /// `RootFilter` is a predicate that checks if a file can belong to a root. If /// several filters match a file (nested dirs), the most nested one wins. pub(crate) struct RootFilter { root: PathBuf, filter: fn(&Path, &RelativePath) -> bool, + excluded_dirs: Vec, } impl RootFilter { - fn new(root: PathBuf) -> RootFilter { + fn new(root: PathBuf, excluded_dirs: Vec) -> RootFilter { RootFilter { root, filter: default_filter, + excluded_dirs, } } /// Check if this root can contain `path`. NB: even if this returns @@ -56,6 +59,17 @@ impl RootFilter { } Some(rel_path) } + + pub(crate) fn entry_filter<'a>(&'a self) -> impl FnMut(&DirEntry) -> bool + 'a { + move |entry: &DirEntry| { + if entry.path().is_dir() && self.excluded_dirs.iter().any(|it| it == entry.path()) { + // do not walk nested roots + false + } else { + self.can_contain(entry.path()).is_some() + } + } + } } pub(crate) fn default_filter(path: &Path, rel_path: &RelativePath) -> bool { @@ -94,10 +108,22 @@ pub(crate) struct Roots { } impl Roots { - pub(crate) fn new() -> Roots { - Roots { - roots: Arena::default(), + pub(crate) fn new(mut paths: Vec) -> Roots { + let mut roots = Arena::default(); + // A hack to make nesting work. + paths.sort_by_key(|it| Reverse(it.as_os_str().len())); + for (i, path) in paths.iter().enumerate() { + let nested_roots = paths[..i] + .iter() + .filter(|it| it.starts_with(path)) + .map(|it| it.clone()) + .collect::>(); + + let root_filter = Arc::new(RootFilter::new(path.clone(), nested_roots)); + + roots.alloc(root_filter.clone()); } + Roots { roots } } pub(crate) fn find(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf)> { self.roots @@ -135,36 +161,22 @@ impl fmt::Debug for Vfs { impl Vfs { pub fn new(roots: Vec) -> (Vfs, Vec) { - let mut root_paths = roots; - let worker = io::Worker::start(); - - let mut roots = Roots::new(); + let roots = Arc::new(Roots::new(roots)); + let worker = io::Worker::start(roots.clone()); let mut root2files = FxHashMap::default(); - // A hack to make nesting work. - root_paths.sort_by_key(|it| Reverse(it.as_os_str().len())); - for (i, path) in root_paths.iter().enumerate() { - let root_filter = Arc::new(RootFilter::new(path.clone())); - - let root = roots.alloc(root_filter.clone()); + for (root, filter) in roots.iter() { root2files.insert(root, Default::default()); - - let nested_roots = root_paths[..i] - .iter() - .filter(|it| it.starts_with(path)) - .map(|it| it.clone()) - .collect::>(); - - let task = io::Task::AddRoot { - root, - path: path.clone(), - root_filter, - nested_roots, - }; - worker.sender().send(task).unwrap(); + worker + .sender() + .send(io::Task::AddRoot { + root, + filter: filter.clone(), + }) + .unwrap(); } let res = Vfs { - roots: Arc::new(roots), + roots, files: Arena::default(), root2files, worker, @@ -225,90 +237,46 @@ impl Vfs { pub fn handle_task(&mut self, task: io::TaskResult) { match task { - TaskResult::AddRoot(task) => { - let mut files = Vec::new(); + TaskResult::BulkLoadRoot { root, files } => { + let mut cur_files = Vec::new(); // While we were scanning the root in the backgound, a file might have // been open in the editor, so we need to account for that. - let exising = self.root2files[&task.root] + let exising = self.root2files[&root] .iter() .map(|&file| (self.files[file].path.clone(), file)) .collect::>(); - for (path, text) in task.files { + for (path, text) in files { if let Some(&file) = exising.get(&path) { let text = Arc::clone(&self.files[file].text); - files.push((file, path, text)); + cur_files.push((file, path, text)); continue; } let text = Arc::new(text); - let file = self.add_file(task.root, path.clone(), Arc::clone(&text), false); - files.push((file, path, text)); + let file = self.add_file(root, path.clone(), Arc::clone(&text), false); + cur_files.push((file, path, text)); } let change = VfsChange::AddRoot { - root: task.root, - files, + root, + files: cur_files, }; self.pending_changes.push(change); } - TaskResult::HandleChange(change) => match &change { - WatcherChange::Create(path) if path.is_dir() => { - if let Some((root, _path, _file)) = self.find_root(&path) { - let root_filter = self.roots[root].clone(); - self.worker - .sender() - .send(Task::Watch { - dir: path.to_path_buf(), - root_filter, - }) - .unwrap() - } - } - WatcherChange::Create(path) - | WatcherChange::Remove(path) - | WatcherChange::Write(path) => { - if self.should_handle_change(&path) { - self.worker.sender().send(Task::LoadChange(change)).unwrap() - } - } - WatcherChange::Rescan => { - // TODO we should reload all files - } - }, - TaskResult::LoadChange(change) => match change { - WatcherChangeData::Create { path, text } - | WatcherChangeData::Write { path, text } => { - if let Some((root, path, file)) = self.find_root(&path) { - if let Some(file) = file { - self.do_change_file(file, text, false); - } else { - self.do_add_file(root, path, text, false); - } - } - } - WatcherChangeData::Remove { path } => { - if let Some((root, path, file)) = self.find_root(&path) { - if let Some(file) = file { - self.do_remove_file(root, path, file, false); - } - } - } - }, - } - } - - fn should_handle_change(&self, path: &Path) -> bool { - if let Some((_root, _rel_path, file)) = self.find_root(&path) { - if let Some(file) = file { - if self.files[file].is_overlayed { - // file is overlayed - log::debug!("skipping overlayed \"{}\"", path.display()); - return false; + TaskResult::AddSingleFile { root, path, text } => { + self.do_add_file(root, path, text, false); + } + TaskResult::ChangeSingleFile { root, path, text } => { + if let Some(file) = self.find_file(root, &path) { + self.do_change_file(file, text, false); + } else { + self.do_add_file(root, path, text, false); + } + } + TaskResult::RemoveSingleFile { root, path } => { + if let Some(file) = self.find_file(root, &path) { + self.do_remove_file(root, path, file, false); } } - true - } else { - // file doesn't belong to any root - false } } @@ -434,11 +402,15 @@ impl Vfs { fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option)> { let (root, path) = self.roots.find(&path)?; - let file = self.root2files[&root] + let file = self.find_file(root, &path); + Some((root, path, file)) + } + + fn find_file(&self, root: VfsRoot, path: &RelativePath) -> Option { + self.root2files[&root] .iter() .map(|&it| it) - .find(|&file| self.files[file].path == path); - Some((root, path, file)) + .find(|&file| self.files[file].path == path) } } diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index bf44e97c57..8562c56b96 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -75,27 +75,31 @@ fn test_vfs_works() -> std::io::Result<()> { } fs::write(&dir.path().join("a/b/baz.rs"), "quux").unwrap(); - // 2 tasks per change, HandleChange and then LoadChange - process_tasks(&mut vfs, 2); + process_tasks(&mut vfs, 1); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::ChangeFile { text, .. }], assert_eq!(text.as_str(), "quux") ); - vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); + vfs.add_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string()); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::ChangeFile { text, .. }], assert_eq!(text.as_str(), "m") ); + // changing file on disk while overlayed doesn't generate a VfsChange + fs::write(&dir.path().join("a/b/baz.rs"), "corge").unwrap(); + process_tasks(&mut vfs, 1); + assert_match!(vfs.commit_changes().as_slice(), []); + // removing overlay restores data on disk vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs")); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::ChangeFile { text, .. }], - assert_eq!(text.as_str(), "quux") + assert_eq!(text.as_str(), "corge") ); vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string()); @@ -117,7 +121,7 @@ fn test_vfs_works() -> std::io::Result<()> { fs::create_dir_all(dir.path().join("a/sub1/sub2")).unwrap(); fs::write(dir.path().join("a/sub1/sub2/new.rs"), "new hello").unwrap(); - process_tasks(&mut vfs, 3); + process_tasks(&mut vfs, 1); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::AddFile { text, path, .. }], @@ -132,7 +136,7 @@ fn test_vfs_works() -> std::io::Result<()> { &dir.path().join("a/sub1/sub2/new1.rs"), ) .unwrap(); - process_tasks(&mut vfs, 4); + process_tasks(&mut vfs, 2); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::RemoveFile { @@ -150,17 +154,16 @@ fn test_vfs_works() -> std::io::Result<()> { ); fs::remove_file(&dir.path().join("a/sub1/sub2/new1.rs")).unwrap(); - process_tasks(&mut vfs, 2); + process_tasks(&mut vfs, 1); assert_match!( vfs.commit_changes().as_slice(), [VfsChange::RemoveFile { path, .. }], assert_eq!(path, "sub1/sub2/new1.rs") ); - fs::create_dir_all(dir.path().join("a/target")).unwrap(); // should be ignored + fs::create_dir_all(dir.path().join("a/target")).unwrap(); fs::write(&dir.path().join("a/target/new.rs"), "ignore me").unwrap(); - process_tasks(&mut vfs, 1); // 1 task because no LoadChange will happen, just HandleChange for dir creation assert_match!( vfs.task_receiver().try_recv(), From 410a3ae6e847b59f9930ce4d6bf9f3c5f1d72167 Mon Sep 17 00:00:00 2001 From: Bernardo Date: Fri, 25 Jan 2019 22:13:55 +0100 Subject: [PATCH 24/24] use entry file_type, improve test --- crates/ra_vfs/src/io/watcher.rs | 6 +++--- crates/ra_vfs/src/lib.rs | 3 ++- crates/ra_vfs/tests/vfs.rs | 18 +++++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/ra_vfs/src/io/watcher.rs b/crates/ra_vfs/src/io/watcher.rs index 1d7ce21367..ff6775f594 100644 --- a/crates/ra_vfs/src/io/watcher.rs +++ b/crates/ra_vfs/src/io/watcher.rs @@ -65,7 +65,7 @@ impl Watcher { { match res { Ok(entry) => { - if entry.path().is_dir() { + if entry.file_type().is_dir() { watch_one(self.watcher.as_ref(), entry.path()); } } @@ -172,11 +172,11 @@ impl WatcherWorker { let filter = &self.roots[root]; for res in WalkDir::new(dir) .into_iter() - .filter_entry(|entry| filter.can_contain(entry.path()).is_some()) + .filter_entry(filter.entry_filter()) { match res { Ok(entry) => { - if entry.path().is_dir() { + if entry.file_type().is_dir() { watch_one(self.watcher.as_ref(), entry.path()); } else { // emit only for files otherwise we will cause watch_recursive to be called again with a dir that we are already watching diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs index 661892f8a2..d1b0222e79 100644 --- a/crates/ra_vfs/src/lib.rs +++ b/crates/ra_vfs/src/lib.rs @@ -62,7 +62,8 @@ impl RootFilter { pub(crate) fn entry_filter<'a>(&'a self) -> impl FnMut(&DirEntry) -> bool + 'a { move |entry: &DirEntry| { - if entry.path().is_dir() && self.excluded_dirs.iter().any(|it| it == entry.path()) { + if entry.file_type().is_dir() && self.excluded_dirs.iter().any(|it| it == entry.path()) + { // do not walk nested roots false } else { diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs index 8562c56b96..357e1c775e 100644 --- a/crates/ra_vfs/tests/vfs.rs +++ b/crates/ra_vfs/tests/vfs.rs @@ -1,12 +1,16 @@ -use std::{collections::HashSet, fs}; +use std::{collections::HashSet, fs, time::Duration}; -use flexi_logger::Logger; +// use flexi_logger::Logger; +use crossbeam_channel::RecvTimeoutError; use ra_vfs::{Vfs, VfsChange}; use tempfile::tempdir; fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { for _ in 0..num_tasks { - let task = vfs.task_receiver().recv().unwrap(); + let task = vfs + .task_receiver() + .recv_timeout(Duration::from_secs(3)) + .unwrap(); log::debug!("{:?}", task); vfs.handle_task(task); } @@ -14,7 +18,7 @@ fn process_tasks(vfs: &mut Vfs, num_tasks: u32) { macro_rules! assert_match { ($x:expr, $pat:pat) => { - assert_match!($x, $pat, assert!(true)) + assert_match!($x, $pat, ()) }; ($x:expr, $pat:pat, $assert:expr) => { match $x { @@ -26,7 +30,7 @@ macro_rules! assert_match { #[test] fn test_vfs_works() -> std::io::Result<()> { - Logger::with_str("vfs=debug,ra_vfs=debug").start().unwrap(); + // Logger::with_str("vfs=debug,ra_vfs=debug").start().unwrap(); let files = [ ("a/foo.rs", "hello"), @@ -166,8 +170,8 @@ fn test_vfs_works() -> std::io::Result<()> { fs::write(&dir.path().join("a/target/new.rs"), "ignore me").unwrap(); assert_match!( - vfs.task_receiver().try_recv(), - Err(crossbeam_channel::TryRecvError::Empty) + vfs.task_receiver().recv_timeout(Duration::from_millis(300)), // slightly more than watcher debounce delay + Err(RecvTimeoutError::Timeout) ); vfs.shutdown().unwrap();