introduce ra_vfs_glob crate

It manages exclusion rules for the vfs crate

Cargo.lock

@@ -117,6 +117,14 @@ dependencies = [
  "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "bstr"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@@ -478,6 +486,11 @@ dependencies = [
  "yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "format-buf"
 version = "1.0.0"
@@ -571,6 +584,18 @@ name = "glob"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "globset"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.1"
@@ -1145,6 +1170,7 @@ dependencies = [
  "ra_ide_api 0.1.0",
  "ra_project_model 0.1.0",
  "ra_vfs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ra_vfs_glob 0.1.0",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1250,6 +1276,7 @@ dependencies = [
  "ra_syntax 0.1.0",
  "ra_text_edit 0.1.0",
  "ra_vfs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ra_vfs_glob 0.1.0",
  "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1301,7 +1328,6 @@ dependencies = [
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "ra_arena 0.1.0",
  "ra_db 0.1.0",
- "relative-path 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1369,6 +1395,14 @@ dependencies = [
  "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ra_vfs_glob"
+version = "0.1.0"
+dependencies = [
+ "globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ra_vfs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rand"
 version = "0.6.5"
@@ -2116,6 +2150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
 "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
 "checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09"
+"checksum bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e0a692f1c740e7e821ca71a22cf99b9b2322dfa94d10f71443befb1797b3946a"
 "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7d64d04786e0f528460fc884753cf8dddcc466be308f6026f8e355c41a0e4101"
@@ -2158,6 +2193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum filetime 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2f8c63033fcba1f51ef744505b3cad42510432b904c062afa67ad7ece008429d"
 "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
 "checksum flexi_logger 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2be2727ae1f60711d1991788605245e70cc500bd1d92edb909af160634e1a869"
+"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum format-buf 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f7aea5a5909a74969507051a3b17adc84737e31a5f910559892aedce026f4d53"
 "checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674"
 "checksum fsevent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6"
@@ -2170,6 +2206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum getrandom 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "34f33de6f0ae7c9cb5e574502a562e2b512799e32abb801cd1e79ad952b62b49"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
 "checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+"checksum globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2"
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e"
 "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"

crates/ra_batch/Cargo.toml

@@ -9,6 +9,7 @@ log = "0.4.5"
 rustc-hash = "1.0"
 
 ra_vfs = "0.2.0"
+ra_vfs_glob = { path = "../ra_vfs_glob" }
 ra_db = { path = "../ra_db" }
 ra_ide_api = { path = "../ra_ide_api" }
 ra_hir = { path = "../ra_hir" }

crates/ra_batch/src/lib.rs

@@ -1,5 +1,3 @@
-mod vfs_filter;
-
 use std::{collections::HashSet, error::Error, path::Path};
 
 use rustc_hash::FxHashMap;
@@ -7,8 +5,8 @@ use rustc_hash::FxHashMap;
 use ra_db::{CrateGraph, FileId, SourceRootId};
 use ra_ide_api::{AnalysisChange, AnalysisHost};
 use ra_project_model::{PackageRoot, ProjectWorkspace};
-use ra_vfs::{Vfs, VfsChange};
-use vfs_filter::IncludeRustFiles;
+use ra_vfs::{RootEntry, Vfs, VfsChange};
+use ra_vfs_glob::RustPackageFilterBuilder;
 
 type Result<T> = std::result::Result<T, Box<dyn Error + Send + Sync>>;
 
@@ -23,7 +21,19 @@ pub fn load_cargo(root: &Path) -> Result<(AnalysisHost, FxHashMap<SourceRootId,
     let root = std::env::current_dir()?.join(root);
     let ws = ProjectWorkspace::discover(root.as_ref())?;
     let project_roots = ws.to_roots();
-    let (mut vfs, roots) = Vfs::new(IncludeRustFiles::from_roots(project_roots.clone()).collect());
+    let (mut vfs, roots) = Vfs::new(
+        project_roots
+            .iter()
+            .map(|pkg_root| {
+                RootEntry::new(
+                    pkg_root.path().clone(),
+                    RustPackageFilterBuilder::default()
+                        .set_member(pkg_root.is_member())
+                        .into_vfs_filter(),
+                )
+            })
+            .collect(),
+    );
     let crate_graph = ws.to_crate_graph(&mut |path: &Path| {
         let vfs_file = vfs.load(path);
         log::debug!("vfs file {:?} -> {:?}", path, vfs_file);

crates/ra_batch/src/vfs_filter.rs

@@ -1,54 +0,0 @@
-use ra_project_model::PackageRoot;
-use ra_vfs::{Filter, RelativePath, RootEntry};
-use std::path::PathBuf;
-
-/// `IncludeRustFiles` is used to convert
-/// from `PackageRoot` to `RootEntry` for VFS
-pub struct IncludeRustFiles {
-    root: PackageRoot,
-}
-
-impl IncludeRustFiles {
-    pub fn from_roots<R>(roots: R) -> impl Iterator<Item = RootEntry>
-    where
-        R: IntoIterator<Item = PackageRoot>,
-    {
-        roots.into_iter().map(IncludeRustFiles::from_root)
-    }
-
-    pub fn from_root(root: PackageRoot) -> RootEntry {
-        IncludeRustFiles::from(root).into()
-    }
-
-    #[allow(unused)]
-    pub fn external(path: PathBuf) -> RootEntry {
-        IncludeRustFiles::from_root(PackageRoot::new(path, false))
-    }
-
-    pub fn member(path: PathBuf) -> RootEntry {
-        IncludeRustFiles::from_root(PackageRoot::new(path, true))
-    }
-}
-
-impl Filter for IncludeRustFiles {
-    fn include_dir(&self, dir_path: &RelativePath) -> bool {
-        self.root.include_dir(dir_path)
-    }
-
-    fn include_file(&self, file_path: &RelativePath) -> bool {
-        self.root.include_file(file_path)
-    }
-}
-
-impl From<PackageRoot> for IncludeRustFiles {
-    fn from(v: PackageRoot) -> IncludeRustFiles {
-        IncludeRustFiles { root: v }
-    }
-}
-
-impl From<IncludeRustFiles> for RootEntry {
-    fn from(v: IncludeRustFiles) -> RootEntry {
-        let path = v.root.path().clone();
-        RootEntry::new(path, Box::new(v))
-    }
-}

crates/ra_lsp_server/Cargo.toml

@@ -25,6 +25,7 @@ ra_ide_api = { path = "../ra_ide_api" }
 gen_lsp_server = { path = "../gen_lsp_server" }
 ra_project_model = { path = "../ra_project_model" }
 ra_prof = { path = "../ra_prof" }
+ra_vfs_glob = { path = "../ra_vfs_glob" }
 
 [dev-dependencies]
 tempfile = "3"

crates/ra_lsp_server/src/lib.rs

@@ -4,7 +4,6 @@ mod conv;
 mod main_loop;
 mod markdown;
 mod project_model;
-mod vfs_filter;
 pub mod req;
 pub mod init;
 mod world;

crates/ra_lsp_server/src/vfs_filter.rs

@@ -1,54 +0,0 @@
-use ra_project_model::PackageRoot;
-use ra_vfs::{Filter, RelativePath, RootEntry};
-use std::path::PathBuf;
-
-/// `IncludeRustFiles` is used to convert
-/// from `PackageRoot` to `RootEntry` for VFS
-pub struct IncludeRustFiles {
-    root: PackageRoot,
-}
-
-impl IncludeRustFiles {
-    pub fn from_roots<R>(roots: R) -> impl Iterator<Item = RootEntry>
-    where
-        R: IntoIterator<Item = PackageRoot>,
-    {
-        roots.into_iter().map(IncludeRustFiles::from_root)
-    }
-
-    pub fn from_root(root: PackageRoot) -> RootEntry {
-        IncludeRustFiles::from(root).into()
-    }
-
-    #[allow(unused)]
-    pub fn external(path: PathBuf) -> RootEntry {
-        IncludeRustFiles::from_root(PackageRoot::new(path, false))
-    }
-
-    pub fn member(path: PathBuf) -> RootEntry {
-        IncludeRustFiles::from_root(PackageRoot::new(path, true))
-    }
-}
-
-impl Filter for IncludeRustFiles {
-    fn include_dir(&self, dir_path: &RelativePath) -> bool {
-        self.root.include_dir(dir_path)
-    }
-
-    fn include_file(&self, file_path: &RelativePath) -> bool {
-        self.root.include_file(file_path)
-    }
-}
-
-impl std::convert::From<PackageRoot> for IncludeRustFiles {
-    fn from(v: PackageRoot) -> IncludeRustFiles {
-        IncludeRustFiles { root: v }
-    }
-}
-
-impl std::convert::From<IncludeRustFiles> for RootEntry {
-    fn from(v: IncludeRustFiles) -> RootEntry {
-        let path = v.root.path().clone();
-        RootEntry::new(path, Box::new(v))
-    }
-}

crates/ra_lsp_server/src/world.rs

@@ -9,13 +9,13 @@ use parking_lot::RwLock;
 use ra_ide_api::{
     Analysis, AnalysisChange, AnalysisHost, CrateGraph, FileId, LibraryData, SourceRootId,
 };
-use ra_vfs::{Vfs, VfsChange, VfsFile, VfsRoot};
+use ra_vfs::{RootEntry, Vfs, VfsChange, VfsFile, VfsRoot};
+use ra_vfs_glob::RustPackageFilterBuilder;
 use relative_path::RelativePathBuf;
 
 use crate::{
     main_loop::pending_requests::{CompletedRequest, LatestRequests},
     project_model::ProjectWorkspace,
-    vfs_filter::IncludeRustFiles,
     LspError, Result,
 };
 
@@ -61,9 +61,21 @@ impl WorldState {
         let mut change = AnalysisChange::new();
 
         let mut roots = Vec::new();
-        roots.extend(folder_roots.iter().cloned().map(IncludeRustFiles::member));
+        roots.extend(folder_roots.iter().map(|path| {
+            RootEntry::new(
+                path.clone(),
+                RustPackageFilterBuilder::default().set_member(true).into_vfs_filter(),
+            )
+        }));
         for ws in workspaces.iter() {
-            roots.extend(IncludeRustFiles::from_roots(ws.to_roots()));
+            roots.extend(ws.to_roots().into_iter().map(|pkg_root| {
+                RootEntry::new(
+                    pkg_root.path().clone(),
+                    RustPackageFilterBuilder::default()
+                        .set_member(pkg_root.is_member())
+                        .into_vfs_filter(),
+                )
+            }));
         }
 
         let (mut vfs, vfs_roots) = Vfs::new(roots);

crates/ra_project_model/Cargo.toml

@@ -7,7 +7,6 @@ authors = ["rust-analyzer developers"]
 [dependencies]
 log = "0.4.5"
 rustc-hash = "1.0"
-relative-path = "0.4.0"
 
 cargo_metadata = "0.8.0"
 

crates/ra_project_model/src/lib.rs

@@ -10,7 +10,6 @@ use std::{
 };
 
 use ra_db::{CrateGraph, Edition, FileId};
-use relative_path::RelativePath;
 use rustc_hash::FxHashMap;
 use serde_json::from_reader;
 
@@ -54,28 +53,6 @@ impl PackageRoot {
     pub fn is_member(&self) -> bool {
         self.is_member
     }
-
-    pub fn include_dir(&self, dir_path: &RelativePath) -> bool {
-        const COMMON_IGNORED_DIRS: &[&str] = &["node_modules", "target", ".git"];
-        const EXTERNAL_IGNORED_DIRS: &[&str] = &["examples", "tests", "benches"];
-
-        let is_ignored = if self.is_member {
-            dir_path.components().any(|c| COMMON_IGNORED_DIRS.contains(&c.as_str()))
-        } else {
-            dir_path.components().any(|c| {
-                let path = c.as_str();
-                COMMON_IGNORED_DIRS.contains(&path) || EXTERNAL_IGNORED_DIRS.contains(&path)
-            })
-        };
-
-        let hidden = dir_path.components().any(|c| c.as_str().starts_with('.'));
-
-        !is_ignored && !hidden
-    }
-
-    pub fn include_file(&self, file_path: &RelativePath) -> bool {
-        file_path.extension() == Some("rs")
-    }
 }
 
 impl ProjectWorkspace {

crates/ra_vfs_glob/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+edition = "2018"
+name = "ra_vfs_glob"
+version = "0.1.0"
+authors = ["rust-analyzer developers"]
+
+[dependencies]
+ra_vfs = "0.2.0"
+globset = "0.4.4"

crates/ra_vfs_glob/src/lib.rs

@@ -0,0 +1,93 @@
+//! `ra_vfs_glob` crate implements exclusion rules for vfs.
+//!
+//! By default, we include only `.rs` files, and skip some know offenders like
+//! `/target` or `/node_modules` altogether.
+//!
+//! It's also possible to add custom exclusion globs.
+
+use globset::{Glob, GlobSet, GlobSetBuilder};
+use ra_vfs::{Filter, RelativePath};
+
+const ALWAYS_IGNORED: &[&str] = &["target/**", "**/node_modules/**", "**/.git/**"];
+const IGNORED_FOR_NON_MEMBERS: &[&str] = &["examples/**", "tests/**", "benches/**"];
+
+pub struct RustPackageFilterBuilder {
+    is_member: bool,
+    exclude: GlobSetBuilder,
+}
+
+impl Default for RustPackageFilterBuilder {
+    fn default() -> RustPackageFilterBuilder {
+        RustPackageFilterBuilder { is_member: false, exclude: GlobSetBuilder::new() }
+    }
+}
+
+impl RustPackageFilterBuilder {
+    pub fn set_member(mut self, is_member: bool) -> RustPackageFilterBuilder {
+        self.is_member = is_member;
+        self
+    }
+    pub fn exclude(mut self, glob: &str) -> Result<RustPackageFilterBuilder, globset::Error> {
+        self.exclude.add(Glob::new(glob)?);
+        Ok(self)
+    }
+    pub fn into_vfs_filter(self) -> Box<dyn Filter> {
+        let RustPackageFilterBuilder { is_member, mut exclude } = self;
+        for &glob in ALWAYS_IGNORED {
+            exclude.add(Glob::new(glob).unwrap());
+        }
+        if !is_member {
+            for &glob in IGNORED_FOR_NON_MEMBERS {
+                exclude.add(Glob::new(glob).unwrap());
+            }
+        }
+        Box::new(RustPackageFilter { exclude: exclude.build().unwrap() })
+    }
+}
+
+struct RustPackageFilter {
+    exclude: GlobSet,
+}
+
+impl Filter for RustPackageFilter {
+    fn include_dir(&self, dir_path: &RelativePath) -> bool {
+        !self.exclude.is_match(dir_path.as_str())
+    }
+
+    fn include_file(&self, file_path: &RelativePath) -> bool {
+        file_path.extension() == Some("rs")
+    }
+}
+
+#[test]
+fn test_globs() {
+    let filter = RustPackageFilterBuilder::default().set_member(true).into_vfs_filter();
+
+    assert!(filter.include_dir(RelativePath::new("src/tests")));
+    assert!(filter.include_dir(RelativePath::new("src/target")));
+    assert!(filter.include_dir(RelativePath::new("tests")));
+    assert!(filter.include_dir(RelativePath::new("benches")));
+
+    assert!(!filter.include_dir(RelativePath::new("target")));
+    assert!(!filter.include_dir(RelativePath::new("src/foo/.git")));
+    assert!(!filter.include_dir(RelativePath::new("foo/node_modules")));
+
+    let filter = RustPackageFilterBuilder::default().set_member(false).into_vfs_filter();
+
+    assert!(filter.include_dir(RelativePath::new("src/tests")));
+    assert!(filter.include_dir(RelativePath::new("src/target")));
+
+    assert!(!filter.include_dir(RelativePath::new("target")));
+    assert!(!filter.include_dir(RelativePath::new("src/foo/.git")));
+    assert!(!filter.include_dir(RelativePath::new("foo/node_modules")));
+    assert!(!filter.include_dir(RelativePath::new("tests")));
+    assert!(!filter.include_dir(RelativePath::new("benches")));
+
+    let filter = RustPackageFilterBuilder::default()
+        .set_member(true)
+        .exclude("src/llvm-project/**")
+        .unwrap()
+        .into_vfs_filter();
+
+    assert!(!filter.include_dir(RelativePath::new("src/llvm-project/clang")));
+}