indexing infra

Cargo.toml

@@ -1,2 +1,3 @@
 [workspace]
 members = [ "crates/*" ]
+exclude = [ "crates/indxr" ]

code/package.json

@@ -44,6 +44,38 @@
         "key": "ctrl+w",
         "when": "editorTextFocus && editorLangId == rust"
       }
+    ],
+    "problemMatchers": [
+      {
+        "name": "rustc",
+        "fileLocation": [
+          "relative",
+          "${workspaceRoot}"
+        ],
+        "pattern": [
+          {
+            "regexp": "^(warning|warn|error)(\\[(.*)\\])?: (.*)$",
+            "severity": 1,
+            "message": 4,
+            "code": 3
+          },
+          {
+            "regexp": "^([\\s->=]*(.*):(\\d*):(\\d*)|.*)$",
+            "file": 2,
+            "line": 3,
+            "column": 4
+          },
+          {
+            "regexp": "^.*$"
+          },
+          {
+            "regexp": "^([\\s->=]*(.*):(\\d*):(\\d*)|.*)$",
+            "file": 2,
+            "line": 3,
+            "column": 4
+          }
+        ]
+      }
     ]
   }
 }

code/src/extension.ts

@@ -61,8 +61,9 @@ export function deactivate(): Thenable<void> {
 
 function startServer() {
     let run: lc.Executable = {
-        command: "m",
+        command: "cargo",
-        // args: ["run", "--package", "m"],
+        args: ["run", "--package", "m"],
+        // command: "m",
         options: { cwd: "." }
     }
     let serverOptions: lc.ServerOptions = {

crates/libanalysis/src/lib.rs

@@ -1,3 +1,4 @@
+#[macro_use]
 extern crate failure;
 extern crate parking_lot;
 #[macro_use]
@@ -9,12 +10,10 @@ extern crate libeditor;
 use once_cell::sync::OnceCell;
 
 use std::{
-    fs,
     sync::Arc,
     collections::hash_map::HashMap,
     path::{PathBuf, Path},
 };
-use parking_lot::RwLock;
 use libsyntax2::ast;
 use libeditor::LineIndex;
 
@@ -40,22 +39,27 @@ impl WorldState {
         World { data: self.data.clone() }
     }
 
-    pub fn change_overlay(&mut self, path: PathBuf, text: Option<String>) {
+    pub fn change_file(&mut self, path: PathBuf, text: Option<String>) {
+        self.change_files(::std::iter::once((path, text)));
+    }
+
+    pub fn change_files(&mut self, changes: impl Iterator<Item=(PathBuf, Option<String>)>) {
         let data = self.data_mut();
-        data.file_map.get_mut().remove(&path);
+        for (path, text) in changes {
-        if let Some(text) = text {
+            data.file_map.remove(&path);
-            data.mem_map.insert(path, Arc::new(text));
+            if let Some(text) = text {
-        } else {
+                let file_data = FileData::new(text);
-            data.mem_map.remove(&path);
+                data.file_map.insert(path, Arc::new(file_data));
+            } else {
+                data.file_map.remove(&path);
+            }
         }
     }
 
     fn data_mut(&mut self) -> &mut WorldData {
         if Arc::get_mut(&mut self.data).is_none() {
-            let file_map = self.data.file_map.read().clone();
             self.data = Arc::new(WorldData {
-                mem_map: self.data.mem_map.clone(),
+                file_map: self.data.file_map.clone(),
-                file_map: RwLock::new(file_map),
             });
         }
         Arc::get_mut(&mut self.data).unwrap()
@@ -69,7 +73,7 @@ impl World {
         let syntax = data.syntax
             .get_or_init(|| {
                 trace!("parsing: {}", path.display());
-                ast::File::parse(self.file_text(path, &data))
+                ast::File::parse(&data.text)
             }).clone();
         Ok(syntax)
     }
@@ -79,56 +83,38 @@ impl World {
         let index = data.lines
             .get_or_init(|| {
                 trace!("calc line index: {}", path.display());
-                LineIndex::new(self.file_text(path, &data))
+                LineIndex::new(&data.text)
             });
         Ok(index.clone())
     }
 
-    fn file_text<'a>(&'a self, path: &Path, file_data: &'a FileData) -> &'a str {
-        match file_data.text.as_ref() {
-            Some(text) => text.as_str(),
-            None => self.data.mem_map[path].as_str()
-        }
-    }
-
     fn file_data(&self, path: &Path) -> Result<Arc<FileData>> {
-        {
+        match self.data.file_map.get(path) {
-            let guard = self.data.file_map.read();
+            Some(data) => Ok(data.clone()),
-            if let Some(data) = guard.get(path) {
+            None => bail!("unknown file: {}", path.display()),
-                return Ok(data.clone());
-            }
         }
-
-        let text = if self.data.mem_map.contains_key(path) {
-            None
-        } else {
-            trace!("loading file from disk: {}", path.display());
-            Some(fs::read_to_string(path)?)
-        };
-        let res = {
-            let mut guard = self.data.file_map.write();
-            guard.entry(path.to_owned())
-                .or_insert_with(|| Arc::new(FileData {
-                    text,
-                    syntax: OnceCell::new(),
-                    lines: OnceCell::new(),
-                }))
-                .clone()
-        };
-        Ok(res)
     }
 }
 
 
 #[derive(Default, Debug)]
 struct WorldData {
-    mem_map: HashMap<PathBuf, Arc<String>>,
+    file_map: HashMap<PathBuf, Arc<FileData>>,
-    file_map: RwLock<HashMap<PathBuf, Arc<FileData>>>,
 }
 
 #[derive(Debug)]
 struct FileData {
-    text: Option<String>,
+    text: String,
     syntax: OnceCell<ast::File>,
     lines: OnceCell<LineIndex>,
 }
+
+impl FileData {
+    fn new(text: String) -> FileData {
+        FileData {
+            text,
+            syntax: OnceCell::new(),
+            lines: OnceCell::new(),
+        }
+    }
+}

crates/server/Cargo.toml

@@ -15,6 +15,7 @@ flexi_logger = "0.9.1"
 log = "0.4.3"
 url_serde = "0.2.0"
 languageserver-types = "0.49.0"
+walkdir = "2.2.0"
 text_unit = { version = "0.1.2", features = ["serde"] }
 
 libsyntax2 = { path = "../libsyntax2" }

crates/server/src/main.rs

@@ -13,6 +13,7 @@ extern crate threadpool;
 extern crate log;
 extern crate url_serde;
 extern crate flexi_logger;
+extern crate walkdir;
 extern crate libeditor;
 extern crate libanalysis;
 extern crate libsyntax2;
@@ -24,6 +25,9 @@ mod dispatch;
 mod util;
 mod conv;
 mod main_loop;
+mod vfs;
+
+use std::path::PathBuf;
 
 use threadpool::ThreadPool;
 use crossbeam_channel::bounded;
@@ -114,13 +118,29 @@ fn initialized(io: &mut Io) -> Result<()> {
     {
         let mut world = WorldState::new();
         let mut pool = ThreadPool::new(4);
-        let (sender, receiver) = bounded::<Task>(16);
+        let (task_sender, task_receiver) = bounded::<Task>(16);
+        let (fs_events_receiver, watcher) = vfs::watch(vec![
+            PathBuf::from("./")
+        ]);
         info!("lifecycle: handshake finished, server ready to serve requests");
-        let res = main_loop::main_loop(io, &mut world, &mut pool, sender, receiver.clone());
+        let res = main_loop::main_loop(
+            io,
+            &mut world,
+            &mut pool,
+            task_sender,
+            task_receiver.clone(),
+            fs_events_receiver,
+        );
+
         info!("waiting for background jobs to finish...");
-        receiver.for_each(drop);
+        task_receiver.for_each(drop);
         pool.join();
         info!("...background jobs have finished");
+
+        info!("waiting for file watcher to finish...");
+        watcher.stop()?;
+        info!("...file watcher has finished");
+
         res
     }?;
 

crates/server/src/main_loop/mod.rs

@@ -1,6 +1,9 @@
 mod handlers;
 
-use std::collections::HashSet;
+use std::{
+    path::PathBuf,
+    collections::{HashSet, HashMap},
+};
 
 use threadpool::ThreadPool;
 use crossbeam_channel::{Sender, Receiver};
@@ -13,6 +16,7 @@ use {
     Task, Result,
     io::{Io, RawMsg, RawRequest, RawNotification},
     util::FilePath,
+    vfs::{FileEvent, FileEventKind},
     main_loop::handlers::{
         handle_syntax_tree,
         handle_extend_selection,
@@ -28,24 +32,33 @@ pub(super) fn main_loop(
     io: &mut Io,
     world: &mut WorldState,
     pool: &mut ThreadPool,
-    sender: Sender<Task>,
+    task_sender: Sender<Task>,
-    receiver: Receiver<Task>,
+    task_receiver: Receiver<Task>,
+    fs_events_receiver: Receiver<Vec<FileEvent>>,
 ) -> Result<()> {
     info!("server initialized, serving requests");
     let mut next_request_id = 0;
     let mut pending_requests: HashSet<u64> = HashSet::new();
+    let mut mem_map: HashMap<PathBuf, Option<String>> = HashMap::new();
+    let mut fs_events_receiver = Some(&fs_events_receiver);
     loop {
         enum Event {
             Msg(RawMsg),
             Task(Task),
+            Fs(Vec<FileEvent>),
             ReceiverDead,
+            FsWatcherDead,
         }
         let event = select! {
             recv(io.receiver(), msg) => match msg {
                 Some(msg) => Event::Msg(msg),
                 None => Event::ReceiverDead,
             },
-            recv(receiver, task) => Event::Task(task.unwrap()),
+            recv(task_receiver, task) => Event::Task(task.unwrap()),
+            recv(fs_events_receiver, events) => match events {
+                Some(events) => Event::Fs(events),
+                None => Event::FsWatcherDead,
+            }
         };
 
         match event {
@@ -53,6 +66,9 @@ pub(super) fn main_loop(
                 io.cleanup_receiver()?;
                 unreachable!();
             }
+            Event::FsWatcherDead => {
+                fs_events_receiver = None;
+            }
             Event::Task(task) => {
                 match task {
                     Task::Request(mut request) => {
@@ -70,15 +86,36 @@ pub(super) fn main_loop(
                 }
                 continue;
             }
+            Event::Fs(events) => {
+                trace!("fs change, {} events", events.len());
+                let changes = events.into_iter()
+                    .map(|event| {
+                        let text = match event.kind {
+                            FileEventKind::Add(text) => Some(text),
+                            FileEventKind::Remove => None,
+                        };
+                        (event.path, text)
+                    })
+                    .filter_map(|(path, text)| {
+                        if mem_map.contains_key(path.as_path()) {
+                            mem_map.insert(path, text);
+                            None
+                        } else {
+                            Some((path, text))
+                        }
+                    });
+
+                world.change_files(changes);
+            }
             Event::Msg(msg) => {
                 match msg {
                     RawMsg::Request(req) => {
-                        if !on_request(io, world, pool, &sender, req)? {
+                        if !on_request(io, world, pool, &task_sender, req)? {
                             return Ok(());
                         }
                     }
                     RawMsg::Notification(not) => {
-                        on_notification(io, world, pool, &sender, not)?
+                        on_notification(io, world, pool, &task_sender, not, &mut mem_map)?
                     }
                     RawMsg::Response(resp) => {
                         if !pending_requests.remove(&resp.id) {
@@ -160,11 +197,13 @@ fn on_notification(
     pool: &ThreadPool,
     sender: &Sender<Task>,
     not: RawNotification,
+    mem_map: &mut HashMap<PathBuf, Option<String>>,
 ) -> Result<()> {
     let mut not = Some(not);
     dispatch::handle_notification::<req::DidOpenTextDocument, _>(&mut not, |params| {
         let path = params.text_document.file_path()?;
-        world.change_overlay(path, Some(params.text_document.text));
+        mem_map.insert(path.clone(), None);
+        world.change_file(path, Some(params.text_document.text));
         update_file_notifications_on_threadpool(
             pool, world.snapshot(), sender.clone(), params.text_document.uri,
         );
@@ -175,7 +214,7 @@ fn on_notification(
         let text = params.content_changes.pop()
             .ok_or_else(|| format_err!("empty changes"))?
             .text;
-        world.change_overlay(path, Some(text));
+        world.change_file(path, Some(text));
         update_file_notifications_on_threadpool(
             pool, world.snapshot(), sender.clone(), params.text_document.uri,
         );
@@ -183,7 +222,11 @@ fn on_notification(
     })?;
     dispatch::handle_notification::<req::DidCloseTextDocument, _>(&mut not, |params| {
         let path = params.text_document.file_path()?;
-        world.change_overlay(path, None);
+        let text = match mem_map.remove(&path) {
+            Some(text) => text,
+            None => bail!("unmatched close notification"),
+        };
+        world.change_file(path, text);
         let not = req::PublishDiagnosticsParams {
             uri: params.text_document.uri,
             diagnostics: Vec::new(),

crates/server/src/vfs.rs

@@ -0,0 +1,79 @@
+use std::{
+    path::PathBuf,
+    thread,
+    fs,
+};
+
+use crossbeam_channel::{Sender, Receiver, bounded};
+use drop_bomb::DropBomb;
+use walkdir::WalkDir;
+
+use Result;
+
+
+pub struct FileEvent {
+    pub path: PathBuf,
+    pub kind: FileEventKind,
+}
+
+pub enum FileEventKind {
+    Add(String),
+    #[allow(unused)]
+    Remove,
+}
+
+pub struct Watcher {
+    thread: thread::JoinHandle<()>,
+    bomb: DropBomb,
+}
+
+impl Watcher {
+    pub fn stop(mut self) -> Result<()> {
+        self.bomb.defuse();
+        self.thread.join()
+            .map_err(|_| format_err!("file watcher died"))
+    }
+}
+
+pub fn watch(roots: Vec<PathBuf>) -> (Receiver<Vec<FileEvent>>, Watcher) {
+    let (sender, receiver) = bounded(16);
+    let thread = thread::spawn(move || run(roots, sender));
+    (receiver, Watcher {
+        thread,
+        bomb: DropBomb::new("Watcher should be stopped explicitly"),
+    })
+}
+
+fn run(roots: Vec<PathBuf>, sender: Sender<Vec<FileEvent>>) {
+    for root in roots {
+        let mut events = Vec::new();
+        for entry in WalkDir::new(root.as_path()) {
+            let entry = match entry {
+                Ok(entry) => entry,
+                Err(e) => {
+                    warn!("watcher error: {}", e);
+                    continue;
+                }
+            };
+            if !entry.file_type().is_file() {
+                continue;
+            }
+            let path = entry.path();
+            if path.extension().and_then(|os| os.to_str()) != Some("rs") {
+                continue;
+            }
+            let text = match fs::read_to_string(path) {
+                Ok(text) => text,
+                Err(e) => {
+                    warn!("watcher error: {}", e);
+                    continue;
+                }
+            };
+            events.push(FileEvent {
+                path: path.to_owned(),
+                kind: FileEventKind::Add(text),
+            })
+        }
+        sender.send(events)
+    }
+}