Auto merge of #15110 - HKalbasi:run-test-command, r=HKalbasi

internal: Add run-tests command

This command is similar to `cargo test` except that it uses r-a to run tests instead of compiling and running them with rustc. This is slower than `cargo test` and it is only useful for me to see a bird view of what needs to be fixed. The current output is:
```
48 passed, 5028 failed, 2 ignored
All tests            174.74s, 648ginstr
```
48 is very low, but higher than what I originally thought.

Now that there is some passing tests, I can show the plan:

https://github.com/rust-lang/rust-analyzer/assets/45197576/76d7d777-1843-4ca4-b7fe-e463bdade6cb

That is, at the end, I want to be able to immediately re run every test after every change. (0.5s is not really immediate, but it's not finished yet, and it is way better than 8s that running a typical test in r-a will take on my system)

crates/hir-def/src/attr.rs

@@ -272,6 +272,18 @@ impl Attrs {
         self.by_key("proc_macro_derive").exists()
     }
 
+    pub fn is_test(&self) -> bool {
+        self.by_key("test").exists()
+    }
+
+    pub fn is_ignore(&self) -> bool {
+        self.by_key("ignore").exists()
+    }
+
+    pub fn is_bench(&self) -> bool {
+        self.by_key("bench").exists()
+    }
+
     pub fn is_unstable(&self) -> bool {
         self.by_key("unstable").exists()
     }

crates/hir/src/lib.rs

@@ -1927,6 +1927,21 @@ impl Function {
         db.function_data(self.id).has_async_kw()
     }
 
+    /// Does this function have `#[test]` attribute?
+    pub fn is_test(self, db: &dyn HirDatabase) -> bool {
+        db.function_data(self.id).attrs.is_test()
+    }
+
+    /// Does this function have the ignore attribute?
+    pub fn is_ignore(self, db: &dyn HirDatabase) -> bool {
+        db.function_data(self.id).attrs.is_ignore()
+    }
+
+    /// Does this function have `#[bench]` attribute?
+    pub fn is_bench(self, db: &dyn HirDatabase) -> bool {
+        db.function_data(self.id).attrs.is_bench()
+    }
+
     pub fn is_unsafe_to_call(self, db: &dyn HirDatabase) -> bool {
         hir_ty::is_fn_unsafe_to_call(db, self.id)
     }

crates/ide/src/runnables.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 use ast::HasName;
 use cfg::CfgExpr;
-use hir::{AsAssocItem, HasAttrs, HasSource, Semantics};
+use hir::{db::HirDatabase, AsAssocItem, HasAttrs, HasSource, Semantics};
 use ide_assists::utils::test_related_attribute;
 use ide_db::{
     base_db::{FilePosition, FileRange},
@@ -14,7 +14,7 @@ use ide_db::{
 use itertools::Itertools;
 use stdx::{always, format_to};
 use syntax::{
-    ast::{self, AstNode, HasAttrs as _},
+    ast::{self, AstNode},
     SmolStr, SyntaxNode,
 };
 
@@ -307,7 +307,6 @@ pub(crate) fn runnable_fn(
     sema: &Semantics<'_, RootDatabase>,
     def: hir::Function,
 ) -> Option<Runnable> {
-    let func = def.source(sema.db)?;
     let name = def.name(sema.db).to_smol_str();
 
     let root = def.module(sema.db).krate().root_module(sema.db);
@@ -323,10 +322,10 @@ pub(crate) fn runnable_fn(
             canonical_path.map(TestId::Path).unwrap_or(TestId::Name(name))
         };
 
-        if test_related_attribute(&func.value).is_some() {
-            let attr = TestAttr::from_fn(&func.value);
+        if def.is_test(sema.db) {
+            let attr = TestAttr::from_fn(sema.db, def);
             RunnableKind::Test { test_id: test_id(), attr }
-        } else if func.value.has_atom_attr("bench") {
+        } else if def.is_bench(sema.db) {
             RunnableKind::Bench { test_id: test_id() }
         } else {
             return None;
@@ -335,7 +334,7 @@ pub(crate) fn runnable_fn(
 
     let nav = NavigationTarget::from_named(
         sema.db,
-        func.as_ref().map(|it| it as &dyn ast::HasName),
+        def.source(sema.db)?.as_ref().map(|it| it as &dyn ast::HasName),
         SymbolKind::Function,
     );
     let cfg = def.attrs(sema.db).cfg();
@@ -487,12 +486,8 @@ pub struct TestAttr {
 }
 
 impl TestAttr {
-    fn from_fn(fn_def: &ast::Fn) -> TestAttr {
-        let ignore = fn_def
-            .attrs()
-            .filter_map(|attr| attr.simple_name())
-            .any(|attribute_text| attribute_text == "ignore");
-        TestAttr { ignore }
+    fn from_fn(db: &dyn HirDatabase, fn_def: hir::Function) -> TestAttr {
+        TestAttr { ignore: fn_def.is_ignore(db) }
     }
 }
 

crates/rust-analyzer/src/bin/main.rs

@@ -82,6 +82,7 @@ fn main() -> anyhow::Result<()> {
         flags::RustAnalyzerCmd::Search(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?,
+        flags::RustAnalyzerCmd::RunTests(cmd) => cmd.run()?,
     }
     Ok(())
 }

crates/rust-analyzer/src/cli.rs

@@ -10,12 +10,17 @@ mod diagnostics;
 mod ssr;
 mod lsif;
 mod scip;
+mod run_tests;
 
 mod progress_report;
 
 use std::io::Read;
 
+use anyhow::Result;
+use hir::{Module, Name};
+use hir_ty::db::HirDatabase;
 use ide::AnalysisHost;
+use itertools::Itertools;
 use vfs::Vfs;
 
 #[derive(Clone, Copy)]
@@ -70,3 +75,14 @@ fn print_memory_usage(mut host: AnalysisHost, vfs: Vfs) {
 
     eprintln!("{remaining:>8}        Remaining");
 }
+
+fn full_name_of_item(db: &dyn HirDatabase, module: Module, name: Name) -> String {
+    module
+        .path_to_root(db)
+        .into_iter()
+        .rev()
+        .filter_map(|it| it.name(db))
+        .chain(Some(name))
+        .map(|it| it.display(db.upcast()).to_string())
+        .join("::")
+}

crates/rust-analyzer/src/cli/analysis_stats.rs

@@ -34,6 +34,7 @@ use vfs::{AbsPathBuf, Vfs, VfsPath};
 
 use crate::cli::{
     flags::{self, OutputFormat},
+    full_name_of_item,
     load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice},
     print_memory_usage,
     progress_report::ProgressReport,
@@ -274,15 +275,7 @@ impl flags::AnalysisStats {
                 continue
             };
             if verbosity.is_spammy() {
-                let full_name = a
-                    .module(db)
-                    .path_to_root(db)
-                    .into_iter()
-                    .rev()
-                    .filter_map(|it| it.name(db))
-                    .chain(Some(a.name(db)))
-                    .map(|it| it.display(db).to_string())
-                    .join("::");
+                let full_name = full_name_of_item(db, a.module(db), a.name(db));
                 println!("Data layout for {full_name} failed due {e:?}");
             }
             fail += 1;
@@ -304,15 +297,8 @@ impl flags::AnalysisStats {
                 continue;
             };
             if verbosity.is_spammy() {
-                let full_name = c
-                    .module(db)
-                    .path_to_root(db)
-                    .into_iter()
-                    .rev()
-                    .filter_map(|it| it.name(db))
-                    .chain(c.name(db))
-                    .map(|it| it.display(db).to_string())
-                    .join("::");
+                let full_name =
+                    full_name_of_item(db, c.module(db), c.name(db).unwrap_or(Name::missing()));
                 println!("Const eval for {full_name} failed due {e:?}");
             }
             fail += 1;

crates/rust-analyzer/src/cli/flags.rs

@@ -90,6 +90,12 @@ xflags::xflags! {
             optional --skip-const-eval
         }
 
+        /// Run unit tests of the project using mir interpreter
+        cmd run-tests {
+            /// Directory with Cargo.toml.
+            required path: PathBuf
+        }
+
         cmd diagnostics {
             /// Directory with Cargo.toml.
             required path: PathBuf
@@ -147,6 +153,7 @@ pub enum RustAnalyzerCmd {
     Symbols(Symbols),
     Highlight(Highlight),
     AnalysisStats(AnalysisStats),
+    RunTests(RunTests),
     Diagnostics(Diagnostics),
     Ssr(Ssr),
     Search(Search),
@@ -182,16 +189,21 @@ pub struct AnalysisStats {
     pub parallel: bool,
     pub memory_usage: bool,
     pub source_stats: bool,
-    pub skip_lowering: bool,
-    pub skip_inference: bool,
-    pub skip_mir_stats: bool,
-    pub skip_data_layout: bool,
-    pub skip_const_eval: bool,
     pub only: Option<String>,
     pub with_deps: bool,
     pub no_sysroot: bool,
     pub disable_build_scripts: bool,
     pub disable_proc_macros: bool,
+    pub skip_lowering: bool,
+    pub skip_inference: bool,
+    pub skip_mir_stats: bool,
+    pub skip_data_layout: bool,
+    pub skip_const_eval: bool,
+}
+
+#[derive(Debug)]
+pub struct RunTests {
+    pub path: PathBuf,
 }
 
 #[derive(Debug)]
@@ -223,6 +235,7 @@ pub struct Lsif {
 #[derive(Debug)]
 pub struct Scip {
     pub path: PathBuf,
+
     pub output: Option<PathBuf>,
 }
 

crates/rust-analyzer/src/cli/run_tests.rs

@@ -0,0 +1,92 @@
+//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
+
+use hir::{Crate, Module};
+use hir_ty::db::HirDatabase;
+use ide_db::{base_db::SourceDatabaseExt, LineIndexDatabase};
+use profile::StopWatch;
+use project_model::{CargoConfig, RustLibSource};
+use syntax::TextRange;
+
+use crate::cli::{
+    flags, full_name_of_item,
+    load_cargo::load_workspace_at,
+    load_cargo::{LoadCargoConfig, ProcMacroServerChoice},
+    Result,
+};
+
+impl flags::RunTests {
+    pub fn run(self) -> Result<()> {
+        let mut cargo_config = CargoConfig::default();
+        cargo_config.sysroot = Some(RustLibSource::Discover);
+        let load_cargo_config = LoadCargoConfig {
+            load_out_dirs_from_check: true,
+            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
+            prefill_caches: false,
+        };
+        let (host, _vfs, _proc_macro) =
+            load_workspace_at(&self.path, &cargo_config, &load_cargo_config, &|_| {})?;
+        let db = host.raw_database();
+
+        let tests = all_modules(db)
+            .into_iter()
+            .flat_map(|x| x.declarations(db))
+            .filter_map(|x| match x {
+                hir::ModuleDef::Function(f) => Some(f),
+                _ => None,
+            })
+            .filter(|x| x.is_test(db));
+        let span_formatter = |file_id, text_range: TextRange| {
+            let line_col = match db.line_index(file_id).try_line_col(text_range.start()) {
+                None => " (unknown line col)".to_string(),
+                Some(x) => format!("#{}:{}", x.line + 1, x.col),
+            };
+            let path = &db
+                .source_root(db.file_source_root(file_id))
+                .path_for_file(&file_id)
+                .map(|x| x.to_string());
+            let path = path.as_deref().unwrap_or("<unknown file>");
+            format!("file://{path}{line_col}")
+        };
+        let mut pass_count = 0;
+        let mut ignore_count = 0;
+        let mut fail_count = 0;
+        let mut sw_all = StopWatch::start();
+        for test in tests {
+            let full_name = full_name_of_item(db, test.module(db), test.name(db));
+            println!("test {}", full_name);
+            if test.is_ignore(db) {
+                println!("ignored");
+                ignore_count += 1;
+                continue;
+            }
+            let mut sw_one = StopWatch::start();
+            let result = test.eval(db, span_formatter);
+            if result.trim() == "pass" {
+                pass_count += 1;
+            } else {
+                fail_count += 1;
+            }
+            println!("{}", result);
+            eprintln!("{:<20} {}", format!("test {}", full_name), sw_one.elapsed());
+        }
+        println!("{pass_count} passed, {fail_count} failed, {ignore_count} ignored");
+        eprintln!("{:<20} {}", "All tests", sw_all.elapsed());
+        Ok(())
+    }
+}
+
+fn all_modules(db: &dyn HirDatabase) -> Vec<Module> {
+    let mut worklist: Vec<_> = Crate::all(db)
+        .into_iter()
+        .filter(|x| x.origin(db).is_local())
+        .map(|krate| krate.root_module(db))
+        .collect();
+    let mut modules = Vec::new();
+
+    while let Some(module) = worklist.pop() {
+        modules.push(module);
+        worklist.extend(module.children(db));
+    }
+
+    modules
+}