Run rust-analyzer on rustc tests in metrics

.github/workflows/metrics.yaml

@@ -67,7 +67,7 @@ jobs:
   other_metrics:
     strategy:
       matrix:
-        names: [self, ripgrep-13.0.0, webrender-2022, diesel-1.4.8, hyper-0.14.18]
+        names: [self, rustc_tests, ripgrep-13.0.0, webrender-2022, diesel-1.4.8, hyper-0.14.18]
     runs-on: ubuntu-latest
     needs: [setup_cargo, build_metrics]
 
@@ -118,6 +118,11 @@ jobs:
         with:
           name: self-${{ github.sha }}
 
+      - name: Download rustc_tests metrics
+        uses: actions/download-artifact@v3
+        with:
+          name: rustc_tests-${{ github.sha }}
+
       - name: Download ripgrep-13.0.0 metrics
         uses: actions/download-artifact@v3
         with:
@@ -146,7 +151,7 @@ jobs:
           chmod 700 ~/.ssh
 
           git clone --depth 1 git@github.com:rust-analyzer/metrics.git
-          jq -s ".[0] * .[1] * .[2] * .[3] * .[4] * .[5]" build.json self.json ripgrep-13.0.0.json webrender-2022.json diesel-1.4.8.json hyper-0.14.18.json -c >> metrics/metrics.json
+          jq -s ".[0] * .[1] * .[2] * .[3] * .[4] * .[5] * .[6]" build.json self.json rustc_tests.json ripgrep-13.0.0.json webrender-2022.json diesel-1.4.8.json hyper-0.14.18.json -c >> metrics/metrics.json
           cd metrics
           git add .
           git -c user.name=Bot -c user.email=dummy@example.com commit --message 📈

Cargo.lock

@@ -1545,6 +1545,7 @@ dependencies = [
  "triomphe",
  "vfs",
  "vfs-notify",
+ "walkdir",
  "winapi",
  "xflags",
  "xshell",

crates/ide-diagnostics/src/lib.rs

@@ -94,7 +94,7 @@ use syntax::{
 };
 
 // FIXME: Make this an enum
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum DiagnosticCode {
     RustcHardError(&'static str),
     RustcLint(&'static str),
@@ -198,7 +198,7 @@ impl Diagnostic {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub enum Severity {
     Error,
     Warning,

crates/ide/src/lib.rs

@@ -133,7 +133,9 @@ pub use ide_db::{
     symbol_index::Query,
     RootDatabase, SymbolKind,
 };
-pub use ide_diagnostics::{Diagnostic, DiagnosticsConfig, ExprFillDefaultMode, Severity};
+pub use ide_diagnostics::{
+    Diagnostic, DiagnosticCode, DiagnosticsConfig, ExprFillDefaultMode, Severity,
+};
 pub use ide_ssr::SsrError;
 pub use syntax::{TextRange, TextSize};
 pub use text_edit::{Indel, TextEdit};

crates/rust-analyzer/Cargo.toml

@@ -42,6 +42,7 @@ tracing-tree.workspace = true
 triomphe.workspace = true
 nohash-hasher.workspace = true
 always-assert = "0.1.2"
+walkdir = "2.3.2"
 
 cfg.workspace = true
 flycheck.workspace = true

crates/rust-analyzer/src/bin/main.rs

@@ -87,6 +87,7 @@ fn main() -> anyhow::Result<()> {
         flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::RunTests(cmd) => cmd.run()?,
+        flags::RustAnalyzerCmd::RustcTests(cmd) => cmd.run()?,
     }
     Ok(())
 }

crates/rust-analyzer/src/cli.rs

@@ -10,6 +10,7 @@ mod ssr;
 mod lsif;
 mod scip;
 mod run_tests;
+mod rustc_tests;
 
 mod progress_report;
 

crates/rust-analyzer/src/cli/flags.rs

@@ -98,6 +98,15 @@ xflags::xflags! {
             required path: PathBuf
         }
 
+        /// Run unit tests of the project using mir interpreter
+        cmd rustc-tests {
+            /// Directory with Cargo.toml.
+            required rustc_repo: PathBuf
+
+            /// Only run tests with filter as substring
+            optional --filter path: String
+        }
+
         cmd diagnostics {
             /// Directory with Cargo.toml.
             required path: PathBuf
@@ -159,6 +168,7 @@ pub enum RustAnalyzerCmd {
     Highlight(Highlight),
     AnalysisStats(AnalysisStats),
     RunTests(RunTests),
+    RustcTests(RustcTests),
     Diagnostics(Diagnostics),
     Ssr(Ssr),
     Search(Search),
@@ -211,6 +221,12 @@ pub struct RunTests {
     pub path: PathBuf,
 }
 
+#[derive(Debug)]
+pub struct RustcTests {
+    pub rustc_repo: PathBuf,
+    pub filter: Option<String>,
+}
+
 #[derive(Debug)]
 pub struct Diagnostics {
     pub path: PathBuf,

crates/rust-analyzer/src/cli/rustc_tests.rs

@@ -0,0 +1,236 @@
+//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter.
+
+use std::{
+    cell::RefCell, collections::HashMap, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf,
+};
+
+use hir::Crate;
+use ide::{AnalysisHost, Change, DiagnosticCode, DiagnosticsConfig};
+use profile::StopWatch;
+use project_model::{CargoConfig, ProjectWorkspace, RustLibSource, Sysroot};
+
+use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
+use triomphe::Arc;
+use vfs::{AbsPathBuf, FileId};
+use walkdir::WalkDir;
+
+use crate::cli::{flags, report_metric, Result};
+
+struct Tester {
+    host: AnalysisHost,
+    root_file: FileId,
+    pass_count: u64,
+    ignore_count: u64,
+    fail_count: u64,
+    stopwatch: StopWatch,
+}
+
+fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode {
+    thread_local! {
+        static LEAK_STORE: RefCell<HashMap<String, DiagnosticCode>> = RefCell::new(HashMap::new());
+    }
+    LEAK_STORE.with_borrow_mut(|s| match s.get(code) {
+        Some(c) => *c,
+        None => {
+            let v = DiagnosticCode::RustcHardError(format!("E{code}").leak());
+            s.insert(code.to_owned(), v);
+            v
+        }
+    })
+}
+
+fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> HashMap<DiagnosticCode, usize> {
+    let text = read_to_string(p).unwrap();
+    let mut result = HashMap::new();
+    {
+        let mut text = &*text;
+        while let Some(p) = text.find("error[E") {
+            text = &text[p + 7..];
+            let code = string_to_diagnostic_code_leaky(&text[..4]);
+            *result.entry(code).or_insert(0) += 1;
+        }
+    }
+    result
+}
+
+impl Tester {
+    fn new() -> Result<Self> {
+        let tmp_file = AbsPathBuf::assert("/tmp/ra-rustc-test.rs".into());
+        std::fs::write(&tmp_file, "")?;
+        let mut cargo_config = CargoConfig::default();
+        cargo_config.sysroot = Some(RustLibSource::Discover);
+        let workspace = ProjectWorkspace::DetachedFiles {
+            files: vec![tmp_file.clone()],
+            sysroot: Ok(
+                Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env).unwrap()
+            ),
+            rustc_cfg: vec![],
+        };
+        let load_cargo_config = LoadCargoConfig {
+            load_out_dirs_from_check: false,
+            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
+            prefill_caches: false,
+        };
+        let (host, _vfs, _proc_macro) =
+            load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?;
+        let db = host.raw_database();
+        let krates = Crate::all(db);
+        let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap();
+        let root_file = root_crate.root_file(db);
+        Ok(Self {
+            host,
+            root_file,
+            pass_count: 0,
+            ignore_count: 0,
+            fail_count: 0,
+            stopwatch: StopWatch::start(),
+        })
+    }
+
+    fn test(&mut self, p: PathBuf) {
+        if p.parent().unwrap().file_name().unwrap() == "auxiliary" {
+            // These are not tests
+            return;
+        }
+        if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) {
+            println!("{p:?} IGNORE");
+            self.ignore_count += 1;
+            return;
+        }
+        let stderr_path = p.with_extension("stderr");
+        let expected = if stderr_path.exists() {
+            detect_errors_from_rustc_stderr_file(stderr_path)
+        } else {
+            HashMap::new()
+        };
+        let text = read_to_string(&p).unwrap();
+        let mut change = Change::new();
+        // Ignore unstable tests, since they move too fast and we do not intend to support all of them.
+        let mut ignore_test = text.contains("#![feature");
+        // Ignore test with extern crates, as this infra don't support them yet.
+        ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:");
+        // Ignore test with extern modules similarly.
+        ignore_test |= text.contains("mod ");
+        // These should work, but they don't, and I don't know why, so ignore them.
+        ignore_test |= text.contains("extern crate proc_macro");
+        let should_have_no_error = text.contains("// check-pass")
+            || text.contains("// build-pass")
+            || text.contains("// run-pass");
+        change.change_file(self.root_file, Some(Arc::from(text)));
+        self.host.apply_change(change);
+        let diagnostic_config = DiagnosticsConfig::test_sample();
+        let diags = self
+            .host
+            .analysis()
+            .diagnostics(&diagnostic_config, ide::AssistResolveStrategy::None, self.root_file)
+            .unwrap();
+        let mut actual = HashMap::new();
+        for diag in diags {
+            if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) {
+                continue;
+            }
+            if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) {
+                continue;
+            }
+            *actual.entry(diag.code).or_insert(0) += 1;
+        }
+        // Ignore tests with diagnostics that we don't emit.
+        ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k));
+        if ignore_test {
+            println!("{p:?} IGNORE");
+            self.ignore_count += 1;
+        } else if actual == expected {
+            println!("{p:?} PASS");
+            self.pass_count += 1;
+        } else {
+            println!("{p:?} FAIL");
+            println!("actual   (r-a)   = {:?}", actual);
+            println!("expected (rustc) = {:?}", expected);
+            self.fail_count += 1;
+        }
+    }
+
+    fn report(&mut self) {
+        println!(
+            "Pass count = {}, Fail count = {}, Ignore count = {}",
+            self.pass_count, self.fail_count, self.ignore_count
+        );
+        println!("Testing time and memory = {}", self.stopwatch.elapsed());
+        report_metric("rustc failed tests", self.fail_count, "#");
+        report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms");
+    }
+}
+
+/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them.
+const IGNORED_TESTS: &[&str] = &[
+    "trait-with-missing-associated-type-restriction.rs", // #15646
+    "trait-with-missing-associated-type-restriction-fixable.rs", // #15646
+    "resolve-self-in-impl.rs",
+    "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs
+    "issue-26056.rs",
+    "float-field.rs",
+    "invalid_operator_trait.rs",
+    "type-alias-impl-trait-assoc-dyn.rs",
+    "deeply-nested_closures.rs",    // exponential time
+    "hang-on-deeply-nested-dyn.rs", // exponential time
+    "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0
+    "issue-16098.rs",      // Huge recursion limit for macros?
+    "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0
+];
+
+const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[
+    DiagnosticCode::RustcHardError("E0023"),
+    DiagnosticCode::RustcHardError("E0046"),
+    DiagnosticCode::RustcHardError("E0063"),
+    DiagnosticCode::RustcHardError("E0107"),
+    DiagnosticCode::RustcHardError("E0117"),
+    DiagnosticCode::RustcHardError("E0133"),
+    DiagnosticCode::RustcHardError("E0210"),
+    DiagnosticCode::RustcHardError("E0268"),
+    DiagnosticCode::RustcHardError("E0308"),
+    DiagnosticCode::RustcHardError("E0384"),
+    DiagnosticCode::RustcHardError("E0407"),
+    DiagnosticCode::RustcHardError("E0432"),
+    DiagnosticCode::RustcHardError("E0451"),
+    DiagnosticCode::RustcHardError("E0507"),
+    DiagnosticCode::RustcHardError("E0583"),
+    DiagnosticCode::RustcHardError("E0559"),
+    DiagnosticCode::RustcHardError("E0616"),
+    DiagnosticCode::RustcHardError("E0618"),
+    DiagnosticCode::RustcHardError("E0624"),
+    DiagnosticCode::RustcHardError("E0774"),
+    DiagnosticCode::RustcHardError("E0767"),
+    DiagnosticCode::RustcHardError("E0777"),
+];
+
+impl flags::RustcTests {
+    pub fn run(self) -> Result<()> {
+        let mut tester = Tester::new()?;
+        let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui"));
+        for i in walk_dir {
+            let i = i?;
+            let p = i.into_path();
+            if let Some(f) = &self.filter {
+                if !p.as_os_str().to_string_lossy().contains(f) {
+                    continue;
+                }
+            }
+            if p.extension().map_or(true, |x| x != "rs") {
+                continue;
+            }
+            if let Err(e) = std::panic::catch_unwind({
+                let tester = AssertUnwindSafe(&mut tester);
+                let p = p.clone();
+                move || {
+                    let tester = tester;
+                    tester.0.test(p);
+                }
+            }) {
+                println!("panic detected at test {:?}", p);
+                std::panic::resume_unwind(e);
+            }
+        }
+        tester.report();
+        Ok(())
+    }
+}

xtask/src/flags.rs

@@ -110,6 +110,7 @@ pub struct PublishReleaseNotes {
 #[derive(Debug)]
 pub enum MeasurementType {
     Build,
+    RustcTests,
     AnalyzeSelf,
     AnalyzeRipgrep,
     AnalyzeWebRender,
@@ -122,6 +123,7 @@ impl FromStr for MeasurementType {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "build" => Ok(Self::Build),
+            "rustc_tests" => Ok(Self::RustcTests),
             "self" => Ok(Self::AnalyzeSelf),
             "ripgrep-13.0.0" => Ok(Self::AnalyzeRipgrep),
             "webrender-2022" => Ok(Self::AnalyzeWebRender),
@@ -135,6 +137,7 @@ impl AsRef<str> for MeasurementType {
     fn as_ref(&self) -> &str {
         match self {
             Self::Build => "build",
+            Self::RustcTests => "rustc_tests",
             Self::AnalyzeSelf => "self",
             Self::AnalyzeRipgrep => "ripgrep-13.0.0",
             Self::AnalyzeWebRender => "webrender-2022",

xtask/src/metrics.rs

@@ -36,6 +36,9 @@ impl flags::Metrics {
                     MeasurementType::Build => {
                         metrics.measure_build(sh)?;
                     }
+                    MeasurementType::RustcTests => {
+                        metrics.measure_rustc_tests(sh)?;
+                    }
                     MeasurementType::AnalyzeSelf => {
                         metrics.measure_analysis_stats_self(sh)?;
                     }
@@ -50,6 +53,7 @@ impl flags::Metrics {
             }
             None => {
                 metrics.measure_build(sh)?;
+                metrics.measure_rustc_tests(sh)?;
                 metrics.measure_analysis_stats_self(sh)?;
                 metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeRipgrep.as_ref())?;
                 metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeWebRender.as_ref())?;
@@ -78,6 +82,19 @@ impl Metrics {
         self.report("build", time.as_millis() as u64, "ms".into());
         Ok(())
     }
+
+    fn measure_rustc_tests(&mut self, sh: &Shell) -> anyhow::Result<()> {
+        eprintln!("\nMeasuring rustc tests");
+
+        cmd!(sh, "git clone https://github.com/rust-lang/rust").run()?;
+
+        let output = cmd!(sh, "./target/release/rust-analyzer rustc-tests ./rust").read()?;
+        for (metric, value, unit) in parse_metrics(&output) {
+            self.report(metric, value, unit.into());
+        }
+        Ok(())
+    }
+
     fn measure_analysis_stats_self(&mut self, sh: &Shell) -> anyhow::Result<()> {
         self.measure_analysis_stats_path(sh, "self", ".")
     }