Auto merge of #4837 - flip1995:integration, r=phansch

RIIR: Integration tests

In #4825 the `rust-lang/chalk` test failed because the output was too large. I didn't want to completely disabling the output, since showing the backtrace of an ICE directly in travis is pretty useful. Since finding strings in command outputs is easier in Rust, than in bash, I just RIIRed it.

This and also rewriting our tests in Rust may help with trying out new CI platforms (cc #4577)

changelog: none

.travis.yml

@@ -61,14 +61,12 @@ matrix:
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=rust-lang/cargo
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME: Output too large
-    # - env: INTEGRATION=rust-lang-nursery/chalk
-    #   if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/chalk
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=Geal/nom
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME blocked on https://github.com/rust-lang/rust-clippy/issues/4727
-    #- env: INTEGRATION=rust-lang/rustfmt
-    #  if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/rustfmt
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=hyperium/hyper
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=bluss/rust-itertools
@@ -127,7 +125,7 @@ before_script:
 script:
   - |
     if [[ -n ${INTEGRATION} ]]; then
-      ./ci/integration-tests.sh && sleep 5
+      cargo test --test integration --features integration && sleep 5
     else
       ./ci/base-tests.sh && sleep 5
     fi

Cargo.toml

@@ -38,6 +38,8 @@ clippy_lints = { version = "0.0.212", path = "clippy_lints" }
 regex = "1"
 semver = "0.9"
 rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
+git2 = { version = "0.10", optional = true }
+tempfile = { version = "3.1.0", optional = true }
 
 [dev-dependencies]
 cargo_metadata = "0.9.0"
@@ -58,3 +60,4 @@ rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
 
 [features]
 deny-warnings = []
+integration = ["git2", "tempfile"]

ci/integration-tests.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-if [[ -z "$INTEGRATION" ]]; then
-    exit 0
-fi
-
-CARGO_TARGET_DIR=$(pwd)/target/
-export CARGO_TARGET_DIR
-
-rm ~/.cargo/bin/cargo-clippy
-cargo install --force --debug --path .
-
-echo "Running integration test for crate ${INTEGRATION}"
-
-git clone --depth=1 "https://github.com/${INTEGRATION}.git" checkout
-cd checkout
-
-# run clippy on a project, try to be verbose and trigger as many warnings
-# as possible for greater coverage
-# NOTE: we use `tee` to print any warnings and errors to stdout
-#       to avoid build timeout in Travis
-RUST_BACKTRACE=full \
-cargo clippy \
-    --all-targets \
-    --all-features \
-    -- \
-    --cap-lints warn \
-    -W clippy::pedantic \
-    -W clippy::nursery \
-    2>&1 | tee clippy_output
-
-cargo uninstall clippy
-
-if grep -q "internal compiler error\|query stack during panic\|E0463" clippy_output; then
-    exit 1
-fi

tests/integration.rs

@@ -0,0 +1,75 @@
+#![cfg(feature = "integration")]
+
+use git2::Repository;
+use tempfile;
+
+use std::env;
+use std::process::Command;
+
+#[cfg_attr(feature = "integration", test)]
+fn integration_test() {
+    let repo_name = env::var("INTEGRATION").expect("`INTEGRATION` var not set");
+    let repo_url = format!("https://github.com/{}", repo_name);
+    let crate_name = repo_name
+        .split('/')
+        .nth(1)
+        .expect("repo name should have format `<org>/<name>`");
+
+    let repo_dir = tempfile::tempdir()
+        .expect("couldn't create temp dir")
+        .path()
+        .join(crate_name);
+
+    Repository::clone(&repo_url, &repo_dir).expect("clone of repo failed");
+
+    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let target_dir = std::path::Path::new(&root_dir).join("target");
+    let clippy_binary = target_dir.join(env!("PROFILE")).join("cargo-clippy");
+
+    let output = Command::new(clippy_binary)
+        .current_dir(repo_dir)
+        .env("RUST_BACKTRACE", "full")
+        .env("CARGO_TARGET_DIR", target_dir)
+        .args(&[
+            "clippy",
+            "--all-targets",
+            "--all-features",
+            "--",
+            "--cap-lints",
+            "warn",
+            "-Wclippy::pedantic",
+            "-Wclippy::nursery",
+        ])
+        .output()
+        .expect("unable to run clippy");
+
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    if stderr.contains("internal compiler error") {
+        let backtrace_start = stderr
+            .find("thread 'rustc' panicked at")
+            .expect("start of backtrace not found");
+        let backtrace_end = stderr
+            .rfind("error: internal compiler error")
+            .expect("end of backtrace not found");
+
+        panic!(
+            "internal compiler error\nBacktrace:\n\n{}",
+            &stderr[backtrace_start..backtrace_end]
+        );
+    } else if stderr.contains("query stack during panic") {
+        panic!("query stack during panic in the output");
+    } else if stderr.contains("E0463") {
+        panic!("error: E0463");
+    }
+
+    match output.status.code() {
+        Some(code) => {
+            if code == 0 {
+                println!("Compilation successful");
+            } else {
+                eprintln!("Compilation failed. Exit code: {}", code);
+            }
+        },
+        None => panic!("Process terminated by signal"),
+    }
+}