Auto merge of #6401 - ebroto:pin_to_a_nightly, r=ebroto

📌 Pin Clippy to a nightly 📌

changelog: Pin Clippy to a specific nightly version (No more master/custom toolchain required to compile Clippy)

Addresses partially #5561. As proposed there in [this comment](https://github.com/rust-lang/rust-clippy/issues/5561#issuecomment-623109095), this kicks off the process, to help us get acquainted with how the syncs should work, before working on improving the tooling.

Open questions:
* When performing a rustup, we will need to exclude the commits that were merged that same day, or else wait until that nightly is released. I did not update the documentation about this part, mainly because I'm not sure about how to do that.
* When should we perform the rustups now? My first idea is to do it at the same time we do the clippyups, to have a clear cadence and to avoid the two copies of the repo to diverge enough to make the process painful.
* Who does the rustups now? If we follow my previous idea and do both rustup and clippyup at the same time, it would be more work for `@flip1995` who currently does the clippyups. I would prefer to establish some kind of rotation to spead the work. Other ideas?
* I'm not sure if this affects the release process in any way.
* ???

`@rust-lang/clippy` thoughts?

r? `@flip1995`

.github/workflows/clippy.yml

@@ -35,29 +35,11 @@ jobs:
       with:
         github_token: "${{ secrets.github_token }}"
 
-    - name: rust-toolchain
-      uses: actions-rs/toolchain@v1.0.6
-      with:
-        toolchain: nightly
-        target: x86_64-unknown-linux-gnu
-        profile: minimal
-
     - name: Checkout
       uses: actions/checkout@v2.3.3
 
-    - name: Run cargo update
+    - name: Install toolchain
-      run: cargo update
+      run: rustup show active-toolchain
-
-    - name: Cache cargo dir
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo
-        key: ${{ runner.os }}-x86_64-unknown-linux-gnu-${{ hashFiles('Cargo.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-x86_64-unknown-linux-gnu
-
-    - name: Master Toolchain Setup
-      run: bash setup-toolchain.sh
 
     # Run
     - name: Set LD_LIBRARY_PATH (Linux)
@@ -66,13 +48,13 @@ jobs:
         echo "LD_LIBRARY_PATH=${SYSROOT}/lib${LD_LIBRARY_PATH+:${LD_LIBRARY_PATH}}" >> $GITHUB_ENV
 
     - name: Build
-      run: cargo build --features deny-warnings
+      run: cargo build --features deny-warnings,internal-lints
 
     - name: Test
-      run: cargo test --features deny-warnings
+      run: cargo test --features deny-warnings,internal-lints
 
     - name: Test clippy_lints
-      run: cargo test --features deny-warnings
+      run: cargo test --features deny-warnings,internal-lints
       working-directory: clippy_lints
 
     - name: Test rustc_tools_util
@@ -98,9 +80,3 @@ jobs:
         cargo dev new_lint --name new_late_pass --pass late
         cargo check
         git reset --hard HEAD
-
-    # Cleanup
-    - name: Run cargo-cache --autoclean
-      run: |
-        cargo +nightly install cargo-cache --no-default-features --features ci-autoclean cargo-cache
-        cargo cache

.github/workflows/clippy_bors.yml

@@ -23,6 +23,7 @@ jobs:
     - uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
       with:
         github_token: "${{ secrets.github_token }}"
+
     - name: Checkout
       uses: actions/checkout@v2.3.3
       with:
@@ -84,31 +85,11 @@ jobs:
         sudo apt-get install gcc-multilib libssl-dev:i386 libgit2-dev:i386
       if: matrix.host == 'i686-unknown-linux-gnu'
 
-    - name: rust-toolchain
-      uses: actions-rs/toolchain@v1.0.6
-      with:
-        toolchain: nightly
-        target: ${{ matrix.host }}
-        profile: minimal
-
     - name: Checkout
       uses: actions/checkout@v2.3.3
 
-    - name: Run cargo update
+    - name: Install toolchain
-      run: cargo update
+      run: rustup show active-toolchain
-
-    - name: Cache cargo dir
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo
-        key: ${{ runner.os }}-${{ matrix.host }}-${{ hashFiles('Cargo.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ matrix.host }}
-
-    - name: Master Toolchain Setup
-      run: bash setup-toolchain.sh
-      env:
-        HOST_TOOLCHAIN: ${{ matrix.host }}
 
     # Run
     - name: Set LD_LIBRARY_PATH (Linux)
@@ -128,13 +109,13 @@ jobs:
         SYSROOT=$(rustc --print sysroot)
         echo "$SYSROOT/bin" >> $GITHUB_PATH
 
-    - name: Build with internal lints
+    - name: Build
       run: cargo build --features deny-warnings,internal-lints
 
-    - name: Test with internal lints
+    - name: Test
       run: cargo test --features deny-warnings,internal-lints
 
-    - name: Test clippy_lints with internal lints
+    - name: Test clippy_lints
       run: cargo test --features deny-warnings,internal-lints
       working-directory: clippy_lints
 
@@ -155,12 +136,6 @@ jobs:
       env:
         OS: ${{ runner.os }}
 
-    # Cleanup
-    - name: Run cargo-cache --autoclean
-      run: |
-        cargo +nightly install cargo-cache --no-default-features --features ci-autoclean cargo-cache
-        cargo cache
-
   integration_build:
     needs: changelog
     runs-on: ubuntu-latest
@@ -171,29 +146,11 @@ jobs:
       with:
         github_token: "${{ secrets.github_token }}"
 
-    - name: rust-toolchain
-      uses: actions-rs/toolchain@v1.0.6
-      with:
-        toolchain: nightly
-        target: x86_64-unknown-linux-gnu
-        profile: minimal
-
     - name: Checkout
       uses: actions/checkout@v2.3.3
 
-    - name: Run cargo update
+    - name: Install toolchain
-      run: cargo update
+      run: rustup show active-toolchain
-
-    - name: Cache cargo dir
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo
-        key: ${{ runner.os }}-x86_64-unknown-linux-gnu-${{ hashFiles('Cargo.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-x86_64-unknown-linux-gnu
-
-    - name: Master Toolchain Setup
-      run: bash setup-toolchain.sh
 
     # Run
     - name: Build Integration Test
@@ -214,11 +171,6 @@ jobs:
         name: target
         path: target
 
-    # Cleanup
-    - name: Run cargo-cache --autoclean
-      run: |
-        cargo +nightly install cargo-cache --no-default-features --features ci-autoclean cargo-cache
-        cargo cache
   integration:
     needs: integration_build
     strategy:
@@ -252,29 +204,11 @@ jobs:
       with:
         github_token: "${{ secrets.github_token }}"
 
-    - name: rust-toolchain
-      uses: actions-rs/toolchain@v1.0.6
-      with:
-        toolchain: nightly
-        target: x86_64-unknown-linux-gnu
-        profile: minimal
-
     - name: Checkout
       uses: actions/checkout@v2.3.3
 
-    - name: Run cargo update
+    - name: Install toolchain
-      run: cargo update
+      run: rustup show active-toolchain
-
-    - name: Cache cargo dir
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo
-        key: ${{ runner.os }}-x86_64-unknown-linux-gnu-${{ hashFiles('Cargo.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-x86_64-unknown-linux-gnu
-
-    - name: Master Toolchain Setup
-      run: bash setup-toolchain.sh
 
     # Download
     - name: Download target dir
@@ -288,16 +222,11 @@ jobs:
 
     # Run
     - name: Test ${{ matrix.integration }}
-      run: $CARGO_TARGET_DIR/debug/integration
+      run: |
+        RUSTUP_TOOLCHAIN="$(rustup show active-toolchain | grep -o -E "nightly-[0-9]{4}-[0-9]{2}-[0-9]{2}")" \
+          $CARGO_TARGET_DIR/debug/integration
       env:
         INTEGRATION: ${{ matrix.integration }}
-        RUSTUP_TOOLCHAIN: master
-
-    # Cleanup
-    - name: Run cargo-cache --autoclean
-      run: |
-        cargo +nightly install cargo-cache --no-default-features --features ci-autoclean cargo-cache
-        cargo cache
 
   # These jobs doesn't actually test anything, but they're only used to tell
   # bors the build completed, as there is no practical way to detect when a

.github/workflows/clippy_dev.yml

@@ -22,6 +22,12 @@ jobs:
 
     steps:
     # Setup
+    - name: Checkout
+      uses: actions/checkout@v2.3.3
+
+    - name: remove toolchain file
+      run: rm rust-toolchain
+
     - name: rust-toolchain
       uses: actions-rs/toolchain@v1.0.6
       with:
@@ -29,9 +35,7 @@ jobs:
         target: x86_64-unknown-linux-gnu
         profile: minimal
         components: rustfmt
-
+        default: true
-    - name: Checkout
-      uses: actions/checkout@v2.3.3
 
     # Run
     - name: Build

CONTRIBUTING.md

@@ -182,18 +182,26 @@ That's why the `else_if_without_else` example uses the `register_early_pass` fun
 [early_lint_pass]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/trait.EarlyLintPass.html
 [late_lint_pass]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/trait.LateLintPass.html
 
-## Fixing build failures caused by Rust
+## Syncing changes between Clippy and [`rust-lang/rust`]
 
-Clippy currently gets built with `rustc` of the `rust-lang/rust` `master`
+Clippy currently gets built with a pinned nightly version.
-branch. Most of the times we have to adapt to the changes and only very rarely
-there's an actual bug in Rust.
 
-If you decide to make Clippy work again with a Rust commit that breaks it, you
+In the `rust-lang/rust` repository, where rustc resides, there's a copy of Clippy
-have to sync the `rust-lang/rust-clippy` repository with the `subtree` copy of
+that compiler hackers modify from time to time to adapt to changes in the unstable
-Clippy in the `rust-lang/rust` repository.
+API of the compiler.
 
-For general information about `subtree`s in the Rust repository see [Rust's
+We need to sync these changes back to this repository periodically, and the changes
-`CONTRIBUTING.md`][subtree].
+made to this repository in the meantime also need to be synced to the `rust-lang/rust` repository.
+
+To avoid flooding the `rust-lang/rust` PR queue, this two-way sync process is done
+in a bi-weekly basis if there's no urgent changes. This is done starting on the day of
+the Rust stable release and then every other week. That way we guarantee that we keep
+this repo up to date with the latest compiler API, and every feature in Clippy is available
+for 2 weeks in nightly, before it can get to beta. For reference, the first sync
+following this cadence was performed the 2020-08-27.
+
+This process is described in detail in the following sections. For general information
+about `subtree`s in the Rust repository see [Rust's `CONTRIBUTING.md`][subtree].
 
 ### Patching git-subtree to work with big repos
 
@@ -222,13 +230,14 @@ This shell has a hardcoded recursion limit set to 1000. In order to make this pr
 you need to force the script to run `bash` instead. You can do this by editing the first
 line of the `git-subtree` script and changing `sh` to `bash`.
 
-### Performing the sync
+### Performing the sync from [`rust-lang/rust`] to Clippy
 
 Here is a TL;DR version of the sync process (all of the following commands have
 to be run inside the `rust` directory):
 
-1. Clone the [`rust-lang/rust`] repository
+1. Clone the [`rust-lang/rust`] repository or make sure it is up to date.
-2. Sync the changes to the rust-copy of Clippy to your Clippy fork:
+2. Checkout the commit from the latest available nightly. You can get it using `rustup check`.
+3. Sync the changes to the rust-copy of Clippy to your Clippy fork:
     ```bash
     # Make sure to change `your-github-name` to your github name in the following command
     git subtree push -P src/tools/clippy git@github.com:your-github-name/rust-clippy sync-from-rust
@@ -246,17 +255,11 @@ to be run inside the `rust` directory):
     git checkout sync-from-rust
     git merge upstream/master
     ```
-3. Open a PR to `rust-lang/rust-clippy` and wait for it to get merged (to
+4. Open a PR to `rust-lang/rust-clippy` and wait for it to get merged (to
    accelerate the process ping the `@rust-lang/clippy` team in your PR and/or
    ~~annoy~~ ask them in the [Zulip] stream.)
 
-### Syncing back changes in Clippy to [`rust-lang/rust`]
+### Performing the sync from Clippy to [`rust-lang/rust`]
-
-To avoid flooding the [`rust-lang/rust`] PR queue, changes in Clippy's repo are synced back
-in a bi-weekly basis if there's no urgent changes. This is done starting on the day of
-the Rust stable release and then every other week. That way we guarantee that
-every feature in Clippy is available for 2 weeks in nightly, before it can get to beta.
-For reference, the first sync following this cadence was performed the 2020-08-27.
 
 All of the following commands have to be run inside the `rust` directory.
 

doc/basics.md

@@ -1,16 +1,14 @@
 # Basics for hacking on Clippy
 
 This document explains the basics for hacking on Clippy. Besides others, this
-includes how to set-up the development environment, how to build and how to test
+includes how to build and test Clippy. For a more in depth description on
-Clippy. For a more in depth description on the codebase take a look at [Adding
+the codebase take a look at [Adding Lints] or [Common Tools].
-Lints] or [Common Tools].
 
 [Adding Lints]: https://github.com/rust-lang/rust-clippy/blob/master/doc/adding_lints.md
 [Common Tools]: https://github.com/rust-lang/rust-clippy/blob/master/doc/common_tools_writing_lints.md
 
 - [Basics for hacking on Clippy](#basics-for-hacking-on-clippy)
   - [Get the code](#get-the-code)
-  - [Setup](#setup)
   - [Building and Testing](#building-and-testing)
   - [`cargo dev`](#cargo-dev)
   - [PR](#pr)
@@ -38,29 +36,9 @@ git rebase upstream/master
 git push
 ```
 
-## Setup
-
-Next we need to setup the toolchain to compile Clippy. Since Clippy heavily
-relies on compiler internals it is build with the latest rustc master. To get
-this toolchain, you can just use the `setup-toolchain.sh` script or use
-`rustup-toolchain-install-master`:
-
-```bash
-bash setup-toolchain.sh
-# OR
-cargo install rustup-toolchain-install-master
-# For better IDE integration also add `-c rustfmt -c rust-src` (optional)
-rustup-toolchain-install-master -f -n master -c rustc-dev -c llvm-tools
-rustup override set master
-```
-
-_Note:_ Sometimes you may get compiler errors when building Clippy, even if you
-didn't change anything. Normally those will be fixed by a maintainer in a few hours. 
-
 ## Building and Testing
 
-Once the `master` toolchain is installed, you can build and test Clippy like
+You can build and test Clippy like every other Rust project:
-every other Rust project:
 
 ```bash
 cargo build  # builds Clippy

rust-toolchain

@@ -1 +1,3 @@
-nightly
+[toolchain]
+channel = "nightly-2020-12-09"
+components = ["llvm-tools-preview", "rustc-dev", "rust-src", "rustfmt"]

setup-toolchain.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Set up the appropriate rustc toolchain
-
-set -e
-
-cd "$(dirname "$0")"
-
-RTIM_PATH=$(command -v rustup-toolchain-install-master) || INSTALLED=false
-CARGO_HOME=${CARGO_HOME:-$HOME/.cargo}
-
-# Check if RTIM is not installed or installed in other locations not in ~/.cargo/bin
-if [[ "$INSTALLED" == false || "$RTIM_PATH" == $CARGO_HOME/bin/rustup-toolchain-install-master ]]; then
-    cargo +nightly install rustup-toolchain-install-master
-else
-    VERSION=$(rustup-toolchain-install-master -V | grep -o "[0-9.]*")
-    REMOTE=$(cargo +nightly search rustup-toolchain-install-master | grep -o "[0-9.]*")
-    echo "info: skipping updating rustup-toolchain-install-master at $RTIM_PATH"
-    echo "      current version : $VERSION"
-    echo "      remote version  : $REMOTE"
-fi
-
-RUST_COMMIT=$(git ls-remote https://github.com/rust-lang/rust master | awk '{print $1}')
-
-if rustc +master -Vv 2>/dev/null | grep -q "$RUST_COMMIT"; then
-    echo "info: master toolchain is up-to-date"
-    exit 0
-fi
-
-if [[ -n "$HOST_TOOLCHAIN" ]]; then
-    TOOLCHAIN=('--host' "$HOST_TOOLCHAIN")
-else
-    TOOLCHAIN=()
-fi
-
-rustup-toolchain-install-master -f -n master "${TOOLCHAIN[@]}" -c rustc-dev -c llvm-tools -- "$RUST_COMMIT"
-rustup override set master

tests/integration.rs

@@ -72,6 +72,8 @@ fn integration_test() {
         panic!("incompatible crate versions");
     } else if stderr.contains("failed to run `rustc` to learn about target-specific information") {
         panic!("couldn't find librustc_driver, consider setting `LD_LIBRARY_PATH`");
+    } else if stderr.contains("toolchain") && stderr.contains("is not installed") {
+        panic!("missing required toolchain");
     }
 
     match output.status.code() {