update_lints rewrite: Add structure and --print-only

Cargo.toml

@@ -38,6 +38,11 @@ name = "clippy-driver"
 test = false
 path = "src/driver.rs"
 
+[[bin]]
+name = "clippy-dev"
+test = false
+path = "src/main.rs"
+
 [dependencies]
 # begin automatic update
 clippy_lints = { version = "0.0.212", path = "clippy_lints" }
@@ -46,6 +51,7 @@ regex = "1"
 semver = "0.9"
 
 [dev-dependencies]
+clippy_dev = { version = "0.0.1", path = "clippy_dev" }
 cargo_metadata = "0.6"
 compiletest_rs = "0.3.7"
 lazy_static = "1.0"

clippy_dev/.gitignore

@@ -0,0 +1,38 @@
+# Used by Travis to be able to push:
+/.github/deploy_key
+out
+
+# Compiled files
+*.o
+*.d
+*.so
+*.rlib
+*.dll
+*.pyc
+*.rmeta
+
+# Executables
+*.exe
+
+# Generated by Cargo
+Cargo.lock
+/target
+/clippy_lints/target
+/clippy_workspace_tests/target
+
+# Generated by dogfood
+/target_recur/
+
+# gh pages docs
+util/gh-pages/lints.json
+
+# rustfmt backups
+*.rs.bk
+
+helper.txt
+*.iml
+.vscode
+.idea
+
+# Used by the Clippy build script
+min_version.txt

clippy_dev/Cargo.lock

@@ -0,0 +1,215 @@
+[[package]]
+name = "aho-corasick"
+version = "0.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "clap"
+version = "2.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "clippy_dev"
+version = "0.0.1"
+dependencies = [
+ "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "libc"
+version = "0.2.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "memchr"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "redox_termios"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "strsim"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "termion"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thread_local"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ucd-util"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "unreachable"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "utf8-ranges"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "vec_map"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "void"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c6d463cbe7ed28720b5b489e7c083eeb8f90d08be2a0d6bb9e1ffea9ce1afa"
+"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1"
+"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789"
+"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
+"checksum lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fb497c35d362b6a331cfd94956a07fc2c78a4604cdbee844a81170386b996dd3"
+"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1"
+"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
+"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
+"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
+"checksum regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5bbbea44c5490a1e84357ff28b7d518b4619a159fed5d25f6c1de2d19cc42814"
+"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d"
+"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
+"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
+"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
+"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
+"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
+"checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
+"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
+"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
+"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
+"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
+"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

clippy_dev/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "clippy_dev"
+version = "0.0.1"
+authors = ["Philipp Hansch <dev@phansch.net>"]
+
+[dependencies]
+clap = "~2.32"
+regex = "1"
+lazy_static = "1.0"

clippy_dev/src/lib.rs

@@ -0,0 +1,144 @@
+extern crate regex;
+#[macro_use]
+extern crate lazy_static;
+
+use regex::Regex;
+use std::ffi::OsStr;
+use std::fs;
+use std::io::prelude::*;
+
+lazy_static! {
+    static ref DEC_CLIPPY_LINT_RE: Regex = Regex::new(r#"declare_clippy_lint!\s*[\{(]\s*pub\s+(?P<name>[A-Z_][A-Z_0-9]*)\s*,\s*(?P<cat>[a-z_]+)\s*,\s*"(?P<desc>(?:[^"\\]+|\\(?s).(?-s))*)"\s*[})]"#).unwrap();
+    static ref DEC_DEPRECATED_LINT_RE: Regex = Regex::new(r#"declare_deprecated_lint!\s*[{(]\s*pub\s+(?P<name>[A-Z_][A-Z_0-9]*)\s*,\s*"(?P<desc>(?:[^"\\]+|\\(?s).(?-s))*)"\s*[})]"#).unwrap();
+    static ref NL_ESCAPE_RE: Regex = Regex::new(r#"\\\n\s*"#).unwrap();
+    pub static ref DOCS_LINK: String = "https://rust-lang-nursery.github.io/rust-clippy/master/index.html".to_string();
+}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct Lint {
+    pub name: String,
+    pub group: String,
+    pub desc: String,
+    pub deprecation: Option<String>,
+    pub module: String,
+}
+
+impl Lint {
+    pub fn new(name: &str, group: &str, desc: &str, deprecation: Option<&str>, module: &str) -> Lint {
+        Lint {
+            name: name.to_lowercase(),
+            group: group.to_string(),
+            desc: NL_ESCAPE_RE.replace(&desc.replace("\\\"", "\""), "").to_string(),
+            deprecation: deprecation.map(|d| d.to_string()),
+            module: module.to_string(),
+        }
+    }
+
+    pub fn active_lints(lints: &[Lint]) -> Vec<Lint> {
+        lints.iter().filter(|l| l.deprecation.is_none()).cloned().collect::<Vec<Lint>>()
+    }
+
+    pub fn in_lint_group(group: &str, lints: &[Lint]) -> Vec<Lint> {
+        lints.iter().filter(|l| l.group == group).cloned().collect::<Vec<Lint>>()
+    }
+}
+
+pub fn collect_all() -> Vec<Lint> {
+    let mut lints = vec![];
+    for direntry in lint_files() {
+        lints.append(&mut collect_from_file(&direntry));
+    }
+    lints
+}
+
+fn collect_from_file(direntry: &fs::DirEntry) -> Vec<Lint> {
+    let mut file = fs::File::open(direntry.path()).unwrap();
+    let mut content = String::new();
+    file.read_to_string(&mut content).unwrap();
+    parse_contents(&content, direntry.path().file_stem().unwrap().to_str().unwrap())
+}
+
+fn parse_contents(content: &str, filename: &str) -> Vec<Lint> {
+    let mut lints: Vec<Lint> = DEC_CLIPPY_LINT_RE
+        .captures_iter(&content)
+        .map(|m| Lint::new(&m["name"], &m["cat"], &m["desc"], None, filename))
+        .collect();
+    let mut deprecated = DEC_DEPRECATED_LINT_RE
+        .captures_iter(&content)
+        .map(|m| Lint::new( &m["name"], "Deprecated", &m["desc"], Some(&m["desc"]), filename))
+        .collect();
+    lints.append(&mut deprecated);
+    lints
+}
+
+/// Collects all .rs files in the `clippy_lints/src` directory
+fn lint_files() -> Vec<fs::DirEntry> {
+    let paths = fs::read_dir("../clippy_lints/src").unwrap();
+    paths
+        .filter_map(|f| f.ok())
+        .filter(|f| f.path().extension() == Some(OsStr::new("rs")))
+        .collect::<Vec<fs::DirEntry>>()
+}
+
+#[test]
+fn test_parse_contents() {
+    let result = parse_contents(
+        r#"
+declare_clippy_lint! {
+    pub PTR_ARG,
+    style,
+    "really long \
+     text"
+}
+
+declare_clippy_lint!{
+    pub DOC_MARKDOWN,
+    pedantic,
+    "single line"
+}
+
+/// some doc comment
+declare_deprecated_lint! {
+    pub SHOULD_ASSERT_EQ,
+    "`assert!()` will be more flexible with RFC 2011"
+}
+    "#,
+    "module_name");
+
+    let expected = vec![
+        Lint::new("ptr_arg", "style", "really long text", None, "module_name"),
+        Lint::new("doc_markdown", "pedantic", "single line", None, "module_name"),
+        Lint::new(
+            "should_assert_eq",
+            "Deprecated",
+            "`assert!()` will be more flexible with RFC 2011",
+            Some("`assert!()` will be more flexible with RFC 2011"),
+            "module_name"
+        ),
+    ];
+    assert_eq!(expected, result);
+}
+
+#[test]
+fn test_active_lints() {
+    let lints = vec![
+        Lint::new("should_assert_eq", "Deprecated", "abc", Some("Reason"), "module_name"),
+        Lint::new("should_assert_eq2", "Not Deprecated", "abc", None, "module_name")
+    ];
+    let expected = vec![
+        Lint::new("should_assert_eq2", "Not Deprecated", "abc", None, "module_name")
+    ];
+    assert_eq!(expected, Lint::active_lints(&lints));
+}
+
+#[test]
+fn test_in_lint_group() {
+    let lints = vec![
+        Lint::new("ptr_arg", "style", "really long text", None, "module_name"),
+        Lint::new("doc_markdown", "pedantic", "single line", None, "module_name"),
+    ];
+    let expected = vec![
+        Lint::new("ptr_arg", "style", "really long text", None, "module_name")
+    ];
+    assert_eq!(expected, Lint::in_lint_group("style", &lints));
+}

clippy_dev/src/main.rs

@@ -0,0 +1,53 @@
+extern crate clap;
+extern crate clippy_dev;
+extern crate regex;
+
+use clap::{App, Arg, SubCommand};
+use clippy_dev::*;
+
+fn main() {
+    let matches = App::new("Clippy developer tooling")
+        .subcommand(
+            SubCommand::with_name("update_lints")
+                .about("Update the lint list")
+                .arg(
+                    Arg::with_name("print-only")
+                        .long("print-only")
+                        .short("p")
+                        .help("Print a table of lints to STDOUT. Does not modify any files."),
+                )
+        )
+        .get_matches();
+
+    if let Some(matches) = matches.subcommand_matches("update_lints") {
+        if matches.is_present("print-only") {
+            print_lints();
+        }
+    }
+}
+
+fn print_lints() {
+    let lint_list = collect_all();
+    let print_clippy_lint_groups: [&str; 7] = [
+        "correctness",
+        "style",
+        "complexity",
+        "perf",
+        "pedantic",
+        "nursery",
+        "restriction"
+    ];
+    // We could use itertools' group_by to make this much more concise:
+    for group in &print_clippy_lint_groups {
+        println!("\n## {}", group);
+
+        let mut group_lints = Lint::in_lint_group(group, &lint_list);
+        group_lints.sort_by(|a, b| a.name.cmp(&b.name));
+
+        for lint in group_lints {
+            if lint.deprecation.is_some() { continue; }
+            println!("* [{}]({}#{}) ({})", lint.name, clippy_dev::DOCS_LINK.clone(), lint.name, lint.desc);
+        }
+    }
+    println!("there are {} lints", Lint::active_lints(&lint_list).len());
+}

util/dev

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cd clippy_dev && cargo run -- $@

util/update_lints.py

@@ -7,6 +7,7 @@
 import os
 import re
 import sys
+from subprocess import call
 
 declare_deprecated_lint_re = re.compile(r'''
     declare_deprecated_lint! \s* [{(] \s*
@@ -166,19 +167,7 @@ def main(print_only=False, check=False):
         all_lints += value
 
     if print_only:
-        print_clippy_lint_groups = [
+        call(["./util/dev", "update_lints", "--print-only"])
-            "correctness",
-            "style",
-            "complexity",
-            "perf",
-            "pedantic",
-            "nursery",
-            "restriction"
-        ]
-        for group in print_clippy_lint_groups:
-            sys.stdout.write('\n## ' + group + '\n')
-            for (_, name, _, descr) in sorted(clippy_lints[group]):
-                sys.stdout.write('* [' + name + '](https://rust-lang-nursery.github.io/rust-clippy/master/index.html#' + name + ') (' + descr + ')\n')
         return
 
     # update the lint counter in README.md