Make Python utils more idiomatic, use better names, fix -c mode of update_wiki.

.gitignore

@@ -7,6 +7,7 @@ out
 *.so
 *.rlib
 *.dll
+*.pyc
 
 # Executables
 *.exe

util/export.py

@@ -1,79 +1,29 @@
 #!/usr/bin/env python
 # Build the gh-pages
 
-import json
-import os
 import re
 import sys
+import json
 
+from lintlib import parse_all, log
 
-level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
-conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
-confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
 lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
 
-conf_template = """
+CONF_TEMPLATE = """\
 This lint has the following configuration variables:
 
-* `%s: %s`: %s (defaults to `%s`).
+* `%s: %s`: %s (defaults to `%s`)."""
-"""
 
 
-# TODO: actual logging
+def parse_lint_def(lint):
-def warn(*args):
+    lint_dict = {}
-    print(args)
+    lint_dict['id'] = lint.name
-
+    lint_dict['level'] = lint.level
-
+    lint_dict['docs'] = {}
-def debug(*args):
-    print(args)
-
-
-def info(*args):
-    print(args)
-
-
-def parse_path(p="clippy_lints/src"):
-    lints = []
-    for f in os.listdir(p):
-        if f.endswith(".rs"):
-            parse_file(lints, os.path.join(p, f))
-
-    conf = parse_conf(p)
-    info(conf)
-
-    for lint_id in conf:
-        lint = next(l for l in lints if l['id'] == lint_id)
-        if lint:
-            lint['docs']['Configuration'] = (conf_template % conf[lint_id]).strip()
-
-    return lints
-
-
-def parse_conf(p):
-    c = {}
-    with open(p + '/utils/conf.rs') as f:
-        f = f.read()
-
-        m = re.search(conf_re, f)
-        m = m.groups()[0]
-
-        m = re.findall(confvar_re, m)
-
-        for (lint, doc, name, default, ty) in m:
-            c[lint.lower()] = (name, ty, doc, default)
-
-    return c
-
-
-def parseLintDef(level, comment, name):
-    lint = {}
-    lint['id'] = name
-    lint['level'] = level
-    lint['docs'] = {}
 
     last_section = None
 
-    for line in comment:
+    for line in lint.doc:
         if len(line.strip()) == 0:
             continue
 
@@ -86,77 +36,29 @@ def parseLintDef(level, comment, name):
             text = line
 
         if not last_section:
-            warn("Skipping comment line as it was not preceded by a heading")
+            log.warn("Skipping comment line as it was not preceded by a heading")
-            debug("in lint `%s`, line `%s`" % name, line)
+            log.debug("in lint `%s`, line `%s`", lint.name, line)
 
-        lint['docs'][last_section] = (lint['docs'].get(last_section, "") + "\n" + text).strip()
+        lint_dict['docs'][last_section] = \
+            (lint_dict['docs'].get(last_section, "") + "\n" + text).strip()
 
-    return lint
+    return lint_dict
-
-
-def parse_file(d, f):
-    last_comment = []
-    comment = True
-
-    with open(f) as rs:
-        for line in rs:
-            if comment:
-                if line.startswith("///"):
-                    if line.startswith("/// "):
-                        last_comment.append(line[4:])
-                    else:
-                        last_comment.append(line[3:])
-                elif line.startswith("declare_lint!"):
-                    comment = False
-                    deprecated = False
-                    restriction = False
-                elif line.startswith("declare_restriction_lint!"):
-                    comment = False
-                    deprecated = False
-                    restriction = True
-                elif line.startswith("declare_deprecated_lint!"):
-                    comment = False
-                    deprecated = True
-                else:
-                    last_comment = []
-            if not comment:
-                l = line.strip()
-                m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l)
-
-                if m:
-                    name = m.group(1).lower()
-
-                    # Intentionally either a never looping or infinite loop
-                    while not deprecated and not restriction:
-                        m = re.search(level_re, line)
-                        if m:
-                            level = m.group(0)
-                            break
-
-                        line = next(rs)
-
-                    if deprecated:
-                        level = "Deprecated"
-                    elif restriction:
-                        level = "Allow"
-
-                    info("found %s with level %s in %s" % (name, level, f))
-                    d.append(parseLintDef(level, last_comment, name=name))
-                    last_comment = []
-                    comment = True
-                if "}" in l:
-                    warn("Warning: Missing Lint-Name in", f)
-                    comment = True
 
 
 def main():
-    lints = parse_path()
+    lintlist, configs = parse_all()
-    info("got %s lints" % len(lints))
+    lints = {}
+    for lint in lintlist:
+        lints[lint.name] = parse_lint_def(lint)
+        if lint.name in configs:
+            lints[lint.name]['docs']['Configuration'] = \
+                CONF_TEMPLATE % configs[lint.name]
+
+    outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
+    with open(outfile, "w") as fp:
+        json.dump(list(lints.values()), fp, indent=2)
+        log.info("wrote JSON for great justice")
 
-    outdir = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
-    with open(outdir, "w") as file:
-        json.dump(lints, file, indent=2)
-        info("wrote JSON for great justice")
 
 if __name__ == "__main__":
     main()

util/lintlib.py

@@ -0,0 +1,95 @@
+# Common utils for the several housekeeping scripts.
+
+import os
+import re
+import collections
+
+import logging as log
+log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s')
+
+Lint = collections.namedtuple('Lint', 'name level doc sourcefile')
+Config = collections.namedtuple('Config', 'name ty doc default')
+
+lintname_re = re.compile(r'''pub\s+([A-Z_][A-Z_0-9]*)''')
+level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
+conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
+confvar_re = re.compile(
+    r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
+
+
+def parse_lints(lints, filepath):
+    last_comment = []
+    comment = True
+
+    with open(filepath) as fp:
+        for line in fp:
+            if comment:
+                if line.startswith("/// "):
+                    last_comment.append(line[4:])
+                elif line.startswith("///"):
+                    last_comment.append(line[3:])
+                elif line.startswith("declare_lint!"):
+                    comment = False
+                    deprecated = False
+                    restriction = False
+                elif line.startswith("declare_restriction_lint!"):
+                    comment = False
+                    deprecated = False
+                    restriction = True
+                elif line.startswith("declare_deprecated_lint!"):
+                    comment = False
+                    deprecated = True
+                else:
+                    last_comment = []
+            if not comment:
+                m = lintname_re.search(line)
+                if m:
+                    name = m.group(1).lower()
+
+                    if deprecated:
+                        level = "Deprecated"
+                    elif restriction:
+                        level = "Allow"
+                    else:
+                        while True:
+                            m = level_re.search(line)
+                            if m:
+                                level = m.group(0)
+                                break
+                            line = next(fp)
+
+                    log.info("found %s with level %s in %s",
+                             name, level, filepath)
+                    lints.append(Lint(name, level, last_comment, filepath))
+                    last_comment = []
+                    comment = True
+                if "}" in line:
+                    log.warn("Warning: missing Lint-Name in %s", filepath)
+                    comment = True
+
+
+def parse_configs(path):
+    configs = {}
+    with open(os.path.join(path, 'utils/conf.rs')) as fp:
+        contents = fp.read()
+
+    match = re.search(conf_re, contents)
+    confvars = re.findall(confvar_re, match.group(1))
+
+    for (lint, doc, name, default, ty) in confvars:
+        configs[lint.lower()] = Config(name, ty, doc, default)
+
+    return configs
+
+
+def parse_all(path="clippy_lints/src"):
+    lints = []
+    for filename in os.listdir(path):
+        if filename.endswith(".rs"):
+            parse_lints(lints, os.path.join(path, filename))
+    log.info("got %s lints", len(lints))
+
+    configs = parse_configs(path)
+    log.info("got %d configs", len(configs))
+
+    return lints, configs

util/update_wiki.py

@@ -3,93 +3,11 @@
 # requires the checked out wiki in ../rust-clippy.wiki/
 # with -c option, print a warning and set exit status 1 if the file would be
 # changed.
-import os
+
 import re
 import sys
 
-
+from lintlib import log, parse_all
-level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
-conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
-confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
-
-
-def parse_path(p="clippy_lints/src"):
-    d = {}
-    for f in os.listdir(p):
-        if f.endswith(".rs"):
-            parse_file(d, os.path.join(p, f))
-    return (d, parse_conf(p))
-
-
-def parse_conf(p):
-    c = {}
-    with open(p + '/utils/conf.rs') as f:
-        f = f.read()
-
-        m = re.search(conf_re, f)
-        m = m.groups()[0]
-
-        m = re.findall(confvar_re, m)
-
-        for (lint, doc, name, default, ty) in m:
-            c[lint.lower()] = (name, ty, doc, default)
-
-    return c
-
-
-def parse_file(d, f):
-    last_comment = []
-    comment = True
-
-    with open(f) as rs:
-        for line in rs:
-            if comment:
-                if line.startswith("///"):
-                    if line.startswith("/// "):
-                        last_comment.append(line[4:])
-                    else:
-                        last_comment.append(line[3:])
-                elif line.startswith("declare_lint!"):
-                    comment = False
-                    deprecated = False
-                    restriction = False
-                elif line.startswith("declare_restriction_lint!"):
-                    comment = False
-                    deprecated = False
-                    restriction = True
-                elif line.startswith("declare_deprecated_lint!"):
-                    comment = False
-                    deprecated = True
-                else:
-                    last_comment = []
-            if not comment:
-                l = line.strip()
-                m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l)
-
-                if m:
-                    name = m.group(1).lower()
-
-                    # Intentionally either a never looping or infinite loop
-                    while not deprecated and not restriction:
-                        m = re.search(level_re, line)
-                        if m:
-                            level = m.group(0)
-                            break
-
-                        line = next(rs)
-
-                    if deprecated:
-                        level = "Deprecated"
-                    elif restriction:
-                        level = "Allow"
-
-                    print("found %s with level %s in %s" % (name, level, f))
-                    d[name] = (level, last_comment)
-                    last_comment = []
-                    comment = True
-                if "}" in l:
-                    print("Warning: Missing Lint-Name in", f)
-                    comment = True
 
 PREFIX = """Welcome to the rust-clippy wiki!
 
@@ -108,14 +26,13 @@ errors of missing symbols.
 
 """
 
-
+TEMPLATE = """\n# `%s`
-template = """\n# `%s`
 
 **Default level:** %s
 
 %s"""
 
-conf_template = """
+CONF_TEMPLATE = """
 **Configuration:** This lint has the following configuration variables:
 
 * `%s: %s`: %s (defaults to `%s`).
@@ -129,51 +46,50 @@ def level_message(level):
         return "\n**Those lints are %s by default**:\n\n" % level
 
 
-def write_wiki_page(d, c, f):
+def write_wiki_page(lints, configs, filepath):
-    keys = list(d.keys())
+    lints.sort()
-    keys.sort()
+    with open(filepath, "w") as fp:
-    with open(f, "w") as w:
+        fp.write(PREFIX)
-        w.write(PREFIX)
 
         for level in ('Deny', 'Warn', 'Allow', 'Deprecated'):
-            w.write(level_message(level))
+            fp.write(level_message(level))
-            for k in keys:
+            for lint in lints:
-                if d[k][0] == level:
+                if lint.level == level:
-                    w.write("[`%s`](#%s)\n" % (k, k))
+                    fp.write("[`%s`](#%s)\n" % (lint.name, lint.name))
 
-        w.write(WARNING)
+        fp.write(WARNING)
-        for k in keys:
+        for lint in lints:
-            w.write(template % (k, d[k][0], "".join(d[k][1])))
+            fp.write(TEMPLATE % (lint.name, lint.level, "".join(lint.doc)))
 
-            if k in c:
+            if lint.name in configs:
-                w.write(conf_template % c[k])
+                fp.write(CONF_TEMPLATE % configs[lint.name])
 
 
-def check_wiki_page(d, c, f):
+def check_wiki_page(lints, configs, filepath):
-    errors = []
+    lintdict = dict((lint.name, lint) for lint in lints)
-    with open(f) as w:
+    errors = False
-        for line in w:
+    with open(filepath) as fp:
-            m = re.match("# `([a-z_]+)`", line)
+        for line in fp:
+            m = re.match("# `([a-z_0-9]+)`", line)
             if m:
-                v = d.pop(m.group(1), "()")
+                v = lintdict.pop(m.group(1), None)
-                if v == "()":
+                if v is None:
-                    errors.append("Missing wiki entry: " + m.group(1))
+                    log.error("Spurious wiki entry: %s", m.group(1))
-    keys = list(d.keys())
+                    errors = True
-    keys.sort()
+    for n in sorted(lintdict):
-    for k in keys:
+        log.error("Missing wiki entry: %s", n)
-        errors.append("Spurious wiki entry: " + k)
+        errors = True
     if errors:
-        print("\n".join(errors))
+        return 1
-        sys.exit(1)
 
 
 def main():
-    (d, c) = parse_path()
+    lints, configs = parse_all()
-    print('Found %s lints' % len(d))
     if "-c" in sys.argv:
-        check_wiki_page(d, c, "../rust-clippy.wiki/Home.md")
+        check_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md")
     else:
-        write_wiki_page(d, c, "../rust-clippy.wiki/Home.md")
+        write_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md")
+
 
 if __name__ == "__main__":
     main()