Make Python utils more idiomatic, use better names, fix -c mode of update_wiki.

This commit is contained in:
Georg Brandl 2016-08-06 10:40:35 +02:00
parent b91c1a509e
commit 9a221402e6
4 changed files with 158 additions and 244 deletions

1
.gitignore vendored
View file

@ -7,6 +7,7 @@ out
*.so *.so
*.rlib *.rlib
*.dll *.dll
*.pyc
# Executables # Executables
*.exe *.exe

View file

@ -1,79 +1,29 @@
#!/usr/bin/env python #!/usr/bin/env python
# Build the gh-pages # Build the gh-pages
import json
import os
import re import re
import sys import sys
import json
from lintlib import parse_all, log
level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''') lint_subheadline = re.compile(r'''^\*\*([\w\s]+?)[:?.!]?\*\*(.*)''')
conf_template = """ CONF_TEMPLATE = """\
This lint has the following configuration variables: This lint has the following configuration variables:
* `%s: %s`: %s (defaults to `%s`). * `%s: %s`: %s (defaults to `%s`)."""
"""
# TODO: actual logging def parse_lint_def(lint):
def warn(*args): lint_dict = {}
print(args) lint_dict['id'] = lint.name
lint_dict['level'] = lint.level
lint_dict['docs'] = {}
def debug(*args):
print(args)
def info(*args):
print(args)
def parse_path(p="clippy_lints/src"):
lints = []
for f in os.listdir(p):
if f.endswith(".rs"):
parse_file(lints, os.path.join(p, f))
conf = parse_conf(p)
info(conf)
for lint_id in conf:
lint = next(l for l in lints if l['id'] == lint_id)
if lint:
lint['docs']['Configuration'] = (conf_template % conf[lint_id]).strip()
return lints
def parse_conf(p):
c = {}
with open(p + '/utils/conf.rs') as f:
f = f.read()
m = re.search(conf_re, f)
m = m.groups()[0]
m = re.findall(confvar_re, m)
for (lint, doc, name, default, ty) in m:
c[lint.lower()] = (name, ty, doc, default)
return c
def parseLintDef(level, comment, name):
lint = {}
lint['id'] = name
lint['level'] = level
lint['docs'] = {}
last_section = None last_section = None
for line in comment: for line in lint.doc:
if len(line.strip()) == 0: if len(line.strip()) == 0:
continue continue
@ -86,77 +36,29 @@ def parseLintDef(level, comment, name):
text = line text = line
if not last_section: if not last_section:
warn("Skipping comment line as it was not preceded by a heading") log.warn("Skipping comment line as it was not preceded by a heading")
debug("in lint `%s`, line `%s`" % name, line) log.debug("in lint `%s`, line `%s`", lint.name, line)
lint['docs'][last_section] = (lint['docs'].get(last_section, "") + "\n" + text).strip() lint_dict['docs'][last_section] = \
(lint_dict['docs'].get(last_section, "") + "\n" + text).strip()
return lint return lint_dict
def parse_file(d, f):
last_comment = []
comment = True
with open(f) as rs:
for line in rs:
if comment:
if line.startswith("///"):
if line.startswith("/// "):
last_comment.append(line[4:])
else:
last_comment.append(line[3:])
elif line.startswith("declare_lint!"):
comment = False
deprecated = False
restriction = False
elif line.startswith("declare_restriction_lint!"):
comment = False
deprecated = False
restriction = True
elif line.startswith("declare_deprecated_lint!"):
comment = False
deprecated = True
else:
last_comment = []
if not comment:
l = line.strip()
m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l)
if m:
name = m.group(1).lower()
# Intentionally either a never looping or infinite loop
while not deprecated and not restriction:
m = re.search(level_re, line)
if m:
level = m.group(0)
break
line = next(rs)
if deprecated:
level = "Deprecated"
elif restriction:
level = "Allow"
info("found %s with level %s in %s" % (name, level, f))
d.append(parseLintDef(level, last_comment, name=name))
last_comment = []
comment = True
if "}" in l:
warn("Warning: Missing Lint-Name in", f)
comment = True
def main(): def main():
lints = parse_path() lintlist, configs = parse_all()
info("got %s lints" % len(lints)) lints = {}
for lint in lintlist:
lints[lint.name] = parse_lint_def(lint)
if lint.name in configs:
lints[lint.name]['docs']['Configuration'] = \
CONF_TEMPLATE % configs[lint.name]
outfile = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
with open(outfile, "w") as fp:
json.dump(list(lints.values()), fp, indent=2)
log.info("wrote JSON for great justice")
outdir = sys.argv[1] if len(sys.argv) > 1 else "util/gh-pages/lints.json"
with open(outdir, "w") as file:
json.dump(lints, file, indent=2)
info("wrote JSON for great justice")
if __name__ == "__main__": if __name__ == "__main__":
main() main()

95
util/lintlib.py Normal file
View file

@ -0,0 +1,95 @@
# Common utils for the several housekeeping scripts.
import os
import re
import collections
import logging as log
log.basicConfig(level=log.INFO, format='%(levelname)s: %(message)s')
Lint = collections.namedtuple('Lint', 'name level doc sourcefile')
Config = collections.namedtuple('Config', 'name ty doc default')
lintname_re = re.compile(r'''pub\s+([A-Z_][A-Z_0-9]*)''')
level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
confvar_re = re.compile(
r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
def parse_lints(lints, filepath):
last_comment = []
comment = True
with open(filepath) as fp:
for line in fp:
if comment:
if line.startswith("/// "):
last_comment.append(line[4:])
elif line.startswith("///"):
last_comment.append(line[3:])
elif line.startswith("declare_lint!"):
comment = False
deprecated = False
restriction = False
elif line.startswith("declare_restriction_lint!"):
comment = False
deprecated = False
restriction = True
elif line.startswith("declare_deprecated_lint!"):
comment = False
deprecated = True
else:
last_comment = []
if not comment:
m = lintname_re.search(line)
if m:
name = m.group(1).lower()
if deprecated:
level = "Deprecated"
elif restriction:
level = "Allow"
else:
while True:
m = level_re.search(line)
if m:
level = m.group(0)
break
line = next(fp)
log.info("found %s with level %s in %s",
name, level, filepath)
lints.append(Lint(name, level, last_comment, filepath))
last_comment = []
comment = True
if "}" in line:
log.warn("Warning: missing Lint-Name in %s", filepath)
comment = True
def parse_configs(path):
configs = {}
with open(os.path.join(path, 'utils/conf.rs')) as fp:
contents = fp.read()
match = re.search(conf_re, contents)
confvars = re.findall(confvar_re, match.group(1))
for (lint, doc, name, default, ty) in confvars:
configs[lint.lower()] = Config(name, ty, doc, default)
return configs
def parse_all(path="clippy_lints/src"):
lints = []
for filename in os.listdir(path):
if filename.endswith(".rs"):
parse_lints(lints, os.path.join(path, filename))
log.info("got %s lints", len(lints))
configs = parse_configs(path)
log.info("got %d configs", len(configs))
return lints, configs

View file

@ -3,93 +3,11 @@
# requires the checked out wiki in ../rust-clippy.wiki/ # requires the checked out wiki in ../rust-clippy.wiki/
# with -c option, print a warning and set exit status 1 if the file would be # with -c option, print a warning and set exit status 1 if the file would be
# changed. # changed.
import os
import re import re
import sys import sys
from lintlib import log, parse_all
level_re = re.compile(r'''(Forbid|Deny|Warn|Allow)''')
conf_re = re.compile(r'''define_Conf! {\n([^}]*)\n}''', re.MULTILINE)
confvar_re = re.compile(r'''/// Lint: (\w+). (.*).*\n *\("([^"]*)", (?:[^,]*), (.*) => (.*)\),''')
def parse_path(p="clippy_lints/src"):
d = {}
for f in os.listdir(p):
if f.endswith(".rs"):
parse_file(d, os.path.join(p, f))
return (d, parse_conf(p))
def parse_conf(p):
c = {}
with open(p + '/utils/conf.rs') as f:
f = f.read()
m = re.search(conf_re, f)
m = m.groups()[0]
m = re.findall(confvar_re, m)
for (lint, doc, name, default, ty) in m:
c[lint.lower()] = (name, ty, doc, default)
return c
def parse_file(d, f):
last_comment = []
comment = True
with open(f) as rs:
for line in rs:
if comment:
if line.startswith("///"):
if line.startswith("/// "):
last_comment.append(line[4:])
else:
last_comment.append(line[3:])
elif line.startswith("declare_lint!"):
comment = False
deprecated = False
restriction = False
elif line.startswith("declare_restriction_lint!"):
comment = False
deprecated = False
restriction = True
elif line.startswith("declare_deprecated_lint!"):
comment = False
deprecated = True
else:
last_comment = []
if not comment:
l = line.strip()
m = re.search(r"pub\s+([A-Z_][A-Z_0-9]*)", l)
if m:
name = m.group(1).lower()
# Intentionally either a never looping or infinite loop
while not deprecated and not restriction:
m = re.search(level_re, line)
if m:
level = m.group(0)
break
line = next(rs)
if deprecated:
level = "Deprecated"
elif restriction:
level = "Allow"
print("found %s with level %s in %s" % (name, level, f))
d[name] = (level, last_comment)
last_comment = []
comment = True
if "}" in l:
print("Warning: Missing Lint-Name in", f)
comment = True
PREFIX = """Welcome to the rust-clippy wiki! PREFIX = """Welcome to the rust-clippy wiki!
@ -108,14 +26,13 @@ errors of missing symbols.
""" """
TEMPLATE = """\n# `%s`
template = """\n# `%s`
**Default level:** %s **Default level:** %s
%s""" %s"""
conf_template = """ CONF_TEMPLATE = """
**Configuration:** This lint has the following configuration variables: **Configuration:** This lint has the following configuration variables:
* `%s: %s`: %s (defaults to `%s`). * `%s: %s`: %s (defaults to `%s`).
@ -129,51 +46,50 @@ def level_message(level):
return "\n**Those lints are %s by default**:\n\n" % level return "\n**Those lints are %s by default**:\n\n" % level
def write_wiki_page(d, c, f): def write_wiki_page(lints, configs, filepath):
keys = list(d.keys()) lints.sort()
keys.sort() with open(filepath, "w") as fp:
with open(f, "w") as w: fp.write(PREFIX)
w.write(PREFIX)
for level in ('Deny', 'Warn', 'Allow', 'Deprecated'): for level in ('Deny', 'Warn', 'Allow', 'Deprecated'):
w.write(level_message(level)) fp.write(level_message(level))
for k in keys: for lint in lints:
if d[k][0] == level: if lint.level == level:
w.write("[`%s`](#%s)\n" % (k, k)) fp.write("[`%s`](#%s)\n" % (lint.name, lint.name))
w.write(WARNING) fp.write(WARNING)
for k in keys: for lint in lints:
w.write(template % (k, d[k][0], "".join(d[k][1]))) fp.write(TEMPLATE % (lint.name, lint.level, "".join(lint.doc)))
if k in c: if lint.name in configs:
w.write(conf_template % c[k]) fp.write(CONF_TEMPLATE % configs[lint.name])
def check_wiki_page(d, c, f): def check_wiki_page(lints, configs, filepath):
errors = [] lintdict = dict((lint.name, lint) for lint in lints)
with open(f) as w: errors = False
for line in w: with open(filepath) as fp:
m = re.match("# `([a-z_]+)`", line) for line in fp:
m = re.match("# `([a-z_0-9]+)`", line)
if m: if m:
v = d.pop(m.group(1), "()") v = lintdict.pop(m.group(1), None)
if v == "()": if v is None:
errors.append("Missing wiki entry: " + m.group(1)) log.error("Spurious wiki entry: %s", m.group(1))
keys = list(d.keys()) errors = True
keys.sort() for n in sorted(lintdict):
for k in keys: log.error("Missing wiki entry: %s", n)
errors.append("Spurious wiki entry: " + k) errors = True
if errors: if errors:
print("\n".join(errors)) return 1
sys.exit(1)
def main(): def main():
(d, c) = parse_path() lints, configs = parse_all()
print('Found %s lints' % len(d))
if "-c" in sys.argv: if "-c" in sys.argv:
check_wiki_page(d, c, "../rust-clippy.wiki/Home.md") check_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md")
else: else:
write_wiki_page(d, c, "../rust-clippy.wiki/Home.md") write_wiki_page(lints, configs, "../rust-clippy.wiki/Home.md")
if __name__ == "__main__": if __name__ == "__main__":
main() main()