Mirrors/u-boot
2
0
Fork 0
mirror of https://github.com/AsahiLinux/u-boot synced 2024-11-28 15:41:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

buildman: Tidy up pylint problems in boards module

Browse source 
Fix all the pylint warnings. Also tidy up the comments so that they show
type information, as required.

Signed-off-by: Simon Glass <sjg@chromium.org>
This commit is contained in:
Simon Glass 2022-07-11 19:04:05 -06:00 committed by Tom Rini
parent a8a0141bc2
commit 969fd333ba
1 changed files with 204 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

352
tools/buildman/boards.py
View file

@ -24,19 +24,26 @@ from buildman import kconfiglib
OUTPUT_FILE = 'boards.cfg'
CONFIG_DIR = 'configs'
SLEEP_TIME = 0.03
COMMENT_BLOCK = '''#
COMMENT_BLOCK = f'''#
# List of boards
# Automatically generated by %s: don't edit
# Automatically generated by {__file__}: don't edit
#
# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
''' % __file__
'''
def try_remove(f):
"""Remove a file ignoring 'No such file or directory' error."""
def try_remove(fname):
"""Remove a file ignoring 'No such file or directory' error.
Args:
fname (str): Filename to remove
Raises:
OSError: output file exists but could not be removed
"""
try:
os.remove(f)
os.remove(fname)
except OSError as exception:
# Ignore 'No such file or directory' error
if exception.errno != errno.ENOENT:
@ -46,20 +53,30 @@ def try_remove(f):
def output_is_new(output):
"""Check if the output file is up to date.
Looks at defconfig and Kconfig files to make sure none is newer than the
output file. Also ensures that the boards.cfg does not mention any removed
boards.
Args:
output (str): Filename to check
Returns:
True if the given output file exists and is newer than any of
*_defconfig, MAINTAINERS and Kconfig*. False otherwise.
True if the given output file exists and is newer than any of
*_defconfig, MAINTAINERS and Kconfig*. False otherwise.
Raises:
OSError: output file exists but could not be opened
"""
# pylint: disable=too-many-branches
try:
ctime = os.path.getctime(output)
except OSError as exception:
if exception.errno == errno.ENOENT:
# return False on 'No such file or directory' error
return False
else:
raise
raise
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'):
continue
@ -67,7 +84,7 @@ def output_is_new(output):
if ctime < os.path.getctime(filepath):
return False
for (dirpath, dirnames, filenames) in os.walk('.'):
for (dirpath, _, filenames) in os.walk('.'):
for filename in filenames:
if (fnmatch.fnmatch(filename, '*~') or
not fnmatch.fnmatch(filename, 'Kconfig*') and
@ -79,8 +96,8 @@ def output_is_new(output):
# Detect a board that has been removed since the current board database
# was generated
with open(output, encoding="utf-8") as f:
for line in f:
with open(output, encoding="utf-8") as inf:
for line in inf:
if line[0] == '#' or line == '\n':
continue
defconfig = line.split()[6] + '_defconfig'
@ -97,7 +114,7 @@ class Expr:
"""Set up a new Expr object.
Args:
expr: String cotaining regular expression to store
expr (str): String cotaining regular expression to store
"""
self._expr = expr
self._re = re.compile(expr)
@ -106,7 +123,7 @@ class Expr:
"""Check if any of the properties match the regular expression.
Args:
props: List of properties to check
props (list of str): List of properties to check
Returns:
True if any of the properties match the regular expression
"""
@ -132,7 +149,7 @@ class Term:
"""Add an Expr object to the list to check.
Args:
expr: New Expr object to add to the list of those that must
expr (Expr): New Expr object to add to the list of those that must
match for a board to be built.
"""
self._expr_list.append(Expr(expr))
@ -147,7 +164,7 @@ class Term:
Each of the expressions in the term is checked. All must match.
Args:
props: List of properties to check
props (list of str): List of properties to check
Returns:
True if all of the expressions in the Term match, else False
"""
@ -178,6 +195,7 @@ class KconfigScanner:
os.environ['srctree'] = os.getcwd()
os.environ['UBOOTVERSION'] = 'dummy'
os.environ['KCONFIG_OBJDIR'] = ''
self._tmpfile = None
self._conf = kconfiglib.Kconfig(warn=False)
def __del__(self):
@ -188,37 +206,38 @@ class KconfigScanner:
the temporary file might be left over. In that case, it should be
deleted in this destructor.
"""
if hasattr(self, '_tmpfile') and self._tmpfile:
if self._tmpfile:
try_remove(self._tmpfile)
def scan(self, defconfig):
"""Load a defconfig file to obtain board parameters.
Arguments:
defconfig: path to the defconfig file to be processed
Args:
defconfig (str): path to the defconfig file to be processed
Returns:
A dictionary of board parameters. It has a form of:
{
'arch': <arch_name>,
'cpu': <cpu_name>,
'soc': <soc_name>,
'vendor': <vendor_name>,
'board': <board_name>,
'target': <target_name>,
'config': <config_header_name>,
'options': <extra_options>
}
Dictionary of board parameters. It has a form:
{
'arch': <arch_name>,
'cpu': <cpu_name>,
'soc': <soc_name>,
'vendor': <vendor_name>,
'board': <board_name>,
'target': <target_name>,
'config': <config_header_name>,
'options': <extra_options>
}
"""
# strip special prefixes and save it in a temporary file
fd, self._tmpfile = tempfile.mkstemp()
with os.fdopen(fd, 'w') as f:
for line in open(defconfig):
colon = line.find(':CONFIG_')
if colon == -1:
f.write(line)
else:
f.write(line[colon + 1:])
outfd, self._tmpfile = tempfile.mkstemp()
with os.fdopen(outfd, 'w') as outf:
with open(defconfig, encoding='utf-8') as inf:
for line in inf:
colon = line.find(':CONFIG_')
if colon == -1:
outf.write(line)
else:
outf.write(line[colon + 1:])
self._conf.load_config(self._tmpfile)
try_remove(self._tmpfile)
@ -237,7 +256,7 @@ class KconfigScanner:
defconfig = os.path.basename(defconfig)
params['target'], match, rear = defconfig.partition('_defconfig')
assert match and not rear, '%s : invalid defconfig' % defconfig
assert match and not rear, f'{defconfig} : invalid defconfig'
# fix-up for aarch64
if params['arch'] == 'arm' and params['cpu'] == 'armv8':
@ -256,7 +275,15 @@ class KconfigScanner:
class MaintainersDatabase:
"""The database of board status and maintainers."""
"""The database of board status and maintainers.
Properties:
database: dict:
key: Board-target name (e.g. 'snow')
value: tuple:
str: Board status (e.g. 'Active')
str: List of maintainers, separated by :
"""
def __init__(self):
"""Create an empty database."""
@ -269,73 +296,78 @@ class MaintainersDatabase:
Display a warning message and return '-' if status information
is not found.
Args:
target (str): Build-target name
Returns:
'Active', 'Orphan' or '-'.
str: 'Active', 'Orphan' or '-'.
"""
if not target in self.database:
print("WARNING: no status info for '%s'" % target, file=sys.stderr)
print(f"WARNING: no status info for '{target}'", file=sys.stderr)
return '-'
tmp = self.database[target][0]
if tmp.startswith('Maintained'):
return 'Active'
elif tmp.startswith('Supported'):
if tmp.startswith('Supported'):
return 'Active'
elif tmp.startswith('Orphan'):
if tmp.startswith('Orphan'):
return 'Orphan'
else:
print(("WARNING: %s: unknown status for '%s'" %
(tmp, target)), file=sys.stderr)
return '-'
print(f"WARNING: {tmp}: unknown status for '{target}'", file=sys.stderr)
return '-'
def get_maintainers(self, target):
"""Return the maintainers of the given board.
Args:
target (str): Build-target name
Returns:
Maintainers of the board. If the board has two or more maintainers,
they are separated with colons.
str: Maintainers of the board. If the board has two or more
maintainers, they are separated with colons.
"""
if not target in self.database:
print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
print(f"WARNING: no maintainers for '{target}'", file=sys.stderr)
return ''
return ':'.join(self.database[target][1])
def parse_file(self, file):
def parse_file(self, fname):
"""Parse a MAINTAINERS file.
Parse a MAINTAINERS file and accumulates board status and
maintainers information.
Parse a MAINTAINERS file and accumulate board status and maintainers
information in the self.database dict.
Arguments:
file: MAINTAINERS file to be parsed
Args:
fname (str): MAINTAINERS file to be parsed
"""
targets = []
maintainers = []
status = '-'
for line in open(file, encoding="utf-8"):
# Check also commented maintainers
if line[:3] == '#M:':
line = line[1:]
tag, rest = line[:2], line[2:].strip()
if tag == 'M:':
maintainers.append(rest)
elif tag == 'F:':
# expand wildcard and filter by 'configs/*_defconfig'
for f in glob.glob(rest):
front, match, rear = f.partition('configs/')
if not front and match:
front, match, rear = rear.rpartition('_defconfig')
if match and not rear:
targets.append(front)
elif tag == 'S:':
status = rest
elif line == '\n':
for target in targets:
self.database[target] = (status, maintainers)
targets = []
maintainers = []
status = '-'
with open(fname, encoding="utf-8") as inf:
for line in inf:
# Check also commented maintainers
if line[:3] == '#M:':
line = line[1:]
tag, rest = line[:2], line[2:].strip()
if tag == 'M:':
maintainers.append(rest)
elif tag == 'F:':
# expand wildcard and filter by 'configs/*_defconfig'
for item in glob.glob(rest):
front, match, rear = item.partition('configs/')
if not front and match:
front, match, rear = rear.rpartition('_defconfig')
if match and not rear:
targets.append(front)
elif tag == 'S:':
status = rest
elif line == '\n':
for target in targets:
self.database[target] = (status, maintainers)
targets = []
maintainers = []
status = '-'
if targets:
for target in targets:
self.database[target] = (status, maintainers)
@ -353,7 +385,7 @@ class Boards:
The board's target member must not already exist in the board list.
Args:
brd: board to add
brd (Board): board to add
"""
self._boards.append(brd)
@ -363,7 +395,7 @@ class Boards:
Create a Board object for each and add it to our _boards list.
Args:
fname: Filename of boards.cfg file
fname (str): Filename of boards.cfg file
"""
with open(fname, 'r', encoding='utf-8') as inf:
for line in inf:
@ -452,9 +484,10 @@ class Boards:
a board to be selected.
Args:
args: List of command line arguments
args (list of str): List of command line arguments
Returns:
A list of Term objects
list of Term: A list of Term objects
"""
syms = []
for arg in args:
@ -493,11 +526,12 @@ class Boards:
If brds and args are both empty, all boards are selected.
Args:
args: List of strings specifying boards to include, either named,
or by their target, architecture, cpu, vendor or soc. If
empty, all boards are selected.
exclude: List of boards to exclude, regardless of 'args'
brds: List of boards to build
args (list of str): List of strings specifying boards to include,
either named, or by their target, architecture, cpu, vendor or
soc. If empty, all boards are selected.
exclude (list of str): List of boards to exclude, regardless of
'args', or None for none
brds (list of Board): List of boards to build, or None/[] for all
Returns:
Tuple
@ -505,21 +539,21 @@ class Boards:
due to each argument, arranged by argument.
List of errors found
"""
result = OrderedDict()
warnings = []
terms = self._build_terms(args)
def _check_board(brd):
"""Check whether to include or exclude a board
result['all'] = []
for term in terms:
result[str(term)] = []
Checks the various terms and decide whether to build it or not (the
'build_it' variable).
exclude_list = []
if exclude:
for expr in exclude:
exclude_list.append(Expr(expr))
If it is built, add the board to the result[term] list so we know
which term caused it to be built. Add it to result['all'] also.
found = []
for brd in self._boards:
Keep a list of boards we found in 'found', so we can report boards
which appear in self._boards but not in brds.
Args:
brd (Board): Board to check
"""
matching_term = None
build_it = False
if terms:
@ -547,6 +581,23 @@ class Boards:
result[matching_term].append(brd.target)
result['all'].append(brd.target)
result = OrderedDict()
warnings = []
terms = self._build_terms(args)
result['all'] = []
for term in terms:
result[str(term)] = []
exclude_list = []
if exclude:
for expr in exclude:
exclude_list.append(Expr(expr))
found = []
for brd in self._boards:
_check_board(brd)
if brds:
remaining = set(brds) - set(found)
if remaining:
@ -554,37 +605,40 @@ class Boards:
return result, warnings
def scan_defconfigs_for_multiprocess(self, queue, defconfigs):
@classmethod
def scan_defconfigs_for_multiprocess(cls, queue, defconfigs):
"""Scan defconfig files and queue their board parameters
This function is intended to be passed to
multiprocessing.Process() constructor.
This function is intended to be passed to multiprocessing.Process()
constructor.
Arguments:
queue: An instance of multiprocessing.Queue().
The resulting board parameters are written into it.
defconfigs: A sequence of defconfig files to be scanned.
Args:
queue (multiprocessing.Queue): The resulting board parameters are
written into this.
defconfigs (sequence of str): A sequence of defconfig files to be
scanned.
"""
kconf_scanner = KconfigScanner()
for defconfig in defconfigs:
queue.put(kconf_scanner.scan(defconfig))
def read_queues(self, queues, params_list):
@classmethod
def read_queues(cls, queues, params_list):
"""Read the queues and append the data to the paramers list"""
for q in queues:
while not q.empty():
params_list.append(q.get())
for que in queues:
while not que.empty():
params_list.append(que.get())
def scan_defconfigs(self, jobs=1):
"""Collect board parameters for all defconfig files.
This function invokes multiple processes for faster processing.
Arguments:
jobs: The number of jobs to run simultaneously
Args:
jobs (int): The number of jobs to run simultaneously
"""
all_defconfigs = []
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'):
continue
@ -596,42 +650,43 @@ class Boards:
for i in range(jobs):
defconfigs = all_defconfigs[total_boards * i // jobs :
total_boards * (i + 1) // jobs]
q = multiprocessing.Queue(maxsize=-1)
p = multiprocessing.Process(
que = multiprocessing.Queue(maxsize=-1)
proc = multiprocessing.Process(
target=self.scan_defconfigs_for_multiprocess,
args=(q, defconfigs))
p.start()
processes.append(p)
queues.append(q)
args=(que, defconfigs))
proc.start()
processes.append(proc)
queues.append(que)
# The resulting data should be accumulated to this list
params_list = []
# Data in the queues should be retrieved preriodically.
# Otherwise, the queues would become full and subprocesses would get stuck.
while any([p.is_alive() for p in processes]):
while any(p.is_alive() for p in processes):
self.read_queues(queues, params_list)
# sleep for a while until the queues are filled
time.sleep(SLEEP_TIME)
# Joining subprocesses just in case
# (All subprocesses should already have been finished)
for p in processes:
p.join()
for proc in processes:
proc.join()
# retrieve leftover data
self.read_queues(queues, params_list)
return params_list
def insert_maintainers_info(self, params_list):
@classmethod
def insert_maintainers_info(cls, params_list):
"""Add Status and Maintainers information to the board parameters list.
Arguments:
params_list: A list of the board parameters
Args:
params_list (list of dict): A list of the board parameters
"""
database = MaintainersDatabase()
for (dirpath, dirnames, filenames) in os.walk('.'):
for (dirpath, _, filenames) in os.walk('.'):
if 'MAINTAINERS' in filenames:
database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
@ -641,51 +696,52 @@ class Boards:
params['maintainers'] = database.get_maintainers(target)
params_list[i] = params
def format_and_output(self, params_list, output):
@classmethod
def format_and_output(cls, params_list, output):
"""Write board parameters into a file.
Columnate the board parameters, sort lines alphabetically,
and then write them to a file.
Arguments:
params_list: The list of board parameters
output: The path to the output file
Args:
params_list (list of dict): The list of board parameters
output (str): The path to the output file
"""
FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
'options', 'maintainers')
fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
'options', 'maintainers')
# First, decide the width of each column
max_length = dict([ (f, 0) for f in FIELDS])
max_length = {f: 0 for f in fields}
for params in params_list:
for f in FIELDS:
max_length[f] = max(max_length[f], len(params[f]))
for field in fields:
max_length[field] = max(max_length[field], len(params[field]))
output_lines = []
for params in params_list:
line = ''
for f in FIELDS:
for field in fields:
# insert two spaces between fields like column -t would
line += ' ' + params[f].ljust(max_length[f])
line += ' ' + params[field].ljust(max_length[field])
output_lines.append(line.strip())
# ignore case when sorting
output_lines.sort(key=str.lower)
with open(output, 'w', encoding="utf-8") as f:
f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
with open(output, 'w', encoding="utf-8") as outf:
outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
def ensure_board_list(self, output, jobs=1, force=False, quiet=False):
"""Generate a board database file if needed.
Arguments:
output: The name of the output file
jobs: The number of jobs to run simultaneously
force: Force to generate the output even if it is new
quiet: True to avoid printing a message if nothing needs doing
Args:
output (str): The name of the output file
jobs (int): The number of jobs to run simultaneously
force (bool): Force to generate the output even if it is new
quiet (bool): True to avoid printing a message if nothing needs doing
"""
if not force and output_is_new(output):
if not quiet:
print("%s is up to date. Nothing to do." % output)
print(f'{output} is up to date. Nothing to do.')
return
params_list = self.scan_defconfigs(jobs)
self.insert_maintainers_info(params_list)