u-boot/tools/binman/bintool.py
Simon Glass 4583c00236 patman: Move library functions into a library directory
The patman directory has a number of modules which are used by other tools
in U-Boot. This makes it hard to package the tools using pypi since the
common files must be copied along with the tool that uses them.

To address this, move these files into a new u_boot_pylib library. This
can be packaged separately and listed as a dependency of each tool.

Signed-off-by: Simon Glass <sjg@chromium.org>
2023-03-08 11:40:49 -08:00

584 lines
20 KiB
Python

# SPDX-License-Identifier: GPL-2.0+
# Copyright 2022 Google LLC
# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG
# Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
#
"""Base class for all bintools
This defines the common functionality for all bintools, including running
the tool, checking its version and fetching it if needed.
"""
import collections
import glob
import importlib
import multiprocessing
import os
import shutil
import tempfile
import urllib.error
from u_boot_pylib import command
from u_boot_pylib import terminal
from u_boot_pylib import tools
from u_boot_pylib import tout
BINMAN_DIR = os.path.dirname(os.path.realpath(__file__))
# Format string for listing bintools, see also the header in list_all()
FORMAT = '%-16.16s %-12.12s %-26.26s %s'
# List of known modules, to avoid importing the module multiple times
modules = {}
# Possible ways of fetching a tool (FETCH_COUNT is number of ways)
FETCH_ANY, FETCH_BIN, FETCH_BUILD, FETCH_COUNT = range(4)
FETCH_NAMES = {
FETCH_ANY: 'any method',
FETCH_BIN: 'binary download',
FETCH_BUILD: 'build from source'
}
# Status of tool fetching
FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4)
class Bintool:
"""Tool which operates on binaries to help produce entry contents
This is the base class for all bintools
"""
# List of bintools to regard as missing
missing_list = []
# Directory to store tools. Note that this set up by set_tool_dir() which
# must be called before this class is used.
tooldir = ''
def __init__(self, name, desc, version_regex=None, version_args='-V'):
self.name = name
self.desc = desc
self.version_regex = version_regex
self.version_args = version_args
@staticmethod
def find_bintool_class(btype):
"""Look up the bintool class for bintool
Args:
byte: Bintool to use, e.g. 'mkimage'
Returns:
The bintool class object if found, else a tuple:
module name that could not be found
exception received
"""
# Convert something like 'u-boot' to 'u_boot' since we are only
# interested in the type.
module_name = btype.replace('-', '_')
module = modules.get(module_name)
class_name = f'Bintool{module_name}'
# Import the module if we have not already done so
if not module:
try:
module = importlib.import_module('binman.btool.' + module_name)
except ImportError as exc:
try:
# Deal with classes which must be renamed due to conflicts
# with Python libraries
module = importlib.import_module('binman.btool.btool_' +
module_name)
except ImportError:
return module_name, exc
modules[module_name] = module
# Look up the expected class name
return getattr(module, class_name)
@staticmethod
def create(name):
"""Create a new bintool object
Args:
name (str): Bintool to create, e.g. 'mkimage'
Returns:
A new object of the correct type (a subclass of Binutil)
"""
cls = Bintool.find_bintool_class(name)
if isinstance(cls, tuple):
raise ValueError("Cannot import bintool module '%s': %s" % cls)
# Call its constructor to get the object we want.
obj = cls(name)
return obj
@classmethod
def set_tool_dir(cls, pathname):
"""Set the path to use to store and find tools"""
cls.tooldir = pathname
def show(self):
"""Show a line of information about a bintool"""
if self.is_present():
version = self.version()
else:
version = '-'
print(FORMAT % (self.name, version, self.desc,
self.get_path() or '(not found)'))
@classmethod
def set_missing_list(cls, missing_list):
cls.missing_list = missing_list or []
@staticmethod
def get_tool_list(include_testing=False):
"""Get a list of the known tools
Returns:
list of str: names of all tools known to binman
"""
files = glob.glob(os.path.join(BINMAN_DIR, 'btool/*'))
names = [os.path.splitext(os.path.basename(fname))[0]
for fname in files]
names = [name for name in names if name[0] != '_']
names = [name[6:] if name.startswith('btool_') else name
for name in names]
if include_testing:
names.append('_testing')
return sorted(names)
@staticmethod
def list_all():
"""List all the bintools known to binman"""
names = Bintool.get_tool_list()
print(FORMAT % ('Name', 'Version', 'Description', 'Path'))
print(FORMAT % ('-' * 15,'-' * 11, '-' * 25, '-' * 30))
for name in names:
btool = Bintool.create(name)
btool.show()
def is_present(self):
"""Check if a bintool is available on the system
Returns:
bool: True if available, False if not
"""
if self.name in self.missing_list:
return False
return bool(self.get_path())
def get_path(self):
"""Get the path of a bintool
Returns:
str: Path to the tool, if available, else None
"""
return tools.tool_find(self.name)
def fetch_tool(self, method, col, skip_present):
"""Fetch a single tool
Args:
method (FETCH_...): Method to use
col (terminal.Color): Color terminal object
skip_present (boo;): Skip fetching if it is already present
Returns:
int: Result of fetch either FETCHED, FAIL, PRESENT
"""
def try_fetch(meth):
res = None
try:
res = self.fetch(meth)
except urllib.error.URLError as uerr:
message = uerr.reason
print(col.build(col.RED, f'- {message}'))
except ValueError as exc:
print(f'Exception: {exc}')
return res
if skip_present and self.is_present():
return PRESENT
print(col.build(col.YELLOW, 'Fetch: %s' % self.name))
if method == FETCH_ANY:
for try_method in range(1, FETCH_COUNT):
print(f'- trying method: {FETCH_NAMES[try_method]}')
result = try_fetch(try_method)
if result:
break
else:
result = try_fetch(method)
if not result:
return FAIL
if result is not True:
fname, tmpdir = result
dest = os.path.join(self.tooldir, self.name)
os.makedirs(self.tooldir, exist_ok=True)
print(f"- writing to '{dest}'")
shutil.move(fname, dest)
if tmpdir:
shutil.rmtree(tmpdir)
return FETCHED
@staticmethod
def fetch_tools(method, names_to_fetch):
"""Fetch bintools from a suitable place
This fetches or builds the requested bintools so that they can be used
by binman
Args:
names_to_fetch (list of str): names of bintools to fetch
Returns:
True on success, False on failure
"""
def show_status(color, prompt, names):
print(col.build(
color, f'{prompt}:%s{len(names):2}: %s' %
(' ' * (16 - len(prompt)), ' '.join(names))))
col = terminal.Color()
skip_present = False
name_list = names_to_fetch
if len(names_to_fetch) == 1 and names_to_fetch[0] in ['all', 'missing']:
name_list = Bintool.get_tool_list()
if names_to_fetch[0] == 'missing':
skip_present = True
print(col.build(col.YELLOW,
'Fetching tools: %s' % ' '.join(name_list)))
status = collections.defaultdict(list)
for name in name_list:
btool = Bintool.create(name)
result = btool.fetch_tool(method, col, skip_present)
status[result].append(name)
if result == FAIL:
if method == FETCH_ANY:
print('- failed to fetch with all methods')
else:
print(f"- method '{FETCH_NAMES[method]}' is not supported")
if len(name_list) > 1:
if skip_present:
show_status(col.GREEN, 'Already present', status[PRESENT])
show_status(col.GREEN, 'Tools fetched', status[FETCHED])
if status[FAIL]:
show_status(col.RED, 'Failures', status[FAIL])
return not status[FAIL]
def run_cmd_result(self, *args, binary=False, raise_on_error=True):
"""Run the bintool using command-line arguments
Args:
args (list of str): Arguments to provide, in addition to the bintool
name
binary (bool): True to return output as bytes instead of str
raise_on_error (bool): True to raise a ValueError exception if the
tool returns a non-zero return code
Returns:
CommandResult: Resulting output from the bintool, or None if the
tool is not present
"""
if self.name in self.missing_list:
return None
name = os.path.expanduser(self.name) # Expand paths containing ~
all_args = (name,) + args
env = tools.get_env_with_path()
tout.detail(f"bintool: {' '.join(all_args)}")
result = command.run_pipe(
[all_args], capture=True, capture_stderr=True, env=env,
raise_on_error=False, binary=binary)
if result.return_code:
# Return None if the tool was not found. In this case there is no
# output from the tool and it does not appear on the path. We still
# try to run it (as above) since RunPipe() allows faking the tool's
# output
if not any([result.stdout, result.stderr, tools.tool_find(name)]):
tout.info(f"bintool '{name}' not found")
return None
if raise_on_error:
tout.info(f"bintool '{name}' failed")
raise ValueError("Error %d running '%s': %s" %
(result.return_code, ' '.join(all_args),
result.stderr or result.stdout))
if result.stdout:
tout.debug(result.stdout)
if result.stderr:
tout.debug(result.stderr)
return result
def run_cmd(self, *args, binary=False):
"""Run the bintool using command-line arguments
Args:
args (list of str): Arguments to provide, in addition to the bintool
name
binary (bool): True to return output as bytes instead of str
Returns:
str or bytes: Resulting stdout from the bintool
"""
result = self.run_cmd_result(*args, binary=binary)
if result:
return result.stdout
@classmethod
def build_from_git(cls, git_repo, make_target, bintool_path, flags=None):
"""Build a bintool from a git repo
This clones the repo in a temporary directory, builds it with 'make',
then returns the filename of the resulting executable bintool
Args:
git_repo (str): URL of git repo
make_target (str): Target to pass to 'make' to build the tool
bintool_path (str): Relative path of the tool in the repo, after
build is complete
flags (list of str): Flags or variables to pass to make, or None
Returns:
tuple:
str: Filename of fetched file to copy to a suitable directory
str: Name of temp directory to remove, or None
or None on error
"""
tmpdir = tempfile.mkdtemp(prefix='binmanf.')
print(f"- clone git repo '{git_repo}' to '{tmpdir}'")
tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir)
print(f"- build target '{make_target}'")
cmd = ['make', '-C', tmpdir, '-j', f'{multiprocessing.cpu_count()}',
make_target]
if flags:
cmd += flags
tools.run(*cmd)
fname = os.path.join(tmpdir, bintool_path)
if not os.path.exists(fname):
print(f"- File '{fname}' was not produced")
return None
return fname, tmpdir
@classmethod
def fetch_from_url(cls, url):
"""Fetch a bintool from a URL
Args:
url (str): URL to fetch from
Returns:
tuple:
str: Filename of fetched file to copy to a suitable directory
str: Name of temp directory to remove, or None
"""
fname, tmpdir = tools.download(url)
tools.run('chmod', 'a+x', fname)
return fname, tmpdir
@classmethod
def fetch_from_drive(cls, drive_id):
"""Fetch a bintool from Google drive
Args:
drive_id (str): ID of file to fetch. For a URL of the form
'https://drive.google.com/file/d/xxx/view?usp=sharing' the value
passed here should be 'xxx'
Returns:
tuple:
str: Filename of fetched file to copy to a suitable directory
str: Name of temp directory to remove, or None
"""
url = f'https://drive.google.com/uc?export=download&id={drive_id}'
return cls.fetch_from_url(url)
@classmethod
def apt_install(cls, package):
"""Install a bintool using the 'apt' tool
This requires use of servo so may request a password
Args:
package (str): Name of package to install
Returns:
True, assuming it completes without error
"""
args = ['sudo', 'apt', 'install', '-y', package]
print('- %s' % ' '.join(args))
tools.run(*args)
return True
@staticmethod
def WriteDocs(modules, test_missing=None):
"""Write out documentation about the various bintools to stdout
Args:
modules: List of modules to include
test_missing: Used for testing. This is a module to report
as missing
"""
print('''.. SPDX-License-Identifier: GPL-2.0+
Binman bintool Documentation
============================
This file describes the bintools (binary tools) supported by binman. Bintools
are binman's name for external executables that it runs to generate or process
binaries. It is fairly easy to create new bintools. Just add a new file to the
'btool' directory. You can use existing bintools as examples.
''')
modules = sorted(modules)
missing = []
for name in modules:
module = Bintool.find_bintool_class(name)
docs = getattr(module, '__doc__')
if test_missing == name:
docs = None
if docs:
lines = docs.splitlines()
first_line = lines[0]
rest = [line[4:] for line in lines[1:]]
hdr = 'Bintool: %s: %s' % (name, first_line)
print(hdr)
print('-' * len(hdr))
print('\n'.join(rest))
print()
print()
else:
missing.append(name)
if missing:
raise ValueError('Documentation is missing for modules: %s' %
', '.join(missing))
# pylint: disable=W0613
def fetch(self, method):
"""Fetch handler for a bintool
This should be implemented by the base class
Args:
method (FETCH_...): Method to use
Returns:
tuple:
str: Filename of fetched file to copy to a suitable directory
str: Name of temp directory to remove, or None
or True if the file was fetched and already installed
or None if no fetch() implementation is available
Raises:
Valuerror: Fetching could not be completed
"""
print(f"No method to fetch bintool '{self.name}'")
return False
def version(self):
"""Version handler for a bintool
Returns:
str: Version string for this bintool
"""
if self.version_regex is None:
return 'unknown'
import re
result = self.run_cmd_result(self.version_args)
out = result.stdout.strip()
if not out:
out = result.stderr.strip()
if not out:
return 'unknown'
m_version = re.search(self.version_regex, out)
return m_version.group(1) if m_version else out
class BintoolPacker(Bintool):
"""Tool which compression / decompression entry contents
This is a bintools base class for compression / decompression packer
Properties:
name: Name of packer tool
compression: Compression type (COMPRESS_...), value of 'name' property
if none
compress_args: List of positional args provided to tool for compress,
['--compress'] if none
decompress_args: List of positional args provided to tool for
decompress, ['--decompress'] if none
fetch_package: Name of the tool installed using the apt, value of 'name'
property if none
version_regex: Regular expressions to extract the version from tool
version output, '(v[0-9.]+)' if none
"""
def __init__(self, name, compression=None, compress_args=None,
decompress_args=None, fetch_package=None,
version_regex=r'(v[0-9.]+)', version_args='-V'):
desc = '%s compression' % (compression if compression else name)
super().__init__(name, desc, version_regex, version_args)
if compress_args is None:
compress_args = ['--compress']
self.compress_args = compress_args
if decompress_args is None:
decompress_args = ['--decompress']
self.decompress_args = decompress_args
if fetch_package is None:
fetch_package = name
self.fetch_package = fetch_package
def compress(self, indata):
"""Compress data
Args:
indata (bytes): Data to compress
Returns:
bytes: Compressed data
"""
with tempfile.NamedTemporaryFile(prefix='comp.tmp',
dir=tools.get_output_dir()) as tmp:
tools.write_file(tmp.name, indata)
args = self.compress_args + ['--stdout', tmp.name]
return self.run_cmd(*args, binary=True)
def decompress(self, indata):
"""Decompress data
Args:
indata (bytes): Data to decompress
Returns:
bytes: Decompressed data
"""
with tempfile.NamedTemporaryFile(prefix='decomp.tmp',
dir=tools.get_output_dir()) as inf:
tools.write_file(inf.name, indata)
args = self.decompress_args + ['--stdout', inf.name]
return self.run_cmd(*args, binary=True)
def fetch(self, method):
"""Fetch handler
This installs the gzip package using the apt utility.
Args:
method (FETCH_...): Method to use
Returns:
True if the file was fetched and now installed, None if a method
other than FETCH_BIN was requested
Raises:
Valuerror: Fetching could not be completed
"""
if method != FETCH_BIN:
return None
return self.apt_install(self.fetch_package)