buildman: Incorporate the genboardscfg.py tool

Bring this tool into buildman, so we don't have to run it separately. The
board.cfg file is still produced as part of the build, to save time when
doing another build in the same working directory. If it is out of date
with respect to the Kconfig, it is updated.

Time to regenerate on a recent single-thread machine is 4.6s (1.3s on a
32-thread machine), so we do need some sort of cache if we want buildman
to be useful on incremental builds. We could use Python's pickle format
but:

- it seems useful to allow boards.cfg to be regenerated, at least for a
  while, in case other tools use it
- it is possible to grep the file easily, e.g. to find boards which use
  a particular SoC (similar to 'buildman -nv <soc>'

Signed-off-by: Simon Glass <sjg@chromium.org>
Suggested-by: Tom Rini <trini@konsulko.com>

tools/buildman/README

@@ -128,10 +128,10 @@ Selecting which boards to build
 ===============================
 
 Buildman lets you build all boards, or a subset. Specify the subset by passing
-command-line arguments that list the desired board name, architecture name,
-SOC name, or anything else in the boards.cfg file. Multiple arguments are
-allowed. Each argument will be interpreted as a regular expression, so
-behaviour is a superset of exact or substring matching. Examples are:
+command-line arguments that list the desired build target, architecture,
+CPU, board name, vendor, SoC or options. Multiple arguments are allowed. Each
+argument will be interpreted as a regular expression, so behaviour is a superset
+of exact or substring matching. Examples are:
 
 * 'tegra20'      All boards with a Tegra20 SoC
 * 'tegra'        All boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...)
@@ -1054,7 +1054,6 @@ between one commit and the next.
 For example:
 
 $ buildman -b squash brppt1 -sU
-boards.cfg is up to date. Nothing to do.
 Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread)
 01: Migrate bootlimit to Kconfig
 02: Squashed commit of the following:
@@ -1309,6 +1308,18 @@ Some options you might like are:
         break anything. But note this does not check bisectability!
 
 
+Using boards.cfg
+================
+
+This file is no-longer needed by buildman but it is still generated in the
+working directory. This helps avoid a delay on every build, since scanning all
+the Kconfig files takes a few seconds. Use the -R flag to force regeneration
+of the file - in that case buildman exits after writing the file.
+
+You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file,
+since it may be dropped altogether in future.
+
+
 TODO
 ====
 

tools/buildman/boards.py

@@ -1,12 +1,93 @@
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2012 The Chromium OS Authors.
+# Author: Simon Glass <sjg@chromium.org>
+# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
 
 """Maintains a list of boards and allows them to be selected"""
 
 from collections import OrderedDict
+import errno
+import fnmatch
+import glob
+import multiprocessing
+import os
 import re
+import sys
+import tempfile
+import time
 
 from buildman import board
+from buildman import kconfiglib
+
+
+### constant variables ###
+OUTPUT_FILE = 'boards.cfg'
+CONFIG_DIR = 'configs'
+SLEEP_TIME = 0.03
+COMMENT_BLOCK = '''#
+# List of boards
+#   Automatically generated by %s: don't edit
+#
+# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
+
+''' % __file__
+
+
+def try_remove(f):
+    """Remove a file ignoring 'No such file or directory' error."""
+    try:
+        os.remove(f)
+    except OSError as exception:
+        # Ignore 'No such file or directory' error
+        if exception.errno != errno.ENOENT:
+            raise
+
+
+def output_is_new(output):
+    """Check if the output file is up to date.
+
+    Returns:
+      True if the given output file exists and is newer than any of
+      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
+    """
+    try:
+        ctime = os.path.getctime(output)
+    except OSError as exception:
+        if exception.errno == errno.ENOENT:
+            # return False on 'No such file or directory' error
+            return False
+        else:
+            raise
+
+    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
+        for filename in fnmatch.filter(filenames, '*_defconfig'):
+            if fnmatch.fnmatch(filename, '.*'):
+                continue
+            filepath = os.path.join(dirpath, filename)
+            if ctime < os.path.getctime(filepath):
+                return False
+
+    for (dirpath, dirnames, filenames) in os.walk('.'):
+        for filename in filenames:
+            if (fnmatch.fnmatch(filename, '*~') or
+                not fnmatch.fnmatch(filename, 'Kconfig*') and
+                not filename == 'MAINTAINERS'):
+                continue
+            filepath = os.path.join(dirpath, filename)
+            if ctime < os.path.getctime(filepath):
+                return False
+
+    # Detect a board that has been removed since the current board database
+    # was generated
+    with open(output, encoding="utf-8") as f:
+        for line in f:
+            if line[0] == '#' or line == '\n':
+                continue
+            defconfig = line.split()[6] + '_defconfig'
+            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
+                return False
+
+    return True
 
 
 class Expr:
@@ -76,6 +157,190 @@ class Term:
         return True
 
 
+class KconfigScanner:
+
+    """Kconfig scanner."""
+
+    ### constant variable only used in this class ###
+    _SYMBOL_TABLE = {
+        'arch' : 'SYS_ARCH',
+        'cpu' : 'SYS_CPU',
+        'soc' : 'SYS_SOC',
+        'vendor' : 'SYS_VENDOR',
+        'board' : 'SYS_BOARD',
+        'config' : 'SYS_CONFIG_NAME',
+        'options' : 'SYS_EXTRA_OPTIONS'
+    }
+
+    def __init__(self):
+        """Scan all the Kconfig files and create a Kconfig object."""
+        # Define environment variables referenced from Kconfig
+        os.environ['srctree'] = os.getcwd()
+        os.environ['UBOOTVERSION'] = 'dummy'
+        os.environ['KCONFIG_OBJDIR'] = ''
+        self._conf = kconfiglib.Kconfig(warn=False)
+
+    def __del__(self):
+        """Delete a leftover temporary file before exit.
+
+        The scan() method of this class creates a temporay file and deletes
+        it on success.  If scan() method throws an exception on the way,
+        the temporary file might be left over.  In that case, it should be
+        deleted in this destructor.
+        """
+        if hasattr(self, '_tmpfile') and self._tmpfile:
+            try_remove(self._tmpfile)
+
+    def scan(self, defconfig):
+        """Load a defconfig file to obtain board parameters.
+
+        Arguments:
+          defconfig: path to the defconfig file to be processed
+
+        Returns:
+          A dictionary of board parameters.  It has a form of:
+          {
+              'arch': <arch_name>,
+              'cpu': <cpu_name>,
+              'soc': <soc_name>,
+              'vendor': <vendor_name>,
+              'board': <board_name>,
+              'target': <target_name>,
+              'config': <config_header_name>,
+              'options': <extra_options>
+          }
+        """
+        # strip special prefixes and save it in a temporary file
+        fd, self._tmpfile = tempfile.mkstemp()
+        with os.fdopen(fd, 'w') as f:
+            for line in open(defconfig):
+                colon = line.find(':CONFIG_')
+                if colon == -1:
+                    f.write(line)
+                else:
+                    f.write(line[colon + 1:])
+
+        self._conf.load_config(self._tmpfile)
+        try_remove(self._tmpfile)
+        self._tmpfile = None
+
+        params = {}
+
+        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
+        # Set '-' if the value is empty.
+        for key, symbol in list(self._SYMBOL_TABLE.items()):
+            value = self._conf.syms.get(symbol).str_value
+            if value:
+                params[key] = value
+            else:
+                params[key] = '-'
+
+        defconfig = os.path.basename(defconfig)
+        params['target'], match, rear = defconfig.partition('_defconfig')
+        assert match and not rear, '%s : invalid defconfig' % defconfig
+
+        # fix-up for aarch64
+        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
+            params['arch'] = 'aarch64'
+
+        # fix-up options field. It should have the form:
+        # <config name>[:comma separated config options]
+        if params['options'] != '-':
+            params['options'] = params['config'] + ':' + \
+                                params['options'].replace(r'\"', '"')
+        elif params['config'] != params['target']:
+            params['options'] = params['config']
+
+        return params
+
+
+class MaintainersDatabase:
+
+    """The database of board status and maintainers."""
+
+    def __init__(self):
+        """Create an empty database."""
+        self.database = {}
+
+    def get_status(self, target):
+        """Return the status of the given board.
+
+        The board status is generally either 'Active' or 'Orphan'.
+        Display a warning message and return '-' if status information
+        is not found.
+
+        Returns:
+          'Active', 'Orphan' or '-'.
+        """
+        if not target in self.database:
+            print("WARNING: no status info for '%s'" % target, file=sys.stderr)
+            return '-'
+
+        tmp = self.database[target][0]
+        if tmp.startswith('Maintained'):
+            return 'Active'
+        elif tmp.startswith('Supported'):
+            return 'Active'
+        elif tmp.startswith('Orphan'):
+            return 'Orphan'
+        else:
+            print(("WARNING: %s: unknown status for '%s'" %
+                                  (tmp, target)), file=sys.stderr)
+            return '-'
+
+    def get_maintainers(self, target):
+        """Return the maintainers of the given board.
+
+        Returns:
+          Maintainers of the board.  If the board has two or more maintainers,
+          they are separated with colons.
+        """
+        if not target in self.database:
+            print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
+            return ''
+
+        return ':'.join(self.database[target][1])
+
+    def parse_file(self, file):
+        """Parse a MAINTAINERS file.
+
+        Parse a MAINTAINERS file and accumulates board status and
+        maintainers information.
+
+        Arguments:
+          file: MAINTAINERS file to be parsed
+        """
+        targets = []
+        maintainers = []
+        status = '-'
+        for line in open(file, encoding="utf-8"):
+            # Check also commented maintainers
+            if line[:3] == '#M:':
+                line = line[1:]
+            tag, rest = line[:2], line[2:].strip()
+            if tag == 'M:':
+                maintainers.append(rest)
+            elif tag == 'F:':
+                # expand wildcard and filter by 'configs/*_defconfig'
+                for f in glob.glob(rest):
+                    front, match, rear = f.partition('configs/')
+                    if not front and match:
+                        front, match, rear = rear.rpartition('_defconfig')
+                        if match and not rear:
+                            targets.append(front)
+            elif tag == 'S:':
+                status = rest
+            elif line == '\n':
+                for target in targets:
+                    self.database[target] = (status, maintainers)
+                targets = []
+                maintainers = []
+                status = '-'
+        if targets:
+            for target in targets:
+                self.database[target] = (status, maintainers)
+
+
 class Boards:
     """Manage a list of boards."""
     def __init__(self):
@@ -288,3 +553,140 @@ class Boards:
                 warnings.append(f"Boards not found: {', '.join(remaining)}\n")
 
         return result, warnings
+
+    def scan_defconfigs_for_multiprocess(self, queue, defconfigs):
+        """Scan defconfig files and queue their board parameters
+
+        This function is intended to be passed to
+        multiprocessing.Process() constructor.
+
+        Arguments:
+        queue: An instance of multiprocessing.Queue().
+                The resulting board parameters are written into it.
+        defconfigs: A sequence of defconfig files to be scanned.
+        """
+        kconf_scanner = KconfigScanner()
+        for defconfig in defconfigs:
+            queue.put(kconf_scanner.scan(defconfig))
+
+    def read_queues(self, queues, params_list):
+        """Read the queues and append the data to the paramers list"""
+        for q in queues:
+            while not q.empty():
+                params_list.append(q.get())
+
+    def scan_defconfigs(self, jobs=1):
+        """Collect board parameters for all defconfig files.
+
+        This function invokes multiple processes for faster processing.
+
+        Arguments:
+        jobs: The number of jobs to run simultaneously
+        """
+        all_defconfigs = []
+        for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
+            for filename in fnmatch.filter(filenames, '*_defconfig'):
+                if fnmatch.fnmatch(filename, '.*'):
+                    continue
+                all_defconfigs.append(os.path.join(dirpath, filename))
+
+        total_boards = len(all_defconfigs)
+        processes = []
+        queues = []
+        for i in range(jobs):
+            defconfigs = all_defconfigs[total_boards * i // jobs :
+                                        total_boards * (i + 1) // jobs]
+            q = multiprocessing.Queue(maxsize=-1)
+            p = multiprocessing.Process(
+                target=self.scan_defconfigs_for_multiprocess,
+                args=(q, defconfigs))
+            p.start()
+            processes.append(p)
+            queues.append(q)
+
+        # The resulting data should be accumulated to this list
+        params_list = []
+
+        # Data in the queues should be retrieved preriodically.
+        # Otherwise, the queues would become full and subprocesses would get stuck.
+        while any([p.is_alive() for p in processes]):
+            self.read_queues(queues, params_list)
+            # sleep for a while until the queues are filled
+            time.sleep(SLEEP_TIME)
+
+        # Joining subprocesses just in case
+        # (All subprocesses should already have been finished)
+        for p in processes:
+            p.join()
+
+        # retrieve leftover data
+        self.read_queues(queues, params_list)
+
+        return params_list
+
+    def insert_maintainers_info(self, params_list):
+        """Add Status and Maintainers information to the board parameters list.
+
+        Arguments:
+        params_list: A list of the board parameters
+        """
+        database = MaintainersDatabase()
+        for (dirpath, dirnames, filenames) in os.walk('.'):
+            if 'MAINTAINERS' in filenames:
+                database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
+
+        for i, params in enumerate(params_list):
+            target = params['target']
+            params['status'] = database.get_status(target)
+            params['maintainers'] = database.get_maintainers(target)
+            params_list[i] = params
+
+    def format_and_output(self, params_list, output):
+        """Write board parameters into a file.
+
+        Columnate the board parameters, sort lines alphabetically,
+        and then write them to a file.
+
+        Arguments:
+        params_list: The list of board parameters
+        output: The path to the output file
+        """
+        FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
+                'options', 'maintainers')
+
+        # First, decide the width of each column
+        max_length = dict([ (f, 0) for f in FIELDS])
+        for params in params_list:
+            for f in FIELDS:
+                max_length[f] = max(max_length[f], len(params[f]))
+
+        output_lines = []
+        for params in params_list:
+            line = ''
+            for f in FIELDS:
+                # insert two spaces between fields like column -t would
+                line += '  ' + params[f].ljust(max_length[f])
+            output_lines.append(line.strip())
+
+        # ignore case when sorting
+        output_lines.sort(key=str.lower)
+
+        with open(output, 'w', encoding="utf-8") as f:
+            f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
+
+    def ensure_board_list(self, output, jobs=1, force=False, quiet=False):
+        """Generate a board database file if needed.
+
+        Arguments:
+        output: The name of the output file
+        jobs: The number of jobs to run simultaneously
+        force: Force to generate the output even if it is new
+        quiet: True to avoid printing a message if nothing needs doing
+        """
+        if not force and output_is_new(output):
+            if not quiet:
+                print("%s is up to date. Nothing to do." % output)
+            return
+        params_list = self.scan_defconfigs(jobs)
+        self.insert_maintainers_info(params_list)
+        self.format_and_output(params_list, output)

tools/buildman/cmdline.py

@@ -89,6 +89,8 @@ def ParseArgs():
           default=False, help="Use full toolchain path in CROSS_COMPILE")
     parser.add_option('-P', '--per-board-out-dir', action='store_true',
           default=False, help="Use an O= (output) directory per board rather than per thread")
+    parser.add_option('-R', '--regen-board-list', action='store_true',
+          help='Force regeneration of the list of boards, like the old boards.cfg file')
     parser.add_option('-s', '--summary', action='store_true',
           default=False, help='Show a build summary')
     parser.add_option('-S', '--show-sizes', action='store_true',

tools/buildman/control.py

@@ -186,18 +186,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
         if not os.path.exists(options.output_dir):
             os.makedirs(options.output_dir)
         board_file = os.path.join(options.output_dir, 'boards.cfg')
-        our_path = os.path.dirname(os.path.realpath(__file__))
-        genboardscfg = os.path.join(our_path, '../genboardscfg.py')
-        if not os.path.exists(genboardscfg):
-            genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
-        status = subprocess.call([genboardscfg, '-q', '-o', board_file])
-        if status != 0:
-            # Older versions don't support -q
-            status = subprocess.call([genboardscfg, '-o', board_file])
-            if status != 0:
-                sys.exit("Failed to generate boards.cfg")
 
         brds = boards.Boards()
+        brds.ensure_board_list(board_file,
+                               options.threads or multiprocessing.cpu_count(),
+                               force=options.regen_board_list,
+                               quiet=not options.verbose)
+        if options.regen_board_list:
+            return 0
         brds.read_boards(board_file)
 
     exclude = []