mirror of
https://github.com/AsahiLinux/u-boot
synced 2024-11-10 23:24:38 +00:00
d82f539ab9
The hexagon toolchain (4.6.1) from kernel.org, for example, was packaged in
a way that is different from most toolchains. The first entry when unpacking
most toolchain tarballs is:
gcc-<version>-nolib/<targetarch>-<system>
e.g.:
gcc-8.1.0-nolibc/aarch64-linux/
The first entry of the hexagon toolchain, however, is:
gcc-4.6.1-nolibc/
This causes the buildman logic in toolchain.py::ScanPath() to not be able to
find the "*gcc" executable since it looks in gcc-4.6.1-nolib/{.|bin|usr/bin}
instead of gcc-4.6.1/hexagon-linux/{.|bin|usr/bin}. Therefore when buildman
tries to download a set of toolchains that includes hexagon, the script fails.
This update takes the second line of the tarball unpacking (which works for
all the toolchains I've tested from kernel.org) and parses it to take the
first two elements, separated by '/'. It makes this logic a bit more robust.
Signed-off-by: Trevor Woerner <trevor@toganlabs.com>
Reviewed-by: Simon Glass <sjg@chromium.org>
576 lines
20 KiB
Python
576 lines
20 KiB
Python
# SPDX-License-Identifier: GPL-2.0+
|
|
# Copyright (c) 2012 The Chromium OS Authors.
|
|
#
|
|
|
|
import re
|
|
import glob
|
|
from HTMLParser import HTMLParser
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import urllib2
|
|
|
|
import bsettings
|
|
import command
|
|
import terminal
|
|
|
|
(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
|
|
PRIORITY_CALC) = range(4)
|
|
|
|
# Simple class to collect links from a page
|
|
class MyHTMLParser(HTMLParser):
|
|
def __init__(self, arch):
|
|
"""Create a new parser
|
|
|
|
After the parser runs, self.links will be set to a list of the links
|
|
to .xz archives found in the page, and self.arch_link will be set to
|
|
the one for the given architecture (or None if not found).
|
|
|
|
Args:
|
|
arch: Architecture to search for
|
|
"""
|
|
HTMLParser.__init__(self)
|
|
self.arch_link = None
|
|
self.links = []
|
|
self.re_arch = re.compile('[-_]%s-' % arch)
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == 'a':
|
|
for tag, value in attrs:
|
|
if tag == 'href':
|
|
if value and value.endswith('.xz'):
|
|
self.links.append(value)
|
|
if self.re_arch.search(value):
|
|
self.arch_link = value
|
|
|
|
|
|
class Toolchain:
|
|
"""A single toolchain
|
|
|
|
Public members:
|
|
gcc: Full path to C compiler
|
|
path: Directory path containing C compiler
|
|
cross: Cross compile string, e.g. 'arm-linux-'
|
|
arch: Architecture of toolchain as determined from the first
|
|
component of the filename. E.g. arm-linux-gcc becomes arm
|
|
priority: Toolchain priority (0=highest, 20=lowest)
|
|
"""
|
|
def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
|
|
arch=None):
|
|
"""Create a new toolchain object.
|
|
|
|
Args:
|
|
fname: Filename of the gcc component
|
|
test: True to run the toolchain to test it
|
|
verbose: True to print out the information
|
|
priority: Priority to use for this toolchain, or PRIORITY_CALC to
|
|
calculate it
|
|
"""
|
|
self.gcc = fname
|
|
self.path = os.path.dirname(fname)
|
|
|
|
# Find the CROSS_COMPILE prefix to use for U-Boot. For example,
|
|
# 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
|
|
basename = os.path.basename(fname)
|
|
pos = basename.rfind('-')
|
|
self.cross = basename[:pos + 1] if pos != -1 else ''
|
|
|
|
# The architecture is the first part of the name
|
|
pos = self.cross.find('-')
|
|
if arch:
|
|
self.arch = arch
|
|
else:
|
|
self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
|
|
|
|
env = self.MakeEnvironment(False)
|
|
|
|
# As a basic sanity check, run the C compiler with --version
|
|
cmd = [fname, '--version']
|
|
if priority == PRIORITY_CALC:
|
|
self.priority = self.GetPriority(fname)
|
|
else:
|
|
self.priority = priority
|
|
if test:
|
|
result = command.RunPipe([cmd], capture=True, env=env,
|
|
raise_on_error=False)
|
|
self.ok = result.return_code == 0
|
|
if verbose:
|
|
print 'Tool chain test: ',
|
|
if self.ok:
|
|
print "OK, arch='%s', priority %d" % (self.arch,
|
|
self.priority)
|
|
else:
|
|
print 'BAD'
|
|
print 'Command: ', cmd
|
|
print result.stdout
|
|
print result.stderr
|
|
else:
|
|
self.ok = True
|
|
|
|
def GetPriority(self, fname):
|
|
"""Return the priority of the toolchain.
|
|
|
|
Toolchains are ranked according to their suitability by their
|
|
filename prefix.
|
|
|
|
Args:
|
|
fname: Filename of toolchain
|
|
Returns:
|
|
Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
|
|
"""
|
|
priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
|
|
'-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
|
|
'-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
|
|
'-linux-gnueabihf', '-le-linux', '-uclinux']
|
|
for prio in range(len(priority_list)):
|
|
if priority_list[prio] in fname:
|
|
return PRIORITY_CALC + prio
|
|
return PRIORITY_CALC + prio
|
|
|
|
def GetWrapper(self, show_warning=True):
|
|
"""Get toolchain wrapper from the setting file.
|
|
"""
|
|
value = ''
|
|
for name, value in bsettings.GetItems('toolchain-wrapper'):
|
|
if not value:
|
|
print "Warning: Wrapper not found"
|
|
if value:
|
|
value = value + ' '
|
|
|
|
return value
|
|
|
|
def MakeEnvironment(self, full_path):
|
|
"""Returns an environment for using the toolchain.
|
|
|
|
Thie takes the current environment and adds CROSS_COMPILE so that
|
|
the tool chain will operate correctly. This also disables localized
|
|
output and possibly unicode encoded output of all build tools by
|
|
adding LC_ALL=C.
|
|
|
|
Args:
|
|
full_path: Return the full path in CROSS_COMPILE and don't set
|
|
PATH
|
|
"""
|
|
env = dict(os.environ)
|
|
wrapper = self.GetWrapper()
|
|
|
|
if full_path:
|
|
env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
|
|
else:
|
|
env['CROSS_COMPILE'] = wrapper + self.cross
|
|
env['PATH'] = self.path + ':' + env['PATH']
|
|
|
|
env['LC_ALL'] = 'C'
|
|
|
|
return env
|
|
|
|
|
|
class Toolchains:
|
|
"""Manage a list of toolchains for building U-Boot
|
|
|
|
We select one toolchain for each architecture type
|
|
|
|
Public members:
|
|
toolchains: Dict of Toolchain objects, keyed by architecture name
|
|
prefixes: Dict of prefixes to check, keyed by architecture. This can
|
|
be a full path and toolchain prefix, for example
|
|
{'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
|
|
something on the search path, for example
|
|
{'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
|
|
paths: List of paths to check for toolchains (may contain wildcards)
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.toolchains = {}
|
|
self.prefixes = {}
|
|
self.paths = []
|
|
self._make_flags = dict(bsettings.GetItems('make-flags'))
|
|
|
|
def GetPathList(self, show_warning=True):
|
|
"""Get a list of available toolchain paths
|
|
|
|
Args:
|
|
show_warning: True to show a warning if there are no tool chains.
|
|
|
|
Returns:
|
|
List of strings, each a path to a toolchain mentioned in the
|
|
[toolchain] section of the settings file.
|
|
"""
|
|
toolchains = bsettings.GetItems('toolchain')
|
|
if show_warning and not toolchains:
|
|
print ("Warning: No tool chains. Please run 'buildman "
|
|
"--fetch-arch all' to download all available toolchains, or "
|
|
"add a [toolchain] section to your buildman config file "
|
|
"%s. See README for details" %
|
|
bsettings.config_fname)
|
|
|
|
paths = []
|
|
for name, value in toolchains:
|
|
if '*' in value:
|
|
paths += glob.glob(value)
|
|
else:
|
|
paths.append(value)
|
|
return paths
|
|
|
|
def GetSettings(self, show_warning=True):
|
|
"""Get toolchain settings from the settings file.
|
|
|
|
Args:
|
|
show_warning: True to show a warning if there are no tool chains.
|
|
"""
|
|
self.prefixes = bsettings.GetItems('toolchain-prefix')
|
|
self.paths += self.GetPathList(show_warning)
|
|
|
|
def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
|
|
arch=None):
|
|
"""Add a toolchain to our list
|
|
|
|
We select the given toolchain as our preferred one for its
|
|
architecture if it is a higher priority than the others.
|
|
|
|
Args:
|
|
fname: Filename of toolchain's gcc driver
|
|
test: True to run the toolchain to test it
|
|
priority: Priority to use for this toolchain
|
|
arch: Toolchain architecture, or None if not known
|
|
"""
|
|
toolchain = Toolchain(fname, test, verbose, priority, arch)
|
|
add_it = toolchain.ok
|
|
if toolchain.arch in self.toolchains:
|
|
add_it = (toolchain.priority <
|
|
self.toolchains[toolchain.arch].priority)
|
|
if add_it:
|
|
self.toolchains[toolchain.arch] = toolchain
|
|
elif verbose:
|
|
print ("Toolchain '%s' at priority %d will be ignored because "
|
|
"another toolchain for arch '%s' has priority %d" %
|
|
(toolchain.gcc, toolchain.priority, toolchain.arch,
|
|
self.toolchains[toolchain.arch].priority))
|
|
|
|
def ScanPath(self, path, verbose):
|
|
"""Scan a path for a valid toolchain
|
|
|
|
Args:
|
|
path: Path to scan
|
|
verbose: True to print out progress information
|
|
Returns:
|
|
Filename of C compiler if found, else None
|
|
"""
|
|
fnames = []
|
|
for subdir in ['.', 'bin', 'usr/bin']:
|
|
dirname = os.path.join(path, subdir)
|
|
if verbose: print " - looking in '%s'" % dirname
|
|
for fname in glob.glob(dirname + '/*gcc'):
|
|
if verbose: print " - found '%s'" % fname
|
|
fnames.append(fname)
|
|
return fnames
|
|
|
|
def ScanPathEnv(self, fname):
|
|
"""Scan the PATH environment variable for a given filename.
|
|
|
|
Args:
|
|
fname: Filename to scan for
|
|
Returns:
|
|
List of matching pathanames, or [] if none
|
|
"""
|
|
pathname_list = []
|
|
for path in os.environ["PATH"].split(os.pathsep):
|
|
path = path.strip('"')
|
|
pathname = os.path.join(path, fname)
|
|
if os.path.exists(pathname):
|
|
pathname_list.append(pathname)
|
|
return pathname_list
|
|
|
|
def Scan(self, verbose):
|
|
"""Scan for available toolchains and select the best for each arch.
|
|
|
|
We look for all the toolchains we can file, figure out the
|
|
architecture for each, and whether it works. Then we select the
|
|
highest priority toolchain for each arch.
|
|
|
|
Args:
|
|
verbose: True to print out progress information
|
|
"""
|
|
if verbose: print 'Scanning for tool chains'
|
|
for name, value in self.prefixes:
|
|
if verbose: print " - scanning prefix '%s'" % value
|
|
if os.path.exists(value):
|
|
self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
|
|
continue
|
|
fname = value + 'gcc'
|
|
if os.path.exists(fname):
|
|
self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
|
|
continue
|
|
fname_list = self.ScanPathEnv(fname)
|
|
for f in fname_list:
|
|
self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
|
|
if not fname_list:
|
|
raise ValueError, ("No tool chain found for prefix '%s'" %
|
|
value)
|
|
for path in self.paths:
|
|
if verbose: print " - scanning path '%s'" % path
|
|
fnames = self.ScanPath(path, verbose)
|
|
for fname in fnames:
|
|
self.Add(fname, True, verbose)
|
|
|
|
def List(self):
|
|
"""List out the selected toolchains for each architecture"""
|
|
col = terminal.Color()
|
|
print col.Color(col.BLUE, 'List of available toolchains (%d):' %
|
|
len(self.toolchains))
|
|
if len(self.toolchains):
|
|
for key, value in sorted(self.toolchains.iteritems()):
|
|
print '%-10s: %s' % (key, value.gcc)
|
|
else:
|
|
print 'None'
|
|
|
|
def Select(self, arch):
|
|
"""Returns the toolchain for a given architecture
|
|
|
|
Args:
|
|
args: Name of architecture (e.g. 'arm', 'ppc_8xx')
|
|
|
|
returns:
|
|
toolchain object, or None if none found
|
|
"""
|
|
for tag, value in bsettings.GetItems('toolchain-alias'):
|
|
if arch == tag:
|
|
for alias in value.split():
|
|
if alias in self.toolchains:
|
|
return self.toolchains[alias]
|
|
|
|
if not arch in self.toolchains:
|
|
raise ValueError, ("No tool chain found for arch '%s'" % arch)
|
|
return self.toolchains[arch]
|
|
|
|
def ResolveReferences(self, var_dict, args):
|
|
"""Resolve variable references in a string
|
|
|
|
This converts ${blah} within the string to the value of blah.
|
|
This function works recursively.
|
|
|
|
Args:
|
|
var_dict: Dictionary containing variables and their values
|
|
args: String containing make arguments
|
|
Returns:
|
|
Resolved string
|
|
|
|
>>> bsettings.Setup()
|
|
>>> tcs = Toolchains()
|
|
>>> tcs.Add('fred', False)
|
|
>>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
|
|
'second' : '2nd'}
|
|
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
|
|
'this=OBLIQUE_set'
|
|
>>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
|
|
'this=OBLIQUE_setfi2ndrstnd'
|
|
"""
|
|
re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
|
|
|
|
while True:
|
|
m = re_var.search(args)
|
|
if not m:
|
|
break
|
|
lookup = m.group(0)[2:-1]
|
|
value = var_dict.get(lookup, '')
|
|
args = args[:m.start(0)] + value + args[m.end(0):]
|
|
return args
|
|
|
|
def GetMakeArguments(self, board):
|
|
"""Returns 'make' arguments for a given board
|
|
|
|
The flags are in a section called 'make-flags'. Flags are named
|
|
after the target they represent, for example snapper9260=TESTING=1
|
|
will pass TESTING=1 to make when building the snapper9260 board.
|
|
|
|
References to other boards can be added in the string also. For
|
|
example:
|
|
|
|
[make-flags]
|
|
at91-boards=ENABLE_AT91_TEST=1
|
|
snapper9260=${at91-boards} BUILD_TAG=442
|
|
snapper9g45=${at91-boards} BUILD_TAG=443
|
|
|
|
This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
|
|
and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
|
|
|
|
A special 'target' variable is set to the board target.
|
|
|
|
Args:
|
|
board: Board object for the board to check.
|
|
Returns:
|
|
'make' flags for that board, or '' if none
|
|
"""
|
|
self._make_flags['target'] = board.target
|
|
arg_str = self.ResolveReferences(self._make_flags,
|
|
self._make_flags.get(board.target, ''))
|
|
args = arg_str.split(' ')
|
|
i = 0
|
|
while i < len(args):
|
|
if not args[i]:
|
|
del args[i]
|
|
else:
|
|
i += 1
|
|
return args
|
|
|
|
def LocateArchUrl(self, fetch_arch):
|
|
"""Find a toolchain available online
|
|
|
|
Look in standard places for available toolchains. At present the
|
|
only standard place is at kernel.org.
|
|
|
|
Args:
|
|
arch: Architecture to look for, or 'list' for all
|
|
Returns:
|
|
If fetch_arch is 'list', a tuple:
|
|
Machine architecture (e.g. x86_64)
|
|
List of toolchains
|
|
else
|
|
URL containing this toolchain, if avaialble, else None
|
|
"""
|
|
arch = command.OutputOneLine('uname', '-m')
|
|
base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
|
|
versions = ['7.3.0', '6.4.0', '4.9.4']
|
|
links = []
|
|
for version in versions:
|
|
url = '%s/%s/%s/' % (base, arch, version)
|
|
print 'Checking: %s' % url
|
|
response = urllib2.urlopen(url)
|
|
html = response.read()
|
|
parser = MyHTMLParser(fetch_arch)
|
|
parser.feed(html)
|
|
if fetch_arch == 'list':
|
|
links += parser.links
|
|
elif parser.arch_link:
|
|
return url + parser.arch_link
|
|
if fetch_arch == 'list':
|
|
return arch, links
|
|
return None
|
|
|
|
def Download(self, url):
|
|
"""Download a file to a temporary directory
|
|
|
|
Args:
|
|
url: URL to download
|
|
Returns:
|
|
Tuple:
|
|
Temporary directory name
|
|
Full path to the downloaded archive file in that directory,
|
|
or None if there was an error while downloading
|
|
"""
|
|
print 'Downloading: %s' % url
|
|
leaf = url.split('/')[-1]
|
|
tmpdir = tempfile.mkdtemp('.buildman')
|
|
response = urllib2.urlopen(url)
|
|
fname = os.path.join(tmpdir, leaf)
|
|
fd = open(fname, 'wb')
|
|
meta = response.info()
|
|
size = int(meta.getheaders('Content-Length')[0])
|
|
done = 0
|
|
block_size = 1 << 16
|
|
status = ''
|
|
|
|
# Read the file in chunks and show progress as we go
|
|
while True:
|
|
buffer = response.read(block_size)
|
|
if not buffer:
|
|
print chr(8) * (len(status) + 1), '\r',
|
|
break
|
|
|
|
done += len(buffer)
|
|
fd.write(buffer)
|
|
status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
|
|
done * 100 / size)
|
|
status = status + chr(8) * (len(status) + 1)
|
|
print status,
|
|
sys.stdout.flush()
|
|
fd.close()
|
|
if done != size:
|
|
print 'Error, failed to download'
|
|
os.remove(fname)
|
|
fname = None
|
|
return tmpdir, fname
|
|
|
|
def Unpack(self, fname, dest):
|
|
"""Unpack a tar file
|
|
|
|
Args:
|
|
fname: Filename to unpack
|
|
dest: Destination directory
|
|
Returns:
|
|
Directory name of the first entry in the archive, without the
|
|
trailing /
|
|
"""
|
|
stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
|
|
dirs = stdout.splitlines()[1].split('/')[:2]
|
|
return '/'.join(dirs)
|
|
|
|
def TestSettingsHasPath(self, path):
|
|
"""Check if buildman will find this toolchain
|
|
|
|
Returns:
|
|
True if the path is in settings, False if not
|
|
"""
|
|
paths = self.GetPathList(False)
|
|
return path in paths
|
|
|
|
def ListArchs(self):
|
|
"""List architectures with available toolchains to download"""
|
|
host_arch, archives = self.LocateArchUrl('list')
|
|
re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*')
|
|
arch_set = set()
|
|
for archive in archives:
|
|
# Remove the host architecture from the start
|
|
arch = re_arch.match(archive[len(host_arch):])
|
|
if arch:
|
|
arch_set.add(arch.group(1))
|
|
return sorted(arch_set)
|
|
|
|
def FetchAndInstall(self, arch):
|
|
"""Fetch and install a new toolchain
|
|
|
|
arch:
|
|
Architecture to fetch, or 'list' to list
|
|
"""
|
|
# Fist get the URL for this architecture
|
|
col = terminal.Color()
|
|
print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
|
|
url = self.LocateArchUrl(arch)
|
|
if not url:
|
|
print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
|
|
arch)
|
|
return 2
|
|
home = os.environ['HOME']
|
|
dest = os.path.join(home, '.buildman-toolchains')
|
|
if not os.path.exists(dest):
|
|
os.mkdir(dest)
|
|
|
|
# Download the tar file for this toolchain and unpack it
|
|
tmpdir, tarfile = self.Download(url)
|
|
if not tarfile:
|
|
return 1
|
|
print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
|
|
sys.stdout.flush()
|
|
path = self.Unpack(tarfile, dest)
|
|
os.remove(tarfile)
|
|
os.rmdir(tmpdir)
|
|
print
|
|
|
|
# Check that the toolchain works
|
|
print col.Color(col.GREEN, 'Testing')
|
|
dirpath = os.path.join(dest, path)
|
|
compiler_fname_list = self.ScanPath(dirpath, True)
|
|
if not compiler_fname_list:
|
|
print 'Could not locate C compiler - fetch failed.'
|
|
return 1
|
|
if len(compiler_fname_list) != 1:
|
|
print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
|
|
', '.join(compiler_fname_list))
|
|
toolchain = Toolchain(compiler_fname_list[0], True, True)
|
|
|
|
# Make sure that it will be found by buildman
|
|
if not self.TestSettingsHasPath(dirpath):
|
|
print ("Adding 'download' to config file '%s'" %
|
|
bsettings.config_fname)
|
|
bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
|
|
return 0
|