From 2945eb73dd988cbd9775395128935256deb9a96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Br=C3=BCns?= Date: Thu, 11 Aug 2016 22:52:03 +0200 Subject: [PATCH 01/21] sandbox: document support of block device emulation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Brüns Acked-by: Simon Glass Changed 'Sandbox' to 'sandbox' in subject: Signed-off-by: Simon Glass --- board/sandbox/README.sandbox | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/board/sandbox/README.sandbox b/board/sandbox/README.sandbox index ed820d338e..02d8ab3d09 100644 --- a/board/sandbox/README.sandbox +++ b/board/sandbox/README.sandbox @@ -320,6 +320,25 @@ CONFIG_SPI_IDLE_VAL The idle value on the SPI bus +Block Device Emulation +---------------------- + +U-Boot can use raw disk images for block device emulation. To e.g. list +the contents of the root directory on the second partion of the image +"disk.raw", you can use the following commands: + +=>host bind 0 ./disk.raw +=>ls host 0:2 + +A disk image can be created using the following commands: + +$> truncate -s 1200M ./disk.raw +$> echo -e "label: gpt\n,64M,U\n,,L" | /usr/sbin/sfdisk ./disk.raw +$> lodev=`sudo losetup -P -f --show ./disk.raw` +$> sudo mkfs.vfat -n EFI -v ${lodev}p1 +$> sudo mkfs.ext4 -L ROOT -v ${lodev}p2 + + Writing Sandbox Drivers ----------------------- From 49afb37988a397ce215e99563e0ab5d2970c6a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Br=C3=BCns?= Date: Thu, 11 Aug 2016 22:52:04 +0200 Subject: [PATCH 02/21] sandbox: Add "host size" hostfs command for fs test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This complements the size/fatsize/ext4size commands added in commit cf6598193aed5de8855eaf70c1994f2bc437255a load, save and ls are already implemented for hostfs, now tests can cover the same operations on hostfs and emulated block devices. Signed-off-by: Stefan Brüns Acked-by: Simon Glass --- cmd/host.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/host.c b/cmd/host.c index 8d84415301..b427e541f1 100644 --- a/cmd/host.c +++ b/cmd/host.c @@ -25,6 +25,12 @@ static int do_host_ls(cmd_tbl_t *cmdtp, int flag, int argc, return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX); } +static int do_host_size(cmd_tbl_t *cmdtp, int flag, int argc, + char * const argv[]) +{ + return do_size(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX); +} + static int do_host_save(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]) { @@ -138,6 +144,7 @@ static cmd_tbl_t cmd_host_sub[] = { U_BOOT_CMD_MKENT(load, 7, 0, do_host_load, "", ""), U_BOOT_CMD_MKENT(ls, 3, 0, do_host_ls, "", ""), U_BOOT_CMD_MKENT(save, 6, 0, do_host_save, "", ""), + U_BOOT_CMD_MKENT(size, 3, 0, do_host_size, "", ""), U_BOOT_CMD_MKENT(bind, 3, 0, do_host_bind, "", ""), U_BOOT_CMD_MKENT(info, 3, 0, do_host_info, "", ""), U_BOOT_CMD_MKENT(dev, 0, 1, do_host_dev, "", ""), @@ -174,6 +181,7 @@ U_BOOT_CMD( "host ls hostfs - - list files on host\n" "host save hostfs - [] - " "save a file to host\n" + "host size hostfs - - determine size of file on host" "host bind [] - bind \"host\" device to file\n" "host info [] - show device binding & info\n" "host dev [] - Set or retrieve the current host device\n" From 785f1548a9e74cf4796c96e6f32ed67b25f79b81 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:00 -0600 Subject: [PATCH 03/21] patman: Adjust command.Output() to raise an error by default It is more useful to have this method raise an error when something goes wrong. Make this the default and adjust the few callers that don't want to use it this way. Signed-off-by: Simon Glass --- tools/buildman/control.py | 2 +- tools/patman/checkpatch.py | 3 ++- tools/patman/command.py | 5 +++-- tools/patman/gitutil.py | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/buildman/control.py b/tools/buildman/control.py index b86d7b3c1f..0b6ab03b4c 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -237,7 +237,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, options.step = len(series.commits) - 1 gnu_make = command.Output(os.path.join(options.git, - 'scripts/show-gnu-make')).rstrip() + 'scripts/show-gnu-make'), raise_on_error=False).rstrip() if not gnu_make: sys.exit('GNU Make not found') diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index 34a3bd22b0..3eef6de221 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -63,7 +63,8 @@ def CheckPatch(fname, verbose=False): result.problems = [] chk = FindCheckPatch() item = {} - result.stdout = command.Output(chk, '--no-tree', fname) + result.stdout = command.Output(chk, '--no-tree', fname, + raise_on_error=False) #pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) #stdout, stderr = pipe.communicate() diff --git a/tools/patman/command.py b/tools/patman/command.py index d586f11158..d1f0ca505c 100644 --- a/tools/patman/command.py +++ b/tools/patman/command.py @@ -104,8 +104,9 @@ def RunPipe(pipe_list, infile=None, outfile=None, raise Exception("Error running '%s'" % user_pipestr) return result -def Output(*cmd): - return RunPipe([cmd], capture=True, raise_on_error=False).stdout +def Output(*cmd, **kwargs): + raise_on_error = kwargs.get('raise_on_error', True) + return RunPipe([cmd], capture=True, raise_on_error=raise_on_error).stdout def OutputOneLine(*cmd, **kwargs): raise_on_error = kwargs.pop('raise_on_error', True) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index e088baeb81..bb7c9e08bc 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -391,7 +391,8 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, """ to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error) if not to: - git_config_to = command.Output('git', 'config', 'sendemail.to') + git_config_to = command.Output('git', 'config', 'sendemail.to', + raise_on_error=False) if not git_config_to: print ("No recipient.\n" "Please add something like this to a commit\n" From 58593115453199eb136f1c625bc29e44d057c07e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:01 -0600 Subject: [PATCH 04/21] dtoc: Move the struct import into the correct order This should be in with the other system includes. Move it. Signed-off-by: Simon Glass --- tools/dtoc/dtoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index ec80abe717..e9ab46f706 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -9,6 +9,7 @@ import copy from optparse import OptionError, OptionParser import os +import struct import sys import fdt_util @@ -29,8 +30,6 @@ except ImportError: from fdt_fallback import Fdt import fdt_fallback as fdt -import struct - # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ '#address-cells', From ba482585661431dde74eb69c7a94ac2d5a772d9f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:02 -0600 Subject: [PATCH 05/21] dtoc: Move the fdt library selection into fdt_select Rather than have dtc worry about which fdt library to use, move this into a helper file. Add a function which creates a new Fdt object and scans it, regardless of the implementation. Signed-off-by: Simon Glass --- tools/dtoc/dtoc.py | 18 +++--------------- tools/dtoc/fdt_select.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 tools/dtoc/fdt_select.py diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index e9ab46f706..10f6a12578 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -12,23 +12,12 @@ import os import struct import sys -import fdt_util - # Bring in the patman libraries our_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(our_path, '../patman')) -# Bring in either the normal fdt library (which relies on libfdt) or the -# fallback one (which uses fdtget and is slower). Both provide the same -# interfface for this file to use. -try: - from fdt import Fdt - import fdt - have_libfdt = True -except ImportError: - have_libfdt = False - from fdt_fallback import Fdt - import fdt_fallback as fdt +import fdt_select +import fdt_util # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ @@ -177,8 +166,7 @@ class DtbPlatdata: Once this is done, self.fdt.GetRoot() can be called to obtain the device tree root node, and progress from there. """ - self.fdt = Fdt(self._dtb_fname) - self.fdt.Scan() + self.fdt = fdt_select.FdtScan(self._dtb_fname) def ScanTree(self): """Scan the device tree for useful information diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py new file mode 100644 index 0000000000..5aff297c8f --- /dev/null +++ b/tools/dtoc/fdt_select.py @@ -0,0 +1,23 @@ +#!/usr/bin/python +# +# Copyright (C) 2016 Google, Inc +# Written by Simon Glass +# +# SPDX-License-Identifier: GPL-2.0+ +# + +# Bring in either the normal fdt library (which relies on libfdt) or the +# fallback one (which uses fdtget and is slower). Both provide the same +# interface for this file to use. +try: + import fdt + have_libfdt = True +except ImportError: + have_libfdt = False + import fdt_fallback as fdt + +def FdtScan(fname): + """Returns a new Fdt object from the implementation we are using""" + dtb = fdt.Fdt(fname) + dtb.Scan() + return dtb From 66051b1f59dfba48566dc3e0eee4c093e1104590 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:03 -0600 Subject: [PATCH 06/21] dtoc: Rename fdt.py to fdt_normal.py In preparation for creating an Fdt base class, rename this file to indicate it is the normal Fdt implementation. Signed-off-by: Simon Glass --- tools/dtoc/{fdt.py => fdt_normal.py} | 0 tools/dtoc/fdt_select.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tools/dtoc/{fdt.py => fdt_normal.py} (100%) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt_normal.py similarity index 100% rename from tools/dtoc/fdt.py rename to tools/dtoc/fdt_normal.py diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py index 5aff297c8f..681dfbfda0 100644 --- a/tools/dtoc/fdt_select.py +++ b/tools/dtoc/fdt_select.py @@ -10,7 +10,7 @@ # fallback one (which uses fdtget and is slower). Both provide the same # interface for this file to use. try: - import fdt + import fdt_normal as fdt have_libfdt = True except ImportError: have_libfdt = False From a06a34b2031e0797892e188595bfb305cd9719ab Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:04 -0600 Subject: [PATCH 07/21] dtoc: Create a base class for Fdt At present we have two separate implementations of the Fdt library, one which uses fdtget/fdtput and one which uses libfdt (via swig). Before adding more functionality it makes sense to create a base class for these. This will allow common functions to be shared, and make the Fdt API a little clearer. Create a new fdt.py file with the base class, and adjust fdt_normal.py and fdt_fallback.py to use it. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 68 ++++++++++++++++++++++++++++++++++++++ tools/dtoc/fdt_fallback.py | 61 ++++++++++++++++++++-------------- tools/dtoc/fdt_normal.py | 58 ++++++++++++++++++++------------ tools/dtoc/fdt_select.py | 9 +++-- 4 files changed, 148 insertions(+), 48 deletions(-) create mode 100644 tools/dtoc/fdt.py diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py new file mode 100644 index 0000000000..413d45daa4 --- /dev/null +++ b/tools/dtoc/fdt.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# Copyright (C) 2016 Google, Inc +# Written by Simon Glass +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import struct +import sys + +import fdt_util + +# This deals with a device tree, presenting it as an assortment of Node and +# Prop objects, representing nodes and properties, respectively. This file +# contains the base classes and defines the high-level API. Most of the +# implementation is in the FdtFallback and FdtNormal subclasses. See +# fdt_select.py for how to create an Fdt object. + +def CheckErr(errnum, msg): + if errnum: + raise ValueError('Error %d: %s: %s' % + (errnum, libfdt.fdt_strerror(errnum), msg)) + +class PropBase: + """A device tree property + + Properties: + name: Property name (as per the device tree) + value: Property value as a string of bytes, or a list of strings of + bytes + type: Value type + """ + def __init__(self, node, offset, name): + self._node = node + self._offset = offset + self.name = name + self.value = None + +class NodeBase: + """A device tree node + + Properties: + offset: Integer offset in the device tree + name: Device tree node tname + path: Full path to node, along with the node name itself + _fdt: Device tree object + subnodes: A list of subnodes for this node, each a Node object + props: A dict of properties for this node, each a Prop object. + Keyed by property name + """ + def __init__(self, fdt, offset, name, path): + self._fdt = fdt + self._offset = offset + self.name = name + self.path = path + self.subnodes = [] + self.props = {} + +class Fdt: + """Provides simple access to a flat device tree blob. + + Properties: + fname: Filename of fdt + _root: Root of device tree (a Node object) + """ + def __init__(self, fname): + self._fname = fname diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 9ed11e4cbf..be3b5badc9 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -7,6 +7,8 @@ # import command +import fdt +from fdt import Fdt, NodeBase, PropBase import fdt_util import sys @@ -17,7 +19,7 @@ import sys # is not very efficient for larger trees. The tool is called once for each # node and property in the tree. -class Prop: +class Prop(PropBase): """A device tree property Properties: @@ -26,14 +28,14 @@ class Prop: bytes type: Value type """ - def __init__(self, name, byte_list_str): - self.name = name - self.value = None + def __init__(self, node, name, byte_list_str): + PropBase.__init__(self, node, 0, name) if not byte_list_str.strip(): self.type = fdt_util.TYPE_BOOL return - bytes = [chr(int(byte, 16)) for byte in byte_list_str.strip().split(' ')] - self.type, self.value = fdt_util.BytesToValue(''.join(bytes)) + self.bytes = [chr(int(byte, 16)) + for byte in byte_list_str.strip().split(' ')] + self.type, self.value = fdt_util.BytesToValue(''.join(self.bytes)) def GetPhandle(self): """Get a (single) phandle value from a property @@ -77,7 +79,7 @@ class Prop: self.value.append(val) -class Node: +class Node(NodeBase): """A device tree node Properties: @@ -88,12 +90,8 @@ class Node: props: A dict of properties for this node, each a Prop object. Keyed by property name """ - def __init__(self, fdt, name, path): - self.name = name - self.path = path - self._fdt = fdt - self.subnodes = [] - self.props = {} + def __init__(self, fdt, offset, name, path): + NodeBase.__init__(self, fdt, offset, name, path) def Scan(self): """Scan a node's properties and subnodes @@ -102,35 +100,34 @@ class Node: searching into subnodes so that the entire tree is built. """ for name, byte_list_str in self._fdt.GetProps(self.path).iteritems(): - prop = Prop(name, byte_list_str) + prop = Prop(self, name, byte_list_str) self.props[name] = prop for name in self._fdt.GetSubNodes(self.path): sep = '' if self.path[-1] == '/' else '/' path = self.path + sep + name - node = Node(self._fdt, name, path) + node = Node(self._fdt, 0, name, path) self.subnodes.append(node) node.Scan() -class Fdt: - """Provides simple access to a flat device tree blob. +class FdtFallback(Fdt): + """Provides simple access to a flat device tree blob using fdtget/fdtput Properties: - fname: Filename of fdt - _root: Root of device tree (a Node object) + See superclass """ def __init__(self, fname): - self.fname = fname + Fdt.__init__(self, fname) def Scan(self): """Scan a device tree, building up a tree of Node objects This fills in the self._root property """ - self._root = Node(self, '/', '/') + self._root = Node(self, 0, '/', '/') self._root.Scan() def GetRoot(self): @@ -153,7 +150,7 @@ class Fdt: Raises: CmdError: if the node does not exist. """ - out = command.Output('fdtget', self.fname, '-l', node) + out = command.Output('fdtget', self._fname, '-l', node) return out.strip().splitlines() def GetProps(self, node, convert_dashes=False): @@ -171,7 +168,7 @@ class Fdt: Raises: CmdError: if the node does not exist. """ - out = command.Output('fdtget', self.fname, node, '-p') + out = command.Output('fdtget', self._fname, node, '-p') props = out.strip().splitlines() props_dict = {} for prop in props: @@ -204,10 +201,26 @@ class Fdt: Raises: CmdError: if the property does not exist and no default is provided. """ - args = [self.fname, node, prop, '-t', 'bx'] + args = [self._fname, node, prop, '-t', 'bx'] if default is not None: args += ['-d', str(default)] if typespec is not None: args += ['-t%s' % typespec] out = command.Output('fdtget', *args) return out.strip() + + @classmethod + def Node(self, fdt, offset, name, path): + """Create a new node + + This is used by Fdt.Scan() to create a new node using the correct + class. + + Args: + fdt: Fdt object + offset: Offset of node + name: Node name + path: Full path to node + """ + node = Node(fdt, offset, name, path) + return node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 1d913a925e..ca5335ba5b 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -6,9 +6,13 @@ # SPDX-License-Identifier: GPL-2.0+ # +import struct +import sys + +import fdt +from fdt import Fdt, NodeBase, PropBase import fdt_util import libfdt -import sys # This deals with a device tree, presenting it as a list of Node and Prop # objects, representing nodes and properties, respectively. @@ -16,7 +20,7 @@ import sys # This implementation uses a libfdt Python library to access the device tree, # so it is fairly efficient. -class Prop: +class Prop(PropBase): """A device tree property Properties: @@ -25,9 +29,9 @@ class Prop: bytes type: Value type """ - def __init__(self, name, bytes): - self.name = name - self.value = None + def __init__(self, node, offset, name, bytes): + PropBase.__init__(self, node, offset, name) + self.bytes = bytes if not bytes: self.type = fdt_util.TYPE_BOOL self.value = True @@ -76,7 +80,7 @@ class Prop: self.value.append(val) -class Node: +class Node(NodeBase): """A device tree node Properties: @@ -89,12 +93,7 @@ class Node: Keyed by property name """ def __init__(self, fdt, offset, name, path): - self.offset = offset - self.name = name - self.path = path - self._fdt = fdt - self.subnodes = [] - self.props = {} + NodeBase.__init__(self, fdt, offset, name, path) def Scan(self): """Scan a node's properties and subnodes @@ -104,7 +103,7 @@ class Node: """ self.props = self._fdt.GetProps(self.path) - offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.offset) + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) while offset >= 0: sep = '' if self.path[-1] == '/' else '/' name = libfdt.Name(self._fdt.GetFdt(), offset) @@ -116,17 +115,17 @@ class Node: offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) -class Fdt: - """Provides simple access to a flat device tree blob. +class FdtNormal(Fdt): + """Provides simple access to a flat device tree blob using libfdt. Properties: - fname: Filename of fdt - _root: Root of device tree (a Node object) + _fdt: Device tree contents (bytearray) + _cached_offsets: True if all the nodes have a valid _offset property, + False if something has changed to invalidate the offsets """ - def __init__(self, fname): - self.fname = fname - with open(fname) as fd: + Fdt.__init__(self, fname) + with open(self._fname) as fd: self._fdt = fd.read() def GetFdt(self): @@ -173,8 +172,25 @@ class Fdt: poffset = libfdt.fdt_first_property_offset(self._fdt, offset) while poffset >= 0: dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset) - prop = Prop(libfdt.String(self._fdt, dprop.nameoff), libfdt.Data(dprop)) + prop = Prop(node, poffset, libfdt.String(self._fdt, dprop.nameoff), + libfdt.Data(dprop)) props_dict[prop.name] = prop poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) return props_dict + + @classmethod + def Node(self, fdt, offset, name, path): + """Create a new node + + This is used by Fdt.Scan() to create a new node using the correct + class. + + Args: + fdt: Fdt object + offset: Offset of node + name: Node name + path: Full path to node + """ + node = Node(fdt, offset, name, path) + return node diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py index 681dfbfda0..18a36d88a0 100644 --- a/tools/dtoc/fdt_select.py +++ b/tools/dtoc/fdt_select.py @@ -10,14 +10,17 @@ # fallback one (which uses fdtget and is slower). Both provide the same # interface for this file to use. try: - import fdt_normal as fdt + import fdt_normal have_libfdt = True except ImportError: have_libfdt = False - import fdt_fallback as fdt + import fdt_fallback def FdtScan(fname): """Returns a new Fdt object from the implementation we are using""" - dtb = fdt.Fdt(fname) + if have_libfdt: + dtb = fdt_normal.FdtNormal(fname) + else: + dtb = fdt_fallback.FdtFallback(fname) dtb.Scan() return dtb From bc1dea3656e55d91f7a3c1339d53fc9def3bbf31 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:05 -0600 Subject: [PATCH 08/21] dtoc: Move BytesToValue() and GetEmpty() into PropBase These functions are currently in a separate fdt_util file. Since they are only used from PropBase and subclasses, it makes sense for them to be in the PropBase class. Move these functions into fdt.py along with the list of types. Signed-off-by: Simon Glass --- tools/dtoc/dtoc.py | 17 +++++----- tools/dtoc/fdt.py | 66 ++++++++++++++++++++++++++++++++++++++ tools/dtoc/fdt_fallback.py | 4 +-- tools/dtoc/fdt_normal.py | 6 ++-- tools/dtoc/fdt_util.py | 66 -------------------------------------- 5 files changed, 80 insertions(+), 79 deletions(-) diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index 10f6a12578..518aa51216 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -16,6 +16,7 @@ import sys our_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(our_path, '../patman')) +import fdt import fdt_select import fdt_util @@ -33,10 +34,10 @@ PROP_IGNORE_LIST = [ # C type declarations for the tyues we support TYPE_NAMES = { - fdt_util.TYPE_INT: 'fdt32_t', - fdt_util.TYPE_BYTE: 'unsigned char', - fdt_util.TYPE_STRING: 'const char *', - fdt_util.TYPE_BOOL: 'bool', + fdt.TYPE_INT: 'fdt32_t', + fdt.TYPE_BYTE: 'unsigned char', + fdt.TYPE_STRING: 'const char *', + fdt.TYPE_BOOL: 'bool', }; STRUCT_PREFIX = 'dtd_' @@ -138,13 +139,13 @@ class DtbPlatdata: type: Data type (fdt_util) value: Data value, as a string of bytes """ - if type == fdt_util.TYPE_INT: + if type == fdt.TYPE_INT: return '%#x' % fdt_util.fdt32_to_cpu(value) - elif type == fdt_util.TYPE_BYTE: + elif type == fdt.TYPE_BYTE: return '%#x' % ord(value[0]) - elif type == fdt_util.TYPE_STRING: + elif type == fdt.TYPE_STRING: return '"%s"' % value - elif type == fdt_util.TYPE_BOOL: + elif type == fdt.TYPE_BOOL: return 'true' def GetCompatName(self, node): diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 413d45daa4..329d03cc6b 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -17,6 +17,9 @@ import fdt_util # implementation is in the FdtFallback and FdtNormal subclasses. See # fdt_select.py for how to create an Fdt object. +# A list of types we support +(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4) + def CheckErr(errnum, msg): if errnum: raise ValueError('Error %d: %s: %s' % @@ -37,6 +40,69 @@ class PropBase: self.name = name self.value = None + def BytesToValue(self, bytes): + """Converts a string of bytes into a type and value + + Args: + A string containing bytes + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: string value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte string + TYPE_BYTE: a byte stored as a single-byte string + """ + size = len(bytes) + strings = bytes.split('\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not strings[-1]: + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + if ch < ' ' or ch > '~': + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + return TYPE_STRING, strings[0] + else: + return TYPE_STRING, strings[:-1] + if size % 4: + if size == 1: + return TYPE_BYTE, bytes[0] + else: + return TYPE_BYTE, list(bytes) + val = [] + for i in range(0, size, 4): + val.append(bytes[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + def GetEmpty(self, type): + """Get an empty / zero value of the given type + + Returns: + A single value of the given type + """ + if type == TYPE_BYTE: + return chr(0) + elif type == TYPE_INT: + return struct.pack(' len(self.value): - val = fdt_util.GetEmpty(self.type) + val = self.GetEmpty(self.type) while len(self.value) < len(newprop.value): self.value.append(val) diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 929b524fcf..6b572483e7 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -8,72 +8,6 @@ import struct -# A list of types we support -(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4) - -def BytesToValue(bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - -def GetEmpty(type): - """Get an empty / zero value of the given type - - Returns: - A single value of the given type - """ - if type == TYPE_BYTE: - return chr(0) - elif type == TYPE_INT: - return struct.pack(' Date: Mon, 25 Jul 2016 18:59:06 -0600 Subject: [PATCH 09/21] dtoc: Move Widen() and GetPhandle() into the base class These functions are identical in both subclasses. Move them into the base class. Note: In fact there is a bug in one version, which was fixed by this patch: https://patchwork.ozlabs.org/patch/651697/ Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 41 +++++++++++++++++++++++++++++++++++++ tools/dtoc/fdt_fallback.py | 41 ------------------------------------- tools/dtoc/fdt_normal.py | 42 -------------------------------------- 3 files changed, 41 insertions(+), 83 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 329d03cc6b..964ef7cbb4 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -40,6 +40,47 @@ class PropBase: self.name = name self.value = None + def GetPhandle(self): + """Get a (single) phandle value from a property + + Gets the phandle valuie from a property and returns it as an integer + """ + return fdt_util.fdt32_to_cpu(self.value[:4]) + + def Widen(self, newprop): + """Figure out which property type is more general + + Given a current property and a new property, this function returns the + one that is less specific as to type. The less specific property will + be ble to represent the data in the more specific property. This is + used for things like: + + node1 { + compatible = "fred"; + value = <1>; + }; + node1 { + compatible = "fred"; + value = <1 2>; + }; + + He we want to use an int array for 'value'. The first property + suggests that a single int is enough, but the second one shows that + it is not. Calling this function with these two propertes would + update the current property to be like the second, since it is less + specific. + """ + if newprop.type < self.type: + self.type = newprop.type + + if type(newprop.value) == list and type(self.value) != list: + self.value = [self.value] + + if type(self.value) == list and len(newprop.value) > len(self.value): + val = self.GetEmpty(self.type) + while len(self.value) < len(newprop.value): + self.value.append(val) + def BytesToValue(self, bytes): """Converts a string of bytes into a type and value diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 798c51097e..84a3db1e78 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -37,47 +37,6 @@ class Prop(PropBase): for byte in byte_list_str.strip().split(' ')] self.type, self.value = self.BytesToValue(''.join(self.bytes)) - def GetPhandle(self): - """Get a (single) phandle value from a property - - Gets the phandle valuie from a property and returns it as an integer - """ - return fdt_util.fdt32_to_cpu(self.value[:4]) - - def Widen(self, newprop): - """Figure out which property type is more general - - Given a current property and a new property, this function returns the - one that is less specific as to type. The less specific property will - be ble to represent the data in the more specific property. This is - used for things like: - - node1 { - compatible = "fred"; - value = <1>; - }; - node1 { - compatible = "fred"; - value = <1 2>; - }; - - He we want to use an int array for 'value'. The first property - suggests that a single int is enough, but the second one shows that - it is not. Calling this function with these two propertes would - update the current property to be like the second, since it is less - specific. - """ - if newprop.type < self.type: - self.type = newprop.type - - if type(newprop.value) == list and type(self.value) != list: - self.value = newprop.value - - if type(self.value) == list and len(newprop.value) > len(self.value): - val = fdt_util.GetEmpty(self.type) - while len(self.value) < len(newprop.value): - self.value.append(val) - class Node(NodeBase): """A device tree node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index c7c86b8a88..6f019c1f12 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -38,48 +38,6 @@ class Prop(PropBase): return self.type, self.value = self.BytesToValue(bytes) - def GetPhandle(self): - """Get a (single) phandle value from a property - - Gets the phandle valuie from a property and returns it as an integer - """ - return fdt_util.fdt32_to_cpu(self.value[:4]) - - def Widen(self, newprop): - """Figure out which property type is more general - - Given a current property and a new property, this function returns the - one that is less specific as to type. The less specific property will - be ble to represent the data in the more specific property. This is - used for things like: - - node1 { - compatible = "fred"; - value = <1>; - }; - node1 { - compatible = "fred"; - value = <1 2>; - }; - - He we want to use an int array for 'value'. The first property - suggests that a single int is enough, but the second one shows that - it is not. Calling this function with these two propertes would - update the current property to be like the second, since it is less - specific. - """ - if newprop.type < self.type: - self.type = newprop.type - - if type(newprop.value) == list and type(self.value) != list: - self.value = [self.value] - - if type(self.value) == list and len(newprop.value) > len(self.value): - val = self.GetEmpty(self.type) - while len(self.value) < len(newprop.value): - self.value.append(val) - - class Node(NodeBase): """A device tree node From f7a2aeeeb8d4bbbb9bfd788903942062c8535efb Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:07 -0600 Subject: [PATCH 10/21] dtoc: Move a few more common functions into fdt.py Some functions have the same code in the subclasses. Move these into the superclass to avoid duplication. Signed-off-by: Simon Glass --- tools/dtoc/fdt.py | 57 ++++++++++++++++++++++++++++++++++++++ tools/dtoc/fdt_fallback.py | 16 ----------- tools/dtoc/fdt_normal.py | 16 ----------- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 964ef7cbb4..c0ce5af8ac 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -164,6 +164,26 @@ class NodeBase: self.subnodes = [] self.props = {} + def _FindNode(self, name): + """Find a node given its name + + Args: + name: Node name to look for + Returns: + Node object if found, else None + """ + for subnode in self.subnodes: + if subnode.name == name: + return subnode + return None + + def Scan(self): + """Scan the subnodes of a node + + This should be implemented by subclasses + """ + raise NotImplementedError() + class Fdt: """Provides simple access to a flat device tree blob. @@ -173,3 +193,40 @@ class Fdt: """ def __init__(self, fname): self._fname = fname + + def Scan(self, root='/'): + """Scan a device tree, building up a tree of Node objects + + This fills in the self._root property + + Args: + root: Ignored + + TODO(sjg@chromium.org): Implement the 'root' parameter + """ + self._root = self.Node(self, 0, '/', '/') + self._root.Scan() + + def GetRoot(self): + """Get the root Node of the device tree + + Returns: + The root Node object + """ + return self._root + + def GetNode(self, path): + """Look up a node from its path + + Args: + path: Path to look up, e.g. '/microcode/update@0' + Returns: + Node object, or None if not found + """ + node = self._root + for part in path.split('/')[1:]: + node = node._FindNode(part) + if not node: + return None + return node + diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 84a3db1e78..5b0f2a181b 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -81,22 +81,6 @@ class FdtFallback(Fdt): def __init__(self, fname): Fdt.__init__(self, fname) - def Scan(self): - """Scan a device tree, building up a tree of Node objects - - This fills in the self._root property - """ - self._root = Node(self, 0, '/', '/') - self._root.Scan() - - def GetRoot(self): - """Get the root Node of the device tree - - Returns: - The root Node object - """ - return self._root - def GetSubNodes(self, node): """Returns a list of sub-nodes of a given node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 6f019c1f12..861f60c7d0 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -94,22 +94,6 @@ class FdtNormal(Fdt): """ return self._fdt - def Scan(self): - """Scan a device tree, building up a tree of Node objects - - This fills in the self._root property - """ - self._root = Node(self, 0, '/', '/') - self._root.Scan() - - def GetRoot(self): - """Get the root Node of the device tree - - Returns: - The root Node object - """ - return self._root - def GetProps(self, node): """Get all properties from a node. From 1f1864b408b74fe04d787d9f8941f7b18b9ded66 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:08 -0600 Subject: [PATCH 11/21] patman: Add a tools library for using temporary files For tools which want to use input files and temporary output, it is useful to have the handling of these dealt with in one place. Add a new library which allows input files to be read, and output files to be written, all based on a common directory structure. Signed-off-by: Simon Glass --- tools/patman/tools.py | 120 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tools/patman/tools.py diff --git a/tools/patman/tools.py b/tools/patman/tools.py new file mode 100644 index 0000000000..ba24853030 --- /dev/null +++ b/tools/patman/tools.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2016 Google, Inc +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import os +import shutil +import tempfile + +import tout + +outdir = None +indirs = None +preserve_outdir = False + +def PrepareOutputDir(dirname, preserve=False): + """Select an output directory, ensuring it exists. + + This either creates a temporary directory or checks that the one supplied + by the user is valid. For a temporary directory, it makes a note to + remove it later if required. + + Args: + dirname: a string, name of the output directory to use to store + intermediate and output files. If is None - create a temporary + directory. + preserve: a Boolean. If outdir above is None and preserve is False, the + created temporary directory will be destroyed on exit. + + Raises: + OSError: If it cannot create the output directory. + """ + global outdir, preserve_outdir + + preserve_outdir = dirname or preserve + if dirname: + outdir = dirname + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError as err: + raise CmdError("Cannot make output directory '%s': '%s'" % + (outdir, err.strerror)) + tout.Debug("Using output directory '%s'" % outdir) + else: + outdir = tempfile.mkdtemp(prefix='binman.') + tout.Debug("Using temporary directory '%s'" % outdir) + +def _RemoveOutputDir(): + global outdir + + shutil.rmtree(outdir) + tout.Debug("Deleted temporary directory '%s'" % outdir) + outdir = None + +def FinaliseOutputDir(): + global outdir, preserve_outdir + + """Tidy up: delete output directory if temporary and not preserved.""" + if outdir and not preserve_outdir: + _RemoveOutputDir() + +def GetOutputFilename(fname): + """Return a filename within the output directory. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the output directory + """ + return os.path.join(outdir, fname) + +def _FinaliseForTest(): + """Remove the output directory (for use by tests)""" + global outdir + + if outdir: + _RemoveOutputDir() + +def SetInputDirs(dirname): + """Add a list of input directories, where input files are kept. + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + indir = dirname + tout.Debug("Using input directories %s" % indir) + +def GetInputFilename(fname): + """Return a filename for use as input. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the input directory + """ + if not indir: + return fname + for dirname in indir: + pathname = os.path.join(dirname, fname) + if os.path.exists(pathname): + return pathname + + raise ValueError("Filename '%s' not found in input path (%s)" % + (fname, ','.join(indir))) + +def Align(pos, align): + if align: + mask = align - 1 + pos = (pos + mask) & ~mask + return pos + +def NotPowerOfTwo(num): + return num and (num & (num - 1)) From 0faf6144fd1f6443a52abb0d80a6ca1238ecc029 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:09 -0600 Subject: [PATCH 12/21] patman: Add a library to handle logging and progress When tools want to display information of varying levels of importance, it helps to provide the user with control over the verbosity of these messages. Progress messages work best if they are displayed and then removed from the display when no-longer relevant. Add a new tout library (terminal out) to handle these tasks. Signed-off-by: Simon Glass --- tools/patman/tout.py | 166 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 tools/patman/tout.py diff --git a/tools/patman/tout.py b/tools/patman/tout.py new file mode 100644 index 0000000000..c5fbd80dbc --- /dev/null +++ b/tools/patman/tout.py @@ -0,0 +1,166 @@ +# Copyright (c) 2016 Google, Inc +# +# SPDX-License-Identifier: GPL-2.0+ +# +# Terminal output logging. +# + +import sys + +import terminal + +# Output verbosity levels that we support +ERROR = 0 +WARNING = 1 +NOTICE = 2 +INFO = 3 +DEBUG = 4 + +""" +This class handles output of progress and other useful information +to the user. It provides for simple verbosity level control and can +output nothing but errors at verbosity zero. + +The idea is that modules set up an Output object early in their years and pass +it around to other modules that need it. This keeps the output under control +of a single class. + +Public properties: + verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug +""" +def __enter__(): + return + +def __exit__(unused1, unused2, unused3): + """Clean up and remove any progress message.""" + ClearProgress() + return False + +def UserIsPresent(): + """This returns True if it is likely that a user is present. + + Sometimes we want to prompt the user, but if no one is there then this + is a waste of time, and may lock a script which should otherwise fail. + + Returns: + True if it thinks the user is there, and False otherwise + """ + return stdout_is_tty and verbose > 0 + +def ClearProgress(): + """Clear any active progress message on the terminal.""" + if verbose > 0 and stdout_is_tty: + _stdout.write('\r%s\r' % (" " * len (_progress))) + _stdout.flush() + +def Progress(msg, warning=False, trailer='...'): + """Display progress information. + + Args: + msg: Message to display. + warning: True if this is a warning.""" + ClearProgress() + if verbose > 0: + _progress = msg + trailer + if stdout_is_tty: + col = _color.YELLOW if warning else _color.GREEN + _stdout.write('\r' + _color.Color(col, _progress)) + _stdout.flush() + else: + _stdout.write(_progress + '\n') + +def _Output(level, msg, color=None): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + error: True if this is an error message, else False. + """ + if verbose >= level: + ClearProgress() + if color: + msg = _color.Color(color, msg) + _stdout.write(msg + '\n') + +def DoOutput(level, msg): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + """ + _Output(level, msg) + +def Error(msg): + """Display an error message + + Args: + msg; Message to display. + """ + _Output(0, msg, _color.RED) + +def Warning(msg): + """Display a warning message + + Args: + msg; Message to display. + """ + _Output(1, msg, _color.YELLOW) + +def Notice(msg): + """Display an important infomation message + + Args: + msg; Message to display. + """ + _Output(2, msg) + +def Info(msg): + """Display an infomation message + + Args: + msg; Message to display. + """ + _Output(3, msg) + +def Debug(msg): + """Display a debug message + + Args: + msg; Message to display. + """ + _Output(4, msg) + +def UserOutput(msg): + """Display a message regardless of the current output level. + + This is used when the output was specifically requested by the user. + Args: + msg; Message to display. + """ + _Output(0, msg) + +def Init(_verbose=WARNING, stdout=sys.stdout): + """Initialize a new output object. + + Args: + verbose: Verbosity level (0-4). + stdout: File to use for stdout. + """ + global verbose, _progress, _color, _stdout, stdout_is_tty + + verbose = _verbose + _progress = '' # Our last progress message + _color = terminal.Color() + _stdout = stdout + + # TODO(sjg): Move this into Chromite libraries when we have them + stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + +def Uninit(): + ClearProgress() + +Init() From 355c67c35a8ce5aa9e9e2e2e8df99413c8215093 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:10 -0600 Subject: [PATCH 13/21] dtoc: Allow the device tree to be compiled from source If a source device tree is provide to the Fdt() constructors, compile it automatically. This will be used in tests, where we want to build a particular test .dts file and check that it works correctly in binman. Signed-off-by: Simon Glass --- tools/dtoc/fdt_fallback.py | 2 ++ tools/dtoc/fdt_normal.py | 7 +++++-- tools/dtoc/fdt_util.py | 41 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 5b0f2a181b..f76f42a101 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -80,6 +80,8 @@ class FdtFallback(Fdt): def __init__(self, fname): Fdt.__init__(self, fname) + if self._fname: + self._fname = fdt_util.EnsureCompiled(self._fname) def GetSubNodes(self, node): """Returns a list of sub-nodes of a given node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 861f60c7d0..d9ba4aca80 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -83,8 +83,11 @@ class FdtNormal(Fdt): """ def __init__(self, fname): Fdt.__init__(self, fname) - with open(self._fname) as fd: - self._fdt = fd.read() + if self._fname: + self._fname = fdt_util.EnsureCompiled(self._fname) + + with open(self._fname) as fd: + self._fdt = fd.read() def GetFdt(self): """Get the contents of the FDT diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 6b572483e7..3e25a8b980 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -6,7 +6,12 @@ # SPDX-License-Identifier: GPL-2.0+ # +import os import struct +import tempfile + +import command +import tools def fdt32_to_cpu(val): """Convert a device tree cell to an integer @@ -18,3 +23,39 @@ def fdt32_to_cpu(val): A native-endian integer value """ return struct.unpack(">I", val)[0] + +def EnsureCompiled(fname): + """Compile an fdt .dts source file into a .dtb binary blob if needed. + + Args: + fname: Filename (if .dts it will be compiled). It not it will be + left alone + + Returns: + Filename of resulting .dtb file + """ + _, ext = os.path.splitext(fname) + if ext != '.dts': + return fname + + dts_input = tools.GetOutputFilename('source.dts') + dtb_output = tools.GetOutputFilename('source.dtb') + + search_paths = [os.path.join(os.getcwd(), 'include')] + root, _ = os.path.splitext(fname) + args = ['-E', '-P', '-x', 'assembler-with-cpp', '-D__ASSEMBLY__'] + args += ['-Ulinux'] + for path in search_paths: + args.extend(['-I', path]) + args += ['-o', dts_input, fname] + command.Run('cc', *args) + + # If we don't have a directory, put it in the tools tempdir + search_list = [] + for path in search_paths: + search_list.extend(['-i', path]) + args = ['-I', 'dts', '-o', dtb_output, '-O', 'dtb'] + args.extend(search_list) + args.append(dts_input) + command.Run('dtc', *args) + return dtb_output From 6b93c55f59aba3a3fff36b771ca75739e31ba7db Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:11 -0600 Subject: [PATCH 14/21] dtoc: Drop the convert_dash parameter to GetProps() This is not used anywhere in dtoc, so drop it. Signed-off-by: Simon Glass --- tools/dtoc/fdt_fallback.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index f76f42a101..1c8c9c7a6d 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -98,12 +98,11 @@ class FdtFallback(Fdt): out = command.Output('fdtget', self._fname, '-l', node) return out.strip().splitlines() - def GetProps(self, node, convert_dashes=False): + def GetProps(self, node): """Get all properties from a node Args: node: full path to node name to look in - convert_dashes: True to convert - to _ in node names Returns: A dictionary containing all the properties, indexed by node name. @@ -118,8 +117,6 @@ class FdtFallback(Fdt): props_dict = {} for prop in props: name = prop - if convert_dashes: - prop = re.sub('-', '_', prop) props_dict[prop] = self.GetProp(node, name) return props_dict From 346179f0d3383e88f7df117b13820df70f68c74a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:12 -0600 Subject: [PATCH 15/21] dtoc: Prepare for supporting changing of device trees For binman we need to support deleting properties in the device tree. This will change the offsets of nodes after the deletion. In preparation, add code to keep track of when the offsets are invalid, and regenerate them. Signed-off-by: Simon Glass --- tools/dtoc/fdt_normal.py | 46 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index d9ba4aca80..4a667a115d 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -53,15 +53,24 @@ class Node(NodeBase): def __init__(self, fdt, offset, name, path): NodeBase.__init__(self, fdt, offset, name, path) + def Offset(self): + """Returns the offset of a node, after checking the cache + + This should be used instead of self._offset directly, to ensure that + the cache does not contain invalid offsets. + """ + self._fdt.CheckCache() + return self._offset + def Scan(self): """Scan a node's properties and subnodes This fills in the props and subnodes properties, recursively searching into subnodes so that the entire tree is built. """ - self.props = self._fdt.GetProps(self.path) + self.props = self._fdt.GetProps(self, self.path) - offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.Offset()) while offset >= 0: sep = '' if self.path[-1] == '/' else '/' name = libfdt.Name(self._fdt.GetFdt(), offset) @@ -72,6 +81,19 @@ class Node(NodeBase): node.Scan() offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) + def Refresh(self, my_offset): + """Fix up the _offset for each node, recursively + + Note: This does not take account of property offsets - these will not + be updated. + """ + if self._offset != my_offset: + #print '%s: %d -> %d\n' % (self.path, self._offset, my_offset) + self._offset = my_offset + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) + for subnode in self.subnodes: + subnode.Refresh(offset) + offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) class FdtNormal(Fdt): """Provides simple access to a flat device tree blob using libfdt. @@ -83,6 +105,7 @@ class FdtNormal(Fdt): """ def __init__(self, fname): Fdt.__init__(self, fname) + self._cached_offsets = False if self._fname: self._fname = fdt_util.EnsureCompiled(self._fname) @@ -97,7 +120,7 @@ class FdtNormal(Fdt): """ return self._fdt - def GetProps(self, node): + def GetProps(self, node, path): """Get all properties from a node. Args: @@ -110,7 +133,7 @@ class FdtNormal(Fdt): Raises: ValueError: if the node does not exist. """ - offset = libfdt.fdt_path_offset(self._fdt, node) + offset = libfdt.fdt_path_offset(self._fdt, path) if offset < 0: libfdt.Raise(offset) props_dict = {} @@ -124,6 +147,21 @@ class FdtNormal(Fdt): poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) return props_dict + def Invalidate(self): + """Mark our offset cache as invalid""" + self._cached_offsets = False + + def CheckCache(self): + """Refresh the offset cache if needed""" + if self._cached_offsets: + return + self.Refresh() + self._cached_offsets = True + + def Refresh(self): + """Refresh the offset cache""" + self._root.Refresh(0) + @classmethod def Node(self, fdt, offset, name, path): """Create a new node From 0170804f60b19a2033ac39964fcd192a0c7eda42 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:13 -0600 Subject: [PATCH 16/21] dtoc: Move to using bytearray Since we want to be able to change the in-memory device tree using libfdt, use a bytearray instead of a string. This makes interfacing from Python easier. Signed-off-by: Simon Glass --- lib/libfdt/libfdt.swig | 8 ++++++++ tools/dtoc/fdt_normal.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 14f583dfbe..26d42fc5d6 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -75,6 +75,14 @@ struct fdt_property { } %} +%typemap(in) (const void *) { + if (!PyByteArray_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " + "$argnum"" of type '" "$type""'"); + } + $1 = (void *) PyByteArray_AsString($input); +} + const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int checklen); int fdt_path_offset(const void *fdt, const char *path); int fdt_first_property_offset(const void *fdt, int nodeoffset); diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 4a667a115d..eb45742a10 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -110,7 +110,7 @@ class FdtNormal(Fdt): self._fname = fdt_util.EnsureCompiled(self._fname) with open(self._fname) as fd: - self._fdt = fd.read() + self._fdt = bytearray(fd.read()) def GetFdt(self): """Get the contents of the FDT From 2a70d897ed68fd521411a10831ac05e1ffdd3d41 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:14 -0600 Subject: [PATCH 17/21] dtoc: Support deleting device tree properties Add support for deleting a device tree property. With the fallback implementation this uses fdtput. With libfdt it uses the API call and updates the offsets afterwards. Signed-off-by: Simon Glass --- lib/libfdt/libfdt.swig | 12 ++++++++++++ tools/dtoc/fdt.py | 10 ++++++++++ tools/dtoc/fdt_fallback.py | 13 +++++++++++++ tools/dtoc/fdt_normal.py | 20 ++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 26d42fc5d6..ce516fddf2 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -95,3 +95,15 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *OUTPUT); const char *fdt_string(const void *fdt, int stroffset); int fdt_first_subnode(const void *fdt, int offset); int fdt_next_subnode(const void *fdt, int offset); + +%typemap(in) (void *) { + if (!PyByteArray_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " + "$argnum"" of type '" "$type""'"); + } + $1 = PyByteArray_AsString($input); +} + +int fdt_delprop(void *fdt, int nodeoffset, const char *name); + +const char *fdt_strerror(int errval); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index c0ce5af8ac..f01c7b18ea 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -184,6 +184,16 @@ class NodeBase: """ raise NotImplementedError() + def DeleteProp(self, prop_name): + """Delete a property of a node + + This should be implemented by subclasses + + Args: + prop_name: Name of the property to delete + """ + raise NotImplementedError() + class Fdt: """Provides simple access to a flat device tree blob. diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 1c8c9c7a6d..0c0ebbcf47 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -70,6 +70,19 @@ class Node(NodeBase): node.Scan() + def DeleteProp(self, prop_name): + """Delete a property of a node + + The property is deleted using fdtput. + + Args: + prop_name: Name of the property to delete + Raises: + CommandError if the property does not exist + """ + args = [self._fdt._fname, '-d', self.path, prop_name] + command.Output('fdtput', *args) + del self.props[prop_name] class FdtFallback(Fdt): """Provides simple access to a flat device tree blob using fdtget/fdtput diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index eb45742a10..52d80555ab 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -20,6 +20,11 @@ import libfdt # This implementation uses a libfdt Python library to access the device tree, # so it is fairly efficient. +def CheckErr(errnum, msg): + if errnum: + raise ValueError('Error %d: %s: %s' % + (errnum, libfdt.fdt_strerror(errnum), msg)) + class Prop(PropBase): """A device tree property @@ -95,6 +100,21 @@ class Node(NodeBase): subnode.Refresh(offset) offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) + def DeleteProp(self, prop_name): + """Delete a property of a node + + The property is deleted and the offset cache is invalidated. + + Args: + prop_name: Name of the property to delete + Raises: + ValueError if the property does not exist + """ + CheckErr(libfdt.fdt_delprop(self._fdt.GetFdt(), self.Offset(), prop_name), + "Node '%s': delete property: '%s'" % (self.path, prop_name)) + del self.props[prop_name] + self._fdt.Invalidate() + class FdtNormal(Fdt): """Provides simple access to a flat device tree blob using libfdt. From da5f74998b9e5e6b706608a4ca625ef0ee195150 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:15 -0600 Subject: [PATCH 18/21] dtoc: Support packing the device tree After any node/property deletion the device tree can be packed to remove spare space. Add a way to perform this operation. Note that for fdt_fallback, fdtput automatically packs the device tree after deletion, so no action is required here. Signed-off-by: Simon Glass --- lib/libfdt/libfdt.swig | 1 + tools/dtoc/fdt.py | 16 ++++++++++++++++ tools/dtoc/fdt_normal.py | 11 +++++++++++ 3 files changed, 28 insertions(+) diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index ce516fddf2..0cb7977ffe 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -107,3 +107,4 @@ int fdt_next_subnode(const void *fdt, int offset); int fdt_delprop(void *fdt, int nodeoffset, const char *name); const char *fdt_strerror(int errval); +int fdt_pack(void *fdt); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index f01c7b18ea..403eb1fe6d 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -240,3 +240,19 @@ class Fdt: return None return node + def Flush(self): + """Flush device tree changes back to the file + + If the device tree has changed in memory, write it back to the file. + Subclasses can implement this if needed. + """ + pass + + def Pack(self): + """Pack the device tree down to its minimum size + + When nodes and properties shrink or are deleted, wasted space can + build up in the device tree binary. Subclasses can implement this + to remove that spare space. + """ + pass diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 52d80555ab..f2cf608b1e 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -140,6 +140,17 @@ class FdtNormal(Fdt): """ return self._fdt + def Flush(self): + """Flush device tree changes back to the file""" + with open(self._fname, 'wb') as fd: + fd.write(self._fdt) + + def Pack(self): + """Pack the device tree down to its minimum size""" + CheckErr(libfdt.fdt_pack(self._fdt), 'pack') + fdt_len = libfdt.fdt_totalsize(self._fdt) + del self._fdt[fdt_len:] + def GetProps(self, node, path): """Get all properties from a node. From babdbde68f1b993289462394f209f4010c761246 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:16 -0600 Subject: [PATCH 19/21] dtoc: Support finding the offset of a property Add a way to find the byte offset of a property within the device tree. This is only supported with the normal libfdt implementation since fdtget does not provide this information. Signed-off-by: Simon Glass --- lib/libfdt/libfdt.swig | 3 +++ tools/dtoc/fdt.py | 11 +++++++++++ tools/dtoc/fdt_normal.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 0cb7977ffe..b24c72b1a2 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -108,3 +108,6 @@ int fdt_delprop(void *fdt, int nodeoffset, const char *name); const char *fdt_strerror(int errval); int fdt_pack(void *fdt); + +int fdt_totalsize(const void *fdt); +int fdt_off_dt_struct(const void *fdt); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 403eb1fe6d..816fdbe525 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -144,6 +144,17 @@ class PropBase: else: return True + def GetOffset(self): + """Get the offset of a property + + This can be implemented by subclasses. + + Returns: + The offset of the property (struct fdt_property) within the + file, or None if not known. + """ + return None + class NodeBase: """A device tree node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index f2cf608b1e..aae258e412 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -43,6 +43,14 @@ class Prop(PropBase): return self.type, self.value = self.BytesToValue(bytes) + def GetOffset(self): + """Get the offset of a property + + Returns: + The offset of the property (struct fdt_property) within the file + """ + return self._node._fdt.GetStructOffset(self._offset) + class Node(NodeBase): """A device tree node @@ -193,6 +201,16 @@ class FdtNormal(Fdt): """Refresh the offset cache""" self._root.Refresh(0) + def GetStructOffset(self, offset): + """Get the file offset of a given struct offset + + Args: + offset: Offset within the 'struct' region of the device tree + Returns: + Position of @offset within the device tree binary + """ + return libfdt.fdt_off_dt_struct(self._fdt) + offset + @classmethod def Node(self, fdt, offset, name, path): """Create a new node From 20024daee58906712f71b927bd86951d1ddb469d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:17 -0600 Subject: [PATCH 20/21] dtoc: Correct quotes in fdt_util The style is to use single quotes for strings where possible. Adjust this function. Signed-off-by: Simon Glass --- tools/dtoc/fdt_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 3e25a8b980..32f41d72d7 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -22,7 +22,7 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - return struct.unpack(">I", val)[0] + return struct.unpack('>I', val)[0] def EnsureCompiled(fname): """Compile an fdt .dts source file into a .dtb binary blob if needed. From 8f224b3734d042884a8981a14db64c48e87b87a2 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:18 -0600 Subject: [PATCH 21/21] dtoc: Add methods for reading data from properties Provide easy helpers for reading integer, string and boolean values from device-tree properties. Signed-off-by: Simon Glass --- tools/dtoc/fdt_util.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 32f41d72d7..3a10838109 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -59,3 +59,28 @@ def EnsureCompiled(fname): args.append(dts_input) command.Run('dtc', *args) return dtb_output + +def GetInt(node, propname, default=None): + prop = node.props.get(propname) + if not prop: + return default + value = fdt32_to_cpu(prop.value) + if type(value) == type(list): + raise ValueError("Node '%s' property '%' has list value: expecting" + "a single integer" % (node.name, propname)) + return value + +def GetString(node, propname, default=None): + prop = node.props.get(propname) + if not prop: + return default + value = prop.value + if type(value) == type(list): + raise ValueError("Node '%s' property '%' has list value: expecting" + "a single string" % (node.name, propname)) + return value + +def GetBool(node, propname, default=False): + if propname in node.props: + return True + return default