u-boot/tools/binman/cbfs_util_test.py
Simon Glass e073d4e14f binman: Add support for fixed-offset files in CBFS
A feature of CBFS is that it allows files to be positioned at particular
offset (as with binman in general). This is useful to support
execute-in-place (XIP) code, since this may not be relocatable.

Add a new cbfs-offset property to control this.

Signed-off-by: Simon Glass <sjg@chromium.org>
2019-07-24 12:53:46 -07:00

625 lines
24 KiB
Python
Executable file

#!/usr/bin/env python
# SPDX-License-Identifier: GPL-2.0+
# Copyright 2019 Google LLC
# Written by Simon Glass <sjg@chromium.org>
"""Tests for cbfs_util
These create and read various CBFSs and compare the results with expected
values and with cbfstool
"""
from __future__ import print_function
import io
import os
import shutil
import struct
import tempfile
import unittest
import cbfs_util
from cbfs_util import CbfsWriter
import elf
import test_util
import tools
U_BOOT_DATA = b'1234'
U_BOOT_DTB_DATA = b'udtb'
COMPRESS_DATA = b'compress xxxxxxxxxxxxxxxxxxxxxx data'
class TestCbfs(unittest.TestCase):
"""Test of cbfs_util classes"""
#pylint: disable=W0212
@classmethod
def setUpClass(cls):
# Create a temporary directory for test files
cls._indir = tempfile.mkdtemp(prefix='cbfs_util.')
tools.SetInputDirs([cls._indir])
# Set up some useful data files
TestCbfs._make_input_file('u-boot.bin', U_BOOT_DATA)
TestCbfs._make_input_file('u-boot.dtb', U_BOOT_DTB_DATA)
TestCbfs._make_input_file('compress', COMPRESS_DATA)
# Set up a temporary output directory, used by the tools library when
# compressing files
tools.PrepareOutputDir(None)
cls.have_cbfstool = True
try:
tools.Run('which', 'cbfstool')
except:
cls.have_cbfstool = False
cls.have_lz4 = True
try:
tools.Run('lz4', '--no-frame-crc', '-c',
tools.GetInputFilename('u-boot.bin'))
except:
cls.have_lz4 = False
@classmethod
def tearDownClass(cls):
"""Remove the temporary input directory and its contents"""
if cls._indir:
shutil.rmtree(cls._indir)
cls._indir = None
tools.FinaliseOutputDir()
@classmethod
def _make_input_file(cls, fname, contents):
"""Create a new test input file, creating directories as needed
Args:
fname: Filename to create
contents: File contents to write in to the file
Returns:
Full pathname of file created
"""
pathname = os.path.join(cls._indir, fname)
tools.WriteFile(pathname, contents)
return pathname
def _check_hdr(self, data, size, offset=0, arch=cbfs_util.ARCHITECTURE_X86):
"""Check that the CBFS has the expected header
Args:
data: Data to check
size: Expected ROM size
offset: Expected offset to first CBFS file
arch: Expected architecture
Returns:
CbfsReader object containing the CBFS
"""
cbfs = cbfs_util.CbfsReader(data)
self.assertEqual(cbfs_util.HEADER_MAGIC, cbfs.magic)
self.assertEqual(cbfs_util.HEADER_VERSION2, cbfs.version)
self.assertEqual(size, cbfs.rom_size)
self.assertEqual(0, cbfs.boot_block_size)
self.assertEqual(cbfs_util.ENTRY_ALIGN, cbfs.align)
self.assertEqual(offset, cbfs.cbfs_offset)
self.assertEqual(arch, cbfs.arch)
return cbfs
def _check_uboot(self, cbfs, ftype=cbfs_util.TYPE_RAW, offset=0x38,
data=U_BOOT_DATA, cbfs_offset=None):
"""Check that the U-Boot file is as expected
Args:
cbfs: CbfsReader object to check
ftype: Expected file type
offset: Expected offset of file
data: Expected data in file
cbfs_offset: Expected CBFS offset for file's data
Returns:
CbfsFile object containing the file
"""
self.assertIn('u-boot', cbfs.files)
cfile = cbfs.files['u-boot']
self.assertEqual('u-boot', cfile.name)
self.assertEqual(offset, cfile.offset)
if cbfs_offset is not None:
self.assertEqual(cbfs_offset, cfile.cbfs_offset)
self.assertEqual(data, cfile.data)
self.assertEqual(ftype, cfile.ftype)
self.assertEqual(cbfs_util.COMPRESS_NONE, cfile.compress)
self.assertEqual(len(data), cfile.memlen)
return cfile
def _check_dtb(self, cbfs, offset=0x38, data=U_BOOT_DTB_DATA,
cbfs_offset=None):
"""Check that the U-Boot dtb file is as expected
Args:
cbfs: CbfsReader object to check
offset: Expected offset of file
data: Expected data in file
cbfs_offset: Expected CBFS offset for file's data
"""
self.assertIn('u-boot-dtb', cbfs.files)
cfile = cbfs.files['u-boot-dtb']
self.assertEqual('u-boot-dtb', cfile.name)
self.assertEqual(offset, cfile.offset)
if cbfs_offset is not None:
self.assertEqual(cbfs_offset, cfile.cbfs_offset)
self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
self.assertEqual(cbfs_util.TYPE_RAW, cfile.ftype)
self.assertEqual(cbfs_util.COMPRESS_NONE, cfile.compress)
self.assertEqual(len(U_BOOT_DTB_DATA), cfile.memlen)
def _check_raw(self, data, size, offset=0, arch=cbfs_util.ARCHITECTURE_X86):
"""Check that two raw files are added as expected
Args:
data: Data to check
size: Expected ROM size
offset: Expected offset to first CBFS file
arch: Expected architecture
"""
cbfs = self._check_hdr(data, size, offset=offset, arch=arch)
self._check_uboot(cbfs)
self._check_dtb(cbfs)
def _get_expected_cbfs(self, size, arch='x86', compress=None, base=None):
"""Get the file created by cbfstool for a particular scenario
Args:
size: Size of the CBFS in bytes
arch: Architecture of the CBFS, as a string
compress: Compression to use, e.g. cbfs_util.COMPRESS_LZMA
base: Base address of file, or None to put it anywhere
Returns:
Resulting CBFS file, or None if cbfstool is not available
"""
if not self.have_cbfstool or not self.have_lz4:
return None
cbfs_fname = os.path.join(self._indir, 'test.cbfs')
cbfs_util.cbfstool(cbfs_fname, 'create', '-m', arch, '-s', '%#x' % size)
if base:
base = [(1 << 32) - size + b for b in base]
cbfs_util.cbfstool(cbfs_fname, 'add', '-n', 'u-boot', '-t', 'raw',
'-c', compress and compress[0] or 'none',
'-f', tools.GetInputFilename(
compress and 'compress' or 'u-boot.bin'),
base=base[0] if base else None)
cbfs_util.cbfstool(cbfs_fname, 'add', '-n', 'u-boot-dtb', '-t', 'raw',
'-c', compress and compress[1] or 'none',
'-f', tools.GetInputFilename(
compress and 'compress' or 'u-boot.dtb'),
base=base[1] if base else None)
return cbfs_fname
def _compare_expected_cbfs(self, data, cbfstool_fname):
"""Compare against what cbfstool creates
This compares what binman creates with what cbfstool creates for what
is proportedly the same thing.
Args:
data: CBFS created by binman
cbfstool_fname: CBFS created by cbfstool
"""
if not self.have_cbfstool or not self.have_lz4:
return
expect = tools.ReadFile(cbfstool_fname)
if expect != data:
tools.WriteFile('/tmp/expect', expect)
tools.WriteFile('/tmp/actual', data)
print('diff -y <(xxd -g1 /tmp/expect) <(xxd -g1 /tmp/actual) | colordiff')
self.fail('cbfstool produced a different result')
def test_cbfs_functions(self):
"""Test global functions of cbfs_util"""
self.assertEqual(cbfs_util.ARCHITECTURE_X86, cbfs_util.find_arch('x86'))
self.assertIsNone(cbfs_util.find_arch('bad-arch'))
self.assertEqual(cbfs_util.COMPRESS_LZMA, cbfs_util.find_compress('lzma'))
self.assertIsNone(cbfs_util.find_compress('bad-comp'))
def test_cbfstool_failure(self):
"""Test failure to run cbfstool"""
if not self.have_cbfstool:
self.skipTest('No cbfstool available')
try:
# In verbose mode this test fails since stderr is not captured. Fix
# this by turning off verbosity.
old_verbose = cbfs_util.VERBOSE
cbfs_util.VERBOSE = False
with test_util.capture_sys_output() as (_stdout, stderr):
with self.assertRaises(Exception) as e:
cbfs_util.cbfstool('missing-file', 'bad-command')
finally:
cbfs_util.VERBOSE = old_verbose
self.assertIn('Unknown command', stderr.getvalue())
self.assertIn('Failed to run', str(e.exception))
def test_cbfs_raw(self):
"""Test base handling of a Coreboot Filesystem (CBFS)"""
size = 0xb0
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
data = cbw.get_data()
self._check_raw(data, size)
cbfs_fname = self._get_expected_cbfs(size=size)
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_invalid_file_type(self):
"""Check handling of an invalid file type when outputiing a CBFS"""
size = 0xb0
cbw = CbfsWriter(size)
cfile = cbw.add_file_raw('u-boot', U_BOOT_DATA)
# Change the type manually before generating the CBFS, and make sure
# that the generator complains
cfile.ftype = 0xff
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('Unknown type 0xff when writing', str(e.exception))
def test_cbfs_invalid_file_type_on_read(self):
"""Check handling of an invalid file type when reading the CBFS"""
size = 0xb0
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
# Read in the first file header
cbr = cbfs_util.CbfsReader(data, read=False)
with io.BytesIO(data) as fd:
self.assertTrue(cbr._find_and_read_header(fd, len(data)))
pos = fd.tell()
hdr_data = fd.read(cbfs_util.FILE_HEADER_LEN)
magic, size, ftype, attr, offset = struct.unpack(
cbfs_util.FILE_HEADER_FORMAT, hdr_data)
# Create a new CBFS with a change to the file type
ftype = 0xff
newdata = data[:pos]
newdata += struct.pack(cbfs_util.FILE_HEADER_FORMAT, magic, size, ftype,
attr, offset)
newdata += data[pos + cbfs_util.FILE_HEADER_LEN:]
# Read in this CBFS and make sure that the reader complains
with self.assertRaises(ValueError) as e:
cbfs_util.CbfsReader(newdata)
self.assertIn('Unknown type 0xff when reading', str(e.exception))
def test_cbfs_no_space(self):
"""Check handling of running out of space in the CBFS"""
size = 0x60
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('No space for header', str(e.exception))
def test_cbfs_no_space_skip(self):
"""Check handling of running out of space in CBFS with file header"""
size = 0x5c
cbw = CbfsWriter(size, arch=cbfs_util.ARCHITECTURE_PPC64)
cbw._add_fileheader = True
cbw.add_file_raw('u-boot', U_BOOT_DATA)
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('No space for data before offset', str(e.exception))
def test_cbfs_no_space_pad(self):
"""Check handling of running out of space in CBFS with file header"""
size = 0x70
cbw = CbfsWriter(size)
cbw._add_fileheader = True
cbw.add_file_raw('u-boot', U_BOOT_DATA)
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('No space for data before pad offset', str(e.exception))
def test_cbfs_bad_header_ptr(self):
"""Check handling of a bad master-header pointer"""
size = 0x70
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
# Add one to the pointer to make it invalid
newdata = data[:-4] + struct.pack('<I', cbw._header_offset + 1)
# We should still be able to find the master header by searching
with test_util.capture_sys_output() as (stdout, _stderr):
cbfs = cbfs_util.CbfsReader(newdata)
self.assertIn('Relative offset seems wrong', stdout.getvalue())
self.assertIn('u-boot', cbfs.files)
self.assertEqual(size, cbfs.rom_size)
def test_cbfs_bad_header(self):
"""Check handling of a bad master header"""
size = 0x70
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
# Drop most of the header and try reading the modified CBFS
newdata = data[:cbw._header_offset + 4]
with test_util.capture_sys_output() as (stdout, _stderr):
with self.assertRaises(ValueError) as e:
cbfs_util.CbfsReader(newdata)
self.assertIn('Relative offset seems wrong', stdout.getvalue())
self.assertIn('Cannot find master header', str(e.exception))
def test_cbfs_bad_file_header(self):
"""Check handling of a bad file header"""
size = 0x70
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
# Read in the CBFS master header (only), then stop
cbr = cbfs_util.CbfsReader(data, read=False)
with io.BytesIO(data) as fd:
self.assertTrue(cbr._find_and_read_header(fd, len(data)))
pos = fd.tell()
# Remove all but 4 bytes of the file headerm and try to read the file
newdata = data[:pos + 4]
with test_util.capture_sys_output() as (stdout, _stderr):
with io.BytesIO(newdata) as fd:
fd.seek(pos)
self.assertEqual(False, cbr._read_next_file(fd))
self.assertIn('File header at 0 ran out of data', stdout.getvalue())
def test_cbfs_bad_file_string(self):
"""Check handling of an incomplete filename string"""
size = 0x70
cbw = CbfsWriter(size)
cbw.add_file_raw('16-characters xx', U_BOOT_DATA)
data = cbw.get_data()
# Read in the CBFS master header (only), then stop
cbr = cbfs_util.CbfsReader(data, read=False)
with io.BytesIO(data) as fd:
self.assertTrue(cbr._find_and_read_header(fd, len(data)))
pos = fd.tell()
# Create a new CBFS with only the first 16 bytes of the file name, then
# try to read the file
newdata = data[:pos + cbfs_util.FILE_HEADER_LEN + 16]
with test_util.capture_sys_output() as (stdout, _stderr):
with io.BytesIO(newdata) as fd:
fd.seek(pos)
self.assertEqual(False, cbr._read_next_file(fd))
self.assertIn('String at %x ran out of data' %
cbfs_util.FILE_HEADER_LEN, stdout.getvalue())
def test_cbfs_debug(self):
"""Check debug output"""
size = 0x70
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
try:
cbfs_util.DEBUG = True
with test_util.capture_sys_output() as (stdout, _stderr):
cbfs_util.CbfsReader(data)
self.assertEqual('name u-boot\ndata %s\n' % U_BOOT_DATA,
stdout.getvalue())
finally:
cbfs_util.DEBUG = False
def test_cbfs_bad_attribute(self):
"""Check handling of bad attribute tag"""
if not self.have_lz4:
self.skipTest('lz4 --no-frame-crc not available')
size = 0x140
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', COMPRESS_DATA, None,
compress=cbfs_util.COMPRESS_LZ4)
data = cbw.get_data()
# Search the CBFS for the expected compression tag
with io.BytesIO(data) as fd:
while True:
pos = fd.tell()
tag, = struct.unpack('>I', fd.read(4))
if tag == cbfs_util.FILE_ATTR_TAG_COMPRESSION:
break
# Create a new CBFS with the tag changed to something invalid
newdata = data[:pos] + struct.pack('>I', 0x123) + data[pos + 4:]
with test_util.capture_sys_output() as (stdout, _stderr):
cbfs_util.CbfsReader(newdata)
self.assertEqual('Unknown attribute tag 123\n', stdout.getvalue())
def test_cbfs_missing_attribute(self):
"""Check handling of an incomplete attribute tag"""
if not self.have_lz4:
self.skipTest('lz4 --no-frame-crc not available')
size = 0x140
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', COMPRESS_DATA, None,
compress=cbfs_util.COMPRESS_LZ4)
data = cbw.get_data()
# Read in the CBFS master header (only), then stop
cbr = cbfs_util.CbfsReader(data, read=False)
with io.BytesIO(data) as fd:
self.assertTrue(cbr._find_and_read_header(fd, len(data)))
pos = fd.tell()
# Create a new CBFS with only the first 4 bytes of the compression tag,
# then try to read the file
tag_pos = pos + cbfs_util.FILE_HEADER_LEN + cbfs_util.FILENAME_ALIGN
newdata = data[:tag_pos + 4]
with test_util.capture_sys_output() as (stdout, _stderr):
with io.BytesIO(newdata) as fd:
fd.seek(pos)
self.assertEqual(False, cbr._read_next_file(fd))
self.assertIn('Attribute tag at %x ran out of data' % tag_pos,
stdout.getvalue())
def test_cbfs_file_master_header(self):
"""Check handling of a file containing a master header"""
size = 0x100
cbw = CbfsWriter(size)
cbw._add_fileheader = True
cbw.add_file_raw('u-boot', U_BOOT_DATA)
data = cbw.get_data()
cbr = cbfs_util.CbfsReader(data)
self.assertIn('u-boot', cbr.files)
self.assertEqual(size, cbr.rom_size)
def test_cbfs_arch(self):
"""Test on non-x86 architecture"""
size = 0x100
cbw = CbfsWriter(size, arch=cbfs_util.ARCHITECTURE_PPC64)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
data = cbw.get_data()
self._check_raw(data, size, offset=0x40,
arch=cbfs_util.ARCHITECTURE_PPC64)
# Compare against what cbfstool creates
cbfs_fname = self._get_expected_cbfs(size=size, arch='ppc64')
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_stage(self):
"""Tests handling of a Coreboot Filesystem (CBFS)"""
if not elf.ELF_TOOLS:
self.skipTest('Python elftools not available')
elf_fname = os.path.join(self._indir, 'cbfs-stage.elf')
elf.MakeElf(elf_fname, U_BOOT_DATA, U_BOOT_DTB_DATA)
size = 0xb0
cbw = CbfsWriter(size)
cbw.add_file_stage('u-boot', tools.ReadFile(elf_fname))
data = cbw.get_data()
cbfs = self._check_hdr(data, size)
load = 0xfef20000
entry = load + 2
cfile = self._check_uboot(cbfs, cbfs_util.TYPE_STAGE, offset=0x28,
data=U_BOOT_DATA + U_BOOT_DTB_DATA)
self.assertEqual(entry, cfile.entry)
self.assertEqual(load, cfile.load)
self.assertEqual(len(U_BOOT_DATA) + len(U_BOOT_DTB_DATA),
cfile.data_len)
# Compare against what cbfstool creates
if self.have_cbfstool:
cbfs_fname = os.path.join(self._indir, 'test.cbfs')
cbfs_util.cbfstool(cbfs_fname, 'create', '-m', 'x86', '-s',
'%#x' % size)
cbfs_util.cbfstool(cbfs_fname, 'add-stage', '-n', 'u-boot',
'-f', elf_fname)
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_raw_compress(self):
"""Test base handling of compressing raw files"""
if not self.have_lz4:
self.skipTest('lz4 --no-frame-crc not available')
size = 0x140
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', COMPRESS_DATA, None,
compress=cbfs_util.COMPRESS_LZ4)
cbw.add_file_raw('u-boot-dtb', COMPRESS_DATA, None,
compress=cbfs_util.COMPRESS_LZMA)
data = cbw.get_data()
cbfs = self._check_hdr(data, size)
self.assertIn('u-boot', cbfs.files)
cfile = cbfs.files['u-boot']
self.assertEqual(cfile.name, 'u-boot')
self.assertEqual(cfile.offset, 56)
self.assertEqual(cfile.data, COMPRESS_DATA)
self.assertEqual(cfile.ftype, cbfs_util.TYPE_RAW)
self.assertEqual(cfile.compress, cbfs_util.COMPRESS_LZ4)
self.assertEqual(cfile.memlen, len(COMPRESS_DATA))
self.assertIn('u-boot-dtb', cbfs.files)
cfile = cbfs.files['u-boot-dtb']
self.assertEqual(cfile.name, 'u-boot-dtb')
self.assertEqual(cfile.offset, 56)
self.assertEqual(cfile.data, COMPRESS_DATA)
self.assertEqual(cfile.ftype, cbfs_util.TYPE_RAW)
self.assertEqual(cfile.compress, cbfs_util.COMPRESS_LZMA)
self.assertEqual(cfile.memlen, len(COMPRESS_DATA))
cbfs_fname = self._get_expected_cbfs(size=size, compress=['lz4', 'lzma'])
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_raw_space(self):
"""Test files with unused space in the CBFS"""
size = 0xf0
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
data = cbw.get_data()
self._check_raw(data, size)
cbfs_fname = self._get_expected_cbfs(size=size)
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_offset(self):
"""Test a CBFS with files at particular offsets"""
size = 0x200
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA, 0x40)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA, 0x140)
data = cbw.get_data()
cbfs = self._check_hdr(data, size)
self._check_uboot(cbfs, ftype=cbfs_util.TYPE_RAW, offset=0x40,
cbfs_offset=0x40)
self._check_dtb(cbfs, offset=0x40, cbfs_offset=0x140)
cbfs_fname = self._get_expected_cbfs(size=size, base=(0x40, 0x140))
self._compare_expected_cbfs(data, cbfs_fname)
def test_cbfs_invalid_file_type_header(self):
"""Check handling of an invalid file type when outputting a header"""
size = 0xb0
cbw = CbfsWriter(size)
cfile = cbw.add_file_raw('u-boot', U_BOOT_DATA, 0)
# Change the type manually before generating the CBFS, and make sure
# that the generator complains
cfile.ftype = 0xff
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('Unknown file type 0xff', str(e.exception))
def test_cbfs_offset_conflict(self):
"""Test a CBFS with files that want to overlap"""
size = 0x200
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA, 0x40)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA, 0x80)
with self.assertRaises(ValueError) as e:
cbw.get_data()
self.assertIn('No space for data before pad offset', str(e.exception))
def test_cbfs_check_offset(self):
"""Test that we can discover the offset of a file after writing it"""
size = 0xb0
cbw = CbfsWriter(size)
cbw.add_file_raw('u-boot', U_BOOT_DATA)
cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
data = cbw.get_data()
cbfs = cbfs_util.CbfsReader(data)
self.assertEqual(0x38, cbfs.files['u-boot'].cbfs_offset)
self.assertEqual(0x78, cbfs.files['u-boot-dtb'].cbfs_offset)
if __name__ == '__main__':
unittest.main()