diff --git a/.github/workflows/custom.yml b/.github/workflows/custom.yml index 916b7e81bc..c34f1b26c1 100644 --- a/.github/workflows/custom.yml +++ b/.github/workflows/custom.yml @@ -89,14 +89,7 @@ jobs: - name: Install essential packages run: | sudo apt-get update - sudo apt-get install swig - echo 'switch to python2 as default' - sudo rm /usr/bin/python - wget https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tar.xz - tar -xJf Python-2.7.18.tar.xz - cd Python-2.7.18 - ./configure && sudo make && sudo make install - sudo ln -s /usr/bin/python2 /usr/bin/python + sudo apt-get install swig python3-dev - name: Download OpenWrt toolchain run: | diff --git a/scripts/dtc/README b/scripts/dtc/README new file mode 100644 index 0000000000..a48312a422 --- /dev/null +++ b/scripts/dtc/README @@ -0,0 +1,106 @@ +The source tree contains the Device Tree Compiler (dtc) toolchain for +working with device tree source and binary files and also libfdt, a +utility library for reading and manipulating the binary format. + +DTC and LIBFDT are maintained by: + +David Gibson +Jon Loeliger + + +Python library +-------------- + +A Python library is also available. To build this you will need to install +swig and Python development files. On Debian distributions: + + sudo apt-get install swig python3-dev + +The library provides an Fdt class which you can use like this: + +$ PYTHONPATH=../pylibfdt python3 +>>> import libfdt +>>> fdt = libfdt.Fdt(open('test_tree1.dtb', mode='rb').read()) +>>> node = fdt.path_offset('/subnode@1') +>>> print(node) +124 +>>> prop_offset = fdt.first_property_offset(node) +>>> prop = fdt.get_property_by_offset(prop_offset) +>>> print('%s=%s' % (prop.name, prop.as_str())) +compatible=subnode1 +>>> node2 = fdt.path_offset('/') +>>> print(fdt.getprop(node2, 'compatible').as_str()) +test_tree1 + +You will find tests in tests/pylibfdt_tests.py showing how to use each +method. Help is available using the Python help command, e.g.: + + $ cd pylibfdt + $ python3 -c "import libfdt; help(libfdt)" + +If you add new features, please check code coverage: + + $ sudo apt-get install python3-coverage + $ cd tests + # It's just 'coverage' on most other distributions + $ python3-coverage run pylibfdt_tests.py + $ python3-coverage html + # Open 'htmlcov/index.html' in your browser + + +The library can be installed with pip from a local source tree: + + pip install . [--user|--prefix=/path/to/install_dir] + +Or directly from a remote git repo: + + pip install git+git://git.kernel.org/pub/scm/utils/dtc/dtc.git@main + +The install depends on libfdt shared library being installed on the host system +first. Generally, using --user or --prefix is not necessary and pip will use the +default location for the Python installation which varies if the user is root or +not. + +You can also install everything via make if you like, but pip is recommended. + +To install both libfdt and pylibfdt you can use: + + make install [PREFIX=/path/to/install_dir] + +To disable building the python library, even if swig and Python are available, +use: + + make NO_PYTHON=1 + + +More work remains to support all of libfdt, including access to numeric +values. + + +Adding a new function to libfdt.h +--------------------------------- + +The shared library uses libfdt/version.lds to list the exported functions, so +add your new function there. Check that your function works with pylibfdt. If +it cannot be supported, put the declaration in libfdt.h behind #ifndef SWIG so +that swig ignores it. + + +Tests +----- + +Test files are kept in the tests/ directory. Use 'make check' to build and run +all tests. + +If you want to adjust a test file, be aware that tree_tree1.dts is compiled +and checked against a binary tree from assembler macros in trees.S. So +if you change that file you must change tree.S also. + + +Mailing list +------------ +The following list is for discussion about dtc and libfdt implementation +mailto:devicetree-compiler@vger.kernel.org + +Core device tree bindings are discussed on the devicetree-spec list: +mailto:devicetree-spec@vger.kernel.org diff --git a/scripts/dtc/pylibfdt/Makefile b/scripts/dtc/pylibfdt/Makefile index c769d7db06..e442d5c242 100644 --- a/scripts/dtc/pylibfdt/Makefile +++ b/scripts/dtc/pylibfdt/Makefile @@ -13,19 +13,32 @@ include $(LIBFDT_srcdir)/Makefile.libfdt PYLIBFDT_srcs = $(addprefix $(LIBFDT_srcdir)/,$(LIBFDT_SRCS)) \ $(obj)/libfdt.i +# create a version string compliant with PEP 440 +PEP_VERSION = $(VERSION)$(if $(PATCHLEVEL),.$(PATCHLEVEL)$(if $(SUBLEVEL),.$(SUBLEVEL)))$(subst -,,$(EXTRAVERSION)) + quiet_cmd_pymod = PYMOD $@ cmd_pymod = unset CROSS_COMPILE; unset CFLAGS; \ CC="$(HOSTCC)" LDSHARED="$(HOSTCC) -shared " \ LDFLAGS="$(HOSTLDFLAGS)" \ - VERSION="u-boot-$(UBOOTVERSION)" \ + VERSION="$(PEP_VERSION)" \ CPPFLAGS="$(HOSTCFLAGS) -I$(LIBFDT_srcdir)" OBJDIR=$(obj) \ SOURCES="$(PYLIBFDT_srcs)" \ SWIG_OPTS="-I$(LIBFDT_srcdir) -I$(LIBFDT_srcdir)/.." \ - $(PYTHON) $< --quiet build_ext --inplace + $(PYTHON3) $< --quiet build_ext --inplace -$(obj)/_libfdt.so: $(src)/setup.py $(PYLIBFDT_srcs) FORCE +rebuild: $(src)/setup.py $(PYLIBFDT_srcs) + @# Remove the library since otherwise Python doesn't seem to regenerate + @# the libfdt.py file if it is missing. + @rm -f $(obj)/_libfdt*.so $(call if_changed,pymod) + @# Rename the file to _libfdt.so so this Makefile doesn't run every time + @if [ ! -e $(obj)/_libfdt.so ]; then \ + mv $(obj)/_libfdt*.so $(obj)/_libfdt.so; \ + fi + +$(obj)/_libfdt.so $(obj)/libfdt.py &: rebuild + @: -always += _libfdt.so +always += _libfdt.so libfdt.py clean-files += libfdt.i _libfdt.so libfdt.py libfdt_wrap.c diff --git a/scripts/dtc/pylibfdt/libfdt.i_shipped b/scripts/dtc/pylibfdt/libfdt.i_shipped index 0f17c5879e..ada728bfd7 100644 --- a/scripts/dtc/pylibfdt/libfdt.i_shipped +++ b/scripts/dtc/pylibfdt/libfdt.i_shipped @@ -7,6 +7,10 @@ %module libfdt +%begin %{ +#define PY_SSIZE_T_CLEAN +%} + %include %{ @@ -18,7 +22,7 @@ * a struct called fdt_property. That struct causes swig to create a class in * libfdt.py called fdt_property(), which confuses things. */ -static int fdt_property_stub(void *fdt, const char *name, const char *val, +static int fdt_property_stub(void *fdt, const char *name, const void *val, int len) { return fdt_property(fdt, name, val, len); @@ -92,7 +96,7 @@ def check_err(val, quiet=()): Raises FdtException if val < 0 """ - if val < 0: + if isinstance(val, int) and val < 0: if -val not in quiet: raise FdtException(val) return val @@ -417,7 +421,7 @@ class FdtRo(object): quiet) if isinstance(pdata, (int)): return pdata - return Property(prop_name, bytearray(pdata[0])) + return Property(prop_name, bytes(pdata[0])) def get_phandle(self, nodeoffset): """Get the phandle of a node @@ -431,6 +435,18 @@ class FdtRo(object): """ return fdt_get_phandle(self._fdt, nodeoffset) + def get_alias(self, name): + """Get the full path referenced by a given alias + + Args: + name: name of the alias to lookup + + Returns: + Full path to the node for the alias named 'name', if it exists + None, if the given alias or the /aliases node does not exist + """ + return fdt_get_alias(self._fdt, name) + def parent_offset(self, nodeoffset, quiet=()): """Get the offset of a node's parent @@ -624,32 +640,54 @@ class Fdt(FdtRo): Raises: FdtException if no parent found or other error occurs """ - val = val.encode('utf-8') + '\0' + val = val.encode('utf-8') + b'\0' return check_err(fdt_setprop(self._fdt, nodeoffset, prop_name, val, len(val)), quiet) - def delprop(self, nodeoffset, prop_name): + def delprop(self, nodeoffset, prop_name, quiet=()): """Delete a property from a node Args: nodeoffset: Node offset containing property to delete prop_name: Name of property to delete + quiet: Errors to ignore (empty to raise on all errors) + + Returns: + Error code, or 0 if OK Raises: FdtError if the property does not exist, or another error occurs """ - return check_err(fdt_delprop(self._fdt, nodeoffset, prop_name)) + return check_err(fdt_delprop(self._fdt, nodeoffset, prop_name), quiet) + + def add_subnode(self, parentoffset, name, quiet=()): + """Add a new subnode to a node + + Args: + parentoffset: Parent offset to add the subnode to + name: Name of node to add + + Returns: + offset of the node created, or negative error code on failure + + Raises: + FdtError if there is not enough space, or another error occurs + """ + return check_err(fdt_add_subnode(self._fdt, parentoffset, name), quiet) - def del_node(self, nodeoffset): + def del_node(self, nodeoffset, quiet=()): """Delete a node Args: - nodeoffset: Node offset containing property to delete + nodeoffset: Offset of node to delete + + Returns: + Error code, or 0 if OK Raises: - FdtError if the node does not exist, or another error occurs + FdtError if an error occurs """ - return check_err(fdt_del_node(self._fdt, nodeoffset)) + return check_err(fdt_del_node(self._fdt, nodeoffset), quiet) class Property(bytearray): @@ -705,8 +743,10 @@ class FdtSw(FdtRo): # First create the device tree with a node and property: sw = FdtSw() - with sw.add_node('node'): - sw.property_u32('reg', 2) + sw.finish_reservemap() + with sw.add_node(''): + with sw.add_node('node'): + sw.property_u32('reg', 2) fdt = sw.as_fdt() # Now we can use it as a real device tree @@ -996,7 +1036,7 @@ typedef uint32_t fdt32_t; fdt_string(fdt1, fdt32_to_cpu($1->nameoff))); buff = PyByteArray_FromStringAndSize( (const char *)($1 + 1), fdt32_to_cpu($1->len)); - resultobj = SWIG_Python_AppendOutput(resultobj, buff); + resultobj = SWIG_AppendOutput(resultobj, buff); } } @@ -1007,17 +1047,24 @@ typedef uint32_t fdt32_t; if (!$1) $result = Py_None; else - $result = Py_BuildValue("s#", $1, *arg4); + %#if PY_VERSION_HEX >= 0x03000000 + $result = Py_BuildValue("y#", $1, *arg4); + %#else + $result = Py_BuildValue("s#", $1, *arg4); + %#endif } /* typemap used for fdt_setprop() */ %typemap(in) (const void *val) { - $1 = PyString_AsString($input); /* char *str */ -} - -/* typemap used for fdt_add_reservemap_entry() */ -%typemap(in) uint64_t { - $1 = PyLong_AsUnsignedLong($input); + %#if PY_VERSION_HEX >= 0x03000000 + if (!PyBytes_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "bytes expected in method '" "$symname" + "', argument " "$argnum"" of type '" "$type""'"); + } + $1 = PyBytes_AsString($input); + %#else + $1 = PyString_AsString($input); /* char *str */ + %#endif } /* typemaps used for fdt_next_node() */ @@ -1028,7 +1075,7 @@ typedef uint32_t fdt32_t; %typemap(argout) int *depth { PyObject *val = Py_BuildValue("i", *arg$argnum); - resultobj = SWIG_Python_AppendOutput(resultobj, val); + resultobj = SWIG_AppendOutput(resultobj, val); } %apply int *depth { int *depth }; @@ -1039,12 +1086,12 @@ typedef uint32_t fdt32_t; } %typemap(argout) uint64_t * { - PyObject *val = PyLong_FromUnsignedLong(*arg$argnum); + PyObject *val = PyLong_FromUnsignedLongLong(*arg$argnum); if (!result) { if (PyTuple_GET_SIZE(resultobj) == 0) resultobj = val; else - resultobj = SWIG_Python_AppendOutput(resultobj, val); + resultobj = SWIG_AppendOutput(resultobj, val); } } @@ -1070,6 +1117,6 @@ int fdt_property_cell(void *fdt, const char *name, uint32_t val); * This function has a stub since the name fdt_property is used for both a * function and a struct, which confuses SWIG. */ -int fdt_property_stub(void *fdt, const char *name, const char *val, int len); +int fdt_property_stub(void *fdt, const char *name, const void *val, int len); %include <../libfdt/libfdt.h> diff --git a/scripts/dtc/pylibfdt/setup.py b/scripts/dtc/pylibfdt/setup.py index 4f7cf042bf..c6fe5a6a44 100755 --- a/scripts/dtc/pylibfdt/setup.py +++ b/scripts/dtc/pylibfdt/setup.py @@ -1,11 +1,13 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 +# SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) """ setup.py file for SWIG libfdt Copyright (C) 2017 Google, Inc. Written by Simon Glass -SPDX-License-Identifier: GPL-2.0+ BSD-2-Clause +This script is modified from the upstream version, to fit in with the U-Boot +build system. Files to be built into the extension are provided in SOURCES C flags to use are provided in CPPFLAGS @@ -18,14 +20,34 @@ ./pylibfdt/setup.py install [--prefix=...] """ -from distutils.core import setup, Extension +from setuptools import setup, Extension +from setuptools.command.build_py import build_py as _build_py import os import re import sys +try: + from setuptools import sic +except ImportError: + pass + +srcdir = os.path.dirname(__file__) + +with open(os.path.join(srcdir, "../README"), "r") as fh: + long_description = fh.read() + # Decodes a Makefile assignment line into key and value (and plus for +=) -RE_KEY_VALUE = re.compile('(?P\w+) *(?P[+])?= *(?P.*)$') +RE_KEY_VALUE = re.compile(r'(?P\w+) *(?P[+])?= *(?P.*)$') + +def get_top_builddir(): + if '--top-builddir' in sys.argv: + index = sys.argv.index('--top-builddir') + sys.argv.pop(index) + return sys.argv.pop(index) + else: + return os.path.join(srcdir, '..') +top_builddir = get_top_builddir() def ParseMakefile(fname): """Parse a Makefile to obtain its variables. @@ -86,7 +108,7 @@ def GetEnvFromMakefiles(): makevars = ParseMakefile(os.path.join(basedir, 'libfdt', 'Makefile.libfdt')) files = makevars['LIBFDT_SRCS'].split() files = [os.path.join(basedir, 'libfdt', fname) for fname in files] - files.append('pylibfdt/libfdt.i') + files.append('libfdt.i') cflags = ['-I%s' % basedir, '-I%s/libfdt' % basedir] objdir = '' return swig_opts, version, files, cflags, objdir @@ -96,7 +118,10 @@ def GetEnvFromMakefiles(): files = os.environ.get('SOURCES', '').split() cflags = os.environ.get('CPPFLAGS', '').split() objdir = os.environ.get('OBJDIR') -version = os.environ.get('VERSION') +try: + version = sic(os.environ.get('VERSION')) +except: + version = os.environ.get('VERSION') swig_opts = os.environ.get('SWIG_OPTS', '').split() # If we were called directly rather than through our Makefile (which is often @@ -107,17 +132,39 @@ def GetEnvFromMakefiles(): libfdt_module = Extension( '_libfdt', - sources = files, - extra_compile_args = cflags, - swig_opts = swig_opts, + sources=files, + include_dirs=[os.path.join(srcdir, 'libfdt')], + library_dirs=[os.path.join(top_builddir, 'libfdt')], + swig_opts=swig_opts, ) +class build_py(_build_py): + def run(self): + self.run_command("build_ext") + return super().run() + setup( name='libfdt', - version= version, - author='Simon Glass ', + version=version, + cmdclass = {'build_py' : build_py}, + author='Simon Glass', + author_email='sjg@chromium.org', description='Python binding for libfdt', ext_modules=[libfdt_module], package_dir={'': objdir}, - py_modules=['pylibfdt/libfdt'], + py_modules=['libfdt'], + + long_description=long_description, + long_description_content_type="text/plain", + url="https://git.kernel.org/pub/scm/utils/dtc/dtc.git", + license="BSD", + license_files=["GPL", "BSD-2-Clause"], + + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", + ], + ) diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 1536e95651..18008b4b4f 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2016 Google, Inc @@ -9,6 +9,8 @@ """See README for more information""" +from __future__ import print_function + import glob import os import sys @@ -67,13 +69,13 @@ def RunTests(debug, args): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) - print result + print(result) for test, err in result.errors: - print test.id(), err + print(test.id(), err) for test, err in result.failures: - print err, result.failures + print(err, result.failures) if result.errors or result.failures: - print 'binman tests FAILED' + print('binman tests FAILED') return 1 return 0 @@ -125,9 +127,9 @@ def RunBinman(options, args): try: ret_code = control.Binman(options, args) except Exception as e: - print 'binman: %s' % e + print('binman: %s' % e) if options.debug: - print + print() traceback.print_exc() ret_code = 1 return ret_code diff --git a/tools/binman/bsection.py b/tools/binman/bsection.py index a0bd1b6d34..b28a9a85e4 100644 --- a/tools/binman/bsection.py +++ b/tools/binman/bsection.py @@ -223,7 +223,7 @@ def GetEntryOffsets(self): """ for entry in self._entries.values(): offset_dict = entry.GetOffsets() - for name, info in offset_dict.iteritems(): + for name, info in offset_dict.items(): self._SetEntryOffsetSize(name, *info) def PackEntries(self): @@ -286,7 +286,7 @@ def BuildSection(self, fd, base_offset): def GetData(self): """Get the contents of the section""" - section_data = chr(self._pad_byte) * self._size + section_data = tools.GetBytes(self._pad_byte, self._size) for entry in self._entries.values(): data = entry.GetData() diff --git a/tools/binman/control.py b/tools/binman/control.py index 2de1c86ecf..3ff257755a 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -5,6 +5,8 @@ # Creates binary images from input files controlled by a description # +from __future__ import print_function + from collections import OrderedDict import os import re @@ -148,9 +150,7 @@ def Binman(options, args): # in binman. dtb_fname = fdt_util.EnsureCompiled(dtb_fname) fname = tools.GetOutputFilename('u-boot-out.dtb') - with open(dtb_fname) as infd: - with open(fname, 'wb') as outfd: - outfd.write(infd.read()) + tools.WriteFile(fname, tools.ReadFile(dtb_fname)) dtb = fdt.FdtScan(fname) # Note the file so that GetFdt() can find it @@ -174,6 +174,7 @@ def Binman(options, args): image.AddMissingProperties() image.ProcessFdt(dtb) + dtb.Sync(auto_resize=True) dtb.Pack() dtb.Flush() @@ -190,6 +191,7 @@ def Binman(options, args): image.SetImagePos() if options.update_fdt: image.SetCalculatedProperties() + dtb.Sync() image.ProcessEntryContents() image.WriteSymbols() image.BuildImage() diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 97df8e32c5..828681d76d 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -59,7 +59,7 @@ def GetSymbols(fname, patterns): flags[1] == 'w') # Sort dict by address - return OrderedDict(sorted(syms.iteritems(), key=lambda x: x[1].address)) + return OrderedDict(sorted(syms.items(), key=lambda x: x[1].address)) def GetSymbolAddress(fname, sym_name): """Get a value of a symbol from an ELF file @@ -98,7 +98,7 @@ def LookupAndWriteSymbols(elf_fname, entry, section): base = syms.get('__image_copy_start') if not base: return - for name, sym in syms.iteritems(): + for name, sym in syms.items(): if name.startswith('_binman'): msg = ("Section '%s': Symbol '%s'\n in entry '%s'" % (section.GetPath(), name, entry.GetPath())) diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index c16f71401d..9f03ceba0f 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -117,7 +117,7 @@ def testNoValue(self): section = FakeSection(sym_value=None) elf_fname = os.path.join(binman_dir, 'test', 'u_boot_binman_syms') syms = elf.LookupAndWriteSymbols(elf_fname, entry, section) - self.assertEqual(chr(255) * 16 + 'a' * 4, entry.data) + self.assertEqual(tools.GetBytes(255, 16) + 'a' * 4, entry.data) def testDebug(self): """Check that enabling debug in the elf module produced debug output""" diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 3f46eecf30..90adca946f 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -33,14 +33,14 @@ def ObtainContents(self): return True def ReadBlobContents(self): - with open(self._pathname) as fd: - # We assume the data is small enough to fit into memory. If this - # is used for large filesystem image that might not be true. - # In that case, Image.BuildImage() could be adjusted to use a - # new Entry method which can read in chunks. Then we could copy - # the data in chunks and avoid reading it all at once. For now - # this seems like an unnecessary complication. - self.SetContents(fd.read()) + # We assume the data is small enough to fit into memory. If this + # is used for large filesystem image that might not be true. + # In that case, Image.BuildImage() could be adjusted to use a + # new Entry method which can read in chunks. Then we could copy + # the data in chunks and avoid reading it all at once. For now + # this seems like an unnecessary complication. + data = tools.ReadFile(self._pathname) + self.SetContents(data) return True def GetDefaultFilename(self): diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index 7210a8324a..8a9359cfbc 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from entry import Entry import fdt_util - +import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value @@ -28,5 +28,5 @@ def __init__(self, section, etype, node): self.fill_value = fdt_util.GetByte(self._node, 'fill-byte', 0) def ObtainContents(self): - self.SetContents(chr(self.fill_value) * self.size) + self.SetContents(tools.GetBytes(self.fill_value, self.size)) return True diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index 8fe10f4713..a94c0fca9d 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -64,7 +64,7 @@ def __init__(self, section, etype, node): self.gbb_flags = 0 flags_node = node.FindNode('flags') if flags_node: - for flag, value in gbb_flag_properties.iteritems(): + for flag, value in gbb_flag_properties.items(): if fdt_util.GetBool(flags_node, flag): self.gbb_flags |= value diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 00b7ac5004..66a296a6f8 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -38,5 +38,5 @@ def ObtainContents(self): bss_size = elf.GetSymbolAddress(fname, '__bss_size') if not bss_size: self.Raise('Expected __bss_size symbol in spl/u-boot-spl') - self.SetContents(chr(0) * bss_size) + self.SetContents(tools.GetBytes(0, bss_size)) return True diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a8456c2615..ef8bc424d2 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -186,7 +186,7 @@ def _DoTestFile(self, fname, debug=False, map=False, update_dtb=False, if update_dtb: args.append('-up') if entry_args: - for arg, value in entry_args.iteritems(): + for arg, value in entry_args.items(): args.append('-a%s=%s' % (arg, value)) return self._DoBinman(*args) diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 55baa3857f..4ce8201d8e 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -11,6 +11,7 @@ import fdt_util import libfdt from libfdt import QUIET_NOTFOUND +import tools # This deals with a device tree, presenting it as an assortment of Node and # Prop objects, representing nodes and properties, respectively. This file @@ -28,6 +29,57 @@ def CheckErr(errnum, msg): raise ValueError('Error %d: %s: %s' % (errnum, libfdt.fdt_strerror(errnum), msg)) + +def BytesToValue(data): + """Converts a string of bytes into a type and value + + Args: + A bytes value (which on Python 2 is an alias for str) + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: str/bytes value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte str/bytes + TYPE_BYTE: a byte stored as a single-byte str/bytes + """ + data = bytes(data) + size = len(data) + strings = data.split(b'\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not len(strings[-1]): + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + if ch < 32 or ch > 127: + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + return TYPE_STRING, strings[0].decode() + else: + return TYPE_STRING, [s.decode() for s in strings[:-1]] + if size % 4: + if size == 1: + return TYPE_BYTE, tools.ToChar(data[0]) + else: + return TYPE_BYTE, [tools.ToChar(ch) for ch in list(data)] + val = [] + for i in range(0, size, 4): + val.append(data[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + class Prop: """A device tree property @@ -37,17 +89,18 @@ class Prop: bytes type: Value type """ - def __init__(self, node, offset, name, bytes): + def __init__(self, node, offset, name, data): self._node = node self._offset = offset self.name = name self.value = None - self.bytes = str(bytes) - if not bytes: + self.bytes = bytes(data) + self.dirty = False + if not data: self.type = TYPE_BOOL self.value = True return - self.type, self.value = self.BytesToValue(bytes) + self.type, self.value = BytesToValue(bytes(data)) def RefreshOffset(self, poffset): self._offset = poffset @@ -86,55 +139,6 @@ def Widen(self, newprop): while len(self.value) < len(newprop.value): self.value.append(val) - def BytesToValue(self, bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - bytes = str(bytes) - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - @classmethod def GetEmpty(self, type): """Get an empty / zero value of the given type @@ -145,7 +149,7 @@ def GetEmpty(self, type): if type == TYPE_BYTE: return chr(0) elif type == TYPE_INT: - return struct.pack('I', 0); elif type == TYPE_STRING: return '' else: @@ -160,6 +164,55 @@ def GetOffset(self): self._node._fdt.CheckCache() return self._node._fdt.GetStructOffset(self._offset) + def SetInt(self, val): + """Set the integer value of the property + + The device tree is marked dirty so that the value will be written to + the block on the next sync. + + Args: + val: Integer value (32-bit, single cell) + """ + self.bytes = struct.pack('>I', val); + self.value = self.bytes + self.type = TYPE_INT + self.dirty = True + + def SetData(self, bytes): + """Set the value of a property as bytes + + Args: + bytes: New property value to set + """ + self.bytes = bytes + self.type, self.value = BytesToValue(bytes) + self.dirty = True + + def Sync(self, auto_resize=False): + """Sync property changes back to the device tree + + This updates the device tree blob with any changes to this property + since the last sync. + + Args: + auto_resize: Resize the device tree automatically if it does not + have enough space for the update + + Raises: + FdtException if auto_resize is False and there is not enough space + """ + if self._offset is None or self.dirty: + node = self._node + fdt_obj = node._fdt._fdt_obj + if auto_resize: + while fdt_obj.setprop(node.Offset(), self.name, self.bytes, + (libfdt.NOSPACE,)) == -libfdt.NOSPACE: + fdt_obj.resize(fdt_obj.totalsize() + 1024) + fdt_obj.setprop(node.Offset(), self.name, self.bytes) + else: + fdt_obj.setprop(node.Offset(), self.name, self.bytes) + + class Node: """A device tree node @@ -284,25 +337,133 @@ def AddZeroProp(self, prop_name): Args: prop_name: Name of property """ - fdt_obj = self._fdt._fdt_obj - if fdt_obj.setprop_u32(self.Offset(), prop_name, 0, - (libfdt.NOSPACE,)) == -libfdt.NOSPACE: - fdt_obj.resize(fdt_obj.totalsize() + 1024) - fdt_obj.setprop_u32(self.Offset(), prop_name, 0) - self.props[prop_name] = Prop(self, -1, prop_name, '\0' * 4) - self._fdt.Invalidate() + self.props[prop_name] = Prop(self, None, prop_name, + tools.GetBytes(0, 4)) + + def AddEmptyProp(self, prop_name, len): + """Add a property with a fixed data size, for filling in later + + The device tree is marked dirty so that the value will be written to + the blob on the next sync. + + Args: + prop_name: Name of property + len: Length of data in property + """ + value = tools.GetBytes(0, len) + self.props[prop_name] = Prop(self, None, prop_name, value) def SetInt(self, prop_name, val): """Update an integer property int the device tree. This is not allowed to change the size of the FDT. + The device tree is marked dirty so that the value will be written to + the blob on the next sync. + Args: prop_name: Name of property val: Value to set """ - fdt_obj = self._fdt._fdt_obj - fdt_obj.setprop_u32(self.Offset(), prop_name, val) + self.props[prop_name].SetInt(val) + + def SetData(self, prop_name, val): + """Set the data value of a property + + The device tree is marked dirty so that the value will be written to + the blob on the next sync. + + Args: + prop_name: Name of property to set + val: Data value to set + """ + self.props[prop_name].SetData(val) + + def SetString(self, prop_name, val): + """Set the string value of a property + + The device tree is marked dirty so that the value will be written to + the blob on the next sync. + + Args: + prop_name: Name of property to set + val: String value to set (will be \0-terminated in DT) + """ + if type(val) == str: + val = val.encode('utf-8') + self.props[prop_name].SetData(val + b'\0') + + def AddString(self, prop_name, val): + """Add a new string property to a node + + The device tree is marked dirty so that the value will be written to + the blob on the next sync. + + Args: + prop_name: Name of property to add + val: String value of property + """ + if sys.version_info[0] >= 3: # pragma: no cover + val = bytes(val, 'utf-8') + self.props[prop_name] = Prop(self, None, prop_name, val + b'\0') + + def AddSubnode(self, name): + """Add a new subnode to the node + + Args: + name: name of node to add + + Returns: + New subnode that was created + """ + path = self.path + '/' + name + subnode = Node(self._fdt, self, None, name, path) + self.subnodes.append(subnode) + return subnode + + def Sync(self, auto_resize=False): + """Sync node changes back to the device tree + + This updates the device tree blob with any changes to this node and its + subnodes since the last sync. + + Args: + auto_resize: Resize the device tree automatically if it does not + have enough space for the update + + Raises: + FdtException if auto_resize is False and there is not enough space + """ + if self._offset is None: + # The subnode doesn't exist yet, so add it + fdt_obj = self._fdt._fdt_obj + if auto_resize: + while True: + offset = fdt_obj.add_subnode(self.parent._offset, self.name, + (libfdt.NOSPACE,)) + if offset != -libfdt.NOSPACE: + break + fdt_obj.resize(fdt_obj.totalsize() + 1024) + else: + offset = fdt_obj.add_subnode(self.parent._offset, self.name) + self._offset = offset + + # Sync subnodes in reverse so that we don't disturb node offsets for + # nodes that are earlier in the DT. This avoids an O(n^2) rescan of + # node offsets. + for node in reversed(self.subnodes): + node.Sync(auto_resize) + + # Sync properties now, whose offsets should not have been disturbed. + # We do this after subnodes, since this disturbs the offsets of these + # properties. Note that new properties will have an offset of None here, + # which Python 3 cannot sort against int. So use a large value instead + # to ensure that the new properties are added first. + prop_list = sorted(self.props.values(), + key=lambda prop: prop._offset or 1 << 31, + reverse=True) + for prop in prop_list: + prop.Sync(auto_resize) class Fdt: @@ -319,9 +480,23 @@ def __init__(self, fname): if self._fname: self._fname = fdt_util.EnsureCompiled(self._fname) - with open(self._fname) as fd: + with open(self._fname, 'rb') as fd: self._fdt_obj = libfdt.Fdt(fd.read()) + @staticmethod + def FromData(data): + """Create a new Fdt object from the given data + + Args: + data: Device-tree data blob + + Returns: + Fdt object containing the data + """ + fdt = Fdt(None) + fdt._fdt_obj = libfdt.Fdt(bytes(data)) + return fdt + def LookupPhandle(self, phandle): """Look up a phandle @@ -381,6 +556,19 @@ def Flush(self): with open(self._fname, 'wb') as fd: fd.write(self._fdt_obj.as_bytearray()) + def Sync(self, auto_resize=False): + """Make sure any DT changes are written to the blob + + Args: + auto_resize: Resize the device tree automatically if it does not + have enough space for the update + + Raises: + FdtException if auto_resize is False and there is not enough space + """ + self._root.Sync(auto_resize) + self.Invalidate() + def Pack(self): """Pack the device tree down to its minimum size @@ -396,7 +584,7 @@ def GetContents(self): Returns: The FDT contents as a string of bytes """ - return self._fdt_obj.as_bytearray() + return bytes(self._fdt_obj.as_bytearray()) def GetFdtObj(self): """Get the contents of the FDT diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 5fbfc8877b..f47879ac00 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -16,14 +16,6 @@ import command import tools -VERSION3 = sys.version_info > (3, 0) - -def get_plain_bytes(val): - """Handle Python 3 strings""" - if isinstance(val, bytes): - val = val.decode('utf-8') - return val.encode('raw_unicode_escape') - def fdt32_to_cpu(val): """Convert a device tree cell to an integer @@ -33,9 +25,6 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - if VERSION3: - # This code is not reached in Python 2 - val = get_plain_bytes(val) # pragma: no cover return struct.unpack('>I', val)[0] def fdt_cells_to_cpu(val, cells): @@ -45,11 +34,11 @@ def fdt_cells_to_cpu(val, cells): Value to convert (array of one or more 4-character strings) Return: - A native-endian long value + A native-endian integer value """ if not cells: return 0 - out = long(fdt32_to_cpu(val[0])) + out = int(fdt32_to_cpu(val[0])) if cells == 2: out = out << 32 | fdt32_to_cpu(val[1]) return out diff --git a/tools/dtoc/test_fdt.py b/tools/dtoc/test_fdt.py index e88d19f80e..4b9081369b 100755 --- a/tools/dtoc/test_fdt.py +++ b/tools/dtoc/test_fdt.py @@ -45,7 +45,7 @@ def _GetPropertyValue(dtb, node, prop_name): # Add 12, which is sizeof(struct fdt_property), to get to start of data offset = prop.GetOffset() + 12 data = dtb.GetContents()[offset:offset + len(prop.value)] - return prop, [chr(x) for x in data] + return prop, [tools.ToChar(x) for x in data] class TestFdt(unittest.TestCase): @@ -277,7 +277,7 @@ def testGetEmpty(self): """Tests the GetEmpty() function for the various supported types""" self.assertEqual(True, fdt.Prop.GetEmpty(fdt.TYPE_BOOL)) self.assertEqual(chr(0), fdt.Prop.GetEmpty(fdt.TYPE_BYTE)) - self.assertEqual(chr(0) * 4, fdt.Prop.GetEmpty(fdt.TYPE_INT)) + self.assertEqual(tools.GetBytes(0, 4), fdt.Prop.GetEmpty(fdt.TYPE_INT)) self.assertEqual('', fdt.Prop.GetEmpty(fdt.TYPE_STRING)) def testGetOffset(self): @@ -337,6 +337,7 @@ def testAdd(self): self.node.AddZeroProp('one') self.node.AddZeroProp('two') self.node.AddZeroProp('three') + self.dtb.Sync(auto_resize=True) # Updating existing properties should be OK, since the device-tree size # does not change @@ -344,11 +345,75 @@ def testAdd(self): self.node.SetInt('one', 1) self.node.SetInt('two', 2) self.node.SetInt('three', 3) + self.dtb.Sync(auto_resize=False) # This should fail since it would need to increase the device-tree size + self.node.AddZeroProp('four') with self.assertRaises(libfdt.FdtException) as e: - self.node.SetInt('four', 4) + self.dtb.Sync(auto_resize=False) self.assertIn('FDT_ERR_NOSPACE', str(e.exception)) + self.dtb.Sync(auto_resize=True) + + def testAddNode(self): + self.fdt.pack() + self.node.AddSubnode('subnode') + with self.assertRaises(libfdt.FdtException) as e: + self.dtb.Sync(auto_resize=False) + self.assertIn('FDT_ERR_NOSPACE', str(e.exception)) + + self.dtb.Sync(auto_resize=True) + offset = self.fdt.path_offset('/spl-test/subnode') + self.assertTrue(offset > 0) + + def testAddMore(self): + """Test various other methods for adding and setting properties""" + self.node.AddZeroProp('one') + self.dtb.Sync(auto_resize=True) + data = self.fdt.getprop(self.node.Offset(), 'one') + self.assertEqual(0, fdt32_to_cpu(data)) + + self.node.SetInt('one', 1) + self.dtb.Sync(auto_resize=False) + data = self.fdt.getprop(self.node.Offset(), 'one') + self.assertEqual(1, fdt32_to_cpu(data)) + + val = '123' + chr(0) + '456' + self.node.AddString('string', val) + self.dtb.Sync(auto_resize=True) + data = self.fdt.getprop(self.node.Offset(), 'string') + self.assertEqual(tools.ToBytes(val) + b'\0', data) + + self.fdt.pack() + self.node.SetString('string', val + 'x') + with self.assertRaises(libfdt.FdtException) as e: + self.dtb.Sync(auto_resize=False) + self.assertIn('FDT_ERR_NOSPACE', str(e.exception)) + self.node.SetString('string', val[:-1]) + + prop = self.node.props['string'] + prop.SetData(tools.ToBytes(val)) + self.dtb.Sync(auto_resize=False) + data = self.fdt.getprop(self.node.Offset(), 'string') + self.assertEqual(tools.ToBytes(val), data) + + self.node.AddEmptyProp('empty', 5) + self.dtb.Sync(auto_resize=True) + prop = self.node.props['empty'] + prop.SetData(tools.ToBytes(val)) + self.dtb.Sync(auto_resize=False) + data = self.fdt.getprop(self.node.Offset(), 'empty') + self.assertEqual(tools.ToBytes(val), data) + + self.node.SetData('empty', b'123') + self.assertEqual(b'123', prop.bytes) + + def testFromData(self): + dtb2 = fdt.Fdt.FromData(self.dtb.GetContents()) + self.assertEqual(dtb2.GetContents(), self.dtb.GetContents()) + + self.node.AddEmptyProp('empty', 5) + self.dtb.Sync(auto_resize=True) + self.assertTrue(dtb2.GetContents() != self.dtb.GetContents()) class TestFdtUtil(unittest.TestCase): @@ -436,9 +501,6 @@ def testEnsureCompiled(self): dtb = fdt_util.EnsureCompiled('tools/dtoc/dtoc_test_simple.dts') self.assertEqual(dtb, fdt_util.EnsureCompiled(dtb)) - def testGetPlainBytes(self): - self.assertEqual('fred', fdt_util.get_plain_bytes('fred')) - def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index d79e716074..31481157fc 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -159,7 +159,6 @@ def testBasic(self): os.remove(cc_file) lines = out[0].splitlines() - #print '\n'.join(lines) self.assertEqual('Cleaned %s patches' % len(series.commits), lines[0]) self.assertEqual('Change log missing for v2', lines[1]) self.assertEqual('Change log missing for v3', lines[2]) @@ -223,7 +222,6 @@ def testBasic(self): ''' lines = open(cover_fname).read().splitlines() - #print '\n'.join(lines) self.assertEqual( 'Subject: [RFC PATCH v3 0/2] test: A test patch series', lines[3]) @@ -231,7 +229,6 @@ def testBasic(self): for i, fname in enumerate(args): lines = open(fname).read().splitlines() - #print '\n'.join(lines) subject = [line for line in lines if line.startswith('Subject')] self.assertEqual('Subject: [RFC %d/%d]' % (i + 1, count), subject[0][:18]) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 9905bb0bbd..fb9e67c0f0 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -12,6 +12,7 @@ import checkpatch import settings +import tools # True to use --no-decorate - we check this in Setup() use_no_decorate = True @@ -325,6 +326,7 @@ def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): raw += LookupEmail(item, alias, raise_on_error=raise_on_error) result = [] for item in raw: + item = tools.FromUnicode(item) if not item in result: result.append(item) if tag: @@ -395,11 +397,11 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, git_config_to = command.Output('git', 'config', 'sendemail.to', raise_on_error=False) if not git_config_to: - print ("No recipient.\n" - "Please add something like this to a commit\n" - "Series-to: Fred Bloggs \n" - "Or do something like this\n" - "git config sendemail.to u-boot@lists.denx.de") + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs \n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") return cc = BuildEmailList(list(set(series.get('cc')) - set(series.get('to'))), '--cc', alias, raise_on_error) diff --git a/tools/patman/series.py b/tools/patman/series.py index 2735afaf88..0b71a896cb 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -11,6 +11,7 @@ import gitutil import settings import terminal +import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', @@ -249,7 +250,7 @@ def MakeCcFile(self, process_tags, cover_fname, raise_on_error, cover_cc = gitutil.BuildEmailList(self.get('cover_cc', '')) cover_cc = [m.encode('utf-8') if type(m) != str else m for m in cover_cc] - cc_list = ', '.join([x.decode('utf-8') + cc_list = ', '.join([tools.ToUnicode(x) for x in set(cover_cc + all_ccs)]) print(cover_fname, cc_list.encode('utf-8'), file=fd) diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index 0e79af871a..82ef808480 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -3,6 +3,8 @@ # Copyright (c) 2016 Google, Inc # +from __future__ import print_function + from contextlib import contextmanager import glob import os @@ -54,18 +56,18 @@ def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None): missing_list = required missing_list.difference_update(test_set) if missing_list: - print 'Missing tests for %s' % (', '.join(missing_list)) - print stdout + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) ok = False coverage = lines[-1].split(' ')[-1] ok = True - print coverage + print(coverage) if coverage != '100%': - print stdout - print ("Type 'python-coverage html' to get a report in " - 'htmlcov/index.html') - print 'Coverage error: %s, but should be 100%%' % coverage + print(stdout) + print("Type 'python-coverage html' to get a report in " + 'htmlcov/index.html') + print('Coverage error: %s, but should be 100%%' % coverage) ok = False if not ok: raise ValueError('Test coverage failure') diff --git a/tools/patman/tools.py b/tools/patman/tools.py index e80481438b..91f498891f 100644 --- a/tools/patman/tools.py +++ b/tools/patman/tools.py @@ -6,6 +6,7 @@ import command import os import shutil +import sys import tempfile import tout @@ -191,3 +192,105 @@ def WriteFile(fname, data): #(fname, len(data), len(data))) with open(Filename(fname), 'wb') as fd: fd.write(data) + +def GetBytes(byte, size): + """Get a string of bytes of a given size + + This handles the unfortunate different between Python 2 and Python 2. + + Args: + byte: Numeric byte value to use + size: Size of bytes/string to return + + Returns: + A bytes type with 'byte' repeated 'size' times + """ + if sys.version_info[0] >= 3: + data = bytes([byte]) * size + else: + data = chr(byte) * size + return data + +def ToUnicode(val): + """Make sure a value is a unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it returns a unicode object. + + Args: + val: string or unicode object + + Returns: + unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, unicode) else val.decode('utf-8') + +def FromUnicode(val): + """Make sure a value is a non-unicode string + + This allows some amount of compatibility between Python 2 and Python3. For + the former, it converts a unicode object to a string. + + Args: + val: string or unicode object + + Returns: + non-unicode version of val + """ + if sys.version_info[0] >= 3: + return val + return val if isinstance(val, str) else val.encode('utf-8') + +def ToByte(ch): + """Convert a character to an ASCII value + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts the argument to + an ASCII value in either case. + + Args: + ch: A string (Python 2) or byte (Python 3) value + + Returns: + integer ASCII value for ch + """ + return ord(ch) if type(ch) == str else ch + +def ToChar(byte): + """Convert a byte to a character + + This is useful because in Python 2 bytes is an alias for str, but in + Python 3 they are separate types. This function converts an ASCII value to + a value with the appropriate type in either case. + + Args: + byte: A byte or str value + """ + return chr(byte) if type(byte) != str else byte + +def ToChars(byte_list): + """Convert a list of bytes to a str/bytes type + + Args: + byte_list: List of ASCII values representing the string + + Returns: + string made by concatenating all the ASCII values + """ + return ''.join([chr(byte) for byte in byte_list]) + +def ToBytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert value + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + if sys.version_info[0] >= 3: + return string.encode('utf-8') + return string