From b40c69a6971b6002d6a8694d45f74a8af4770002 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 23 Jul 2018 13:32:53 -0700 Subject: [PATCH 001/688] scripts: Remove gcc-wrapper.py This adds an unnecessary dependency on Python when regular -Werror will do fine. Change-Id: I81f0e52e7270923f7e3d30a79ec9e9205d47ae0b Signed-off-by: Nathan Chancellor --- Makefile | 8 +--- scripts/gcc-wrapper.py | 95 ------------------------------------------ 2 files changed, 1 insertion(+), 102 deletions(-) delete mode 100755 scripts/gcc-wrapper.py diff --git a/Makefile b/Makefile index d32900fe677c..787eecdd41f5 100644 --- a/Makefile +++ b/Makefile @@ -339,10 +339,8 @@ include scripts/Kbuild.include # Make variables (CC, etc...) AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld -REAL_CC = $(CROSS_COMPILE)gcc -LDGOLD = $(CROSS_COMPILE)ld.gold -REAL_CC = $(CROSS_COMPILE)gcc CC = $(CROSS_COMPILE)gcc +LDGOLD = $(CROSS_COMPILE)ld.gold CPP = $(CC) -E AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm @@ -358,10 +356,6 @@ PERL = perl PYTHON = python CHECK = sparse -# Use the wrapper for the compiler. This wrapper scans for new -# warnings and causes the build to stop upon encountering them -CC = $(PYTHON) $(srctree)/scripts/gcc-wrapper.py $(REAL_CC) - CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void $(CF) NOSTDINC_FLAGS = diff --git a/scripts/gcc-wrapper.py b/scripts/gcc-wrapper.py deleted file mode 100755 index 3d1d6fbcaa04..000000000000 --- a/scripts/gcc-wrapper.py +++ /dev/null @@ -1,95 +0,0 @@ -#! /usr/bin/env python2 -# -*- coding: utf-8 -*- - -# Copyright (c) 2011-2017, The Linux Foundation. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of The Linux Foundation nor -# the names of its contributors may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Invoke gcc, looking for warnings, and causing a failure if there are -# non-whitelisted warnings. - -import errno -import re -import os -import sys -import subprocess - -# Note that gcc uses unicode, which may depend on the locale. TODO: -# force LANG to be set to en_US.UTF-8 to get consistent warnings. - -allowed_warnings = set([ - ]) - -# Capture the name of the object file, can find it. -ofile = None - -warning_re = re.compile(r'''(.*/|)([^/]+\.[a-z]+:\d+):(\d+:)? warning:''') -def interpret_warning(line): - """Decode the message from gcc. The messages we care about have a filename, and a warning""" - line = line.rstrip('\n') - m = warning_re.match(line) - if m and m.group(2) not in allowed_warnings: - print >> sys.stderr, "error, forbidden warning:", m.group(2) - - # If there is a warning, remove any object if it exists. - if ofile: - try: - os.remove(ofile) - except OSError: - pass - sys.exit(1) - -def run_gcc(): - args = sys.argv[1:] - # Look for -o - try: - i = args.index('-o') - global ofile - ofile = args[i+1] - except (ValueError, IndexError): - pass - - compiler = sys.argv[0] - - try: - proc = subprocess.Popen(args, stderr=subprocess.PIPE) - for line in proc.stderr: - print >> sys.stderr, line, - interpret_warning(line) - - result = proc.wait() - except OSError as e: - result = e.errno - if result == errno.ENOENT: - print >> sys.stderr, args[0] + ':',e.strerror - print >> sys.stderr, 'Is your PATH set correctly?' - else: - print >> sys.stderr, ' '.join(args), str(e) - - return result - -if __name__ == '__main__': - status = run_gcc() - sys.exit(status) From af7339407461cb7cc3cb6df81baedd604c229937 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 26 Feb 2020 15:23:36 -0800 Subject: [PATCH 002/688] BACKPORT: Documentation/llvm: add documentation on building w/ Clang/LLVM Added to kbuild documentation. Provides more official info on building kernels with Clang and LLVM than our wiki. Suggested-by: Kees Cook Reviewed-by: Kees Cook Reviewed-by: Nathan Chancellor Reviewed-by: Sedat Dilek Signed-off-by: Nick Desaulniers Signed-off-by: Masahiro Yamada (cherry picked from commit fcf1b6a35c16ac500fa908a4022238e5d666eabf) Change-Id: I33bc4a97cf9e9ef923712bb490df0bceac81dade --- Documentation/kbuild/00-INDEX | 2 + Documentation/kbuild/llvm.txt | 83 +++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 Documentation/kbuild/llvm.txt diff --git a/Documentation/kbuild/00-INDEX b/Documentation/kbuild/00-INDEX index 8c5e6aa78004..b367200c94b5 100644 --- a/Documentation/kbuild/00-INDEX +++ b/Documentation/kbuild/00-INDEX @@ -1,5 +1,7 @@ 00-INDEX - this file: info on the kernel build process +llvm.txt + - how to compile Linux with llvm headers_install.txt - how to export Linux headers for use by userspace kbuild.txt diff --git a/Documentation/kbuild/llvm.txt b/Documentation/kbuild/llvm.txt new file mode 100644 index 000000000000..0c30444a7368 --- /dev/null +++ b/Documentation/kbuild/llvm.txt @@ -0,0 +1,83 @@ +Building Linux with Clang/LLVM +============================== + +This document covers how to build the Linux kernel with Clang and LLVM +utilities. + +About +----- + +The Linux kernel has always traditionally been compiled with GNU +toolchains such as GCC and binutils. Ongoing work has allowed for +[Clang](https://clang.llvm.org/) and [LLVM](https://llvm.org/) utilities +to be used as viable substitutes. Distributions such as +[Android](https://www.android.com/), +[ChromeOS](https://www.chromium.org/chromium-os), and +[OpenMandriva](https://www.openmandriva.org/) use Clang built kernels. +[LLVM is a collection of toolchain components implemented in terms of +C++ objects](https://www.aosabook.org/en/llvm.html). Clang is a +front-end to LLVM that supports C and the GNU C extensions required by +the kernel, and is pronounced "klang," not "see-lang." + +Clang +----- + +The compiler used can be swapped out via [CC=]{.title-ref} command line +argument to [make]{.title-ref}. [CC=]{.title-ref} should be set when +selecting a config and during a build. + +> make CC=clang defconfig +> +> make CC=clang + +Cross Compiling +--------------- + +A single Clang compiler binary will typically contain all supported +backends, which can help simplify cross compiling. + +> ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make CC=clang + +[CROSS\_COMPILE]{.title-ref} is not used to prefix the Clang compiler +binary, instead [CROSS\_COMPILE]{.title-ref} is used to set a command +line flag: [--target ]{.title-ref}. For example: + +> clang --target aarch64-linux-gnu foo.c + +LLVM Utilities +-------------- + +LLVM has substitutes for GNU binutils utilities. These can be invoked as +additional parameters to [make]{.title-ref}. + +> make CC=clang AS=clang LD=ld.lld AR=llvm-ar NM=llvm-nm STRIP=llvm-strip \\ +> OBJCOPY=llvm-objcopy OBJDUMP=llvm-objdump OBJSIZE=llvm-objsize \\ +> READELF=llvm-readelf HOSTCC=clang HOSTCXX=clang++ HOSTAR=llvm-ar \\ +> HOSTLD=ld.lld + +Getting Help +------------ + +- [Website](https://clangbuiltlinux.github.io/) +- [Mailing + List](https://groups.google.com/forum/#!forum/clang-built-linux): + +- [Issue Tracker](https://github.com/ClangBuiltLinux/linux/issues) +- IRC: \#clangbuiltlinux on chat.freenode.net +- [Telegram](https://t.me/ClangBuiltLinux): \@ClangBuiltLinux +- [Wiki](https://github.com/ClangBuiltLinux/linux/wiki) +- [Beginner + Bugs](https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) + +Getting LLVM +------------ + +- +- +- +- +- +- +- +- +- From 6b6881b49f02cfa370b8d23489c98cc2aaaab393 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 2 Apr 2020 10:38:42 -0700 Subject: [PATCH 003/688] BACKPORT: Documentation/llvm: fix the name of llvm-size The tool is called llvm-size, not llvm-objsize. Fixes: fcf1b6a35c16 ("Documentation/llvm: add documentation on building w/ Clang/LLVM") Signed-off-by: Fangrui Song Reviewed-by: Nick Desaulniers Reviewed-by: Nathan Chancellor Signed-off-by: Masahiro Yamada (cherry picked from commit 0f44fbc162b737ff6251ae248184390ae2279fee) Change-Id: Ib6ff2a256b55983d0a3e9bb10f53f7a5c3c214d2 --- Documentation/kbuild/llvm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/kbuild/llvm.txt b/Documentation/kbuild/llvm.txt index 0c30444a7368..d469accd5fa3 100644 --- a/Documentation/kbuild/llvm.txt +++ b/Documentation/kbuild/llvm.txt @@ -51,7 +51,7 @@ LLVM has substitutes for GNU binutils utilities. These can be invoked as additional parameters to [make]{.title-ref}. > make CC=clang AS=clang LD=ld.lld AR=llvm-ar NM=llvm-nm STRIP=llvm-strip \\ -> OBJCOPY=llvm-objcopy OBJDUMP=llvm-objdump OBJSIZE=llvm-objsize \\ +> OBJCOPY=llvm-objcopy OBJDUMP=llvm-objdump OBJSIZE=llvm-size \\ > READELF=llvm-readelf HOSTCC=clang HOSTCXX=clang++ HOSTAR=llvm-ar \\ > HOSTLD=ld.lld From 73d166d08b04ca887f5e9d6adc785b760271eaca Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 8 Apr 2020 10:36:22 +0900 Subject: [PATCH 004/688] BACKPORT: kbuild: replace AS=clang with LLVM_IAS=1 commit 7e20e47c70f810d678d02941fa3c671209c4ca97 upstream. The 'AS' variable is unused for building the kernel. Only the remaining usage is to turn on the integrated assembler. A boolean flag is a better fit for this purpose. AS=clang was added for experts. So, I replaced it with LLVM_IAS=1, breaking the backward compatibility. Suggested-by: Nick Desaulniers Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Reviewed-by: Nick Desaulniers Signed-off-by: Nick Desaulniers Signed-off-by: Greg Kroah-Hartman (cherry-picked from commit 7e20e47c70f810d678d02941fa3c671209c4ca97) Change-Id: I1dcf901b2543ee6a95cfffddd2ced50b0a1ae14c --- Documentation/kbuild/llvm.txt | 5 ++++- Makefile | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/kbuild/llvm.txt b/Documentation/kbuild/llvm.txt index d469accd5fa3..51c009baa35a 100644 --- a/Documentation/kbuild/llvm.txt +++ b/Documentation/kbuild/llvm.txt @@ -50,11 +50,14 @@ LLVM Utilities LLVM has substitutes for GNU binutils utilities. These can be invoked as additional parameters to [make]{.title-ref}. -> make CC=clang AS=clang LD=ld.lld AR=llvm-ar NM=llvm-nm STRIP=llvm-strip \\ +> make CC=clang LD=ld.lld AR=llvm-ar NM=llvm-nm STRIP=llvm-strip \\ > OBJCOPY=llvm-objcopy OBJDUMP=llvm-objdump OBJSIZE=llvm-size \\ > READELF=llvm-readelf HOSTCC=clang HOSTCXX=clang++ HOSTAR=llvm-ar \\ > HOSTLD=ld.lld +Currently, the integrated assembler is disabled by default. You can pass +`LLVM_IAS=1` to enable it. + Getting Help ------------ diff --git a/Makefile b/Makefile index 787eecdd41f5..d75c82c97e3b 100644 --- a/Makefile +++ b/Makefile @@ -517,7 +517,9 @@ endif ifneq ($(GCC_TOOLCHAIN),) CLANG_FLAGS += --gcc-toolchain=$(GCC_TOOLCHAIN) endif +ifneq ($(LLVM_IAS),1) CLANG_FLAGS += -no-integrated-as +endif CLANG_FLAGS += -Werror=unknown-warning-option KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_AFLAGS += $(CLANG_FLAGS) From 764a8e82ee933b18e6cfcc7084264beb4cfd59d2 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 8 Apr 2020 10:36:23 +0900 Subject: [PATCH 005/688] BACKPORT: kbuild: support LLVM=1 to switch the default tools to Clang/LLVM commit a0d1c951ef08ed24f35129267e3595d86f57f5d3 upstream. As Documentation/kbuild/llvm.rst implies, building the kernel with a full set of LLVM tools gets very verbose and unwieldy. Provide a single switch LLVM=1 to use Clang and LLVM tools instead of GCC and Binutils. You can pass it from the command line or as an environment variable. Please note LLVM=1 does not turn on the integrated assembler. You need to pass LLVM_IAS=1 to use it. When the upstream kernel is ready for the integrated assembler, I think we can make it default. We discussed what we need, and we agreed to go with a simple boolean flag that switches both target and host tools: https://lkml.org/lkml/2020/3/28/494 https://lkml.org/lkml/2020/4/3/43 Some items discussed, but not adopted: - LLVM_DIR When multiple versions of LLVM are installed, I just thought supporting LLVM_DIR=/path/to/my/llvm/bin/ might be useful. CC = $(LLVM_DIR)clang LD = $(LLVM_DIR)ld.lld ... However, we can handle this by modifying PATH. So, we decided to not do this. - LLVM_SUFFIX Some distributions (e.g. Debian) package specific versions of LLVM with naming conventions that use the version as a suffix. CC = clang$(LLVM_SUFFIX) LD = ld.lld(LLVM_SUFFIX) ... will allow a user to pass LLVM_SUFFIX=-11 to use clang-11 etc., but the suffixed versions in /usr/bin/ are symlinks to binaries in /usr/lib/llvm-#/bin/, so this can also be handled by PATH. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Tested-by: Nathan Chancellor # build Tested-by: Nick Desaulniers Reviewed-by: Nick Desaulniers Signed-off-by: Nick Desaulniers [nd: conflict in exported vars list from not backporting commit e83b9f55448a ("kbuild: add ability to generate BTF type info for vmlinux")] [nd: hunk against Documentation/kbuild/kbuild.rst dropped due to not backporting commit cd238effefa2 ("docs: kbuild: convert docs to ReST and rename to *.rst")] Signed-off-by: Greg Kroah-Hartman (cherry-picked from commit 62353048e2d4e398cfc22dbee3e1a7f6f535dcc1) Change-Id: Ib62c20d4d01b0b95bdcd5e73fa18f32cdb8f585c --- Documentation/kbuild/kbuild.txt | 5 +++++ Makefile | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Documentation/kbuild/kbuild.txt b/Documentation/kbuild/kbuild.txt index 0ff6a466a05b..a6aea8f9a161 100644 --- a/Documentation/kbuild/kbuild.txt +++ b/Documentation/kbuild/kbuild.txt @@ -238,3 +238,8 @@ KBUILD_VMLINUX_MAIN All object files for the main part of vmlinux. KBUILD_VMLINUX_INIT and KBUILD_VMLINUX_MAIN together specify all the object files used to link vmlinux. + +LLVM +---- +If this variable is set to 1, Kbuild will use Clang and LLVM utilities instead +of GCC and GNU binutils to build the kernel. diff --git a/Makefile b/Makefile index d75c82c97e3b..2a43456bb360 100644 --- a/Makefile +++ b/Makefile @@ -301,8 +301,14 @@ CONFIG_SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \ else if [ -x /bin/bash ]; then echo /bin/bash; \ else echo sh; fi ; fi) +ifneq ($(LLVM),) +HOSTCC = clang +HOSTCXX = clang++ +else HOSTCC = gcc HOSTCXX = g++ +endif + HOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu89 HOSTCXXFLAGS = -O2 @@ -337,16 +343,28 @@ scripts/Kbuild.include: ; include scripts/Kbuild.include # Make variables (CC, etc...) +LDGOLD = $(CROSS_COMPILE)ld.gold +CPP = $(CC) -E +ifneq ($(LLVM),) +CC = clang +LD = ld.lld +AR = llvm-ar +NM = llvm-nm +OBJCOPY = llvm-objcopy +OBJDUMP = llvm-objdump +READELF = llvm-readelf +OBJSIZE = llvm-size +STRIP = llvm-strip +else AS = $(CROSS_COMPILE)as LD = $(CROSS_COMPILE)ld CC = $(CROSS_COMPILE)gcc -LDGOLD = $(CROSS_COMPILE)ld.gold -CPP = $(CC) -E AR = $(CROSS_COMPILE)ar NM = $(CROSS_COMPILE)nm STRIP = $(CROSS_COMPILE)strip OBJCOPY = $(CROSS_COMPILE)objcopy OBJDUMP = $(CROSS_COMPILE)objdump +endif DTC = scripts/dtc/dtc AWK = awk GENKSYMS = scripts/genksyms/genksyms From c8ce7c33ed6f1e23daadd21599a5540cec5dba7b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 23 Aug 2018 17:48:45 +0100 Subject: [PATCH 006/688] BACKPORT: crypto: arm64/aes-modes - get rid of literal load of addend vector commit ed6ed11830a9ded520db31a6e2b69b6b0a1eb0e2 upstream. Replace the literal load of the addend vector with a sequence that performs each add individually. This sequence is only 2 instructions longer than the original, and 2% faster on Cortex-A53. This is an improvement by itself, but also works around a Clang issue, whose integrated assembler does not implement the GNU ARM asm syntax completely, and does not support the =literal notation for FP registers (more info at https://bugs.llvm.org/show_bug.cgi?id=38642) Cc: Nick Desaulniers Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Signed-off-by: Herbert Xu Signed-off-by: Greg Kroah-Hartman Change-Id: Ic8f7adcd28bd2da57b465a8e11e9d55b5669a539 --- arch/arm64/crypto/aes-modes.S | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 838dad5c209f..067b4e222f74 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -297,17 +297,19 @@ AES_ENTRY(aes_ctr_encrypt) eor v1.16b, v1.16b, v3.16b st1 {v0.16b-v1.16b}, [x0], #32 #else - ldr q8, =0x30000000200000001 /* addends 1,2,3[,0] */ - dup v7.4s, w8 + add w7, w6, #1 mov v0.16b, v4.16b - add v7.4s, v7.4s, v8.4s + add w8, w6, #2 mov v1.16b, v4.16b - rev32 v8.16b, v7.16b + add w9, w6, #3 mov v2.16b, v4.16b + rev w7, w7 mov v3.16b, v4.16b - mov v1.s[3], v8.s[0] - mov v2.s[3], v8.s[1] - mov v3.s[3], v8.s[2] + rev w8, w8 + mov v1.s[3], w7 + rev w9, w9 + mov v2.s[3], w8 + mov v3.s[3], w9 ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ do_encrypt_block4x eor v0.16b, v5.16b, v0.16b From 4c07bd2c7ec02524dd2622c7e62c332ac4d75a7e Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 10 Dec 2018 13:39:48 +0000 Subject: [PATCH 007/688] BACKPORT: arm64: Fix minor issues with the dcache_by_line_op macro [ Upstream commit 33309ecda0070506c49182530abe7728850ebe78 ] The dcache_by_line_op macro suffers from a couple of small problems: First, the GAS directives that are currently being used rely on assembler behavior that is not documented, and probably not guaranteed to produce the correct behavior going forward. As a result, we end up with some undefined symbols in cache.o: $ nm arch/arm64/mm/cache.o ... U civac ... U cvac U cvap U cvau This is due to the fact that the comparisons used to select the operation type in the dcache_by_line_op macro are comparing symbols not strings, and even though it seems that GAS is doing the right thing here (undefined symbols by the same name are equal to each other), it seems unwise to rely on this. Second, when patching in a DC CVAP instruction on CPUs that support it, the fallback path consists of a DC CVAU instruction which may be affected by CPU errata that require ARM64_WORKAROUND_CLEAN_CACHE. Solve these issues by unrolling the various maintenance routines and using the conditional directives that are documented as operating on strings. To avoid the complexity of nested alternatives, we move the DC CVAP patching to __clean_dcache_area_pop, falling back to a branch to __clean_dcache_area_poc if DCPOP is not supported by the CPU. Reported-by: Ard Biesheuvel Suggested-by: Robin Murphy Signed-off-by: Will Deacon Signed-off-by: Sasha Levin Change-Id: I006dd10a0f3706b9302832d04c65262861239709 --- arch/arm64/include/asm/assembler.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 7d5bc694c43c..a053e07e59b8 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -386,21 +386,33 @@ alternative_endif * size: size of the region * Corrupts: kaddr, size, tmp1, tmp2 */ + .macro __dcache_op_workaround_clean_cache, op, kaddr +alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE + dc \op, \kaddr +alternative_else + dc civac, \kaddr +alternative_endif + .endm + .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 dcache_line_size \tmp1, \tmp2 add \size, \kaddr, \size sub \tmp2, \tmp1, #1 bic \kaddr, \kaddr, \tmp2 9998: - .if (\op == cvau || \op == cvac) -alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE - dc \op, \kaddr -alternative_else - dc civac, \kaddr -alternative_endif + .ifc \op, cvau + __dcache_op_workaround_clean_cache \op, \kaddr + .else + .ifc \op, cvac + __dcache_op_workaround_clean_cache \op, \kaddr + .else + .ifc \op, cvap + sys 3, c7, c12, 1, \kaddr // dc cvap .else dc \op, \kaddr .endif + .endif + .endif add \kaddr, \kaddr, \tmp1 cmp \kaddr, \size b.lo 9998b From b2bbb6ceb8548452a8b17d3ff8e4950710d51111 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 29 Oct 2020 11:19:51 -0700 Subject: [PATCH 008/688] BACKPORT: arm64: Change .weak to SYM_FUNC_START_WEAK_PI for arch/arm64/lib/mem*.S commit ec9d78070de986ecf581ea204fd322af4d2477ec upstream. Commit 39d114ddc682 ("arm64: add KASAN support") added .weak directives to arch/arm64/lib/mem*.S instead of changing the existing SYM_FUNC_START_PI macros. This can lead to the assembly snippet `.weak memcpy ... .globl memcpy` which will produce a STB_WEAK memcpy with GNU as but STB_GLOBAL memcpy with LLVM's integrated assembler before LLVM 12. LLVM 12 (since https://reviews.llvm.org/D90108) will error on such an overridden symbol binding. Use the appropriate SYM_FUNC_START_WEAK_PI instead. Fixes: 39d114ddc682 ("arm64: add KASAN support") Reported-by: Sami Tolvanen Signed-off-by: Fangrui Song Tested-by: Sami Tolvanen Tested-by: Nick Desaulniers Reviewed-by: Nick Desaulniers Cc: Link: https://lore.kernel.org/r/20201029181951.1866093-1-maskray@google.com Signed-off-by: Will Deacon [nd: backport to adjust for missing: commit 3ac0f4526dfb ("arm64: lib: Use modern annotations for assembly functions") commit 35e61c77ef38 ("arm64: asm: Add new-style position independent function annotations")] Signed-off-by: Nick Desaulniers Change-Id: Ibebbfa15f3337b5a2ac88ba683c5e429758d7a98 --- arch/arm64/lib/memcpy.S | 3 +-- arch/arm64/lib/memmove.S | 3 +-- arch/arm64/lib/memset.S | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index 67613937711f..dfedd4ab1a76 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -68,9 +68,8 @@ stp \ptr, \regB, [\regC], \val .endm - .weak memcpy ENTRY(__memcpy) -ENTRY(memcpy) +WEAK(memcpy) #include "copy_template.S" ret ENDPIPROC(memcpy) diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S index a5a4459013b1..e3de8f05c21a 100644 --- a/arch/arm64/lib/memmove.S +++ b/arch/arm64/lib/memmove.S @@ -57,9 +57,8 @@ C_h .req x12 D_l .req x13 D_h .req x14 - .weak memmove ENTRY(__memmove) -ENTRY(memmove) +WEAK(memmove) cmp dstin, src b.lo __memcpy add tmp1, src, count diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S index f2670a9f218c..316263c47c00 100644 --- a/arch/arm64/lib/memset.S +++ b/arch/arm64/lib/memset.S @@ -54,9 +54,8 @@ dst .req x8 tmp3w .req w9 tmp3 .req x9 - .weak memset ENTRY(__memset) -ENTRY(memset) +WEAK(memset) mov dst, dstin /* Preserve return value. */ and A_lw, val, #255 orr A_lw, A_lw, A_lw, lsl #8 From d57f0917e8453e73f63a3cd5d093375d02f4f3df Mon Sep 17 00:00:00 2001 From: Alex Matveev Date: Mon, 20 Nov 2017 21:30:38 +0000 Subject: [PATCH 009/688] arm64: crypto: fix typo in aes_sub() Clang's integrated assembler can't parse "v0.4s[0]" argument of the UMOV instruction. And, as per ARM ARM, this is incorrect usage: UMOV , .[] ... For the 32-bit variant: is an element size specifier, encoded in the "imm5" field. It can have the following values: B when imm5 = xxxx1 H when imm5 = xxx10 S when imm5 = xx100 Signed-off-by: Alex Matveev Change-Id: If8d1f7dffb449d06f1c5bbefba88e488e6883f40 --- arch/arm64/crypto/aes-ce-cipher-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/crypto/aes-ce-cipher-core.c b/arch/arm64/crypto/aes-ce-cipher-core.c index 9f4191774b35..3ec77b767420 100644 --- a/arch/arm64/crypto/aes-ce-cipher-core.c +++ b/arch/arm64/crypto/aes-ce-cipher-core.c @@ -131,7 +131,7 @@ static u32 aes_sub(u32 input) __asm__("dup v1.4s, %w[in] ;" "movi v0.16b, #0 ;" "aese v0.16b, v1.16b ;" - "umov %w[out], v0.4s[0] ;" + "umov %w[out], v0.s[0] ;" : [out] "=r"(ret) : [in] "r"(input) From 9a1965e1e88d385e4628c108cdc9cb89bf15f1d1 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 21 Jun 2020 21:48:35 +0300 Subject: [PATCH 010/688] kbuild: clear LDFLAGS in the top Makefile The kernel needs to be compiled as a LP64 binary for ARM64, even when using a compiler that defaults to code-generation for the ILP32 ABI. Consequently, we need to explicitly pass '-mabi=lp64' (supported on gcc-4.9 and newer). Signed-off-by: Andrew Pinski Signed-off-by: Philipp Tomsich Signed-off-by: Christoph Muellner Signed-off-by: Yury Norov Reviewed-by: David Daney Signed-off-by: Catalin Marinas Change-Id: Ife00e01e3e3a6d85841b88c2f1a042617bd61500 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 2a43456bb360..2f4fc4d309d7 100644 --- a/Makefile +++ b/Makefile @@ -416,6 +416,7 @@ KBUILD_AFLAGS_MODULE := -DMODULE KBUILD_CFLAGS_MODULE := -DMODULE KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds GCC_PLUGINS_CFLAGS := +LDFLAGS := CLANG_FLAGS := # Read KERNELRELEASE from include/config/kernel.release (if it exists) From 8c5d678be0bde547fb05d6f2fa86fdd32eeb9349 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 3 Jul 2018 10:22:00 +0900 Subject: [PATCH 011/688] arm64: add endianness option to LDFLAGS instead of LD With the recent syntax extension, Kconfig is now able to evaluate the compiler / toolchain capability. However, accumulating flags to 'LD' is not compatible with the way it works; 'LD' must be passed to Kconfig to call $(ld-option,...) from Kconfig files. If you tweak 'LD' in arch Makefile depending on CONFIG_CPU_BIG_ENDIAN, this would end up with circular dependency between Makefile and Kconfig. Acked-by: Will Deacon Signed-off-by: Masahiro Yamada Signed-off-by: Catalin Marinas Change-Id: I8a7654684975d45e979917e3b1c4b6249dec02ec --- arch/arm64/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 99d7f5127ef0..c29406dee249 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -103,12 +103,12 @@ endif ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian AS += -EB -LD += -EB +LDFLAGS += -EB -maarch64elfb UTS_MACHINE := aarch64_be else KBUILD_CPPFLAGS += -mlittle-endian AS += -EL -LD += -EL +LDFLAGS += -EL -maarch64elf UTS_MACHINE := aarch64 endif From 0ac9bb1b2101c8570057646fcc6007d43b0c332d Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Fri, 13 Jul 2018 08:30:33 -0700 Subject: [PATCH 012/688] arm64: build with baremetal linker target instead of Linux when available Not all toolchains have the baremetal elf targets, RedHat/Fedora ones in particular. So, probe for whether it's available and use the previous (linux) targets if it isn't. Reported-by: Laura Abbott Tested-by: Laura Abbott Acked-by: Masahiro Yamada Cc: Paul Kocialkowski Signed-off-by: Olof Johansson Signed-off-by: Will Deacon Change-Id: Icf5462a8318b347cf11559c1654886c48c7a62b5 --- arch/arm64/Makefile | 6 ++++-- scripts/Kbuild.include | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index c29406dee249..1ed7554ed08b 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -103,12 +103,14 @@ endif ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian AS += -EB -LDFLAGS += -EB -maarch64elfb +# Prefer the baremetal ELF build target, but not all toolchains include +LDFLAGS += -EB $(call ld-option, -maarch64elfb, -maarch64linuxb) UTS_MACHINE := aarch64_be else KBUILD_CPPFLAGS += -mlittle-endian AS += -EL -LDFLAGS += -EL -maarch64elf +# Same as above, prefer ELF but fall back to linux target if needed. +LDFLAGS += -EL $(call ld-option, -maarch64elf, -maarch64linux) UTS_MACHINE := aarch64 endif diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 937f719e3033..b64a07fbd62c 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -197,8 +197,8 @@ cc-ldoption = $(call try-run,\ $(CC) $(CLANG_TARGET) $(1) $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) -nostdlib -x c /dev/null -o "$$TMP",$(1),$(2)) # ld-option -# Usage: LDFLAGS += $(call ld-option, -X) -ld-option = $(call try-run, $(LD) $(LDFLAGS) $(1) -v,$(1),$(2)) +# Usage: LDFLAGS += $(call ld-option, -X, -Y) +ld-option = $(call try-run, $(LD) $(LDFLAGS) $(1) -v,$(1),$(2),$(3)) # ar-option # Usage: KBUILD_ARFLAGS := $(call ar-option,D) From f43bde45bcf1b0181880799b107d43644b64cfa1 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 15 May 2019 11:24:41 -0700 Subject: [PATCH 013/688] lkdtm: support llvm-objcopy commit e9e08a07385e08f1a7f85c5d1e345c21c9564963 upstream. With CONFIG_LKDTM=y and make OBJCOPY=llvm-objcopy, llvm-objcopy errors: llvm-objcopy: error: --set-section-flags=.text conflicts with --rename-section=.text=.rodata Rather than support setting flags then renaming sections vs renaming then setting flags, it's simpler to just change both at the same time via --rename-section. Adding the load flag is required for GNU objcopy to mark .rodata Type as PROGBITS after the rename. This can be verified with: $ readelf -S drivers/misc/lkdtm/rodata_objcopy.o ... Section Headers: [Nr] Name Type Address Offset Size EntSize Flags Link Info Align ... [ 1] .rodata PROGBITS 0000000000000000 00000040 0000000000000004 0000000000000000 A 0 0 4 ... Which shows that .text is now renamed .rodata, the alloc flag A is set, the type is PROGBITS, and the section is not flagged as writeable W. Cc: stable@vger.kernel.org Link: https://sourceware.org/bugzilla/show_bug.cgi?id=24554 Link: https://github.com/ClangBuiltLinux/linux/issues/448 Reported-by: Nathan Chancellor Suggested-by: Alan Modra Suggested-by: Jordan Rupprect Suggested-by: Kees Cook Acked-by: Kees Cook Reviewed-by: Nathan Chancellor Signed-off-by: Nick Desaulniers Signed-off-by: Greg Kroah-Hartman Change-Id: I1a3d5129e96e909e6d3751752a0bf474474664d9 --- drivers/misc/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 86cade73835a..e75a95452d32 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -85,8 +85,7 @@ KCOV_INSTRUMENT_lkdtm_rodata.o := n OBJCOPYFLAGS := OBJCOPYFLAGS_lkdtm_rodata_objcopy.o := \ - --set-section-flags .text=alloc,readonly \ - --rename-section .text=.rodata + --rename-section .text=.rodata,alloc,readonly,load targets += lkdtm_rodata.o lkdtm_rodata_objcopy.o $(obj)/lkdtm_rodata_objcopy.o: $(obj)/lkdtm_rodata.o FORCE $(call if_changed,objcopy) From c269ef409c756d9c19ee3f85d4275df62956e679 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 3 Dec 2018 20:58:05 +0100 Subject: [PATCH 014/688] arm64: relocatable: fix inconsistencies in linker script and options commit 3bbd3db86470c701091fb1d67f1fab6621debf50 upstream. readelf complains about the section layout of vmlinux when building with CONFIG_RELOCATABLE=y (for KASLR): readelf: Warning: [21]: Link field (0) should index a symtab section. readelf: Warning: [21]: Info field (0) should index a relocatable section. Also, it seems that our use of '-pie -shared' is contradictory, and thus ambiguous. In general, the way KASLR is wired up at the moment is highly tailored to how ld.bfd happens to implement (and conflate) PIE executables and shared libraries, so given the current effort to support other toolchains, let's fix some of these issues as well. - Drop the -pie linker argument and just leave -shared. In ld.bfd, the differences between them are unclear (except for the ELF type of the produced image [0]) but lld chokes on seeing both at the same time. - Rename the .rela output section to .rela.dyn, as is customary for shared libraries and PIE executables, so that it is not misidentified by readelf as a static relocation section (producing the warnings above). - Pass the -z notext and -z norelro options to explicitly instruct the linker to permit text relocations, and to omit the RELRO program header (which requires a certain section layout that we don't adhere to in the kernel). These are the defaults for current versions of ld.bfd. - Discard .eh_frame and .gnu.hash sections to avoid them from being emitted between .head.text and .text, screwing up the section layout. These changes only affect the ELF image, and produce the same binary image. [0] b9dce7f1ba01 ("arm64: kernel: force ET_DYN ELF type for ...") Cc: Nick Desaulniers Cc: Peter Smith Tested-by: Nick Desaulniers Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon Signed-off-by: Greg Kroah-Hartman Change-Id: I9fe6c4b09993a97051ea856a5053b220f2722872 --- arch/arm64/Makefile | 2 +- arch/arm64/kernel/vmlinux.lds.S | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 1ed7554ed08b..7513b23ea1dd 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -18,7 +18,7 @@ ifeq ($(CONFIG_RELOCATABLE), y) # Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour # for relative relocs, since this leads to better Image compression # with the relocation offsets always being zero. -LDFLAGS_vmlinux += -pie -shared -Bsymbolic \ +LDFLAGS_vmlinux += -shared -Bsymbolic -z notext -z norelro \ $(call ld-option, --no-apply-dynamic-relocs) endif diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index c1947a3067e8..010a2ca70b58 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -122,7 +122,8 @@ SECTIONS *(.discard) *(.discard.*) *(.interp .dynamic) - *(.dynsym .dynstr .hash) + *(.dynsym .dynstr .hash .gnu.hash) + *(.eh_frame) } . = KIMAGE_VADDR + TEXT_OFFSET; @@ -193,12 +194,12 @@ SECTIONS .altinstr_replacement : { KEEP(*(.altinstr_replacement)) } - .rela : ALIGN(8) { + .rela.dyn : ALIGN(8) { *(.rela .rela*) } - __rela_offset = ABSOLUTE(ADDR(.rela) - KIMAGE_VADDR); - __rela_size = SIZEOF(.rela); + __rela_offset = ABSOLUTE(ADDR(.rela.dyn) - KIMAGE_VADDR); + __rela_size = SIZEOF(.rela.dyn); . = ALIGN(SEGMENT_ALIGN); __init_end = .; From 53966441215ae2e1a550a56d98ebe4a3a645d052 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 16 Oct 2020 10:53:39 -0700 Subject: [PATCH 015/688] arm64: link with -z norelro regardless of CONFIG_RELOCATABLE commit 3b92fa7485eba16b05166fddf38ab42f2ff6ab95 upstream. With CONFIG_EXPERT=y, CONFIG_KASAN=y, CONFIG_RANDOMIZE_BASE=n, CONFIG_RELOCATABLE=n, we observe the following failure when trying to link the kernel image with LD=ld.lld: error: section: .exit.data is not contiguous with other relro sections ld.lld defaults to -z relro while ld.bfd defaults to -z norelro. This was previously fixed, but only for CONFIG_RELOCATABLE=y. Fixes: 3bbd3db86470 ("arm64: relocatable: fix inconsistencies in linker script and options") Signed-off-by: Nick Desaulniers Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20201016175339.2429280-1-ndesaulniers@google.com Signed-off-by: Will Deacon Signed-off-by: Greg Kroah-Hartman Change-Id: Idf6fa5186e771d62b969c371ae4b1b8d4bc154ea --- arch/arm64/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 7513b23ea1dd..3de0369f40f7 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -10,7 +10,7 @@ # # Copyright (C) 1995-2001 by Russell King -LDFLAGS_vmlinux :=--no-undefined -X +LDFLAGS_vmlinux :=--no-undefined -X -z norelro CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) GZFLAGS :=-9 @@ -18,7 +18,7 @@ ifeq ($(CONFIG_RELOCATABLE), y) # Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour # for relative relocs, since this leads to better Image compression # with the relocation offsets always being zero. -LDFLAGS_vmlinux += -shared -Bsymbolic -z notext -z norelro \ +LDFLAGS_vmlinux += -shared -Bsymbolic -z notext \ $(call ld-option, --no-apply-dynamic-relocs) endif From 5262ef9fd088b12b97a84a4ac1321f66fa573f90 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Tue, 12 Feb 2019 10:58:47 -0800 Subject: [PATCH 016/688] kbuild: allow lld to be used with CONFIG_LTO_CLANG Bug: 63740206 Bug: 117299373 Change-Id: Ic9c8ca03fd082a8404905718f5312a3f497efa5a Signed-off-by: Sami Tolvanen --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2f4fc4d309d7..5cd7a96c0da3 100644 --- a/Makefile +++ b/Makefile @@ -657,12 +657,14 @@ all: vmlinux # Make toolchain changes before including arch/$(SRCARCH)/Makefile to ensure # ar/cc/ld-* macros return correct values. ifdef CONFIG_LTO_CLANG +ifneq ($(ld-name),lld) # use GNU gold with LLVMgold for LTO linking, and LD for vmlinux_link LDFINAL_vmlinux := $(LD) LD := $(LDGOLD) LDFLAGS += -plugin LLVMgold.so LDFLAGS += -plugin-opt=-function-sections LDFLAGS += -plugin-opt=-data-sections +endif # use llvm-ar for building symbol tables from IR files, and llvm-dis instead # of objdump for processing symbol versions and exports LLVM_AR := llvm-ar @@ -1217,8 +1219,10 @@ ifdef CONFIG_LTO_CLANG ifneq ($(call clang-ifversion, -ge, 0500, y), y) @echo Cannot use CONFIG_LTO_CLANG: requires clang 5.0 or later >&2 && exit 1 endif - ifneq ($(call gold-ifversion, -ge, 112000000, y), y) - @echo Cannot use CONFIG_LTO_CLANG: requires GNU gold 1.12 or later >&2 && exit 1 + ifneq ($(ld-name),lld) + ifneq ($(call gold-ifversion, -ge, 112000000, y), y) + @echo Cannot use CONFIG_LTO_CLANG: requires GNU gold 1.12 or later >&2 && exit 1 + endif endif endif # Make sure compiler supports LTO flags From 5b7256082a6fdbfc747cae0706e852dd5b738cca Mon Sep 17 00:00:00 2001 From: UtsavBalar1231 Date: Tue, 22 Nov 2022 23:02:49 +0530 Subject: [PATCH 017/688] scripts: Use python rewrite in libfdt for mkdtimg --- scripts/Makefile.lib | 2 +- scripts/dtc/libfdt/mkdtboimg.py | 950 ++++++++++++++++++++++++++++++++ 2 files changed, 951 insertions(+), 1 deletion(-) create mode 100644 scripts/dtc/libfdt/mkdtboimg.py diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 8627e3da5af6..4711e1689032 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -329,7 +329,7 @@ dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp) # mkdtimg #---------------------------------------------------------------------------- quiet_cmd_mkdtimg = DTBOIMG $@ -cmd_mkdtimg = mkdtimg cfg_create $@ $< +cmd_mkdtimg = python2 $(srctree)/scripts/dtc/libfdt/mkdtboimg.py cfg_create $@ $< # cat # --------------------------------------------------------------------------- diff --git a/scripts/dtc/libfdt/mkdtboimg.py b/scripts/dtc/libfdt/mkdtboimg.py new file mode 100644 index 000000000000..e7ed21c9022b --- /dev/null +++ b/scripts/dtc/libfdt/mkdtboimg.py @@ -0,0 +1,950 @@ +#! /usr/bin/env python +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +"""Tool for packing multiple DTB/DTBO files into a single image""" + +import argparse +import os +from array import array +from collections import namedtuple +import struct +from sys import stdout +import zlib + +class CompressionFormat(object): + """Enum representing DT compression format for a DT entry. + """ + NO_COMPRESSION = 0x00 + ZLIB_COMPRESSION = 0x01 + GZIP_COMPRESSION = 0x02 + +class DtEntry(object): + """Provides individual DT image file arguments to be added to a DTBO. + Attributes: + _REQUIRED_KEYS: 'keys' needed to be present in the dictionary passed to instantiate + an object of this class. + _COMPRESSION_FORMAT_MASK: Mask to retrieve compression info for DT entry from flags field + when a DTBO header of version 1 is used. + """ + _COMPRESSION_FORMAT_MASK = 0x0f + REQUIRED_KEYS = ('dt_file', 'dt_size', 'dt_offset', 'id', 'rev', 'flags', + 'custom0', 'custom1', 'custom2') + + @staticmethod + def __get_number_or_prop(arg): + """Converts string to integer or reads the property from DT image. + Args: + arg: String containing the argument provided on the command line. + Returns: + An integer property read from DT file or argument string + converted to integer + """ + + if not arg or arg[0] == '+' or arg[0] == '-': + raise ValueError('Invalid argument passed to DTImage') + if arg[0] == '/': + # TODO(b/XXX): Use pylibfdt to get property value from DT + raise ValueError('Invalid argument passed to DTImage') + else: + base = 10 + if arg.startswith('0x') or arg.startswith('0X'): + base = 16 + elif arg.startswith('0'): + base = 8 + return int(arg, base) + + def __init__(self, **kwargs): + """Constructor for DtEntry object. + Initializes attributes from dictionary object that contains + values keyed with names equivalent to the class's attributes. + Args: + kwargs: Dictionary object containing values to instantiate + class members with. Expected keys in dictionary are from + the tuple (_REQUIRED_KEYS) + """ + + missing_keys = set(self.REQUIRED_KEYS) - set(kwargs) + if missing_keys: + raise ValueError('Missing keys in DtEntry constructor: %r' % + sorted(missing_keys)) + + self.__dt_file = kwargs['dt_file'] + self.__dt_offset = kwargs['dt_offset'] + self.__dt_size = kwargs['dt_size'] + self.__id = self.__get_number_or_prop(kwargs['id']) + self.__rev = self.__get_number_or_prop(kwargs['rev']) + self.__flags = self.__get_number_or_prop(kwargs['flags']) + self.__custom0 = self.__get_number_or_prop(kwargs['custom0']) + self.__custom1 = self.__get_number_or_prop(kwargs['custom1']) + self.__custom2 = self.__get_number_or_prop(kwargs['custom2']) + + def __str__(self): + sb = [] + sb.append('{key:>20} = {value:d}'.format(key='dt_size', + value=self.__dt_size)) + sb.append('{key:>20} = {value:d}'.format(key='dt_offset', + value=self.__dt_offset)) + sb.append('{key:>20} = {value:08x}'.format(key='id', + value=self.__id)) + sb.append('{key:>20} = {value:08x}'.format(key='rev', + value=self.__rev)) + sb.append('{key:>20} = {value:08x}'.format(key='custom[0]', + value=self.__flags)) + sb.append('{key:>20} = {value:08x}'.format(key='custom[1]', + value=self.__custom0)) + sb.append('{key:>20} = {value:08x}'.format(key='custom[2]', + value=self.__custom1)) + sb.append('{key:>20} = {value:08x}'.format(key='custom[3]', + value=self.__custom2)) + return '\n'.join(sb) + + def compression_info(self, version): + """CompressionFormat: compression format for DT image file. + Args: + version: Version of DTBO header, compression is only + supported from version 1. + """ + if version is 0: + return CompressionFormat.NO_COMPRESSION + return self.flags & self._COMPRESSION_FORMAT_MASK + + @property + def dt_file(self): + """file: File handle to the DT image file.""" + return self.__dt_file + + @property + def size(self): + """int: size in bytes of the DT image file.""" + return self.__dt_size + + @size.setter + def size(self, value): + self.__dt_size = value + + @property + def dt_offset(self): + """int: offset in DTBO file for this DT image.""" + return self.__dt_offset + + @dt_offset.setter + def dt_offset(self, value): + self.__dt_offset = value + + @property + def image_id(self): + """int: DT entry _id for this DT image.""" + return self.__id + + @property + def rev(self): + """int: DT entry _rev for this DT image.""" + return self.__rev + + @property + def flags(self): + """int: DT entry _flags for this DT image.""" + return self.__flags + + @property + def custom0(self): + """int: DT entry _custom0 for this DT image.""" + return self.__custom0 + + @property + def custom1(self): + """int: DT entry _custom1 for this DT image.""" + return self.__custom1 + + @property + def custom2(self): + """int: DT entry custom2 for this DT image.""" + return self.__custom2 + + +class Dtbo(object): + """ + Provides parser, reader, writer for dumping and creating Device Tree Blob + Overlay (DTBO) images. + Attributes: + _DTBO_MAGIC: Device tree table header magic. + _ACPIO_MAGIC: Advanced Configuration and Power Interface table header + magic. + _DT_TABLE_HEADER_SIZE: Size of Device tree table header. + _DT_TABLE_HEADER_INTS: Number of integers in DT table header. + _DT_ENTRY_HEADER_SIZE: Size of Device tree entry header within a DTBO. + _DT_ENTRY_HEADER_INTS: Number of integers in DT entry header. + _GZIP_COMPRESSION_WBITS: Argument 'wbits' for gzip compression + _ZLIB_DECOMPRESSION_WBITS: Argument 'wbits' for zlib/gzip compression + """ + + _DTBO_MAGIC = 0xd7b7ab1e + _ACPIO_MAGIC = 0x41435049 + _DT_TABLE_HEADER_SIZE = struct.calcsize('>8I') + _DT_TABLE_HEADER_INTS = 8 + _DT_ENTRY_HEADER_SIZE = struct.calcsize('>8I') + _DT_ENTRY_HEADER_INTS = 8 + _GZIP_COMPRESSION_WBITS = 31 + _ZLIB_DECOMPRESSION_WBITS = 47 + + def _update_dt_table_header(self): + """Converts header entries into binary data for DTBO header. + Packs the current Device tree table header attribute values in + metadata buffer. + """ + struct.pack_into('>8I', self.__metadata, 0, self.magic, + self.total_size, self.header_size, + self.dt_entry_size, self.dt_entry_count, + self.dt_entries_offset, self.page_size, + self.version) + + def _update_dt_entry_header(self, dt_entry, metadata_offset): + """Converts each DT entry header entry into binary data for DTBO file. + Packs the current device tree table entry attribute into + metadata buffer as device tree entry header. + Args: + dt_entry: DtEntry object for the header to be packed. + metadata_offset: Offset into metadata buffer to begin writing. + dtbo_offset: Offset where the DT image file for this dt_entry can + be found in the resulting DTBO image. + """ + struct.pack_into('>8I', self.__metadata, metadata_offset, dt_entry.size, + dt_entry.dt_offset, dt_entry.image_id, dt_entry.rev, + dt_entry.flags, dt_entry.custom0, dt_entry.custom1, + dt_entry.custom2) + + def _update_metadata(self): + """Updates the DTBO metadata. + Initialize the internal metadata buffer and fill it with all Device + Tree table entries and update the DTBO header. + """ + + self.__metadata = array('c', ' ' * self.__metadata_size) + metadata_offset = self.header_size + for dt_entry in self.__dt_entries: + self._update_dt_entry_header(dt_entry, metadata_offset) + metadata_offset += self.dt_entry_size + self._update_dt_table_header() + + def _read_dtbo_header(self, buf): + """Reads DTBO file header into metadata buffer. + Unpack and read the DTBO table header from given buffer. The + buffer size must exactly be equal to _DT_TABLE_HEADER_SIZE. + Args: + buf: Bytebuffer read directly from the file of size + _DT_TABLE_HEADER_SIZE. + """ + (self.magic, self.total_size, self.header_size, + self.dt_entry_size, self.dt_entry_count, self.dt_entries_offset, + self.page_size, self.version) = struct.unpack_from('>8I', buf, 0) + + # verify the header + if self.magic != self._DTBO_MAGIC and self.magic != self._ACPIO_MAGIC: + raise ValueError('Invalid magic number 0x%x in DTBO/ACPIO file' % + (self.magic)) + + if self.header_size != self._DT_TABLE_HEADER_SIZE: + raise ValueError('Invalid header size (%d) in DTBO/ACPIO file' % + (self.header_size)) + + if self.dt_entry_size != self._DT_ENTRY_HEADER_SIZE: + raise ValueError('Invalid DT entry header size (%d) in DTBO/ACPIO file' % + (self.dt_entry_size)) + + def _read_dt_entries_from_metadata(self): + """Reads individual DT entry headers from metadata buffer. + Unpack and read the DTBO DT entry headers from the internal buffer. + The buffer size must exactly be equal to _DT_TABLE_HEADER_SIZE + + (_DT_ENTRY_HEADER_SIZE * dt_entry_count). The method raises exception + if DT entries have already been set for this object. + """ + + if self.__dt_entries: + raise ValueError('DTBO DT entries can be added only once') + + offset = self.dt_entries_offset / 4 + params = {} + params['dt_file'] = None + for i in range(0, self.dt_entry_count): + dt_table_entry = self.__metadata[offset:offset + self._DT_ENTRY_HEADER_INTS] + params['dt_size'] = dt_table_entry[0] + params['dt_offset'] = dt_table_entry[1] + for j in range(2, self._DT_ENTRY_HEADER_INTS): + params[DtEntry.REQUIRED_KEYS[j + 1]] = str(dt_table_entry[j]) + dt_entry = DtEntry(**params) + self.__dt_entries.append(dt_entry) + offset += self._DT_ENTRY_HEADER_INTS + + def _read_dtbo_image(self): + """Parse the input file and instantiate this object.""" + + # First check if we have enough to read the header + file_size = os.fstat(self.__file.fileno()).st_size + if file_size < self._DT_TABLE_HEADER_SIZE: + raise ValueError('Invalid DTBO file') + + self.__file.seek(0) + buf = self.__file.read(self._DT_TABLE_HEADER_SIZE) + self._read_dtbo_header(buf) + + self.__metadata_size = (self.header_size + + self.dt_entry_count * self.dt_entry_size) + if file_size < self.__metadata_size: + raise ValueError('Invalid or truncated DTBO file of size %d expected %d' % + file_size, self.__metadata_size) + + num_ints = (self._DT_TABLE_HEADER_INTS + + self.dt_entry_count * self._DT_ENTRY_HEADER_INTS) + if self.dt_entries_offset > self._DT_TABLE_HEADER_SIZE: + num_ints += (self.dt_entries_offset - self._DT_TABLE_HEADER_SIZE) / 4 + format_str = '>' + str(num_ints) + 'I' + self.__file.seek(0) + self.__metadata = struct.unpack(format_str, + self.__file.read(self.__metadata_size)) + self._read_dt_entries_from_metadata() + + def _find_dt_entry_with_same_file(self, dt_entry): + """Finds DT Entry that has identical backing DT file. + Args: + dt_entry: DtEntry object whose 'dtfile' we find for existence in the + current 'dt_entries'. + Returns: + If a match by file path is found, the corresponding DtEntry object + from internal list is returned. If not, 'None' is returned. + """ + + dt_entry_path = os.path.realpath(dt_entry.dt_file.name) + for entry in self.__dt_entries: + entry_path = os.path.realpath(entry.dt_file.name) + if entry_path == dt_entry_path: + return entry + return None + + def __init__(self, file_handle, dt_type='dtb', page_size=None, version=0): + """Constructor for Dtbo Object + Args: + file_handle: The Dtbo File handle corresponding to this object. + The file handle can be used to write to (in case of 'create') + or read from (in case of 'dump') + """ + + self.__file = file_handle + self.__dt_entries = [] + self.__metadata = None + self.__metadata_size = 0 + + # if page_size is given, assume the object is being instantiated to + # create a DTBO file + if page_size: + if dt_type == 'acpi': + self.magic = self._ACPIO_MAGIC + else: + self.magic = self._DTBO_MAGIC + self.total_size = self._DT_TABLE_HEADER_SIZE + self.header_size = self._DT_TABLE_HEADER_SIZE + self.dt_entry_size = self._DT_ENTRY_HEADER_SIZE + self.dt_entry_count = 0 + self.dt_entries_offset = self._DT_TABLE_HEADER_SIZE + self.page_size = page_size + self.version = version + self.__metadata_size = self._DT_TABLE_HEADER_SIZE + else: + self._read_dtbo_image() + + def __str__(self): + sb = [] + sb.append('dt_table_header:') + _keys = ('magic', 'total_size', 'header_size', 'dt_entry_size', + 'dt_entry_count', 'dt_entries_offset', 'page_size', 'version') + for key in _keys: + if key == 'magic': + sb.append('{key:>20} = {value:08x}'.format(key=key, + value=self.__dict__[key])) + else: + sb.append('{key:>20} = {value:d}'.format(key=key, + value=self.__dict__[key])) + count = 0 + for dt_entry in self.__dt_entries: + sb.append('dt_table_entry[{0:d}]:'.format(count)) + sb.append(str(dt_entry)) + count = count + 1 + return '\n'.join(sb) + + @property + def dt_entries(self): + """Returns a list of DtEntry objects found in DTBO file.""" + return self.__dt_entries + + def compress_dt_entry(self, compression_format, dt_entry_file): + """Compresses a DT entry. + Args: + compression_format: Compression format for DT Entry + dt_entry_file: File handle to read DT entry from. + Returns: + Compressed DT entry and its length. + Raises: + ValueError if unrecognized compression format is found. + """ + compress_zlib = zlib.compressobj() # zlib + compress_gzip = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, + zlib.DEFLATED, self._GZIP_COMPRESSION_WBITS) # gzip + compression_obj_dict = { + CompressionFormat.NO_COMPRESSION: None, + CompressionFormat.ZLIB_COMPRESSION: compress_zlib, + CompressionFormat.GZIP_COMPRESSION: compress_gzip, + } + + if compression_format not in compression_obj_dict: + ValueError("Bad compression format %d" % compression_format) + + if compression_format is CompressionFormat.NO_COMPRESSION: + dt_entry = dt_entry_file.read() + else: + compression_object = compression_obj_dict[compression_format] + dt_entry_file.seek(0) + dt_entry = compression_object.compress(dt_entry_file.read()) + dt_entry += compression_object.flush() + return dt_entry, len(dt_entry) + + def add_dt_entries(self, dt_entries): + """Adds DT image files to the DTBO object. + Adds a list of Dtentry Objects to the DTBO image. The changes are not + committed to the output file until commit() is called. + Args: + dt_entries: List of DtEntry object to be added. + Returns: + A buffer containing all DT entries. + Raises: + ValueError: if the list of DT entries is empty or if a list of DT entries + has already been added to the DTBO. + """ + if not dt_entries: + raise ValueError('Attempted to add empty list of DT entries') + + if self.__dt_entries: + raise ValueError('DTBO DT entries can be added only once') + + dt_entry_count = len(dt_entries) + dt_offset = (self.header_size + + dt_entry_count * self.dt_entry_size) + + dt_entry_buf = "" + for dt_entry in dt_entries: + if not isinstance(dt_entry, DtEntry): + raise ValueError('Adding invalid DT entry object to DTBO') + entry = self._find_dt_entry_with_same_file(dt_entry) + dt_entry_compression_info = dt_entry.compression_info(self.version) + if entry and (entry.compression_info(self.version) + == dt_entry_compression_info): + dt_entry.dt_offset = entry.dt_offset + dt_entry.size = entry.size + else: + dt_entry.dt_offset = dt_offset + compressed_entry, dt_entry.size = self.compress_dt_entry(dt_entry_compression_info, + dt_entry.dt_file) + dt_entry_buf += compressed_entry + dt_offset += dt_entry.size + self.total_size += dt_entry.size + self.__dt_entries.append(dt_entry) + self.dt_entry_count += 1 + self.__metadata_size += self.dt_entry_size + self.total_size += self.dt_entry_size + + return dt_entry_buf + + def extract_dt_file(self, idx, fout, decompress): + """Extract DT Image files embedded in the DTBO file. + Extracts Device Tree blob image file at given index into a file handle. + Args: + idx: Index of the DT entry in the DTBO file. + fout: File handle where the DTB at index idx to be extracted into. + decompress: If a DT entry is compressed, decompress it before writing + it to the file handle. + Raises: + ValueError: if invalid DT entry index or compression format is detected. + """ + if idx > self.dt_entry_count: + raise ValueError('Invalid index %d of DtEntry' % idx) + + size = self.dt_entries[idx].size + offset = self.dt_entries[idx].dt_offset + self.__file.seek(offset, 0) + fout.seek(0) + compression_format = self.dt_entries[idx].compression_info(self.version) + if decompress and compression_format: + if (compression_format == CompressionFormat.ZLIB_COMPRESSION or + compression_format == CompressionFormat.GZIP_COMPRESSION): + fout.write(zlib.decompress(self.__file.read(size), self._ZLIB_DECOMPRESSION_WBITS)) + else: + raise ValueError("Unknown compression format detected") + else: + fout.write(self.__file.read(size)) + + def commit(self, dt_entry_buf): + """Write out staged changes to the DTBO object to create a DTBO file. + Writes a fully instantiated Dtbo Object into the output file using the + file handle present in '_file'. No checks are performed on the object + except for existence of output file handle on the object before writing + out the file. + Args: + dt_entry_buf: Buffer containing all DT entries. + """ + if not self.__file: + raise ValueError('No file given to write to.') + + if not self.__dt_entries: + raise ValueError('No DT image files to embed into DTBO image given.') + + self._update_metadata() + + self.__file.seek(0) + self.__file.write(self.__metadata) + self.__file.write(dt_entry_buf) + self.__file.flush() + + +def parse_dt_entry(global_args, arglist): + """Parse arguments for single DT entry file. + Parses command line arguments for single DT image file while + creating a Device tree blob overlay (DTBO). + Args: + global_args: Dtbo object containing global default values + for DtEntry attributes. + arglist: Command line argument list for this DtEntry. + Returns: + A Namespace object containing all values to instantiate DtEntry object. + """ + + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('dt_file', nargs='?', + type=argparse.FileType('rb'), + default=None) + parser.add_argument('--id', type=str, dest='id', action='store', + default=global_args.global_id) + parser.add_argument('--rev', type=str, dest='rev', + action='store', default=global_args.global_rev) + parser.add_argument('--flags', type=str, dest='flags', + action='store', + default=global_args.global_flags) + parser.add_argument('--custom0', type=str, dest='custom0', + action='store', + default=global_args.global_custom0) + parser.add_argument('--custom1', type=str, dest='custom1', + action='store', + default=global_args.global_custom1) + parser.add_argument('--custom2', type=str, dest='custom2', + action='store', + default=global_args.global_custom2) + return parser.parse_args(arglist) + + +def parse_dt_entries(global_args, arg_list): + """Parse all DT entries from command line. + Parse all DT image files and their corresponding attribute from + command line + Args: + global_args: Argument containing default global values for _id, + _rev and customX. + arg_list: The remainder of the command line after global options + DTBO creation have been parsed. + Returns: + A List of DtEntry objects created after parsing the command line + given in argument. + """ + dt_entries = [] + img_file_idx = [] + idx = 0 + # find all positional arguments (i.e. DT image file paths) + for arg in arg_list: + if not arg.startswith("--"): + img_file_idx.append(idx) + idx = idx + 1 + + if not img_file_idx: + raise ValueError('Input DT images must be provided') + + total_images = len(img_file_idx) + for idx in xrange(total_images): + start_idx = img_file_idx[idx] + if idx == total_images - 1: + argv = arg_list[start_idx:] + else: + end_idx = img_file_idx[idx + 1] + argv = arg_list[start_idx:end_idx] + args = parse_dt_entry(global_args, argv) + params = vars(args) + params['dt_offset'] = 0 + params['dt_size'] = os.fstat(params['dt_file'].fileno()).st_size + dt_entries.append(DtEntry(**params)) + + return dt_entries + +def parse_config_option(line, is_global, dt_keys, global_key_types): + """Parses a single line from the configuration file. + Args: + line: String containing the key=value line from the file. + is_global: Boolean indicating if we should parse global or DT entry + specific option. + dt_keys: Tuple containing all valid DT entry and global option strings + in configuration file. + global_key_types: A dict of global options and their corresponding types. It + contains all exclusive valid global option strings in configuration + file that are not repeated in dt entry options. + Returns: + Returns a tuple for parsed key and value for the option. Also, checks + the key to make sure its valid. + """ + + if line.find('=') == -1: + raise ValueError('Invalid line (%s) in configuration file' % line) + + key, value = (x.strip() for x in line.split('=')) + if is_global and key in global_key_types: + if global_key_types[key] is int: + value = int(value) + elif key not in dt_keys: + raise ValueError('Invalid option (%s) in configuration file' % key) + + return key, value + +def parse_config_file(fin, dt_keys, global_key_types): + """Parses the configuration file for creating DTBO image. + Args: + fin: File handle for configuration file + is_global: Boolean indicating if we should parse global or DT entry + specific option. + dt_keys: Tuple containing all valid DT entry and global option strings + in configuration file. + global_key_types: A dict of global options and their corresponding types. It + contains all exclusive valid global option strings in configuration + file that are not repeated in dt entry options. + Returns: + global_args, dt_args: Tuple of a dictionary with global arguments + and a list of dictionaries for all DT entry specific arguments the + following format. + global_args: + {'id' : , 'rev' : ...} + dt_args: + [{'filename' : 'dt_file_name', 'id' : , + 'rev' : ...}, + {'filename' : 'dt_file_name2', 'id' : , + 'rev' : ...}, ... + ] + """ + + # set all global defaults + global_args = dict((k, '0') for k in dt_keys) + global_args['dt_type'] = 'dtb' + global_args['page_size'] = 2048 + global_args['version'] = 0 + + dt_args = [] + found_dt_entry = False + count = -1 + for line in fin: + line = line.rstrip() + if line.lstrip().startswith('#'): + continue + comment_idx = line.find('#') + line = line if comment_idx == -1 else line[0:comment_idx] + if not line or line.isspace(): + continue + if line.startswith((' ', '\t')) and not found_dt_entry: + # This is a global argument + key, value = parse_config_option(line, True, dt_keys, global_key_types) + global_args[key] = value + elif line.find('=') != -1: + key, value = parse_config_option(line, False, dt_keys, global_key_types) + dt_args[-1][key] = value + else: + found_dt_entry = True + count += 1 + dt_args.append({}) + dt_args[-1]['filename'] = line.strip() + return global_args, dt_args + +def parse_create_args(arg_list): + """Parse command line arguments for 'create' sub-command. + Args: + arg_list: All command line arguments except the outfile file name. + Returns: + The list of remainder of the command line arguments after parsing + for 'create'. + """ + + image_arg_index = 0 + for arg in arg_list: + if not arg.startswith("--"): + break + image_arg_index = image_arg_index + 1 + + argv = arg_list[0:image_arg_index] + remainder = arg_list[image_arg_index:] + parser = argparse.ArgumentParser(prog='create', add_help=False) + parser.add_argument('--dt_type', type=str, dest='dt_type', + action='store', default='dtb') + parser.add_argument('--page_size', type=int, dest='page_size', + action='store', default=2048) + parser.add_argument('--version', type=int, dest='version', + action='store', default=0) + parser.add_argument('--id', type=str, dest='global_id', + action='store', default='0') + parser.add_argument('--rev', type=str, dest='global_rev', + action='store', default='0') + parser.add_argument('--flags', type=str, dest='global_flags', + action='store', default='0') + parser.add_argument('--custom0', type=str, dest='global_custom0', + action='store', default='0') + parser.add_argument('--custom1', type=str, dest='global_custom1', + action='store', default='0') + parser.add_argument('--custom2', type=str, dest='global_custom2', + action='store', default='0') + args = parser.parse_args(argv) + return args, remainder + +def parse_dump_cmd_args(arglist): + """Parse command line arguments for 'dump' sub-command. + Args: + arglist: List of all command line arguments including the outfile + file name if exists. + Returns: + A namespace object of parsed arguments. + """ + + parser = argparse.ArgumentParser(prog='dump') + parser.add_argument('--output', '-o', nargs='?', + type=argparse.FileType('wb'), + dest='outfile', + default=stdout) + parser.add_argument('--dtb', '-b', nargs='?', type=str, + dest='dtfilename') + parser.add_argument('--decompress', action='store_true', dest='decompress') + return parser.parse_args(arglist) + +def parse_config_create_cmd_args(arglist): + """Parse command line arguments for 'cfg_create subcommand. + Args: + arglist: A list of all command line arguments including the + mandatory input configuration file name. + Returns: + A Namespace object of parsed arguments. + """ + parser = argparse.ArgumentParser(prog='cfg_create') + parser.add_argument('conf_file', nargs='?', + type=argparse.FileType('rb'), + default=None) + cwd = os.getcwd() + parser.add_argument('--dtb-dir', '-d', nargs='?', type=str, + dest='dtbdir', default=cwd) + return parser.parse_args(arglist) + +def create_dtbo_image(fout, argv): + """Create Device Tree Blob Overlay image using provided arguments. + Args: + fout: Output file handle to write to. + argv: list of command line arguments. + """ + + global_args, remainder = parse_create_args(argv) + if not remainder: + raise ValueError('List of dtimages to add to DTBO not provided') + dt_entries = parse_dt_entries(global_args, remainder) + dtbo = Dtbo(fout, global_args.dt_type, global_args.page_size, global_args.version) + dt_entry_buf = dtbo.add_dt_entries(dt_entries) + dtbo.commit(dt_entry_buf) + fout.close() + +def dump_dtbo_image(fin, argv): + """Dump DTBO file. + Dump Device Tree Blob Overlay metadata as output and the device + tree image files embedded in the DTBO image into file(s) provided + as arguments + Args: + fin: Input DTBO image files. + argv: list of command line arguments. + """ + dtbo = Dtbo(fin) + args = parse_dump_cmd_args(argv) + if args.dtfilename: + num_entries = len(dtbo.dt_entries) + for idx in range(0, num_entries): + with open(args.dtfilename + '.{:d}'.format(idx), 'wb') as fout: + dtbo.extract_dt_file(idx, fout, args.decompress) + args.outfile.write(str(dtbo) + '\n') + args.outfile.close() + +def create_dtbo_image_from_config(fout, argv): + """Create DTBO file from a configuration file. + Args: + fout: Output file handle to write to. + argv: list of command line arguments. + """ + args = parse_config_create_cmd_args(argv) + if not args.conf_file: + raise ValueError('Configuration file must be provided') + + _DT_KEYS = ('id', 'rev', 'flags', 'custom0', 'custom1', 'custom2') + _GLOBAL_KEY_TYPES = {'dt_type': str, 'page_size': int, 'version': int} + + global_args, dt_args = parse_config_file(args.conf_file, + _DT_KEYS, _GLOBAL_KEY_TYPES) + params = {} + dt_entries = [] + for dt_arg in dt_args: + filepath = args.dtbdir + os.sep + dt_arg['filename'] + params['dt_file'] = open(filepath, 'rb') + params['dt_offset'] = 0 + params['dt_size'] = os.fstat(params['dt_file'].fileno()).st_size + for key in _DT_KEYS: + if key not in dt_arg: + params[key] = global_args[key] + else: + params[key] = dt_arg[key] + dt_entries.append(DtEntry(**params)) + + # Create and write DTBO file + dtbo = Dtbo(fout, global_args['dt_type'], global_args['page_size'], global_args['version']) + dt_entry_buf = dtbo.add_dt_entries(dt_entries) + dtbo.commit(dt_entry_buf) + fout.close() + +def print_default_usage(progname): + """Prints program's default help string. + Args: + progname: This program's name. + """ + sb = [] + sb.append(' ' + progname + ' help all') + sb.append(' ' + progname + ' help \n') + sb.append(' commands:') + sb.append(' help, dump, create, cfg_create') + print('\n'.join(sb)) + +def print_dump_usage(progname): + """Prints usage for 'dump' sub-command. + Args: + progname: This program's name. + """ + sb = [] + sb.append(' ' + progname + ' dump (