From 9c2839bf7ef4c62e66eaad5d32c98a2a31ab9455 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Fri, 24 Feb 2023 10:10:15 +0000 Subject: [PATCH 1/3] run: Gracefully handle malformed output from UART Previously, if an invalid unicode character was received on the UART, Python would raise an exception while trying to decode it to text. Rather than doing that, use the 'replace' policy, where bad characters are replaced with a standin. This issue was seen while running kselftests::kvm::get-reg-list, which outputs a bad character. Signed-off-by: Ryan Roberts --- shrinkwrap/utils/process.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/shrinkwrap/utils/process.py b/shrinkwrap/utils/process.py index 729bd94..d0d9d4f 100644 --- a/shrinkwrap/utils/process.py +++ b/shrinkwrap/utils/process.py @@ -121,7 +121,9 @@ class ProcessManager: # to correctly return the carriage for interactive # terminals. Telnet sometimes gives '\r\r\n' too, which # would be incorrectly translated to '\n\n'. - proc._stdout = io.TextIOWrapper(proc._stdout, newline='') + proc._stdout = io.TextIOWrapper(proc._stdout, + newline='', + errors='replace') # stdout and stderr get merged into pty, so can't tell # them apart. This isn't a problem for the emit build # warnings use case. @@ -132,7 +134,8 @@ class ProcessManager: stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + universal_newlines=True, + errors='replace') proc._stdin = None proc._stdout = proc._popen.stdout -- GitLab From e3c0512ad03c96b9d9a0a5f1562b101646a09259 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Fri, 24 Feb 2023 10:14:55 +0000 Subject: [PATCH 2/3] build: Fix dependency graph resolution for artifacts in build section Previously, ${artifact:} macros were only being looked for in the `params` section of a component. But the documentation says that they are also supported in prebuild/build/postbuild/clean. So if an artifact was directly consumed in one of those sections it would not be considered when building the dependency graph, and therefore there would be a race of production vs consumption of the artifact. Fixes: 79e7a2d ("process, utils: Implement the "resolveb" action for the process command.") Signed-off-by: Ryan Roberts --- shrinkwrap/utils/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 53766a3..6a67a8e 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -434,6 +434,18 @@ def resolveb(config, clivars={}): raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") artifacts.add(m['name']) + for scope in ['prebuild', 'build', 'postbuild', 'clean']: + for s in component[scope]: + for t in _string_tokenize(str(s)): + if t['type'] != 'macro': + continue + m = t['value'] + if m['type'] != 'artifact': + continue + if m['name'] is None: + raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") + artifacts.add(m['name']) + importers[name] = sorted(list(artifacts)) artifacts_exp = {} -- GitLab From 6e7e1b9ab8744862f93c9d7929c854370de49a84 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Fri, 24 Feb 2023 10:36:01 +0000 Subject: [PATCH 3/3] config: Add ability to build kselftests as part of linux-base.yaml By adding the following as a layer above linux-base.yaml: build: linux: prebuild: - export BUILD_KSELFTESTS=true kselftests will now be built, packaged as a tgz and made available in the package and with the ${artifact:KSELFTESTS} macro. Note that not all tests currently build cleanly; there are some x86-specific tests that (obviously) complain, and a few that complain about missing headers (e.g. sys/capability.h). As a result the output is rather noisy. But I think this is par for the course with kselftests. At least all the commonly used tests compile happily. Signed-off-by: Ryan Roberts --- config/linux-base.yaml | 20 +++++++++++++++++++- docker/Dockerfile.slim | 6 ++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/config/linux-base.yaml b/config/linux-base.yaml index 94fb4b1..a011990 100644 --- a/config/linux-base.yaml +++ b/config/linux-base.yaml @@ -13,16 +13,22 @@ description: >- prebuild list to modify the config (e.g. `./scripts/config ...`) or even replace the .config, as required. + A higher level layer can optionally request kselftests to be built by + specifying `export BUILD_KSELFTESTS=true` in the prebuild section. If + specified, the kselftests package is exported in a tgz archive as the + KSELFTESTS artifact. + build: linux: repo: remote: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git revision: v6.1 - toolchain: aarch64-none-elf- + toolchain: aarch64-linux-gnu- prebuild: - export BUILD_KMODULES=false + - export BUILD_KSELFTESTS=false - export ARCH=arm64 - make -j${param:jobs} O=${param:builddir} defconfig @@ -46,6 +52,17 @@ build: - touch ${param:builddir}/modules.tgz - fi + - if [ "$$BUILD_KSELFTESTS" = "true" ]; then + # Make kselftests and package into tgz archive. + - make -j${param:jobs} O=${param:builddir} headers_install + - make -j${param:jobs} O=${param:builddir} -C tools/testing/selftests install INSTALL_PATH=${param:builddir}/kselftests + - tar -caf ${param:builddir}/kselftests.tgz -C ${param:builddir}/kselftests . + - rm -rf ${param:builddir}/kselftests + - else + # Dummy kselftests archive to keep artifacts happy. + - touch ${param:builddir}/kselftests.tgz + - fi + clean: - export ARCH=arm64 - make -j${param:jobs} O=${param:builddir} clean @@ -53,3 +70,4 @@ build: artifacts: KERNEL: ${param:builddir}/arch/arm64/boot/Image KMODULES: ${param:builddir}/modules.tgz + KSELFTESTS: ${param:builddir}/kselftests.tgz diff --git a/docker/Dockerfile.slim b/docker/Dockerfile.slim index 7898a7d..867bdd5 100644 --- a/docker/Dockerfile.slim +++ b/docker/Dockerfile.slim @@ -87,6 +87,12 @@ RUN apt-get install --assume-yes --no-install-recommends --option=debug::pkgProb build-essential \ pkg-config +# Install packages requried by Linux. +RUN apt-get install --assume-yes --no-install-recommends --option=debug::pkgProblemResolver=yes \ + build-essential \ + fakeroot \ + rsync + # TODO: Install any packages required by U-Boot, OP-TEE, Trusty, etc. # Install the aarch64-linux-gnu- toolchain. We use the stock Debian packages for -- GitLab