From f268ec4b2c42d11c1353e7a785cfd9a24f720076 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 17 Jul 2024 11:47:28 +0100 Subject: [PATCH 1/3] lisa._kmod: Fix toolchain-path support FIX Take into account toolchain-path config everywhere necessary and simplify env var flow. --- lisa/_kmod.py | 63 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 33f6ab740..745d935e8 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -254,15 +254,29 @@ def _url_path(url): ) -def _subprocess_log(*args, env=None, extra_env=None, **kwargs): - if env is None: - env = { - k: v - for k, v in os.environ.items() - if k in ('PATH', 'USER', 'TERM', 'TMPDIR') - } +def _filter_env(env): + allowed = { + 'PATH', + 'USER', + 'TERM', + 'TMPDIR', + # Allow any env var set inside our code that is not already in + # os.environ + *( + set(env.keys()) - set(os.environ.keys()) + ) + } + return { + k: v + for k, v in env.items() + if k in allowed + } + + +def _subprocess_log(*args, env=None, **kwargs): + env = env or os.environ + env = _filter_env(env) - env.update(extra_env or {}) with subprocess_detailed_excep(): return subprocess_log(*args, **kwargs, env=env) @@ -1151,6 +1165,7 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): def _prepare_tree(cls, path, cc, cross_compile, abi, build_conf, apply_overlays): logger = cls.get_logger() path = Path(path).resolve() + toolchain_env = cls._make_toolchain_env_from_conf(build_conf) def make(*targets): return _kbuild_make_cmd( @@ -1160,7 +1175,11 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): make_vars=build_conf.get('make-variables', {}), ) - def make_runner(cmd, allow_fail=False, **kwargs): + def make_runner(cmd, allow_fail=False, env=None, **kwargs): + _env = { + **toolchain_env, + **(env or {}), + } def runner(amend_cmd=lambda cmd: cmd): _cmd = amend_cmd(cmd) @@ -1169,6 +1188,7 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): _cmd, logger=logger, level=logging.DEBUG, + env=_env, **kwargs, ) except subprocess.CalledProcessError as e: @@ -1264,7 +1284,8 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): updated = join(updated) _path.write_bytes(updated) - if build_conf['build-env'] == 'alpine': + build_env = build_conf['build-env'] + if build_env == 'alpine': settings = build_conf['build-env-settings']['alpine'] version = settings.get('version', None) alpine_packages = settings.get('packages', None) @@ -1292,7 +1313,7 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): ] for runners in cmds ] - else: + elif build_env == 'host': @contextlib.contextmanager def cmd_cm(cmds): yield [ @@ -1302,6 +1323,8 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): ] for runners in cmds ] + else: + raise ValueError('Unknwon build-env kind: {build_env}') try: config_path = os.environ['KCONFIG_CONFIG'] @@ -1465,10 +1488,12 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): return {**os.environ, **env} @classmethod - def _make_toolchain_env_from_conf(cls, build_conf, env=None): + def _make_toolchain_env_from_conf(cls, build_conf): if build_conf['build-env'] == 'host': toolchain_path = build_conf['build-env-settings']['host'].get('toolchain-path') + env = {'PATH': HOST_PATH} else: + env = {} toolchain_path = None return cls._make_toolchain_env(toolchain_path, env=env) @@ -2377,8 +2402,8 @@ class KmodSrc(Loggable): mod_path=f'/{mod_path.relative_to(chroot)}', make_vars=make_vars, ) - yield (mod_path, _make_build_chroot_cmd(chroot, cmd), {}) - else: + yield (mod_path, _make_build_chroot_cmd(chroot, cmd)) + elif build_conf['build-env'] == 'host': @contextlib.contextmanager def cmd_cm(): with tempfile.TemporaryDirectory() as mod_path: @@ -2388,15 +2413,17 @@ class KmodSrc(Loggable): make_vars=make_vars, ) - env = _KernelBuildEnv._make_toolchain_env_from_conf(build_conf, env={'PATH': HOST_PATH}) - yield (mod_path, cmd, {'PATH': env['PATH']}) + yield (mod_path, cmd) + else: + raise ValueError('Unknwon build-env kind: {build_env}') - with cmd_cm() as (mod_path, cmd, env): + env = _KernelBuildEnv._make_toolchain_env_from_conf(build_conf) + with cmd_cm() as (mod_path, cmd): mod_path = Path(mod_path) populate_mod(mod_path) logger.info(f'Compiling kernel module {self.mod_name}') - _subprocess_log(cmd, logger=logger, level=logging.DEBUG, extra_env=env) + _subprocess_log(cmd, logger=logger, level=logging.DEBUG, env=env) mod_file = find_mod_file(mod_path) with open(mod_file, 'rb') as f: -- GitLab From c37648089ffb7da8b482986f473f87c5b02e7688 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 17 Jul 2024 12:08:21 +0100 Subject: [PATCH 2/3] lisa._kmod: Deal with tarfile API change FIX Deal with TarFile.extractall() raising a DeprecationWarning if "filter" is not specified on recent versions of Python. --- lisa/_kmod.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 745d935e8..ef02cee4a 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -134,6 +134,7 @@ from io import BytesIO from collections.abc import Mapping import typing import fnmatch +import sys from elftools.elf.elffile import ELFFile @@ -148,6 +149,15 @@ import lisa._git as git from lisa.conf import SimpleMultiSrcConf, TopLevelKeyDesc, LevelKeyDesc, KeyDesc, VariadicLevelKeyDesc from lisa._kallsyms import parse_kallsyms + +def _tar_extractall(f, *args, **kwargs): + # Avoid DeprecationWarning, see: + # https://docs.python.org/3/library/tarfile.html#extraction-filters + if sys.version_info[:2] >= (3, 12): + kwargs['filter'] = 'tar' + return f.extractall(*args, **kwargs) + + def _make_vars_cc(make_vars, default=None): try: cc = make_vars['CC'] @@ -493,7 +503,7 @@ def _make_alpine_chroot(version, packages=None, abi=None, bind_paths=None, overl shutil.copyfileobj(url, f) with tarfile.open(tar_path, 'r') as f: - f.extractall(path=path) + _tar_extractall(f, path=path) else: packages = [] @@ -885,8 +895,8 @@ class TarOverlay(_PathOverlayBase): return cls(path) def write_to(self, dst): - with tarfile.open(self.path) as tar: - tar.extractall(dst) + with tarfile.open(self.path) as f: + _tar_extractall(f, dst) class PatchOverlay(OverlayResource): @@ -2189,13 +2199,13 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): with response as url_f, open(tar_path, 'wb') as tar_f: shutil.copyfileobj(url_f, tar_f) - with tarfile.open(tar_path) as tar: + with tarfile.open(tar_path) as f: # Account for a top-level folder in the archive prefix = os.path.commonpath( member.path - for member in tar.getmembers() + for member in f.getmembers() ) - tar.extractall(extract_folder) + _tar_extractall(f, extract_folder) except Exception: with contextlib.suppress(FileNotFoundError): shutil.rmtree(extract_folder, ignore_errors=True) -- GitLab From e8630c68ee65e60a4f0cc6a6c7c2f162ed980f41 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 17 Jul 2024 12:14:33 +0100 Subject: [PATCH 3/3] lisa._kmod: Upgrade default alpine version to 3.20 FEATURE --- lisa/_kmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index ef02cee4a..13d41fe55 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -176,7 +176,7 @@ class KmodVersionError(Exception): pass -_ALPINE_DEFAULT_VERSION = '3.18.3' +_ALPINE_DEFAULT_VERSION = '3.20.0' _ALPINE_ROOTFS_URL = 'https://dl-cdn.alpinelinux.org/alpine/v{minor}/releases/{arch}/alpine-minirootfs-{version}-{arch}.tar.gz' _ALPINE_PACKAGE_INFO_URL = 'https://pkgs.alpinelinux.org/package/v{version}/{repo}/{arch}/{package}' -- GitLab