diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 4805a38c0db6685fef24835fb497f2e2aaf26f1c..eaf6fec447d1ffd757c0f0d77fb81fac607d42fe 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -145,7 +145,7 @@ from lisa.utils import nullcontext, Loggable, LISA_CACHE_HOME, checksum, DirCach from lisa._assets import ASSETS_PATH, HOST_PATH, ABI_BINARIES_FOLDER from lisa._unshare import ensure_root import lisa._git as git -from lisa.conf import SimpleMultiSrcConf, TopLevelKeyDesc, LevelKeyDesc, KeyDesc +from lisa.conf import SimpleMultiSrcConf, TopLevelKeyDesc, LevelKeyDesc, KeyDesc, VariadicLevelKeyDesc class KmodVersionError(Exception): @@ -596,7 +596,7 @@ class OverlayResource(abc.ABC): pass @abc.abstractmethod - def _get_checksum(self): + def _get_key(self): """ Return the checksum of the resource. """ @@ -648,7 +648,7 @@ class _PathOverlayBase(_FileOverlayBase): # This is racy with write_to(), but we are not trying to make something # really secure here, we just want to compute a unique token to be used as # a cache key - def _get_checksum(self): + def _get_key(self): with open(self.path, 'rb') as f: check = checksum(f, 'sha256') return f'{self.__class__.__name__}-{check}' @@ -689,16 +689,20 @@ class _CompressedPathFileOverlay(_PathOverlayBase): class _ContentFileOverlay(_FileOverlayBase): def __init__(self, content): + content = content.encode('utf-8') if isinstance(content, str) else content self.content = content def write_to(self, dst): with open(dst, 'wb') as f: f.write(self.content) - def _get_checksum(self): + def _get_key(self): check = checksum(io.BytesIO(self.content), 'sha256') return f'{self.__class__.__name__}-{check}' + def __str__(self): + return f'{self.__class__.__qualname__}({self.content})' + class TarOverlay(_PathOverlayBase): """ @@ -720,6 +724,32 @@ class TarOverlay(_PathOverlayBase): tar.extractall(dst) +class PatchOverlay(OverlayResource): + """ + Patch to be applied on an existing file. + + :param overlay: Overlay providing the content of the patch. + :type overlay: _FileOverlayBase + """ + def __init__(self, overlay): + self._overlay = overlay + + def write_to(self, dst): + with tempfile.NamedTemporaryFile(mode='w+t') as patch: + self._overlay.write_to(patch.name) + subprocess_log(['patch', '-p0', '-r', '-', '-u', '--forward', dst, patch.name]) + + def _get_key(self): + """ + Return the checksum of the resource. + """ + csum = self._overlay._get_key() + return f'{self.__class__.__name__}-{csum}' + + def __str__(self): + return f'{self.__class__.__qualname__}({self._overlay})' + + class _KernelBuildEnvConf(SimpleMultiSrcConf): STRUCTURE = TopLevelKeyDesc('kernel-build-env-conf', 'Build environment settings', ( @@ -736,6 +766,12 @@ class _KernelBuildEnvConf(SimpleMultiSrcConf): KeyDesc('overlay-backend', 'Backend to use for overlaying folders while building modules. Can be "overlayfs" (overlayfs filesystem, recommended and fastest) or "copy (plain folder copy)', [str]), KeyDesc('make-variables', 'Extra variables to pass to "make" command, such as "CC"', [typing.Dict[str, object]]), + + VariadicLevelKeyDesc('modules', 'modules settings', + LevelKeyDesc('', 'For each module. The module shipped by LISA is "lisa"', ( + KeyDesc('overlays', 'Overlays to apply to the sources of the given module', [typing.Dict[str, OverlayResource]]), + ) + )) ), ) @@ -744,6 +780,22 @@ class _KernelBuildEnvConf(SimpleMultiSrcConf): 'overlay-backend': 'overlayfs', } + def _get_key(self): + return ( + self.get('build-env'), + self.get('build-env-settings').to_map(), + sorted(self.get('make-variables', {}).items()), + ) + + def _get_key_for_kmod(self, kmod): + return ( + self._get_key(), + sorted( + (name, overlay._get_key()) + for name, overlay in self.get('modules', {}).get(kmod.mod_name, {}).get('overlays', {}).items() + ) + ) + class _KernelBuildEnv(Loggable, SerializeViaConstructor): """ @@ -1574,12 +1626,12 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): # unsuitable for compiling a module. key = ( sorted( - overlay._get_checksum() + overlay._get_key() for overlay, dst in overlays.items() ) + [ tree_key, str(cc), - build_conf, + build_conf._get_key(), ] ) @@ -1967,7 +2019,6 @@ class DynamicKmod(Loggable): :type kernel_build_env: lisa._kmod._KernelBuildEnv """ def __init__(self, target, src, kernel_build_env=None): - self.src = src self.target = target if not isinstance(kernel_build_env, _KernelBuildEnv): @@ -1978,6 +2029,30 @@ class DynamicKmod(Loggable): self._kernel_build_env = kernel_build_env + mod_name = src.mod_name + logger = self.logger + overlays = kernel_build_env.conf.get('modules', {}).get(mod_name, {}).get('overlays', {}) + + def apply_overlay(src, name, overlay): + try: + content = src.src[name] + except KeyError: + pass + else: + logger.debug(f'Applying patch to module {mod_name}, file {name}: {overlay}') + with tempfile.NamedTemporaryFile(suffix=name) as f: + path = Path(f.name) + path.write_bytes(content) + overlay.write_to(path) + src.src[name] = path.read_bytes() + + src = copy.deepcopy(src) + if overlays: + for name, overlay in overlays.items(): + apply_overlay(src, name, overlay) + + self.src = src + @property def mod_name(self): return self.src.mod_name @@ -2040,6 +2115,7 @@ class DynamicKmod(Loggable): return bin_ def _do_compile(self, make_vars=None): + kernel_build_env = self.kernel_build_env extra_make_vars = make_vars or {} all_make_vars = { @@ -2053,12 +2129,13 @@ class DynamicKmod(Loggable): if kernel_checksum is None: raise ValueError('kernel build env has no checksum') else: - return ( + key = ( kernel_checksum, - kernel_build_env.conf, + kernel_build_env.conf._get_key_for_kmod(self), src.checksum, all_make_vars, ) + return key def get_bin(kernel_build_env): return src.compile( diff --git a/lisa/conf.py b/lisa/conf.py index 4d65808da1a8d795795161fe7fde89b76ca0a958..882ed2b791ee7c52c182aaa251cb85f6904cd37e 100644 --- a/lisa/conf.py +++ b/lisa/conf.py @@ -121,7 +121,7 @@ class KeyDescBase(abc.ABC): to sanitize user input and generate help snippets used in various places. """ INDENTATION = 4 * ' ' - _VALID_NAME_PATTERN = r'^[a-zA-Z0-9-]+$' + _VALID_NAME_PATTERN = r'^[a-zA-Z0-9-<>]+$' def __init__(self, name, help): # pylint: disable=redefined-builtin @@ -134,7 +134,7 @@ class KeyDescBase(abc.ABC): @classmethod def _check_name(cls, name): if not re.match(cls._VALID_NAME_PATTERN, name): - raise ValueError(f'Invalid key name "{name}". Key names must match: {self._VALID_NAME_PATTERN}') + raise ValueError(f'Invalid key name "{name}". Key names must match: {cls._VALID_NAME_PATTERN}') @property def qualname(self): @@ -228,11 +228,19 @@ class KeyDesc(KeyDescBase): if self._newtype: return self._newtype else: + def filter_compo(compo): + return ''.join( + c + for c in compo + # Keep only characters allowed in identifiers + if c.isidentifier() + ).title() + compos = itertools.chain.from_iterable( x.split('-') for x in self.path[1:] ) - return ''.join(x.title() for x in compos) + return ''.join(map(filter_compo, compos)) def validate_val(self, val): """ @@ -662,6 +670,40 @@ class LevelKeyDesc(KeyDescBase, Mapping): return help_ +class _KeyMap(dict): + def __init__(self, child, content): + self._child = child + super().__init__(content) + + def __missing__(self, _): + return self._child + + +class VariadicLevelKeyDesc(LevelKeyDesc): + """ + Level key descriptor that allows configuration-source-defined sub-level keys. + + :param child: Variadic level. Its name will only be used for documentation + purposes, the configuration instances will be able to hold any string. + :type child: lisa.conf.LevelKeyDesc + + :Variable keyword arguments: Forwarded to :class:`lisa.conf.LevelKeyDesc`. + + This allows "instantiating" a whole sub-configuration for variable level + keys. + """ + def __init__(self, name, help, child, value_path=None): + super().__init__(name=name, help=help, children=[child], value_path=value_path) + + @property + def _child(self): + return self.children[0] + + @property + def _key_map(self): + return _KeyMap(self._child, super()._key_map) + + class DelegatedLevelKeyDesc(LevelKeyDesc): """ Level key descriptor that imports the keys from another @@ -1046,6 +1088,37 @@ class _HashableMultiSrcConf: return False +class _SubLevelMap(dict): + def __init__(self, conf): + self._conf = conf + + # Pre-hit existing known inner levels so that they will be listed when + # converting to a plain dictionary. + for key in conf._structure.keys(): + try: + self[key] + except KeyError: + pass + + def __missing__(self, key): + conf = self._conf + structure = conf._structure + key_desc = structure[key] + + if isinstance(key_desc, LevelKeyDesc): + # Build the tree of objects for nested configuration mappings, + # lazily so that we can accomodate VariadicLevelKeyDesc + new = conf._nested_new( + key_desc_path=key_desc.path, + src_prio=conf._src_prio, + parent=conf, + ) + self[key] = new + return new + else: + raise KeyError(f'No sublevel map for leaf key {key_desc.path} in {conf.__class__.__qualname__}') + + class MultiSrcConf(MultiSrcConfABC, Loggable, Mapping): """ Base class providing layered configuration management. @@ -1146,15 +1219,7 @@ class MultiSrcConf(MultiSrcConfABC, Loggable, Mapping): Hashable proxy, mostly designed to allow instance-oriented lookup in mappings. DO NOT USE IT FOR OTHER PURPOSES. You have been warned. """ - - # Build the tree of objects for nested configuration mappings - for key, key_desc in self._structure.items(): - if isinstance(key_desc, LevelKeyDesc): - self._sublevel_map[key] = self._nested_new( - key_desc_path=key_desc.path, - src_prio=self._src_prio, - parent=self, - ) + self._sublevel_map = _SubLevelMap(self) @property def _structure(self): diff --git a/lisa/target.py b/lisa/target.py index f6e4a0aff6f2e4d12fd8f3920152a5d1c466cfe3..d9f90ac83f986e9b8bda5fdcc65da76f8052ff0f 100644 --- a/lisa/target.py +++ b/lisa/target.py @@ -44,7 +44,7 @@ from devlib.platform.gem5 import Gem5SimulationPlatform from lisa.utils import Loggable, HideExekallID, resolve_dotted_name, get_subclasses, import_all_submodules, LISA_HOME, RESULT_DIR, LATEST_LINK, setup_logging, ArtifactPath, nullcontext, ExekallTaggable, memoized, destroyablecontextmanager, ContextManagerExit, update_params_from from lisa._assets import ASSETS_PATH -from lisa.conf import SimpleMultiSrcConf, KeyDesc, LevelKeyDesc, TopLevelKeyDesc, Configurable, DelegatedLevelKeyDesc +from lisa.conf import SimpleMultiSrcConf, KeyDesc, LevelKeyDesc, TopLevelKeyDesc, Configurable, DelegatedLevelKeyDesc, ConfigKeyError from lisa._kmod import _KernelBuildEnv, DynamicKmod, _KernelBuildEnvConf from lisa.platforms.platinfo import PlatformInfo @@ -699,6 +699,8 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): ) parser.add_argument("--conf", '-c', + action='append', + default=[], help="Path to a TargetConf and PlatformInfo yaml file. Other options will override what is specified in the file." ) @@ -748,18 +750,13 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): platform_info = None if args.conf: - # Tentatively load a PlatformInfo from the conf file - with contextlib.suppress(KeyError, ValueError): - platform_info = PlatformInfo.from_yaml_map(args.conf) - - # Load the TargetConf from the file, and update it with command - # line arguments try: - conf = TargetConf.from_yaml_map(args.conf) - except (KeyError, ValueError): - pass - else: - target_conf.add_src(args.conf, conf) + conf_map = SimpleMultiSrcConf.from_yaml_map_list(args.conf) + except ConfigKeyError as e: + parser.error(f'Could not load {" or ".join(args.conf)}: {e}') + + platform_info = conf_map.get(PlatformInfo) + target_conf = conf_map.get(TargetConf, target_conf) target_conf.add_src('command-line', { k: v for k, v in vars(args).items() diff --git a/lisa/utils.py b/lisa/utils.py index 854cba092276a8164b5bf35c8cef097d36e6b07a..48c9c39be4149a10a017831b56ec56f4abb4aac4 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -49,7 +49,7 @@ from pathlib import Path import importlib import pkgutil import operator -from operator import attrgetter +from operator import attrgetter, itemgetter import threading import itertools import weakref @@ -887,8 +887,20 @@ class Serializable( def _yaml_call_constructor(cls, loader, suffix, node): # Restrict to keyword arguments to have improve stability of # configuration files. - kwargs = loader.construct_mapping(node, deep=True) - return loader.make_python_instance(suffix, node, kwds=kwargs, newobj=False) + conf = loader.construct_mapping(node, deep=True) + + args = {} + kwargs = {} + for name, value in conf.items(): + if isinstance(name, int): + args[name] = value + else: + kwargs[name] = value + + if args: + _, args = zip(*sorted(args.items(), key=itemgetter(0))) + + return loader.make_python_instance(suffix, node, args=args, kwds=kwargs, newobj=False) # Allow !include to use relative paths from the current file. Since we # introduce a global state, we use thread-local storage. diff --git a/tools/lisa-whatsnew b/tools/lisa-whatsnew index 42d8646ded2242e47cd757c05c56578427694e21..410b977fba0e93a23b8d397d20b2766cfbf2a215 100755 --- a/tools/lisa-whatsnew +++ b/tools/lisa-whatsnew @@ -48,8 +48,8 @@ def main(): repo = args.repo since = args.since - since_date = git.git(repo, 'show', '-s', '--format=%ci', since).strip() - since_date_rel = git.git(repo, 'show', '-s', '--format=%cr', since).strip() + since_date = git.git(repo, 'show', '-s', '--format=%ci', since, '--').strip() + since_date_rel = git.git(repo, 'show', '-s', '--format=%cr', since, '--').strip() changelog = make_changelog( repo,