From 5738f36bfe0863dfe688fcba857b10407aaf52c7 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 30 Sep 2022 12:49:09 +0100 Subject: [PATCH 01/10] tools/check-setuppy: Allow watching multiple patterns FEATURE Check setup.py but also check for new CLI tools as will require pip install to run again to create the shim in PATH. --- tools/check-setuppy | 48 ++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/tools/check-setuppy b/tools/check-setuppy index e2b769e13..c55fdc1d4 100755 --- a/tools/check-setuppy +++ b/tools/check-setuppy @@ -36,9 +36,9 @@ def main(): # eachother default_commit_file = os.path.join( ( - os.getenv('LISA_VENV_PATH') - if int(os.getenv('LISA_USE_VENV', '0')) - else os.getenv('LISA_HOME') + os.getenv('LISA_VENV_PATH') + if int(os.getenv('LISA_USE_VENV', '0')) else + os.getenv('LISA_HOME') ), '.lisa-install-commit' ) @@ -63,8 +63,14 @@ def main(): help='Update the commit file using sha1 of given git ref' ) + + import lisa._cli_tools + tools_path = lisa._cli_tools.__path__ + tools_path = tools_path if tools_path else [] + parser.add_argument('--filename-pattern', - default='*setup.py', + nargs='*', + default=['*setup.py', *tools_path], help='git rev-list filename pattern to check' ) @@ -84,17 +90,31 @@ def main(): # no setup already except FileNotFoundError: return 0 + else: + rev_range = '{}..{}'.format(recorded_sha1, args.current_rev) + + def check_pattern(pattern): + commits = call_git('rev-list', rev_range, '--', pattern).splitlines() + return commits + + commits = [ + (pattern, commit) + for pattern in args.filename_pattern + for commit in check_pattern(pattern) + ] + + if commits: + sep = '\n ' + commits = sep.join( + f'{pattern}: {commit}' + for pattern, commit in commits + ) + print(f'Files have been modified in the following commits, re-run lisa-install to update dependencies and install new CLI tools:{sep}{commits}') + print('THIS WILL RESET YOUR VENV !') + return 1 + else: + return 0 - rev_range = '{}..{}'.format(recorded_sha1, args.current_rev) - relevant_commits = call_git('rev-list', rev_range, '--', args.filename_pattern) - - if relevant_commits: - print('{} files have been modified in the following commits, re-run lisa-install to update dependencies:\n{}'.format( - args.filename_pattern, - relevant_commits, - )) - print('THIS WILL RESET YOUR VENV !') - return 1 if __name__ == '__main__': sys.exit(main()) -- GitLab From b2abee7b43582c0ad15667e5e51afcb5c87631ec Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 30 Sep 2022 12:09:52 +0100 Subject: [PATCH 02/10] lisa._kmod: Add DynamicKmod.mod_name property FEATURE Forward the KmodSrc mod_name on DynamicKmod. --- lisa/_kmod.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 549c8ee21..2c0a96523 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -1696,6 +1696,10 @@ class DynamicKmod(Loggable): self._kernel_tree = kernel_tree + @property + def mod_name(self): + return self.src.mod_name + @classmethod def from_target(cls, target, **kwargs): """ -- GitLab From ae76db58f31a2dbc58e2eaedfdec5d6ab0e67d7e Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 17:08:05 +0100 Subject: [PATCH 03/10] lisa.utils: Add destroyablecontextmanager FEATURE Very similar to contextlib.contextmanager but allows graceful handling of destruction without calling __exit__(). Since most context managers only use it with try/yield/finally pattern, this avoids running the "finally" clause in case of destruction without calling __exit__(), just like for context managers not created with contextlib.contextmanager(). --- lisa/utils.py | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/lisa/utils.py b/lisa/utils.py index e3582f0d8..2ba71819f 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -2771,6 +2771,108 @@ def ignore_exceps(exceps, cm, callback=None): yield x +class ContextManagerExit(Exception): + """ + Dummy exception raised in the generator wrapped by + :func:`destroyablecontextmanager` when anything else than + :exc:`GeneratorExit` happened during ``yield``. + """ + + +class ContextManagerExcep(ContextManagerExit): + """ + Exception raised when an exception was raised during ``yield`` in a context + manager created with :func:`destroyablecontextmanager`. + + The ``e`` attribute holds the original exception. + """ + def __init__(self, e): + self.e = e + + +class ContextManagerNoExcep(ContextManagerExit): + """ + Exception raised when no exception was raised during ``yield`` in a context + manager created with :func:`destroyablecontextmanager`. + """ + pass + + +class ContextManagerDestroyed(GeneratorExit): + """ + Exception raised in context managers created by + :func:`destroyablecontextmanager` when no exception was raised during + ``yield`` per say but the context manager was destroyed without calling + ``__exit__``. + """ + + +class _DestroyableCM: + def __init__(self, f): + self._f = f + self._cm = None + + @staticmethod + def _wrap_gen(gen): + e = None + res = gen.send(None) + while True: + try: + yield res + except GeneratorExit: + e = ContextManagerDestroyed() + except BaseException as _e: + e = ContextManagerExcep(_e) + else: + e = ContextManagerNoExcep() + + try: + res = gen.throw(e) + except StopIteration as _e: + return _e.value + + def __enter__(self): + cm = contextlib.contextmanager(lambda: self._wrap_gen(self._f()))() + self._cm = cm + return cm.__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + cm = self._cm + try: + ret = cm.__exit__(exc_type, exc_value, traceback) + finally: + self._cm = None + + return ret + + +def destroyablecontextmanager(f): + """ + Similar to :func:`contextlib.contextmanager` but treats all cases of + ``yield`` as an exception. + + This forces the user to handle them as such, and makes it more apparent + that the ``finally`` clause in ``try/yield/finally`` also catches the case + where the context manager is simply destroyed. + + The user can handle :exc:`ContextManagerExit` to run cleanup code + regardless of exceptions but not when context manager is simply destroyed + without calling ``__exit__()`` (standard behavior of context manager not + created with :func:`contextlib.contextmanager`). + + Handling exceptions is achieved by handling :exc:`ContextManagerExcep`, + with the original exception stored in the ``e`` attribute. + + Handling destruction is achieved with :exc:`ContextManagerDestroyed`. + """ + @functools.wraps(f) + def wrapper(*args, **kwargs): + _f = functools.partial(f, *args, **kwargs) + return _DestroyableCM(_f) + + return wrapper + + class ExekallTaggable: """ Allows tagging the objects produced in exekall expressions ID. -- GitLab From f12d7670bb7791d4bc76450a8bdb0b4656370fa3 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 17:20:19 +0100 Subject: [PATCH 04/10] lisa: Use lisa.utils.destroyablecontextmanager FIX Equivalent to contextlib.contextmanager except that this pattern will not run the cleanup code when context manager is destroyed without calling __exit__() (explicitly or via "with" statement). The usual pattern will run the cleanup code even if __exit__() is not called, as destructing the generator will raise a GeneratorExit in it no matter what: @contextlib.contextmanager def f(): try: yield finally: cleanup() This version will avoid that issue and will therefore behave like context managers directly defined with of __enter__() and __exit__(): @destroyablecontextmanager def f(): try: yield except ContextManagerExit: cleanup() This allows the final user of the context manager to skip cleanup if they want to, which might be useful in some cases. --- lisa/_kmod.py | 20 ++++++++++---------- lisa/notebook.py | 7 +++---- lisa/target.py | 10 +++++----- lisa/wlgen/rta.py | 7 ++++--- lisa/wlgen/workload.py | 6 +++--- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 2c0a96523..3460f0af1 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -139,7 +139,7 @@ from devlib.target import KernelVersion, TypedKernelConfig, KernelConfigTristate from devlib.host import LocalConnection from devlib.exception import TargetStableError -from lisa.utils import nullcontext, Loggable, LISA_CACHE_HOME, checksum, DirCache, chain_cm, memoized, LISA_HOST_ABI, subprocess_log, SerializeViaConstructor +from lisa.utils import nullcontext, Loggable, LISA_CACHE_HOME, checksum, DirCache, chain_cm, memoized, LISA_HOST_ABI, subprocess_log, SerializeViaConstructor, destroyablecontextmanager, ContextManagerExit from lisa._assets import ASSETS_PATH, HOST_PATH from lisa._unshare import ensure_root import lisa._git as git @@ -174,7 +174,7 @@ def _subprocess_log(*args, env=None, extra_env=None, **kwargs): return subprocess_log(*args, **kwargs, env=env) -@contextlib.contextmanager +@destroyablecontextmanager def _make_chroot(make_vars, bind_paths=None, alpine_version='3.16.0', overlay_backend=None): """ Create a chroot folder ready to be used to build a kernel. @@ -319,7 +319,7 @@ def _make_chroot(make_vars, bind_paths=None, alpine_version='3.16.0', overlay_ba populate(key, path, init_cache=False) mount_binds(path, bind_paths) yield path - finally: + except ContextManagerExit: mount_binds(path, bind_paths, mount=False) @@ -332,7 +332,7 @@ def _make_chroot_cmd(chroot, cmd): return ['chroot', chroot, 'sh', '-c', cmd] -@contextlib.contextmanager +@destroyablecontextmanager def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): """ Overlay folders on top of each other. @@ -391,7 +391,7 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): upper=make_dir(temp, 'upper'), ) - @contextlib.contextmanager + @destroyablecontextmanager def do_mount(dirs): dirs['lower'] = ':'.join(map(str, reversed(list(lowers)))) cmd = ['mount', '-t', 'overlay', 'overlay', '-o', 'lowerdir={lower},workdir={work},upperdir={upper}'.format(**dirs), '--', mount_point] @@ -399,7 +399,7 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): try: yield mount_point - finally: + except ContextManagerExit: # Use lazy unmount, so it will not fail if it still in use for # some reason. That said, all supporting folders are going to # be removed so an external user working outside of the "with" @@ -414,7 +414,7 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): if copy_filter is None: copy_filter = lambda src, dst: True - @contextlib.contextmanager + @destroyablecontextmanager def do_copy(dirs): def _python_copytree(src, dst): base_src = Path(src) @@ -481,7 +481,7 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): try: yield mount_point - finally: + except ContextManagerExit: # If the user selected a custom upper layer, sync back the # result in it if upper: @@ -1863,7 +1863,7 @@ class DynamicKmod(Loggable): """ self.target.execute(f'rmmod {quote(self.src.mod_name)}') - @contextlib.contextmanager + @destroyablecontextmanager def run(self, **kwargs): """ Context manager used to run the module by loading it then unloading it. @@ -1878,7 +1878,7 @@ class DynamicKmod(Loggable): x = self.install(**kwargs) try: yield x - finally: + except ContextManagerExit: self.uninstall() diff --git a/lisa/notebook.py b/lisa/notebook.py index 12a3662a3..650ad3bca 100644 --- a/lisa/notebook.py +++ b/lisa/notebook.py @@ -23,7 +23,6 @@ import functools import collections import warnings import importlib -import contextlib import inspect from uuid import uuid4 from itertools import starmap @@ -42,7 +41,7 @@ from cycler import cycler as make_cycler from ipywidgets import widgets, Layout, interact from IPython.display import display -from lisa.utils import is_running_ipython, order_as +from lisa.utils import is_running_ipython, order_as, destroyablecontextmanager, ContextManagerExit pn.extension('tabulator') @@ -540,7 +539,7 @@ def _hv_multi_line_title(fig): return fig.options(hooks=[_hv_multi_line_title_hook]) -@contextlib.contextmanager +@destroyablecontextmanager def _hv_set_backend(backend): """ Context manager to work around this issue: @@ -553,7 +552,7 @@ def _hv_set_backend(backend): # at import time hv.Store.set_current_backend(backend) yield - finally: + except ContextManagerExit: if old_backend: hv.Store.set_current_backend(old_backend) diff --git a/lisa/target.py b/lisa/target.py index ef1bbbd38..6966b9363 100644 --- a/lisa/target.py +++ b/lisa/target.py @@ -40,7 +40,7 @@ from devlib.exception import TargetStableError from devlib.utils.misc import which from devlib.platform.gem5 import Gem5SimulationPlatform -from lisa.utils import Loggable, HideExekallID, resolve_dotted_name, get_subclasses, import_all_submodules, LISA_HOME, RESULT_DIR, LATEST_LINK, setup_logging, ArtifactPath, nullcontext, ExekallTaggable, memoized +from lisa.utils import Loggable, HideExekallID, resolve_dotted_name, get_subclasses, import_all_submodules, LISA_HOME, RESULT_DIR, LATEST_LINK, setup_logging, ArtifactPath, nullcontext, ExekallTaggable, memoized, destroyablecontextmanager, ContextManagerExit from lisa._assets import ASSETS_PATH from lisa.conf import SimpleMultiSrcConf, KeyDesc, LevelKeyDesc, TopLevelKeyDesc,Configurable from lisa._generic import TypedList, TypedDict @@ -939,12 +939,12 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): if isinstance(self.target, devlib.LocalLinuxTarget): exclude.append(str(os.getpid())) - @contextlib.contextmanager + @destroyablecontextmanager def cm(): logger.info(f"Freezing all tasks except: {','.join(exclude)}") try: yield self.cgroups.freeze(exclude) - finally: + except ContextManagerExit: logger.info('Un-freezing userspace tasks') self.cgroups.freeze(thaw=True) @@ -963,13 +963,13 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): logger.warning('Could not disable idle states, cpuidle devlib module is not loaded') cm = nullcontext else: - @contextlib.contextmanager + @destroyablecontextmanager def cm(): try: for cpu in range(self.plat_info['cpus-count']): cpuidle.disable_all(cpu) yield - finally: + except ContextManagerExit: logger.info('Re-enabling idle states for all domains') for cpu in range(self.plat_info['cpus-count']): cpuidle.enable_all(cpu) diff --git a/lisa/wlgen/rta.py b/lisa/wlgen/rta.py index c3e36ee8e..4d44e1c11 100644 --- a/lisa/wlgen/rta.py +++ b/lisa/wlgen/rta.py @@ -193,7 +193,6 @@ from itertools import chain, product, starmap, islice from operator import itemgetter from shlex import quote from statistics import mean -import contextlib from devlib import TargetStableError from devlib.target import KernelConfigTristate @@ -222,6 +221,8 @@ from lisa.utils import ( kwargs_dispatcher, kwargs_forwarded_to, PartialInit, + destroyablecontextmanager, + ContextManagerExit, ) from lisa.wlgen.workload import Workload from lisa.conf import DeferredValueComputationError @@ -743,7 +744,7 @@ class RTA(Workload): self._late_init(conf=conf) return self - @contextlib.contextmanager + @destroyablecontextmanager def _setup(self): logger = self.logger plat_info = self.target.plat_info @@ -809,7 +810,7 @@ class RTA(Workload): try: with capa_cm: yield - finally: + except ContextManagerExit: target.remove(self.remote_json) if self.log_stats: diff --git a/lisa/wlgen/workload.py b/lisa/wlgen/workload.py index 38c596ad2..d74e4b204 100644 --- a/lisa/wlgen/workload.py +++ b/lisa/wlgen/workload.py @@ -30,7 +30,7 @@ import inspect from devlib.utils.misc import list_to_mask -from lisa.utils import ArtifactPath, Loggable, PartialInit, deprecate +from lisa.utils import ArtifactPath, Loggable, PartialInit, deprecate, destroyablecontextmanager, ContextManagerExit class _WorkloadRunCMDecorator: @@ -353,7 +353,7 @@ class Workload(_WorkloadBase, PartialInit, Loggable): def _deployed(self): return self._setup_cm is not None - @contextlib.contextmanager + @destroyablecontextmanager def _setup(self): """ Context manager function called to setup the target before the @@ -375,7 +375,7 @@ class Workload(_WorkloadBase, PartialInit, Loggable): self.target.install_tools(self.REQUIRED_TOOLS) try: yield - finally: + except ContextManagerExit: if self._wipe_run_dir: self.wipe_run_dir() -- GitLab From 78a149293604a68667f2d1a8693999422598f2a2 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 15:39:43 +0100 Subject: [PATCH 05/10] doc/conf.py: Fix holoviews intersphinx inventory location FIX --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index df4130764..031d33020 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -355,7 +355,7 @@ intersphinx_mapping = { 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'matplotlib': ('https://matplotlib.org/stable/', None), 'numpy': ('https://numpy.org/doc/stable/', None), - 'holoviews': ('http://holoviews.org/', None), + 'holoviews': ('https://holoviews.org/', None), # XXX: Doesn't seem to work, might be due to how devlib doc is generated 'devlib': ('https://devlib.readthedocs.io/en/latest/', None), 'wa': ('https://workload-automation.readthedocs.io/en/latest/', None), -- GitLab From 8462750f8f9abe4bebf0d2f3a2539afe72356385 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 14:46:25 +0100 Subject: [PATCH 06/10] lisa.target: Add Target.from_custom_cli(description=...) FEATURE Allow passing custom CLI tool description to argparse. --- lisa/target.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lisa/target.py b/lisa/target.py index 6966b9363..08a621a28 100644 --- a/lisa/target.py +++ b/lisa/target.py @@ -588,7 +588,7 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): return target @classmethod - def from_custom_cli(cls, argv=None, params=None): + def from_custom_cli(cls, argv=None, params=None, description=None): """ Create a Target from command line arguments. @@ -601,14 +601,18 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): ``{param_name: {dict of ArgumentParser.add_argument() options}}``. :type params: dict(str, dict) + :param description: Description passed to the argument parser. If + ``None``, a default one is provided. + :type description: str or None + :return: A tuple ``(args, target)`` .. note:: This method should not be relied upon to implement long-term scripts, it's more designed for quick scripting. """ - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( + + if description is None: + description = textwrap.dedent( """ Connect to a target using the provided configuration in order to run a test. @@ -628,7 +632,13 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): code execution. """.format( script=os.path.basename(sys.argv[0]) - ))) + ) + ) + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=description, + ) parser.add_argument("--conf", '-c', help="Path to a TargetConf and PlatformInfo yaml file. Other options will override what is specified in the file." -- GitLab From 496ef993b20cf87840825bf62e9cf096854765a8 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 14:51:13 +0100 Subject: [PATCH 07/10] lisa.trace: Allow CollectorBase.get_data(path=None) FEATURE If path=None, use the output_path specified to __init__. --- lisa/trace.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lisa/trace.py b/lisa/trace.py index 6abc5fc52..64b27e5b3 100644 --- a/lisa/trace.py +++ b/lisa/trace.py @@ -5415,19 +5415,25 @@ class CollectorBase(Loggable): # How did we get some coconuts ? swallow = self._collector.__exit__(*args, **kwargs) - path = self._output_path - if path is not None: - self.get_data(path) + try: + self.get_data() + except ValueError: + pass return swallow - def get_data(self, path): + def get_data(self, path=None): """ Similar to :meth:`devlib.collector.CollectorBase.get_data` but takes the path directly as a parameter in order to disallow representing an invalid state where no path has been set. """ coll = self._collector + path = path or self._output_path + + if path is None: + raise ValueError('Path cannot be None') + coll.set_output(path) return coll.get_data() -- GitLab From 44278abeec90aa5f5c506fde2601e243f8122d74 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 15:16:25 +0100 Subject: [PATCH 08/10] lisa.target: Fix nargs=argparse.REMAINDER in Target.from_custom_cli() FIX Do not add a leading -- to the parameter name, so that it stays a positional parameter. --- lisa/target.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lisa/target.py b/lisa/target.py index 08a621a28..82fb3539d 100644 --- a/lisa/target.py +++ b/lisa/target.py @@ -669,7 +669,12 @@ class Target(Loggable, HideExekallID, ExekallTaggable, Configurable): params = params or {} for param, settings in params.items(): - parser.add_argument(f'--{param}', **settings) + if settings.get('nargs') == argparse.REMAINDER: + _param = param + else: + _param = f'--{param}' + parser.add_argument(_param, **settings) + custom_params = {k.replace('-', '_') for k in params.keys()} # Options that are not a key in TargetConf must be listed here -- GitLab From a73c27a7cba8ca9fc0b44739b8ce4938ae4f6455 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 15:07:42 +0100 Subject: [PATCH 09/10] lisa._cli_tools: Add lisa_load_kmod.py Add command line tool to compile and load LISA's kernel module using the facilites of the Python API (such as Alpine container build). --- lisa/_cli_tools/lisa_load_kmod.py | 118 ++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100755 lisa/_cli_tools/lisa_load_kmod.py diff --git a/lisa/_cli_tools/lisa_load_kmod.py b/lisa/_cli_tools/lisa_load_kmod.py new file mode 100755 index 000000000..f3123a6db --- /dev/null +++ b/lisa/_cli_tools/lisa_load_kmod.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python3 + +import argparse +import subprocess +import sys +import time +import contextlib +import textwrap +import logging +import tempfile +import shlex + +from lisa.target import Target +from lisa.trace import DmesgCollector +from lisa._kmod import LISAFtraceDynamicKmod +from lisa.utils import ignore_exceps + +def main(): + params = { + 'feature': dict( + action='append', + help='Enable a specific module feature. Can be repeated. By default, the module will try to enable all features and will log in dmesg the ones that failed to enable' + ), + 'cmd': dict( + nargs=argparse.REMAINDER, + help='Load the module, run the given command then unload the module. If not command is provided, just load the module and exit.' + ) + + } + + args, target = Target.from_custom_cli( + description=textwrap.dedent(''' + Compile and load the LISA kernel module, then unloads it when the + command finishes. + + If no command is passed, it will simply compile and load the module then return. + + EXAMPLES + + $ lisa-load-kmod --conf target_conf.yml -- echo hello world + + '''), + params=params, + ) + + features = args.feature + keep_loaded = not bool(args.cmd) + + cmd = args.cmd or [] + if cmd and cmd[0] == '--': + cmd = cmd[1:] + + kmod_params = {} + if features is not None: + kmod_params['features'] = list(features) + + kmod = target.get_kmod(LISAFtraceDynamicKmod) + _kmod_cm = kmod.run(kmod_params=kmod_params) + + if keep_loaded: + @contextlib.contextmanager + def cm(): + logging.info('Compiling and loading kernel module ...') + yield _kmod_cm.__enter__() + logging.info(f'Loaded kernel module as "{kmod.mod_name}"') + else: + @contextlib.contextmanager + def cm(): + with _kmod_cm: + logging.info('Compiling and loading kernel module ...') + try: + yield + finally: + logging.info('Unloading kernel module') + kmod_cm = cm() + + @contextlib.contextmanager + def dmesg_cm(): + with tempfile.NamedTemporaryFile() as f: + dmesg_path = f.name + coll = DmesgCollector(target, output_path=dmesg_path) + + def log_err(when, cm, excep): + logging.error(f'Encounted exceptions while {when}ing dmesg collector: {excep}') + + coll = ignore_exceps(Exception, coll, log_err) + + with coll as coll: + yield + + if coll: + dmesg_entries = [ + entry + for entry in coll.entries + if entry.msg.startswith(kmod.mod_name) + ] + if dmesg_entries: + sep = '\n ' + dmesg = sep.join(map(str, dmesg_entries)) + logging.info(f'Module dmesg output:{sep}{dmesg}') + + def run_cmd(): + if cmd: + pretty_cmd = ' '.join(map(shlex.quote, cmd)) + logging.info(f'Running command: {pretty_cmd}') + return subprocess.run(cmd).returncode + else: + return 0 + + + with dmesg_cm(), kmod_cm: + ret = run_cmd() + + return ret + + +if __name__ == '__main__': + sys.exit(main()) -- GitLab From 19158d95b3e103caf8ae83e994e418bbfd065baa Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 28 Sep 2022 15:39:10 +0100 Subject: [PATCH 10/10] doc/setup.rst: Update kernel module setup section Document the use of lisa-load-kmod --- doc/setup.rst | 54 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/doc/setup.rst b/doc/setup.rst index 824b56dc4..e339ad044 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -190,11 +190,6 @@ shell :ref:`buildroot-commands`. Kernel modules -------------- -The following modules are required to run Lisa tests against some kernels. - -sched_tp -........ - From Linux v5.3, sched_load_cfs_rq and sched_load_se tracepoints are present in mainline as bare tracepoints without any events in tracefs associated with them. @@ -203,8 +198,8 @@ To help expose these tracepoints (and any additional one we might require in the future) as trace events, an external module is required and is provided under the name of sched_tp in $LISA_HOME/tools/kmodules/sched_tp -Building a module ------------------ +Enabling a module +................. LISA Python package will compile and load the module automatically when required for tracing so there is usually no reason to do so manually. The most reliable @@ -243,11 +238,42 @@ way to configure LISA for building the module is: # system depending on config), this should do the trick. # overlay-backend: copy -In case this is still required, the process is standard Linux external module -build step. Helper scripts are provides too. +Automatic route +............... + +Once the kernel and LISA's target have been configured appropriately, the Python +API will build and load the module automatically as required (e.g. when ftrace +events provided by the module are required). + +In order to improve interoperation with other systems, a CLI tool is also +provided to load the module easily: + + .. code-block:: sh + + # Compile and load the module. + lisa-load-kmod --conf target_conf.yml + + # Runs "echo hello world" with the module loaded, then unloads it. + lisa-load-kmod --conf target_conf.yml -- echo hello world + + # See # lisa-load-kmod --help for more options. + + +.. note:: The module name may be different if it was compiled manually vs + compiled via the Python interface due to backward compatiblity + constraints. + + +Manual route +............ + + +In case this is still required, the module is a fairly standard Linux 3rd party +module that can be built as such following the official kernel doc. Helper +scripts are provides too. Build -..... +~~~~~ .. code-block:: sh @@ -264,7 +290,7 @@ This will build the module against the provided kernel tree and install it in could be reused in fresh builds, leading to segfaults and such. Clean -..... +~~~~~ .. code-block:: sh @@ -274,7 +300,7 @@ Highly recommended to clean when switching kernel trees to avoid unintentional breakage for using stale binaries. Pushing the module into the target -.................................. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You need to push the module into your rootfs either by installing it directly there or use commands like ``scp`` to copy it into your device. @@ -284,7 +310,7 @@ there or use commands like ``scp`` to copy it into your device. scp -r /path/to/sched_tp.ko username@ip:/ Loading the module -.................. +~~~~~~~~~~~~~~~~~~ On the target run: @@ -293,7 +319,7 @@ On the target run: modprobe sched_tp Integrating the module in your kernel tree ------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you're rebuilding your kernel tree anyway, it might be easier to integrate the module into your kernel tree as a built-in module so that it's always -- GitLab