From aec7153dfd708bf19033a78a76992e74f50fbdcd Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Tue, 4 Feb 2025 18:57:17 +0000 Subject: [PATCH] lisa.utils: Add DirCache(fmt_version=...) parameter FEATURE Allow each DirCache to specify their own policy for versioning the format of the data they host. This way, a DirCache does not have to be invalidated at each LISA update. Instead, if the format of that category has not changed, it can be re-used, avoiding costly re-population. --- lisa/_kmod.py | 6 +++++- lisa/trace.py | 1 + lisa/utils.py | 31 +++++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 00afd4d8f..a4d0156a0 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -146,7 +146,7 @@ from devlib.target import AndroidTarget, KernelVersion, TypedKernelConfig, Kerne from devlib.host import LocalConnection from devlib.exception import TargetStableError -from lisa.utils import nullcontext, Loggable, LISA_CACHE_HOME, checksum, DirCache, chain_cm, memoized, LISA_HOST_ABI, subprocess_log, SerializeViaConstructor, destroyablecontextmanager, ContextManagerExit, ignore_exceps, get_nested_key, is_link_dead, deduplicate, subprocess_detailed_excep +from lisa.utils import nullcontext, Loggable, checksum, DirCache, chain_cm, memoized, LISA_HOST_ABI, subprocess_log, SerializeViaConstructor, destroyablecontextmanager, ContextManagerExit, ignore_exceps, get_nested_key, is_link_dead, deduplicate, subprocess_detailed_excep from lisa._assets import ASSETS_PATH, HOST_PATH, ABI_BINARIES_FOLDER from lisa._unshare import ensure_root import lisa._git as git @@ -708,6 +708,7 @@ def _make_alpine_chroot(version, packages=None, abi=None, bind_paths=None, overl dir_cache = DirCache( category='alpine_chroot', populate=populate, + fmt_version='1', ) alpine_arch =_abi_to_alpine_arch(abi) key = ( @@ -755,6 +756,7 @@ def _make_alpine_chroot(version, packages=None, abi=None, bind_paths=None, overl rust_dir_cache = DirCache( category='rust_for_alpine_chroot', populate=populate_rust, + fmt_version='1', ) # Add the Alpine key to the Rust key, so that the Rust install is # allowed to depend on the state of the Alpine install (e.g. packages @@ -2461,6 +2463,7 @@ class _KernelBuildEnv(Loggable, SerializeViaConstructor): dir_cache = DirCache( category='kernels', populate=lambda url, path: cls._make_tree(version, path), + fmt_version='1', ) url = cls._get_url(version) @@ -2822,6 +2825,7 @@ class KmodSrc(Loggable): dir_cache = DirCache( category='rust_home', populate=populate, + fmt_version='1', ) key = sorted(rust_spec.items()) rust_home = dir_cache.get_entry(key) diff --git a/lisa/trace.py b/lisa/trace.py index 678e93fc4..3820c537d 100644 --- a/lisa/trace.py +++ b/lisa/trace.py @@ -5235,6 +5235,7 @@ class _Trace(Loggable, _InternalTraceBase): except OSError: dir_cache = DirCache( category='trace_swap', + fmt_version='1', ) swap_dir = str(dir_cache.get_entry(trace_id)) diff --git a/lisa/utils.py b/lisa/utils.py index 3a83b61ca..8622e219e 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -93,7 +93,8 @@ LISA_HOME = os.getenv('LISA_HOME', os.path.abspath('.')) The detected location of your LISA installation """ -def _get_xdg_home(directory): +def _get_xdg_home(directory, fmt_version=None): + fmt_version = fmt_version or f'version-{VERSION_TOKEN}' dir_upper = directory.upper() try: @@ -105,13 +106,13 @@ def _get_xdg_home(directory): try: base = os.environ[f'XDG_{dir_upper}_HOME'] except KeyError: - xdg_home = os.path.join(os.environ['HOME'], '.lisa', directory, VERSION_TOKEN) + xdg_home = os.path.join(os.environ['HOME'], '.lisa', directory, fmt_version) else: - xdg_home = os.path.join(base, 'lisa', VERSION_TOKEN) + xdg_home = os.path.join(base, 'lisa', fmt_version) else: - xdg_home = os.path.join(base, directory, VERSION_TOKEN) + xdg_home = os.path.join(base, directory, fmt_version) else: - xdg_home = os.path.join(base, VERSION_TOKEN) + xdg_home = os.path.join(base, fmt_version) os.makedirs(xdg_home, exist_ok=True) return xdg_home @@ -122,6 +123,8 @@ LISA_CACHE_HOME = _get_xdg_home('cache') Base folder used for caching files. """ +_UNVERSIONED_CACHE_HOME = _get_xdg_home('cache', fmt_version='unversioned') + RESULT_DIR = 'results' LATEST_LINK = 'results_latest' @@ -4077,11 +4080,23 @@ class DirCache(Loggable): which is the same as returning the passed path. :type populate: collections.abc.Callable + :param fmt_version: Version of the format of this cache ``category``. This + allows re-using the cache across multiple versions of :mod:`lisa`, at + the expense of having to manually bump the format version in source + code when the format of the cache changes. This format version is + logically added to each key lookup so that multiple versions of + :mod:`lisa` do not interfere with each other. + :type fmt_version: str or None + The cache is managed in a process-safe way, so that there can be no race between concurrent processes or threads. """ - def __init__(self, category, populate=None): - self._base = Path(LISA_CACHE_HOME, category) + def __init__(self, category, populate=None, fmt_version=None): + base = _UNVERSIONED_CACHE_HOME if fmt_version else LISA_CACHE_HOME + base = Path(base) / category + + self._fmt_version = fmt_version + self._base = base self._populate = populate or (lambda *args, **kwargs: None) self._category = category @@ -4109,7 +4124,7 @@ class DirCache(Loggable): else: return with_typ(repr(x)) - key = normalize(key) + key = normalize((self._fmt_version, key)) key = repr(key).encode('utf-8') h = hashlib.sha256() -- GitLab