From 82f98c9297b4392c3e6c0a9c4338185e7d3cfee3 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 17 Jun 2022 18:37:39 +0100 Subject: [PATCH 1/3] lisa._kmod: Speedup copy overlay backend Speedup the copy overlay backend. --- lisa/_kmod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 0ca0e7d83..2cf44beab 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -434,8 +434,8 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): else: try: dst_mtime = os.path.getmtime(dst) - except OSError: - if os.path.lexists(dst): + except OSError as e: + if not isinstance(e, FileNotFoundError): os.remove(dst) return shutil.copy2(src=src, dst=dst) else: -- GitLab From c01821b2cfe457bb91bcf277a1429e13c438f349 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 17 Jun 2022 20:26:27 +0100 Subject: [PATCH 2/3] lisa._kmod: Speedup copy overlay backend FEATURE Use rsync when available in order to speedup the copy, as rsync is much faster than shutil.copytree() Python function. --- lisa/_kmod.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 2cf44beab..38f4c3a06 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -414,7 +414,7 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): @contextlib.contextmanager def do_copy(dirs): - def _copytree(src, dst): + def _python_copytree(src, dst): base_src = Path(src) base_dst = Path(dst) def copy_file(src, dst): @@ -446,7 +446,6 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): return shutil.copy2(src=src, dst=dst) else: return dst - shutil.copytree( src=str(src), dst=str(dst), @@ -455,6 +454,18 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): dirs_exist_ok=True, copy_function=copy_file, ) + + def _rsync_copytree(src, dst): + subprocess.check_call(['rsync', '-au', '--', f'{src}/', dst]) + + def _copytree(src, dst): + try: + _rsync_copytree(src, dst) + # rsync not installed + except FileNotFoundError: + logger.debug('rsync not installed, falling back on python copy') + _python_copytree(src, dst) + logger.debug(f'Copying trees instead of overlayfs for {mount_point}') for src in lowers: _copytree(src=src, dst=mount_point) -- GitLab From eabd0799c6ca766b4bfa86cc5b70521716cca229 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 17 Jun 2022 23:53:38 +0100 Subject: [PATCH 3/3] lisa._kmod: Fix copy overlay backend FIX The copy overlay backend was not able to restore the deleted state of a file when an upper was reused as a lower, like overlayfs is able to using whiteout. Fix that by only restoring the top-most lower layer when using the copy backemd. This also has the advantage of speeding up the re-use of lowers as only one tree needs to be copied. After this change, lowers created with one backend cannot be mixed with lowers created by another backend, so add the backend to the cache key where appropriate. --- install_base.sh | 2 ++ lisa/_kmod.py | 24 +++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/install_base.sh b/install_base.sh index c875fb14b..735054d7a 100755 --- a/install_base.sh +++ b/install_base.sh @@ -158,6 +158,7 @@ apt_packages=( openssh-client sshpass wget + rsync unzip qemu-user-static kernelshark @@ -179,6 +180,7 @@ apt_packages=( pacman_packages=( coreutils git + rsync openssh sshpass base-devel diff --git a/lisa/_kmod.py b/lisa/_kmod.py index 38f4c3a06..16a3dd3fd 100644 --- a/lisa/_kmod.py +++ b/lisa/_kmod.py @@ -338,7 +338,8 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): Overlay folders on top of each other. :param lowers: List of read-only lower layers. The end of the list takes - precedence. + precedence. Apart from the first item in the list, all items must have + been populated as an "upper" of its preceding lowers. :type lowers: list(str) :param upper: Read-write upper layer taking all the changes made to the @@ -349,11 +350,12 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): * ``overlayfs``: Uses Linux overlayfs mounts. This is the fastest and most space efficient method. - * ``copy``: This uses plain copies to simulate overlayfs. Note that the - simulation is not entirely transparent, as a higher layer is not able - to hide files in lower layers like it can do with overlayfs and - whiteout files. + * ``copy``: This uses plain copies to simulate overlayfs. * ``None``: defaults to ``overlayfs``. + + Note that mixing lowers created with different backends is not + supported. Stick with the same backend when creating all the lowers in + the stack. :type backend: str or None """ logger = logging.getLogger(f'{__name__}.overlay') @@ -467,8 +469,15 @@ def _overlay_folders(lowers, upper=None, backend=None, copy_filter=None): _python_copytree(src, dst) logger.debug(f'Copying trees instead of overlayfs for {mount_point}') - for src in lowers: - _copytree(src=src, dst=mount_point) + + # Apart from the first one in the list, all lowers are expected to + # have been populated as an "upper" of the preceding lowers. + # Therefore, we can simply make a copy of the top-most lower. + # + # This is the only way we can let the user delete a file in an + # upper and subsequently restore that state. + src = lowers[-1] + _copytree(src=src, dst=mount_point) try: yield mount_point @@ -1323,6 +1332,7 @@ class KernelTree(Loggable, SerializeViaConstructor): ) + [ str(tree_key), str(build_env), + str(overlay_backend), ] + [ # We need to take checksum the make variables # as well, as it can influence the kernel tree -- GitLab