From d13dacc0d3ae55c145d2c7fd696f0032199f4cb0 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 29 Jan 2025 17:19:53 +0000 Subject: [PATCH 1/4] external/devlib: Revert to upstream Now that all the PR we needed are merged, revert to using the upstream master branch. --- external/devlib.manifest.yaml | 38 ++++------------------------------- external/subtrees.conf | 9 ++------- 2 files changed, 6 insertions(+), 41 deletions(-) diff --git a/external/devlib.manifest.yaml b/external/devlib.manifest.yaml index 42c72d7db..b99d1fcb8 100644 --- a/external/devlib.manifest.yaml +++ b/external/devlib.manifest.yaml @@ -11,40 +11,10 @@ rebase-conf: ref: master topics: - - - remote: douglas - base: master - tip: fix_signals - - - - remote: douglas - base: master - tip: fix_adb_unroot - - - - remote: douglas - base: master - tip: fix_cpufreq_event - - - - remote: douglas - base: master - tip: fix_kprobe_events2 - - - - remote: douglas - base: master - tip: fix_disconnect - - - - remote: douglas - base: master - tip: android_dmesg - - - - remote: douglas - base: master - tip: fix_ssh_pull + # - + # remote: douglas + # base: master + # tip: a_topic_branch remotes: upstream: diff --git a/external/subtrees.conf b/external/subtrees.conf index 76d1f2dff..34504772c 100644 --- a/external/subtrees.conf +++ b/external/subtrees.conf @@ -1,13 +1,8 @@ [devlib] path = external/devlib -# url = https://github.com/ARM-Software/devlib.git -# ref = master - -url = https://github.com/douglas-raillard-arm/devlib.git -# Dogfooding on our PR -ref = lisa +url = https://github.com/ARM-Software/devlib.git +ref = master -# # See external/devlib.manifest.yaml for instructions on how to build this # branch: # url = https://github.com/douglas-raillard-arm/devlib.git -- GitLab From c0c42ea7548b02b0276dcbb2e996bcb3080d189f Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 3 Mar 2025 11:09:47 +0000 Subject: [PATCH 2/4] external/workload-automation: Revert to upstream Revert to upstream now that all the relevant pull requests have been merged. --- external/subtrees.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/external/subtrees.conf b/external/subtrees.conf index 34504772c..096483ce5 100644 --- a/external/subtrees.conf +++ b/external/subtrees.conf @@ -10,10 +10,10 @@ ref = master [workload-automation] path = external/workload-automation -# url = https://github.com/ARM-Software/workload-automation.git -# ref = master +url = https://github.com/ARM-Software/workload-automation.git +ref = master # See external/workload-automation.manifest.yaml for instructions on how to # build this branch: -url = https://github.com/douglas-raillard-arm/workload-automation.git -ref = lisa +# url = https://github.com/douglas-raillard-arm/workload-automation.git +# ref = lisa -- GitLab From 4accfb274156dbb4e94b0c8d52cad7cd9727bfe1 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 3 Mar 2025 11:10:46 +0000 Subject: [PATCH 3/4] Squashed 'external/devlib/' changes from 100e77ae7..5425f4aff 5425f4aff version: bump dev version fa0d09970 ssh: Fix incorrect method name f2e81a8b5 host: Fix incorrect import c88a5dbb8 devlib: Replace Target.tempfile() by Target.make_temp() 1da260b89 target: Align Target.tempfile() with Target.make_temp() 3e1c928db target: Make Target.tempfile() use Target.tmp_directory e402fc754 target: Make Target.make_temp() use Target.tmp_directory 1ac461ad7 target: Rework temp folder management e551b4620 ftrace: Add write-to-disk mode 9ec36e904 connection: Support all signals in BackgroundCommand.send_signal() eb9e0c987 collector/ftrace: Make emitting cpu_frequency_devlib and extra idle events dependent on the configured events ae8149077 collector/ftrace: Remove cpu_frequency_devlib event in FtraceCollector.stop() 1b6c8069b target: Asyncify Target._prepare_xfer() 4431932e0 target: Reduce the number of commands involved in push/pull 8af9f1a32 target: Use busybox for file transfer operations 1efcfed63 target: Copy symlinks as files when pulling df1b5ef4a ssh: Fix folder pull on SSH connection facd251ed collector/dmesg: Fix dmesg variant detection a3765cc27 target: Remove duplicated disconnection logic 20e5bcd2c utils/android: Restore adb root state when disconnecting f60fa59ac collector/ftrace: Handle missing kprobe_events file REVERT: 100e77ae7 target: Asyncify Target._prepare_xfer() REVERT: 40133abe6 target: Reduce the number of commands involved in push/pull REVERT: 23196b2dd target: Use busybox for file transfer operations REVERT: 7da4b314d target: Copy symlinks as files when pulling REVERT: 5a7e8e3fe ssh: Fix folder pull on SSH connection REVERT: 105c98dca collector/dmesg: Fix dmesg variant detection REVERT: 079c7ef96 target: Remove duplicated disconnection logic REVERT: 9fcc37a30 collector/ftrace: Handle missing kprobe_events file REVERT: 4ad4a446f collector/ftrace: Make emitting cpu_frequency_devlib and extra idle events dependent on the configured events REVERT: 9042ff944 collector/ftrace: Remove cpu_frequency_devlib event in FtraceCollector.stop() REVERT: 5b6066e2b utils/android: Restore adb root state when disconnecting REVERT: e7681ee95 target: Make all Target.__init__() parameters after working_directory keyword-only REVERT: 1d3256be5 target: Align Target.tempfile() with Target.make_temp() REVERT: c38621991 target: Make Target.tempfile() use Target.tmp_directory REVERT: 8d1f9bdf5 target: Make Target.make_temp() use Target.tmp_directory REVERT: ea4bc3b9d target: Rework temp folder management REVERT: 8f14b56a3 ftrace: Add write-to-disk mode REVERT: 7f5dcb410 connection: Support all signals in BackgroundCommand.send_signal() git-subtree-dir: external/devlib git-subtree-split: 5425f4afffa4aab3df75a14a6dae3776b5107b2f --- devlib/host.py | 2 +- devlib/module/cgroups2.py | 95 +++++++++++++++++---------------------- devlib/target.py | 49 ++++++++++---------- devlib/utils/ssh.py | 2 +- devlib/utils/version.py | 2 +- 5 files changed, 67 insertions(+), 83 deletions(-) diff --git a/devlib/host.py b/devlib/host.py index a65b00f9f..a20711cc4 100644 --- a/devlib/host.py +++ b/devlib/host.py @@ -30,7 +30,7 @@ from devlib.connection import ConnectionBase, PopenBackgroundCommand if sys.version_info >= (3, 8): def copy_tree(src, dst): - from shutils import copy, copytree + from shutil import copy, copytree copytree( src, dst, diff --git a/devlib/module/cgroups2.py b/devlib/module/cgroups2.py index 83cbf3948..a632bbee6 100644 --- a/devlib/module/cgroups2.py +++ b/devlib/module/cgroups2.py @@ -335,35 +335,28 @@ def _mount_v2_controllers(target: LinuxTarget): :yield: The path to the root of the mounted V2 controller hierarchy. :rtype: str - - :raises TargetStableError: Occurs in the case where the root directory of the requested CGroup V2 Controller hierarchy + + :raises TargetStableError: Occurs in the case where the root directory of the requested CGroup V2 Controller hierarchy is unable to be created up on the target system. """ - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError: - raise TargetStableError("Un-able to create the root directory of the requested CGroup V2 hierarchy") - - - try: - target.execute( - "{busybox} mount -t cgroup2 none {path}".format( - busybox=quote(target.busybox), path=quote(path) - ), - as_root=True, - ) - yield path - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) + with target.make_temp() as path: + try: + target.execute( + "{busybox} mount -t cgroup2 none {path}".format( + busybox=quote(target.busybox), path=quote(path) + ), + as_root=True, + ) + yield path + finally: + target.execute( + "{busybox} umount {path}".format( + busybox=quote(target.busybox), + path=quote(path), + ), + as_root=True, + ) @contextmanager @@ -379,8 +372,8 @@ def _mount_v1_controllers(target: LinuxTarget, controllers: Set[str]): :yield: A dictionary mapping CGroup controller names to the paths that they're currently mounted at. :rtype: Dict[str,str] - - :raises TargetStableError: Occurs in the case where the root directory of a requested CGroup V1 Controller hierarchy + + :raises TargetStableError: Occurs in the case where the root directory of a requested CGroup V1 Controller hierarchy is unable to be created up on the target system. """ @@ -388,33 +381,25 @@ def _mount_v1_controllers(target: LinuxTarget, controllers: Set[str]): # its mount path. @contextmanager def _mount_controller(controller): + with target.make_temp() as path: + try: + target.execute( + "{busybox} mount -t cgroup -o {controller} none {path}".format( + busybox=quote(target.busybox), + controller=quote(controller), + path=quote(path), + ), + ) - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError as err: - raise TargetStableError("Un-able to create the root directory of the {controller} CGroup V1 hierarchy".format(controller = controller)) - - try: - target.execute( - "{busybox} mount -t cgroup -o {controller} none {path}".format( - busybox=quote(target.busybox), - controller=quote(controller), - path=quote(path), - ), - ) - - yield path - - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) + yield path + finally: + target.execute( + "{busybox} umount {path}".format( + busybox=quote(target.busybox), + path=quote(path), + ), + as_root=True, + ) with ExitStack() as stack: yield { @@ -569,7 +554,7 @@ class _CGroupBase(ABC): ) except TargetStableError: self._set_controller_attribute("cgroup", "procs", pid) - + else: if str(pid) not in member_processes: self._set_controller_attribute("cgroup", "procs", pid) diff --git a/devlib/target.py b/devlib/target.py index 5c6799fc5..48c0acf04 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -312,9 +312,7 @@ class Target(object): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -322,6 +320,7 @@ class Target(object): conn_cls=None, is_container=False, max_async=50, + tmp_directory=None, ): self._lock = threading.RLock() @@ -1337,13 +1336,13 @@ fi @asyn.asyncf async def tempfile(self, prefix=None, suffix=None): prefix = f'{prefix}-' if prefix else '' - sufix = f'-{suffix}' if suffix else '' + suffix = f'-{suffix}' if suffix else '' name = '{prefix}{uuid}{suffix}'.format( prefix=prefix, uuid=uuid.uuid4().hex, suffix=suffix, ) - self.path.join(self.tmp_directory, name) + path = self.path.join(self.tmp_directory, name) if (await self.file_exists.asyn(path)): raise FileExistsError('Path already exists on the target: {}'.format(path)) else: @@ -1811,9 +1810,7 @@ class LinuxTarget(Target): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -1821,19 +1818,21 @@ class LinuxTarget(Target): conn_cls=SshConnection, is_container=False, max_async=50, + tmp_directory=None, ): super(LinuxTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) def wait_boot_complete(self, timeout=10): pass @@ -1901,12 +1900,11 @@ class LinuxTarget(Target): return try: - tmpfile = await self.tempfile.asyn() - cmd = 'DISPLAY=:0.0 scrot {} && {} date -u -Iseconds' - ts = (await self.execute.asyn(cmd.format(quote(tmpfile), quote(self.busybox)))).strip() - filepath = filepath.format(ts=ts) - await self.pull.asyn(tmpfile, filepath) - await self.remove.asyn(tmpfile) + async with self.make_temp(is_directory=False) as tmpfile: + cmd = 'DISPLAY=:0.0 scrot {} && {} date -u -Iseconds' + ts = (await self.execute.asyn(cmd.format(quote(tmpfile), quote(self.busybox)))).strip() + filepath = filepath.format(ts=ts) + await self.pull.asyn(tmpfile, filepath) except TargetStableError as e: if "Can't open X dispay." not in e.message: raise e @@ -2022,9 +2020,7 @@ class AndroidTarget(Target): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -2033,19 +2029,21 @@ class AndroidTarget(Target): package_data_directory="/data/data", is_container=False, max_async=50, + tmp_directory=None, ): super(AndroidTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) self.package_data_directory = package_data_directory self._init_logcat_lock() @@ -3111,9 +3109,7 @@ class LocalLinuxTarget(LinuxTarget): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -3121,19 +3117,21 @@ class LocalLinuxTarget(LinuxTarget): conn_cls=LocalConnection, is_container=False, max_async=50, + tmp_directory=None, ): super(LocalLinuxTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) def _resolve_paths(self): if self.working_directory is None: @@ -3198,7 +3196,6 @@ class ChromeOsTarget(LinuxTarget): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, android_working_directory=None, android_executables_directory=None, @@ -3209,6 +3206,7 @@ class ChromeOsTarget(LinuxTarget): package_data_directory="/data/data", is_container=False, max_async=50, + tmp_directory=None, ): self.supports_android = None @@ -3231,14 +3229,15 @@ class ChromeOsTarget(LinuxTarget): platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=False, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=SshConnection, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) # We can't determine if the target supports android until connected to the linux host so # create unconditionally. diff --git a/devlib/utils/ssh.py b/devlib/utils/ssh.py index caa0f4425..e64f67bd7 100644 --- a/devlib/utils/ssh.py +++ b/devlib/utils/ssh.py @@ -181,7 +181,7 @@ def _resolve_known_hosts(strict_host_check): if isinstance(strict_host_check, (str, os.PathLike)): path = Path(strict_host_check) else: - path = Path('~/.ssh/known_hosts').expandvars() + path = Path('~/.ssh/known_hosts').expanduser() else: path = Path('/dev/null') diff --git a/devlib/utils/version.py b/devlib/utils/version.py index ec6a3f1c5..2409b6783 100644 --- a/devlib/utils/version.py +++ b/devlib/utils/version.py @@ -21,7 +21,7 @@ from subprocess import Popen, PIPE VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev']) -version = VersionTuple(1, 4, 0, 'dev2') +version = VersionTuple(1, 4, 0, 'dev3') def get_devlib_version(): -- GitLab From 0050bf44068bbd831c93aefc8df927287e3b940e Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 3 Mar 2025 11:10:51 +0000 Subject: [PATCH 4/4] Squashed 'external/workload-automation/' changes from 92b047bb8..b03f28d1d b03f28d1d instruments/trace_cmd: Add tracing mode support to TraceCmdInstrument() f125fd340 version: Bump required devlib version 75cfb56b3 Remove dependency on distutils b734e90de ci: Bump python versions and pin ubuntu version 5670e571e workloads/speedometer: Fix SyntaxWarning exceptions in regex pattern REVERT: 92b047bb8 instruments/trace_cmd: Add tracing mode support to TraceCmdInstrument() git-subtree-dir: external/workload-automation git-subtree-split: b03f28d1d5bd4e45bd72f842f05eee7d20e7e64f --- .github/workflows/main.yml | 14 +++++++------- wa/commands/create.py | 19 ++++++++++++++++++- wa/framework/version.py | 2 +- wa/instruments/trace_cmd.py | 1 + wa/utils/misc.py | 6 +++++- wa/workloads/speedometer/__init__.py | 6 +++--- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a85f05ccf..20f213eb7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,13 +13,13 @@ on: jobs: Run-Linters-and-Tests: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.8.18 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8.18 - name: git-bash uses: pkg-src/github-action-git-bash@v1.1 - name: Install dependencies @@ -39,10 +39,10 @@ jobs: nosetests Execute-Test-Workload-and-Process: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7.17, 3.8.18, 3.9.21, 3.10.16, 3.13.2] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -64,10 +64,10 @@ jobs: cd /tmp && wa process -f -p csv idle_workload Test-WA-Commands: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7.17, 3.8.18, 3.9.21, 3.10.16, 3.13.2] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/wa/commands/create.py b/wa/commands/create.py index 289356159..85b4c6cc4 100644 --- a/wa/commands/create.py +++ b/wa/commands/create.py @@ -23,7 +23,6 @@ import re import uuid import getpass from collections import OrderedDict -from distutils.dir_util import copy_tree # pylint: disable=no-name-in-module, import-error from devlib.utils.types import identifier try: @@ -43,6 +42,24 @@ from wa.utils.misc import (ensure_directory_exists as _d, capitalize, from wa.utils.postgres import get_schema, POSTGRES_SCHEMA_DIR from wa.utils.serializer import yaml +if sys.version_info >= (3, 8): + def copy_tree(src, dst): + from shutil import copy, copytree # pylint: disable=import-outside-toplevel + copytree( + src, + dst, + # dirs_exist_ok=True only exists in Python >= 3.8 + dirs_exist_ok=True, + # Align with devlib and only copy the content without metadata + copy_function=copy + ) +else: + def copy_tree(src, dst): + # pylint: disable=import-outside-toplevel, redefined-outer-name + from distutils.dir_util import copy_tree + # Align with devlib and only copy the content without metadata + copy_tree(src, dst, preserve_mode=False, preserve_times=False) + TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates') diff --git a/wa/framework/version.py b/wa/framework/version.py index a329ce9cc..1d22384fd 100644 --- a/wa/framework/version.py +++ b/wa/framework/version.py @@ -23,7 +23,7 @@ VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev']) version = VersionTuple(3, 4, 0, 'dev1') -required_devlib_version = VersionTuple(1, 4, 0, 'dev2') +required_devlib_version = VersionTuple(1, 4, 0, 'dev3') def format_version(v): diff --git a/wa/instruments/trace_cmd.py b/wa/instruments/trace_cmd.py index a24b9b45d..36c30d7a4 100644 --- a/wa/instruments/trace_cmd.py +++ b/wa/instruments/trace_cmd.py @@ -163,6 +163,7 @@ class TraceCmdInstrument(Instrument): distribution's repos may be too old). """), Parameter('mode', kind=str, default='write-to-memory', + allowed_values=['write-to-disk', 'write-to-memory'], description=""" Specifies whether collected traces should be saved in memory or disk. Extensive workloads may hit out of memory issue. Hence, write-to-disk diff --git a/wa/utils/misc.py b/wa/utils/misc.py index 079b48471..75cc3b892 100644 --- a/wa/utils/misc.py +++ b/wa/utils/misc.py @@ -44,7 +44,11 @@ from time import sleep from io import StringIO # pylint: disable=wrong-import-position,unused-import from itertools import chain, cycle -from distutils.spawn import find_executable # pylint: disable=no-name-in-module, import-error + +try: + from shutil import which as find_executable +except ImportError: + from distutils.spawn import find_executable # pylint: disable=no-name-in-module, import-error from dateutil import tz diff --git a/wa/workloads/speedometer/__init__.py b/wa/workloads/speedometer/__init__.py index c225feecc..45c1bdd4b 100755 --- a/wa/workloads/speedometer/__init__.py +++ b/wa/workloads/speedometer/__init__.py @@ -105,9 +105,9 @@ class Speedometer(Workload): # _only_ in the case we didn't see it before. # Since 'index="3"' is always on the left side of the value. regex = re.compile( - '<[^>]*(?Pindex="3"|resource-id="result-number")?[^>]*' - '(?:text|content-desc)="(?P\d+.\d+)"[^>]*' - '(?(Z)|resource-id="result-number")[^>]*\/>' + r'<[^>]*(?Pindex="3"|resource-id="result-number")?[^>]*' + r'(?:text|content-desc)="(?P\d+.\d+)"[^>]*' + r'(?(Z)|resource-id="result-number")[^>]*\/>' ) parameters = [ -- GitLab