diff --git a/external/devlib.manifest.yaml b/external/devlib.manifest.yaml index 42c72d7db0331dd14df66de9e072bd1b9523cb4c..b99d1fcb8e3b0feb4fa9fc51284b578f89c9ac27 100644 --- a/external/devlib.manifest.yaml +++ b/external/devlib.manifest.yaml @@ -11,40 +11,10 @@ rebase-conf: ref: master topics: - - - remote: douglas - base: master - tip: fix_signals - - - - remote: douglas - base: master - tip: fix_adb_unroot - - - - remote: douglas - base: master - tip: fix_cpufreq_event - - - - remote: douglas - base: master - tip: fix_kprobe_events2 - - - - remote: douglas - base: master - tip: fix_disconnect - - - - remote: douglas - base: master - tip: android_dmesg - - - - remote: douglas - base: master - tip: fix_ssh_pull + # - + # remote: douglas + # base: master + # tip: a_topic_branch remotes: upstream: diff --git a/external/devlib/devlib/host.py b/external/devlib/devlib/host.py index a65b00f9ff060a2793140c96b3f38c962ab9c14b..a20711cc4884060bce5027aecd530c544fb1c378 100644 --- a/external/devlib/devlib/host.py +++ b/external/devlib/devlib/host.py @@ -30,7 +30,7 @@ from devlib.connection import ConnectionBase, PopenBackgroundCommand if sys.version_info >= (3, 8): def copy_tree(src, dst): - from shutils import copy, copytree + from shutil import copy, copytree copytree( src, dst, diff --git a/external/devlib/devlib/module/cgroups2.py b/external/devlib/devlib/module/cgroups2.py index 83cbf39487d8d26e0c5e3ba62d0da4dd31fe2f31..a632bbee637e69b3de110caf75cdfde86c10c8d9 100644 --- a/external/devlib/devlib/module/cgroups2.py +++ b/external/devlib/devlib/module/cgroups2.py @@ -335,35 +335,28 @@ def _mount_v2_controllers(target: LinuxTarget): :yield: The path to the root of the mounted V2 controller hierarchy. :rtype: str - - :raises TargetStableError: Occurs in the case where the root directory of the requested CGroup V2 Controller hierarchy + + :raises TargetStableError: Occurs in the case where the root directory of the requested CGroup V2 Controller hierarchy is unable to be created up on the target system. """ - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError: - raise TargetStableError("Un-able to create the root directory of the requested CGroup V2 hierarchy") - - - try: - target.execute( - "{busybox} mount -t cgroup2 none {path}".format( - busybox=quote(target.busybox), path=quote(path) - ), - as_root=True, - ) - yield path - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) + with target.make_temp() as path: + try: + target.execute( + "{busybox} mount -t cgroup2 none {path}".format( + busybox=quote(target.busybox), path=quote(path) + ), + as_root=True, + ) + yield path + finally: + target.execute( + "{busybox} umount {path}".format( + busybox=quote(target.busybox), + path=quote(path), + ), + as_root=True, + ) @contextmanager @@ -379,8 +372,8 @@ def _mount_v1_controllers(target: LinuxTarget, controllers: Set[str]): :yield: A dictionary mapping CGroup controller names to the paths that they're currently mounted at. :rtype: Dict[str,str] - - :raises TargetStableError: Occurs in the case where the root directory of a requested CGroup V1 Controller hierarchy + + :raises TargetStableError: Occurs in the case where the root directory of a requested CGroup V1 Controller hierarchy is unable to be created up on the target system. """ @@ -388,33 +381,25 @@ def _mount_v1_controllers(target: LinuxTarget, controllers: Set[str]): # its mount path. @contextmanager def _mount_controller(controller): + with target.make_temp() as path: + try: + target.execute( + "{busybox} mount -t cgroup -o {controller} none {path}".format( + busybox=quote(target.busybox), + controller=quote(controller), + path=quote(path), + ), + ) - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError as err: - raise TargetStableError("Un-able to create the root directory of the {controller} CGroup V1 hierarchy".format(controller = controller)) - - try: - target.execute( - "{busybox} mount -t cgroup -o {controller} none {path}".format( - busybox=quote(target.busybox), - controller=quote(controller), - path=quote(path), - ), - ) - - yield path - - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) + yield path + finally: + target.execute( + "{busybox} umount {path}".format( + busybox=quote(target.busybox), + path=quote(path), + ), + as_root=True, + ) with ExitStack() as stack: yield { @@ -569,7 +554,7 @@ class _CGroupBase(ABC): ) except TargetStableError: self._set_controller_attribute("cgroup", "procs", pid) - + else: if str(pid) not in member_processes: self._set_controller_attribute("cgroup", "procs", pid) diff --git a/external/devlib/devlib/target.py b/external/devlib/devlib/target.py index 5c6799fc5af584a7f2d89cd03150ccbfdd58e757..48c0acf046b50bfbfd2fafbaaec0317f92991481 100644 --- a/external/devlib/devlib/target.py +++ b/external/devlib/devlib/target.py @@ -312,9 +312,7 @@ class Target(object): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -322,6 +320,7 @@ class Target(object): conn_cls=None, is_container=False, max_async=50, + tmp_directory=None, ): self._lock = threading.RLock() @@ -1337,13 +1336,13 @@ fi @asyn.asyncf async def tempfile(self, prefix=None, suffix=None): prefix = f'{prefix}-' if prefix else '' - sufix = f'-{suffix}' if suffix else '' + suffix = f'-{suffix}' if suffix else '' name = '{prefix}{uuid}{suffix}'.format( prefix=prefix, uuid=uuid.uuid4().hex, suffix=suffix, ) - self.path.join(self.tmp_directory, name) + path = self.path.join(self.tmp_directory, name) if (await self.file_exists.asyn(path)): raise FileExistsError('Path already exists on the target: {}'.format(path)) else: @@ -1811,9 +1810,7 @@ class LinuxTarget(Target): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -1821,19 +1818,21 @@ class LinuxTarget(Target): conn_cls=SshConnection, is_container=False, max_async=50, + tmp_directory=None, ): super(LinuxTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) def wait_boot_complete(self, timeout=10): pass @@ -1901,12 +1900,11 @@ class LinuxTarget(Target): return try: - tmpfile = await self.tempfile.asyn() - cmd = 'DISPLAY=:0.0 scrot {} && {} date -u -Iseconds' - ts = (await self.execute.asyn(cmd.format(quote(tmpfile), quote(self.busybox)))).strip() - filepath = filepath.format(ts=ts) - await self.pull.asyn(tmpfile, filepath) - await self.remove.asyn(tmpfile) + async with self.make_temp(is_directory=False) as tmpfile: + cmd = 'DISPLAY=:0.0 scrot {} && {} date -u -Iseconds' + ts = (await self.execute.asyn(cmd.format(quote(tmpfile), quote(self.busybox)))).strip() + filepath = filepath.format(ts=ts) + await self.pull.asyn(tmpfile, filepath) except TargetStableError as e: if "Can't open X dispay." not in e.message: raise e @@ -2022,9 +2020,7 @@ class AndroidTarget(Target): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -2033,19 +2029,21 @@ class AndroidTarget(Target): package_data_directory="/data/data", is_container=False, max_async=50, + tmp_directory=None, ): super(AndroidTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) self.package_data_directory = package_data_directory self._init_logcat_lock() @@ -3111,9 +3109,7 @@ class LocalLinuxTarget(LinuxTarget): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, - tmp_directory=None, connect=True, modules=None, load_default_modules=True, @@ -3121,19 +3117,21 @@ class LocalLinuxTarget(LinuxTarget): conn_cls=LocalConnection, is_container=False, max_async=50, + tmp_directory=None, ): super(LocalLinuxTarget, self).__init__(connection_settings=connection_settings, platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=connect, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=conn_cls, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) def _resolve_paths(self): if self.working_directory is None: @@ -3198,7 +3196,6 @@ class ChromeOsTarget(LinuxTarget): connection_settings=None, platform=None, working_directory=None, - *, executables_directory=None, android_working_directory=None, android_executables_directory=None, @@ -3209,6 +3206,7 @@ class ChromeOsTarget(LinuxTarget): package_data_directory="/data/data", is_container=False, max_async=50, + tmp_directory=None, ): self.supports_android = None @@ -3231,14 +3229,15 @@ class ChromeOsTarget(LinuxTarget): platform=platform, working_directory=working_directory, executables_directory=executables_directory, - tmp_directory=tmp_directory, connect=False, modules=modules, load_default_modules=load_default_modules, shell_prompt=shell_prompt, conn_cls=SshConnection, is_container=is_container, - max_async=max_async) + max_async=max_async, + tmp_directory=tmp_directory, + ) # We can't determine if the target supports android until connected to the linux host so # create unconditionally. diff --git a/external/devlib/devlib/utils/ssh.py b/external/devlib/devlib/utils/ssh.py index caa0f44251d50782821ec4027c60187f0245fee4..e64f67bd768b210ce321815d637aa824f702bcb5 100644 --- a/external/devlib/devlib/utils/ssh.py +++ b/external/devlib/devlib/utils/ssh.py @@ -181,7 +181,7 @@ def _resolve_known_hosts(strict_host_check): if isinstance(strict_host_check, (str, os.PathLike)): path = Path(strict_host_check) else: - path = Path('~/.ssh/known_hosts').expandvars() + path = Path('~/.ssh/known_hosts').expanduser() else: path = Path('/dev/null') diff --git a/external/devlib/devlib/utils/version.py b/external/devlib/devlib/utils/version.py index ec6a3f1c53a1fe44ba613a5052a9b8e1d7a5ec2c..2409b6783890da56e1fb082e9ededd4f98f0f81e 100644 --- a/external/devlib/devlib/utils/version.py +++ b/external/devlib/devlib/utils/version.py @@ -21,7 +21,7 @@ from subprocess import Popen, PIPE VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev']) -version = VersionTuple(1, 4, 0, 'dev2') +version = VersionTuple(1, 4, 0, 'dev3') def get_devlib_version(): diff --git a/external/subtrees.conf b/external/subtrees.conf index 76d1f2dff1637bba259e2e73b87e534f21e7445d..096483ce5419d087e6e4326a98ef62fd44ecdd14 100644 --- a/external/subtrees.conf +++ b/external/subtrees.conf @@ -1,13 +1,8 @@ [devlib] path = external/devlib -# url = https://github.com/ARM-Software/devlib.git -# ref = master +url = https://github.com/ARM-Software/devlib.git +ref = master -url = https://github.com/douglas-raillard-arm/devlib.git -# Dogfooding on our PR -ref = lisa - -# # See external/devlib.manifest.yaml for instructions on how to build this # branch: # url = https://github.com/douglas-raillard-arm/devlib.git @@ -15,10 +10,10 @@ ref = lisa [workload-automation] path = external/workload-automation -# url = https://github.com/ARM-Software/workload-automation.git -# ref = master +url = https://github.com/ARM-Software/workload-automation.git +ref = master # See external/workload-automation.manifest.yaml for instructions on how to # build this branch: -url = https://github.com/douglas-raillard-arm/workload-automation.git -ref = lisa +# url = https://github.com/douglas-raillard-arm/workload-automation.git +# ref = lisa diff --git a/external/workload-automation/.github/workflows/main.yml b/external/workload-automation/.github/workflows/main.yml index a85f05ccf57f14f548b10f4edb857e6352ddb4b0..20f213eb7217cf9def7ff72808794a3a391d7b2e 100644 --- a/external/workload-automation/.github/workflows/main.yml +++ b/external/workload-automation/.github/workflows/main.yml @@ -13,13 +13,13 @@ on: jobs: Run-Linters-and-Tests: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.8.18 uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.8.18 - name: git-bash uses: pkg-src/github-action-git-bash@v1.1 - name: Install dependencies @@ -39,10 +39,10 @@ jobs: nosetests Execute-Test-Workload-and-Process: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7.17, 3.8.18, 3.9.21, 3.10.16, 3.13.2] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -64,10 +64,10 @@ jobs: cd /tmp && wa process -f -p csv idle_workload Test-WA-Commands: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7.17, 3.8.18, 3.9.21, 3.10.16, 3.13.2] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/external/workload-automation/wa/commands/create.py b/external/workload-automation/wa/commands/create.py index 2893561598fd6725162495453768cba2dbd312e4..85b4c6cc49f01b836af25ea62fa1dea904a7563f 100644 --- a/external/workload-automation/wa/commands/create.py +++ b/external/workload-automation/wa/commands/create.py @@ -23,7 +23,6 @@ import re import uuid import getpass from collections import OrderedDict -from distutils.dir_util import copy_tree # pylint: disable=no-name-in-module, import-error from devlib.utils.types import identifier try: @@ -43,6 +42,24 @@ from wa.utils.misc import (ensure_directory_exists as _d, capitalize, from wa.utils.postgres import get_schema, POSTGRES_SCHEMA_DIR from wa.utils.serializer import yaml +if sys.version_info >= (3, 8): + def copy_tree(src, dst): + from shutil import copy, copytree # pylint: disable=import-outside-toplevel + copytree( + src, + dst, + # dirs_exist_ok=True only exists in Python >= 3.8 + dirs_exist_ok=True, + # Align with devlib and only copy the content without metadata + copy_function=copy + ) +else: + def copy_tree(src, dst): + # pylint: disable=import-outside-toplevel, redefined-outer-name + from distutils.dir_util import copy_tree + # Align with devlib and only copy the content without metadata + copy_tree(src, dst, preserve_mode=False, preserve_times=False) + TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates') diff --git a/external/workload-automation/wa/framework/version.py b/external/workload-automation/wa/framework/version.py index a329ce9cc9ecc2e10f8cfe4da729d4bdfa8651b6..1d22384fd1aa25c03037a2483819f0c9c0698f9d 100644 --- a/external/workload-automation/wa/framework/version.py +++ b/external/workload-automation/wa/framework/version.py @@ -23,7 +23,7 @@ VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev']) version = VersionTuple(3, 4, 0, 'dev1') -required_devlib_version = VersionTuple(1, 4, 0, 'dev2') +required_devlib_version = VersionTuple(1, 4, 0, 'dev3') def format_version(v): diff --git a/external/workload-automation/wa/instruments/trace_cmd.py b/external/workload-automation/wa/instruments/trace_cmd.py index a24b9b45d6f2265aaf4b7829e5261851559f293e..36c30d7a42c783230365f50cffa6180e0f1f3aac 100644 --- a/external/workload-automation/wa/instruments/trace_cmd.py +++ b/external/workload-automation/wa/instruments/trace_cmd.py @@ -163,6 +163,7 @@ class TraceCmdInstrument(Instrument): distribution's repos may be too old). """), Parameter('mode', kind=str, default='write-to-memory', + allowed_values=['write-to-disk', 'write-to-memory'], description=""" Specifies whether collected traces should be saved in memory or disk. Extensive workloads may hit out of memory issue. Hence, write-to-disk diff --git a/external/workload-automation/wa/utils/misc.py b/external/workload-automation/wa/utils/misc.py index 079b484715edd336de391cfe7c88b00df55eb766..75cc3b8923abf28bf2c06bfb593ea46d3cbef73b 100644 --- a/external/workload-automation/wa/utils/misc.py +++ b/external/workload-automation/wa/utils/misc.py @@ -44,7 +44,11 @@ from time import sleep from io import StringIO # pylint: disable=wrong-import-position,unused-import from itertools import chain, cycle -from distutils.spawn import find_executable # pylint: disable=no-name-in-module, import-error + +try: + from shutil import which as find_executable +except ImportError: + from distutils.spawn import find_executable # pylint: disable=no-name-in-module, import-error from dateutil import tz diff --git a/external/workload-automation/wa/workloads/speedometer/__init__.py b/external/workload-automation/wa/workloads/speedometer/__init__.py index c225feecc776d548a4100bcf4713cb12124de4df..45c1bdd4ba582f15b503a4602f4a10e0a1422fa4 100755 --- a/external/workload-automation/wa/workloads/speedometer/__init__.py +++ b/external/workload-automation/wa/workloads/speedometer/__init__.py @@ -105,9 +105,9 @@ class Speedometer(Workload): # _only_ in the case we didn't see it before. # Since 'index="3"' is always on the left side of the value. regex = re.compile( - '<[^>]*(?Pindex="3"|resource-id="result-number")?[^>]*' - '(?:text|content-desc)="(?P\d+.\d+)"[^>]*' - '(?(Z)|resource-id="result-number")[^>]*\/>' + r'<[^>]*(?Pindex="3"|resource-id="result-number")?[^>]*' + r'(?:text|content-desc)="(?P\d+.\d+)"[^>]*' + r'(?(Z)|resource-id="result-number")[^>]*\/>' ) parameters = [