diff --git a/external/devlib/devlib/collector/dmesg.py b/external/devlib/devlib/collector/dmesg.py index faf9682512b4fc5b45f478e6384517c4a630ad9b..b5ba6164c4ffb1f60a9ce71ea0c8a4ffc67055a7 100644 --- a/external/devlib/devlib/collector/dmesg.py +++ b/external/devlib/devlib/collector/dmesg.py @@ -13,6 +13,7 @@ # limitations under the License. # +from __future__ import division import re from itertools import takewhile from datetime import timedelta diff --git a/external/devlib/devlib/collector/ftrace.py b/external/devlib/devlib/collector/ftrace.py index 6de485675df80a38f0da7439c936992caebdf991..292aa8271ab038d386a8e8f5f836a63828b3af22 100644 --- a/external/devlib/devlib/collector/ftrace.py +++ b/external/devlib/devlib/collector/ftrace.py @@ -13,6 +13,7 @@ # limitations under the License. # +from __future__ import division import os import json import time @@ -20,7 +21,7 @@ import re import subprocess import sys import contextlib -from shlex import quote +from pipes import quote from devlib.collector import (CollectorBase, CollectorOutput, CollectorOutputEntry) @@ -237,7 +238,7 @@ class FtraceCollector(CollectorBase): return self.target.read_value(self.available_functions_file).splitlines() def reset(self): - self.target.execute('{} reset -B devlib'.format(self.target_binary), + self.target.execute('{} reset'.format(self.target_binary), as_root=True, timeout=TIMEOUT) if self.functions: self.target.write_value(self.function_profile_file, 0, verify=False) @@ -262,7 +263,7 @@ class FtraceCollector(CollectorBase): self.target.write_value('/proc/sys/kernel/kptr_restrict', 0) self.target.execute( - '{} start -B devlib {buffer_size} {cmdlines_size} {clock} {events} {tracer} {functions}'.format( + '{} start {buffer_size} {cmdlines_size} {clock} {events} {tracer} {functions}'.format( self.target_binary, events=self.event_string, tracer=tracer_string, @@ -307,7 +308,7 @@ class FtraceCollector(CollectorBase): self.stop_time = time.time() if self.automark: self.mark_stop() - self.target.execute('{} stop -B devlib'.format(self.target_binary), + self.target.execute('{} stop'.format(self.target_binary), timeout=TIMEOUT, as_root=True) self._reset_needed = True diff --git a/external/devlib/devlib/derived/energy.py b/external/devlib/devlib/derived/energy.py index 55eae609d7138eb8631d1048b76ce58b7ea0b8e6..768a1ee122da11ae17764c5afbc0e022c9be9579 100644 --- a/external/devlib/devlib/derived/energy.py +++ b/external/devlib/devlib/derived/energy.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import division from collections import defaultdict from devlib.derived import DerivedMeasurements, DerivedMetric diff --git a/external/devlib/devlib/derived/fps.py b/external/devlib/devlib/derived/fps.py index 5663539ed8b0fbbeda0ff973cd3dfd0295508050..b7ef4cab4ba878f7e3670b3ba584b22df3a4963c 100644 --- a/external/devlib/devlib/derived/fps.py +++ b/external/devlib/devlib/derived/fps.py @@ -13,6 +13,7 @@ # limitations under the License. # +from __future__ import division import os try: diff --git a/external/devlib/devlib/host.py b/external/devlib/devlib/host.py index a6796da5fbca40556c02c7c54ec45269951d4108..b2a566a4f4b26113fa1dc636d12eb67cc0e4604d 100644 --- a/external/devlib/devlib/host.py +++ b/external/devlib/devlib/host.py @@ -20,7 +20,7 @@ import subprocess import logging from distutils.dir_util import copy_tree from getpass import getpass -from shlex import quote +from pipes import quote from devlib.exception import ( TargetTransientError, TargetStableError, diff --git a/external/devlib/devlib/instrument/__init__.py b/external/devlib/devlib/instrument/__init__.py index 6dca81cbe89b8243409f037293eda70ebe516141..600b6b6411b88bb39e2eec0b17c226d22aba40b9 100644 --- a/external/devlib/devlib/instrument/__init__.py +++ b/external/devlib/devlib/instrument/__init__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import division import logging import collections diff --git a/external/devlib/devlib/instrument/acmecape.py b/external/devlib/devlib/instrument/acmecape.py index ec0a77bf67b672735b5c03cc6786743f57a5d7ee..4a0a7095f55bf6f8fe165c7a4b59e9e543d4e6d4 100644 --- a/external/devlib/devlib/instrument/acmecape.py +++ b/external/devlib/devlib/instrument/acmecape.py @@ -14,6 +14,7 @@ # #pylint: disable=attribute-defined-outside-init +from __future__ import division import os import sys import time @@ -22,7 +23,7 @@ import shlex from fcntl import fcntl, F_GETFL, F_SETFL from string import Template from subprocess import Popen, PIPE, STDOUT -from shlex import quote +from pipes import quote from devlib import Instrument, CONTINUOUS, MeasurementsCsv from devlib.exception import HostError diff --git a/external/devlib/devlib/instrument/arm_energy_probe.py b/external/devlib/devlib/instrument/arm_energy_probe.py index 80ef643da4357750674238fca932163d1a5a238f..0c5740752dbd6950461ce2c4cc064891355b0284 100644 --- a/external/devlib/devlib/instrument/arm_energy_probe.py +++ b/external/devlib/devlib/instrument/arm_energy_probe.py @@ -30,13 +30,14 @@ # pylint: disable=W0613,E1101,access-member-before-definition,attribute-defined-outside-init +from __future__ import division import os -import shutil -import signal -import tempfile import subprocess -from shlex import quote +import signal +from pipes import quote +import tempfile +import shutil from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv from devlib.exception import HostError diff --git a/external/devlib/devlib/instrument/energy_probe.py b/external/devlib/devlib/instrument/energy_probe.py index b0b51801a4f0cf58f86315617c579a34ff70d591..07fe24bf38e9b2705073adc2d3156ea8101e498d 100644 --- a/external/devlib/devlib/instrument/energy_probe.py +++ b/external/devlib/devlib/instrument/energy_probe.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import division import os import signal import tempfile import struct import subprocess import sys -from shlex import quote +from pipes import quote from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv from devlib.exception import HostError diff --git a/external/devlib/devlib/instrument/frames.py b/external/devlib/devlib/instrument/frames.py index 402c4819455b7fc5faf114dab367c78b221f8483..e9c929ecb25d71a0302ec7bf4e7fadbadc417695 100644 --- a/external/devlib/devlib/instrument/frames.py +++ b/external/devlib/devlib/instrument/frames.py @@ -13,6 +13,7 @@ # limitations under the License. # +from __future__ import division import os from devlib.instrument import (Instrument, CONTINUOUS, diff --git a/external/devlib/devlib/instrument/gem5power.py b/external/devlib/devlib/instrument/gem5power.py index 2a59b6e984cbf6973c90b470941d71c31182ac27..35b338bc53ee8f899515607c634fbb19fa5e7872 100644 --- a/external/devlib/devlib/instrument/gem5power.py +++ b/external/devlib/devlib/instrument/gem5power.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import division + from devlib.platform.gem5 import Gem5SimulationPlatform from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv from devlib.exception import TargetStableError diff --git a/external/devlib/devlib/instrument/hwmon.py b/external/devlib/devlib/instrument/hwmon.py index 7c1cb7d1ac5efe6a461f3bd3ebef879e4977ac8e..8c7f15d08c25e86c7f83f1227311d3e18858b134 100644 --- a/external/devlib/devlib/instrument/hwmon.py +++ b/external/devlib/devlib/instrument/hwmon.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import division import re from devlib.instrument import Instrument, Measurement, INSTANTANEOUS diff --git a/external/devlib/devlib/instrument/netstats/__init__.py b/external/devlib/devlib/instrument/netstats/__init__.py index 74ac9d7ea6113320139d0f4f098cab71d58c0e5f..3db342f8ebbe654c95b52265eef8c2b721edc19c 100644 --- a/external/devlib/devlib/instrument/netstats/__init__.py +++ b/external/devlib/devlib/instrument/netstats/__init__.py @@ -18,7 +18,8 @@ import re import tempfile from datetime import datetime from collections import defaultdict -from itertools import zip_longest + +from future.moves.itertools import zip_longest from devlib.instrument import Instrument, MeasurementsCsv, CONTINUOUS from devlib.exception import TargetStableError, HostError diff --git a/external/devlib/devlib/module/android.py b/external/devlib/devlib/module/android.py index 70564fd057f390b37b7af7bc2306c9f6e834dfcc..c0e1bd5a70a478422d799c94a4ab8472f69b49c8 100644 --- a/external/devlib/devlib/module/android.py +++ b/external/devlib/devlib/module/android.py @@ -22,7 +22,7 @@ import tempfile from devlib.module import FlashModule from devlib.exception import HostError from devlib.utils.android import fastboot_flash_partition, fastboot_command -from devlib.utils.misc import merge_dicts, safe_extract +from devlib.utils.misc import merge_dicts class FastbootFlashModule(FlashModule): @@ -86,7 +86,7 @@ class FastbootFlashModule(FlashModule): self._validate_image_bundle(image_bundle) extract_dir = tempfile.mkdtemp() with tarfile.open(image_bundle) as tar: - safe_extract(tar, path=extract_dir) + tar.extractall(path=extract_dir) files = [tf.name for tf in tar.getmembers()] if self.partitions_file_name not in files: extract_dir = os.path.join(extract_dir, files[0]) diff --git a/external/devlib/devlib/module/cgroups2.py b/external/devlib/devlib/module/cgroups2.py deleted file mode 100644 index 83cbf39487d8d26e0c5e3ba62d0da4dd31fe2f31..0000000000000000000000000000000000000000 --- a/external/devlib/devlib/module/cgroups2.py +++ /dev/null @@ -1,1991 +0,0 @@ -# Copyright 2022 ARM Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" - -Successor to the ``cgroups`` devlib module. - -This one handles both cgroups V1 and V2 transparently with an API matching -cgroup v2 semantic. - -It also handles the cgroup delegation API of systemd. - - -.. code-block:: python - - # Necessary Imports - from devlib import LinuxTarget - from devlib.module.cgroups2 import RequestTree - - # Connecting to target device. Configure appropriately. - - my_target = LinuxTarget(connection_settings={ - "host":"127.0.0.1", - "port":"0000", - "username":"root", - "password":"root" - }) - - # Instantiating the RequestTree object, - # representing a hierarchical CGroup structure consisting - # of a singular parent and child CGroup relationship. - - request = RequestTree( - name="root", - children=[ - RequestTree( - name="child", - controllers={"pids": {"max": 10}} - ) - ], - controllers={"pids": {"max": 20}}, - ) - - # Printing the request to display/inspect the hierarchical - # tree-like structure of the RequestTree object. - - print(request) - - ''' - └──root (pids) {'max': 20} - └──child (pids) {'max': 10} - ''' - - # To set-up either CGroup version hierarchies, ensure the target device is - # appropriately configured alongside a CGroup version appropriate RequestTree object. - - # Setting up the RequestTree object CGroup hierarchy onto target device - # as a V1 hierarchy, and printing the returned ResponseTree object. - - with request.setup_hierarchy(target=my_target, version=1) as CGroup_hierarchy: - print(CGroup_hierarchy) - - ''' - └──root/ pids@/sys/fs/cgroup/pids/system.slice/devlib-42c838fe4f0b4f518825c4e312590113.service/root - └──child/ pids@/sys/fs/cgroup/pids/system.slice/devlib-42c838fe4f0b4f518825c4e312590113.service/root/child - ''' - - # Setting up the RequestTree object CGroup hierarchy onto target device - # as a V1 hierarchy, and adding a process to the 'child' CGroup. - - with request.setup_hierarchy(target=my_target, version=1) as CGroup_hierarchy: - child = CGroup_hierarchy["child"] - child.add_process(1234) - - # Setting up the RequestTree object CGroup hierarchy onto target device - # as a V2 hierarchy, and adding a thread to the 'child' CGroup. - - with request.setup_hierarchy(target=my_target, version=2) as CGroup_hierarchy: - child = CGroup_hierarchy["child"] - child.add_thread(1234) - -""" - -import collections.abc -import itertools -import os -import re -import uuid -from abc import ABC, abstractmethod -from contextlib import ExitStack, contextmanager -from shlex import quote -from typing import Dict, Set, List, Union, Any -from uuid import uuid4 - -from devlib import LinuxTarget -from devlib.exception import ( - TargetStableCalledProcessError, - TargetStableError, -) -from devlib.target import FstabEntry -from devlib.utils.misc import memoized - - -def _is_systemd_online(target: LinuxTarget): - """ - Determines if systemd is activated on the target system. - - :param target: Interface to the target device. - :type target: Target - - :return: Returns ``True`` if systemd is active, ``False`` otherwise. - :rtype: bool - """ - - try: - target.execute("systemctl status 2>&1 >/dev/null") - except TargetStableCalledProcessError: - return False - else: - return True - - -def _read_lines(target: LinuxTarget, path: str): - """ - Reads the lines of a file stored on the target device. - - :param target: Interface to target device. - :type target: Target - - :param path: The path to the file to be read. - :type path: str - - :return: A list of the words/sentences that result from splitting - the read file (trailing and leading white-spaces removed) delimiting on the new-line character. - :rtype: List[str] - """ - - return target.read_value(path=path).split("\n") - - -def _add_controller_versions(controllers: Dict[str, Dict[str, int]]): - """ - Finds the CGroup controller's version and adds it as a ``version`` key. - - :param controllers: A dictionary mapping ``str`` controller names to dictionaries, - where the later dictionary contains ``hierarchy`` and ``num_cgroup`` keys mapped to their - respective suitable ``int`` values. - :type controllers: Dict[str, Dict[str, int]] - - :return: A dictionary mapping ``str`` controller names to dictionaries, - where the later dictionary contains an appended ``version`` key which maps to an ``int`` - value representing the version of the respective controller if applicable. - :rtype: Dict[str, Dict[str,int]] - """ - - # Read how the controller versions can be determined here: - # https://man7.org/linux/man-pages/man7/cgroups.7.html - # (Under NOTES) [Dated 12/08/2022] - - def infer_version(config): - if config["hierarchy"] != 0: - return 1 - elif config["hierarchy"] == 0 and config["num_cgroups"] > 1: - return 2 - else: - return None - - return { - controller: {**config, "version": version} if version is not None else config - for (controller, config, version) in ( - (controller, config, infer_version(config)) - for (controller, config) in controllers.items() - ) - } - - -def _add_controller_mounts( - controllers: Dict[str, Dict[str, int]], target_fs_list: List[FstabEntry] -): - """ - Find the CGroup controller's mount point and adds it as ``mount_point`` key. - - :param controllers: A dictionary mapping ``str`` controller names to dictionaries, - where the later dictionary contains `hierarchy``, ``num_cgroup`` and if appropriate ``version`` - keys mapped to their respective suitable ``int`` values. - :type controllers: Dict[str, Dict[str, int]] - - :param target_fs_list: A list of entries of the NamedTuple type ``FstabEntry``, - where each represents a mounted filesystem on the target device. - :type target_fs: List[FstabEntry] - - :return: A dictionary mapping ``str`` controller names to dictionaries, - where the later dictionary contains an appended ``mount_point`` key which maps to the suitable - ``str`` value of the respective controllers if applicable. - :rtype: Dict[str, Dict[str, Union[str,int]]] - """ - - # Filter the mounted filesystems on the target device, obtaining the respective V1/V2 FstabEntries. - v1_mounts = [fs for fs in target_fs_list if fs.fs_type == "cgroup"] - v2_mounts = [fs for fs in target_fs_list if fs.fs_type == "cgroup2"] - - def _infer_mount(controller: str, configuration: Dict): - controller_version = configuration.get("version") - if controller_version == 1: - for mount in v1_mounts: - if controller in mount.options.strip().split(","): - return mount.mount_point - - elif controller_version == 2: - # If a controller is V2, a V2 hierarchy must exist. Therefore this is a legal - # operation. - return v2_mounts[0].mount_point - - return None - - return { - controller: {**config, "mount_point": path if path is not None else config} - for (controller, config, path) in ( - ( - controller, - config, - _infer_mount(controller=controller, configuration=config), - ) - for (controller, config) in controllers.items() - ) - } - - -def _get_cgroup_controllers(target: LinuxTarget): - """ - Returns the CGroup controllers that are currently enabled on the target device, alongside their appropriate configurations. - - :param target: Interface to target device. - :type target: Target - - :return: A dictionary of controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :rtype: Dict[str, Dict[str,Union[str,int]]] - """ - - # A snippet of the /proc/cgroup is shown below. The column entries are separated - # by '\t'. The regex pattern is structured to match and group these entries. - # - # #subsys_name hierarchy num_cgroups enabled - # cpuset 3 1 1 - - PROC_MOUNT_REGEX = re.compile( - r"^(?!#)(?P.+)\t(?P.+)\t(?P.+)\t(?P.+)" - ) - - proc_cgroup_file = _read_lines(target=target, path="/proc/cgroups") - - def _parse_controllers(controller): - match = PROC_MOUNT_REGEX.match(controller.strip()) - if match: - name = match.group("name") - enabled = int(match.group("enabled")) - hierarchy = int(match.group("hierarchy")) - num_cgroups = int(match.group("num_cgroups")) - # We should ignore disabled controllers. - if enabled != 0: - config = { - "hierarchy": hierarchy, - "num_cgroups": num_cgroups, - } - return (name, config) - return (None, None) - - controllers = dict(map(_parse_controllers, proc_cgroup_file)) - controllers.pop(None) - controllers = _add_controller_versions(controllers=controllers) - controllers = _add_controller_mounts( - controllers=controllers, - target_fs_list=target.list_file_systems(), - ) - - return controllers - - -@contextmanager -def _request_delegation(target: LinuxTarget): - """ - Requests systemd to delegate a subtree CGroup hierarchy to our transient service unit. - - :yield: The Main PID of the delegated transient service unit. - :rtype: int - """ - - service_name = "devlib-" + str(uuid.uuid4().hex) - - try: - target.execute( - 'systemd-run --no-block --property Delegate="yes" --unit {name} --quiet {busybox} sh -c "while true; do sleep 1d; done"'.format( - name=quote(service_name), busybox=quote(target.busybox) - ), - as_root=True, - ) - - pid = int( - target.execute( - "systemctl show --property MainPID --value {name}".format( - name=quote(service_name) - ) - ).strip() - ) - - yield pid - - finally: - target.execute( - "systemctl kill {name}".format(name=quote(service_name)), as_root=True - ) - - -@contextmanager -def _mount_v2_controllers(target: LinuxTarget): - """ - Mounts the V2 unified CGroup controller hierarchy. - - :param target: Interface to target device. - :type target: Target - - :yield: The path to the root of the mounted V2 controller hierarchy. - :rtype: str - - :raises TargetStableError: Occurs in the case where the root directory of the requested CGroup V2 Controller hierarchy - is unable to be created up on the target system. - """ - - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError: - raise TargetStableError("Un-able to create the root directory of the requested CGroup V2 hierarchy") - - - try: - target.execute( - "{busybox} mount -t cgroup2 none {path}".format( - busybox=quote(target.busybox), path=quote(path) - ), - as_root=True, - ) - yield path - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) - - -@contextmanager -def _mount_v1_controllers(target: LinuxTarget, controllers: Set[str]): - """ - Mounts the V1 split CGroup controller hierarchies. - - :param target: Interface to target device. - :type target: Target - - :param controllers: The names of the CGroup controllers required to be mounted. - :type controllers: Set[str] - - :yield: A dictionary mapping CGroup controller names to the paths that they're currently mounted at. - :rtype: Dict[str,str] - - :raises TargetStableError: Occurs in the case where the root directory of a requested CGroup V1 Controller hierarchy - is unable to be created up on the target system. - """ - - # Internal helper function which mounts a single V1 controller hierarchy and returns - # its mount path. - @contextmanager - def _mount_controller(controller): - - path = target.tempfile() - - try: - target.makedirs(path, as_root=True) - except TargetStableCalledProcessError as err: - raise TargetStableError("Un-able to create the root directory of the {controller} CGroup V1 hierarchy".format(controller = controller)) - - try: - target.execute( - "{busybox} mount -t cgroup -o {controller} none {path}".format( - busybox=quote(target.busybox), - controller=quote(controller), - path=quote(path), - ), - ) - - yield path - - finally: - target.execute( - "{busybox} umount {path} && {busybox} rmdir -- {path}".format( - busybox=quote(target.busybox), - path=quote(path), - ), - as_root=True, - ) - - with ExitStack() as stack: - yield { - controller: stack.enter_context(_mount_controller(controller)) - for controller in controllers - } - - -def _validate_requested_hierarchy( - requested_controllers: Set[str], available_controllers: Dict -): - """ - Validates that the requested hierarchy is valid using the controllers available on the target system. - - :param requested_controllers: A set of ``str``, representing the controllers that are requested to be used in the - user defined hierarchy. - :type requested_controllers: Set[str] - - :param available_controllers: A dictionary where the primary keys represent the available CGroup controllers on the target system. - :type available_controllers: Dict - - :raises TargetStableError: Occurs in the case where the requested CGroup hierarchy is unable to be - set up on the target system. - """ - - # Will determine if there are any controllers present within the requested controllers - # and not within the available controllers - - diff = set(requested_controllers) - available_controllers.keys() - - if diff: - raise TargetStableError( - "Unavailable controllers: {missing}".format(missing=" ,".join(diff)) - ) - - -class _CGroupBase(ABC): - """ - The abstract base class that all CGroup class types' subclass. - - :param name: The name assigned to the CGroup. Used to identify the CGroup and define the CGroup directory name. - :type name: str - - :param parent_path: The path to the parent CGroup this CGroup is a child of. - :type parent_path: str - - :param active_controllers: A dictionary of CGroup controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between a specific 'attribute' of the aforementioned - controller and a value for which that controller interface file should be set to. - :type active_controllers: Dict[str, Dict[str, Union[str,int]]] - - :param target: Interface to target device. - :type target: Target - """ - - def __init__( - self, - name: str, - parent_path: str, - active_controllers: Dict[str, Dict[str, str]], - target: LinuxTarget, - ): - self.name = name - self.active_controllers = active_controllers - self.target = target - self._parent_path = parent_path - - @property - def group_path(self): - return self.target.path.join(self._parent_path, self.name) - - def _set_controller_attribute( - self, controller: str, attribute: str, value: Union[int, str], verify=False - ): - """ - Writes the specified ``value`` into the interface file specified by the ``controller`` and ``attribute`` parameters. - In the case where no ``controller`` name is specified, the ``attribute`` argument is assumed to be the name of the - interface file to write to. - - :param controller: The controller we want to select. - :type controller: str - - :param attribute: The specific attribute of the controller we want to alter. - :type attribute: str - - :param value: The value we want to write to the specified interface file. - :type value: str - - :param verify: Whether we want to verify that the value is indeed written to the interface file, defaults to ``False``. - :type verify: bool, optional - """ - - str_value = str(value) - - # Some CGroup interface files don't have a controller name prefix, we accommodate that here. - interface_file = controller + "." + attribute if controller else attribute - - full_path = self.target.path.join(self.group_path, interface_file) - - self.target.write_value(full_path, str_value, verify=verify) - - def _create_directory(self, path: str): - """ - Creates a new directory at the given path, creating the parent directories if required. - If the directory already exists, no exception is thrown. - - :param path: Path to directory to be created. - :type path: str - """ - - self.target.makedirs(path, as_root=True) - - def _delete_directory(self, path: str): - """ - Removes the directory at the given path. - - :param path: Path to the directory to be removed. - :type path: str - """ - - # In this context we can't use the target.remove method since that - # tries to delete the interface/controller files as well which isn't needed nor permitted. - self.target.execute( - "{busybox} rmdir -- {path}".format( - busybox=quote(self.target.busybox), path=quote(path) - ), - as_root=True, - ) - - def _add_process(self, pid: Union[str, int]): - """ - Adds the process associated with the ``pid`` to the CGroup, only if - the process is not already a member of the CGroup. - - :param pid: The PID of the process to be added to the CGroup. - :type pid: Union[str,int] - """ - - if not self.target.file_exists(filepath="/proc/{pid}/status".format(pid=pid)): - return TargetStableError( - "The Process ID: {pid} does not exists.".format(pid=pid) - ) - - # The kernel disallows reading from the cgroup.procs file - # of a threaded CGroup. When trying to add processes to - # threaded CGroups, the process should be added to the CGroup - # regardless. User discretion required. - try: - member_processes = _read_lines( - path=self.target.path.join(self.group_path, "cgroup.procs"), - target=self.target, - ) - except TargetStableError: - self._set_controller_attribute("cgroup", "procs", pid) - - else: - if str(pid) not in member_processes: - self._set_controller_attribute("cgroup", "procs", pid) - - def _get_pid_from_tid(self, tid: int): - """ - Retrieves the ``pid`` (Process ID) that the ``tid`` (Thread ID) is a part of. - - :param tid: The Thread ID of the thread to be added to the CGroup. - :type tid: int - - :return: The ``pid`` (Process ID) associated with the ``tid`` (Thread ID). - :rtype: int - """ - status = _read_lines( - target=self.target, path="/proc/{tid}/status".format(tid=tid) - ) - for line in status: - # the Tgid entry contains the thread group ID, which is the PID of - # the process this thread belongs to. - match = re.match(r"\s*Tgid:\s*(\d+)\s*", line) - if match: - pid = match.group(1) - break - else: - raise TargetStableError( - "Could not get the PID of thread: {tid}".format(tid=tid) - ) - - return int(pid) - - @abstractmethod - def _add_thread(self, tid: int, threaded_domain): - """ - Ensures all sub-classes have the ability to add threads to their CGroups where - their differences dont allow for a common approach. - """ - pass - - @abstractmethod - def _init_cgroup(self): - """ - Ensures all sub-classes are able to initialise their respective CGroup directories - as per defined by their user configurations. - """ - pass - - @abstractmethod - def __enter__(self): - """ - Ensures all sub-classes can be used as context managers. - """ - pass - - @abstractmethod - def __exit__(self, *exc): - """ - Ensures all sub-classes can be used as context managers. - """ - pass - - -class _CGroupV2(_CGroupBase): - """ - A Class representing a CGroup directory within a CGroup V2 hierarchy. - - :param name: The name assigned to the CGroup. Used to identify the CGroup and define the CGroup folder name. - :type name: str - - :param parent_path: The path to the parent CGroup this CGroup is a child of. - :type parent_path: str - - :param active_controllers: A dictionary of controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between a specific 'attribute' of the aforementioned - controller and a value for which that controller interface file should be set to. - :type active_controllers: Dict[str, Dict[str, Union[str,int]]] - - :param subtree_controllers: The controllers that should be delegated to the subtree. - :type subtree_controllers: Set[str] - - :param is_threaded: Whether the CGroup type is threaded, - enables thread level granularity for the CGroup directory and its subtree. - :type is_threaded: bool - - :param target: Interface to target device. - :type target: Target - """ - - def __init__( - self, - name: str, - parent_path: str, - active_controllers: Dict[str, Dict[str, str]], - subtree_controllers: set, - is_threaded: bool, - target: LinuxTarget, - ): - - super().__init__( - name=name, - parent_path=parent_path, - active_controllers=active_controllers, - target=target, - ) - self.subtree_controllers = subtree_controllers - self.is_threaded = is_threaded - - def __enter__(self): - """ - Determines what happens when we enter the context of the CGroup, - in this case creating the required CGroup directory and calling :meth:`_init_cgroup`. - If an exception occurs during this phase, the context will be exited and the exception raised. - - :raises TargetStableError: If an exception occurs when calling :meth:`_init_cgroup`. - - :return: An object reference to itself. - :rtype: :class:`_CGroupV2` - """ - - self._create_directory(path=self.group_path) - try: - self._init_cgroup() - except TargetStableError as err: - self.__exit__(err, type(err), err.__traceback__) - raise - else: - return self - - def __exit__(self, *exc): - self._delete_directory(path=self.group_path) - - def _init_cgroup(self): - """ - Performs the required steps in order to initialize the CGroup to the user specified configuration: - - * Threading the CGroup if required. - * Write the values to be written to the specified controller interfaces files. - * Enable and delegate the controller that the subtree requires. - - :raises TargetStableError: Occurs when domain CGroup V2 controllers have been enabled within a threaded CGroup subtree. - """ - - # Threading the CGroup if required. - if self.is_threaded: - # Transforming a CGroup to type 'threaded' while domain CGroup controllers - # are enabled within the threaded subtree will result in a kernel exception. - # As of Linux Kernel version 4.19, the following controllers - # are threaded: cpu, perf_event, and pids. - try: - self._set_controller_attribute( - "cgroup", "type", "threaded", verify=True - ) - except TargetStableError: - raise TargetStableError( - "Domain CGroup controllers are enabled within a threaded CGroup subtree. Ensure only threaded controllers are enabled in threaded CGroups." - ) - - # Write the values to be written to the specified controller interfaces files. - for controller, configuration in self.active_controllers.items(): - for attr, val in configuration.items(): - self._set_controller_attribute( - controller=controller, attribute=attr, value=val, verify=True - ) - - # Enables/Delegates the required controllers to its subtree hierarchy via cgroup.subtree_control interface file. - for controller in self.subtree_controllers: - self._set_controller_attribute( - controller="cgroup", - attribute="subtree_control", - value="+{cont}".format(cont=controller), - ) - - def _add_thread(self, tid: int, threaded_domain): - """ - Attempts to add the thread associated with ``tid`` to the CGroup. - Due to the requirements imposed by the kernel regarding thread management within a V2 CGroup hierarchy, - the process that the thread associated with ``tid`` is a part of must reside at the root of the threaded - subtree. This method also ensures that this requirement is satisfied by migrating said process to - the CGroup at the root of the threaded sub-tree hierarchy if required, enabling thread level granularity - across the entire subtree. - - :param tid: The TID (Thread ID) of the thread to be added to the CGroup. - :type tid: int - - :param threaded_domain: The :class:`ResponseTree` object representing the threaded domain - of the threaded CGroup subtree. The process will be added to all the CGroups - that the :class:`ResponseTree` represent. - :type threaded_domain: :class:`ResponseTree` - """ - - pid_of_tid = self._get_pid_from_tid(tid=tid) - - for low_level in threaded_domain.low_levels.values(): - low_level._add_process(pid_of_tid) - - self._set_controller_attribute( - controller="cgroup", attribute="threads", value=tid - ) - - -class _CGroupV2Root(_CGroupV2): - """ - A subclass of the :class:`_CGroupV2` class representing a root V2 CGroup directory. - Contains the necessary functionality that enables the setting-up / mounting of a V2 - CGroup hierarchy. - - :param mount_point: The path on which the root of the CGroup V2 hierarchy is mounted on. - :type mount_point: str - - :param subtree_controllers: The controllers that should be delegated to the subtree. - :type subtree_controllers: Set[str] - - :param target: Interface to target device. - :type target: Target - """ - - @classmethod - def _v2_controller_translation( - cls, controllers: Dict[str, Dict[str, Union[str, int]]] - ): - """ - Given the new controller names within V2, rename the controllers to provide CGroupV2 compatibility. - At this point in time, the ``blkio`` controller has been renamed to ``io`` in V2, while the V2 ``cpu`` controller - wraps both ``cpu`` and ``cpuacct`` controllers/sub-systems. - - :param controllers: A dictionary of controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between the ``version`` and `mount_point`` - keys and their respectively obtained values. - :rtype: Dict[str, Dict[str,Union[str,int]]] - - :raises TargetStableError: In the case where the the ``cpu`` and ``cpuacct`` CGroup controllers are in use - under different CGroup version hierarchies. - - :raises TargetStableError: In the case where either ``cpu`` / ``cpuacct`` controller is not enabled on the target device. - - :return: The amended ``controllers`` dictionary with the updated names. - :rtype: Dict[str, Dict[str, Union[str,int]]] - """ - - translation = {} - - if "blkio" in controllers: - translation["io"] = controllers["blkio"] - - if "cpu" in controllers and "cpuacct" in controllers: - if controllers["cpu"].get("version") != controllers["cpuacct"].get( - "version" - ): - raise TargetStableError( - "CPU and CPUACCT controllers differ in versions. Both required to be version 2." - ) - else: - translation["cpu"] = controllers["cpu"] - else: - raise TargetStableError( - "Both CPU and CPUACCT controllers need to be enabled on the system to enable the V2 CPU controller." - ) - - return { - **translation, - **{ - controller: configuration - for controller, configuration in controllers.items() - # We don't to overwrite the performed controller name translation. - if controller not in ["blkio", "cpu", "cpuacct"] - }, - } - - @classmethod - def _get_delegated_sub_path(cls, delegated_pid: int, target: LinuxTarget): - """ - Returns the relative sub-path the delegated root of the V2 hierarchy is mounted on, via the parsing - of the /proc//cgroup file of the delegated process associated with ``delegated_pid``. - - :param delegated_pid: The Main PID of the transient service unit we requested delegation for. - :type delegated_pid: int - - :param target: Interface to target device. - :type target: Target - - :return: The sub-path to the delegate root of the V2 CGroup hierarchy. - :rtype: str - """ - - relative_delegated_mount_paths = _read_lines( - target=target, path="/proc/{pid}/cgroup".format(pid=delegated_pid) - ) - - # Following Regex matches the line that contains the relative sub path. - REL_PATH_REGEX = re.compile(r"0::\/(?P.+)") - - for mount_path in relative_delegated_mount_paths: - m = REL_PATH_REGEX.match(mount_path) - if m: - return m.group("path") - else: - raise TargetStableError( - "A V2 CGroup hierarchy was not delegated by systemd." - ) - - @classmethod - def _get_available_controllers( - cls, controllers: Dict[str, Dict[str, Union[str, int]]] - ): - """ - Returns the CGroup controllers that are currently not in use on the target device, - which can be taken control over and used in a manually mounted V2 hierarchy. - This method will only be called in the absence of systemd. - - :param controllers: A dictionary of CGroup controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :rtype: Dict[str, Dict[str,Union[str,int]]] - - :raises TargetStableError: Occurs in the case where a V2 hierarchy is already mounted on the target device. - We want to bail out in this case. - - :return: The ``controllers`` Dict filtered to just those controllers which are free/un-used. - :rtype: Dict[str, Dict[str, Union[str,int]]] - """ - - # Filters the controllers dict to entries where the version is == 2. - mounted_v2_controllers = { - controller - for controller, configuration in controllers.items() - if (configuration.get("version") == 2) - } - - if mounted_v2_controllers: - raise TargetStableError( - "A V2 CGroup hierarchy is already mounted on the specified target system, therefore unable to mount requested hierarchy" - ) - else: - return { - controller: configuration - for controller, configuration in controllers.items() - if configuration.get("version") is None - } - - @classmethod - def _path_to_delegated_root( - cls, controllers: Dict[str, Dict[str, Union[int, str]]], sub_path: str - ): - """ - Return the full path to the delegated root. This occurs in 2 stages: - - * Initially obtain the path to root mount of the unified V2 hierarchy (usually: ``/sys/fs/cgroup/path/to/root/``). - A subtree with no controllers could be delegated (given a hybrid CGroup hierarchy), - this is verified not be the case. - - * Creating a full path, which consists of the path concatenation of the root mount path - and the delegated subpath. - - :param controllers: A Dictionary of currently mounted controller name keys to Dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type controllers: Dict[str, Dict[str, Union[str,int]]] - - :param sub_path: The relative subpath to the delegated root hierarchy. - :type sub_path: str - - :raises TargetStableError: Occurs in the case where no V2 controllers are active on the target. - - :return: A full path to the delegated root of the V2 CGroup hierarchy. - :rtype: str - """ - - # Filter out non v2 controller mounts and append the "mount_point" to a set - v2_mount_point = { - configuration["mount_point"] - for configuration in controllers.values() - if configuration.get("version") == 2 - } - if not v2_mount_point: - raise TargetStableError( - "No V2 CGroup controllers have been delegated on this target." - ) - else: - # Since there can only be a single V2 hierarchy (ignoring bind mounts), this should be totally legal. - mount_path_to_unified_hierarchy = v2_mount_point.pop() - return str(os.path.join(mount_path_to_unified_hierarchy, sub_path)) - - @classmethod - @contextmanager - def _systemd_offline_mount( - cls, - target: LinuxTarget, - all_controllers: Dict[str, Dict[str, Union[str, int]]], - requested_controllers: Set[str], - ): - """ - Manually mounts the V2 hierarchy on the target device. Occurs in the absence of systemd. - - :param target: Interface to target device. - :type target: Target - - :param all_controllers: A Dictionary of currently mounted controller name keys to Dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type controllers: Dict[str, Dict[str, Union[str,int]]] - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :yield: The path to the root mount point of the unified V2 hierarchy. - :rtype: str - """ - - unused_controllers = _CGroupV2Root._get_available_controllers( - controllers=all_controllers - ) - _validate_requested_hierarchy( - requested_controllers=requested_controllers, - available_controllers=unused_controllers, - ) - - with _mount_v2_controllers(target) as mount_path: - yield mount_path - - @classmethod - @contextmanager - def _systemd_online_setup( - cls, - target: LinuxTarget, - all_controllers: Dict[str, Dict[str, int]], - requested_controllers: Set[str], - ): - """ - Sets up the required V2 hierarchy on the target device. Occurs in the presence of systemd. - - :param target: Interface to target device. - :type target: Target - - :param all_controllers: A Dictionary of currently mounted CGroup controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type all_controllers: Dict[str, Dict[str, Union[str,int]]] - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :yield: The path to the root of the delegated V2 CGroup hierarchy. - :rtype: str - """ - with _request_delegation(target=target) as main_pid: - delegated_sub_path = _CGroupV2Root._get_delegated_sub_path( - delegated_pid=main_pid, target=target - ) - delegated_path = _CGroupV2Root._path_to_delegated_root( - controllers=all_controllers, - sub_path=delegated_sub_path, - ) - - delegated_controllers_path = "{path}/cgroup.controllers".format( - path=delegated_path - ) - - # The controllers that have been delegated are held within - # the 'cgroup.controllers' file. The controller names are stored on a - # single line, requiring us to select the first (and only) element returned - # by _read_file and splitting said element (str) using the white space character - # as the delimiter. - # (The _validate_requested_hierarchy requires the available_controllers argument to be a dict, necessitating this dict structure.) - delegated_controllers = { - controller: None - for controller in _read_lines( - target=target, path=delegated_controllers_path - )[0].split(" ") - } - - _validate_requested_hierarchy( - requested_controllers=requested_controllers, - available_controllers=delegated_controllers, - ) - yield delegated_path - - @classmethod - @contextmanager - def _mount_filesystem(cls, target: LinuxTarget, requested_controllers: Set[str]): - """ - Mounts/Sets-up a V2 hierarchy on the target device, covering contexts where - systemd is both present and absent. - - :param target: Interface to target device. - :type target: Target - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :yield: A path to the root of the V2 hierarchy that has been mounted/delegated for the user. - :rtype: str - """ - - systemd_online = _is_systemd_online(target=target) - controllers = _CGroupV2Root._v2_controller_translation( - _get_cgroup_controllers(target=target) - ) - - if systemd_online: - cm = _CGroupV2Root._systemd_online_setup( - target=target, - all_controllers=controllers, - requested_controllers=requested_controllers, - ) - with cm as mount_path: - yield mount_path - - else: - cm = _CGroupV2Root._systemd_offline_mount( - target=target, - all_controllers=controllers, - requested_controllers=requested_controllers, - ) - with cm as mount_path: - yield mount_path - - def __init__( - self, - mount_point: str, - subtree_controllers: set, - target: LinuxTarget, - ): - - super().__init__( - name="", - parent_path=mount_point, - # Root can not have active controllers. - active_controllers={}, - subtree_controllers=subtree_controllers, - # Root can not be threaded. - is_threaded=False, - target=target, - ) - self.target = target - - def __enter__(self): - """ - Determines what happens when we enter the context of the CGroup, in this case the :meth:`_init_root_cgroup` method - is to be called; initializing the root group to abide by the user defined configuration. - If an exception occurs during this phase, the context will be exited and the exception raised. - - :raises TargetStableError: Occurs when an exception occurs within the :meth:`_init_root_cgroup` method call. - - :return: An object reference to itself. - :rtype: :class:`_CGroupV2Root` - """ - - try: - self._init_root_cgroup() - except TargetStableError as err: - self.__exit__(err, type(err), err.__traceback__) - raise - else: - return self - - def __exit__(self, *exc): - pass - - def _init_root_cgroup(self): - """ - Performs the required actions in order to initialise a Root V2 CGroup. - In the case where systemd is active, there is a required need to create a leaf CGroup from the Root, where the PIDs - systemd has delegated the subtree can be moved to. The reason for this is due to the side effect of being unable to - change the contents of the ``cgroup.subtree_control`` interface file while processes exist within the CGroup. - This process is skipped when initializing a root CGroup on a non-systemd system. - """ - - if _is_systemd_online(target=self.target): - # Create the leaf CGroup directory - group_name = "devlib-" + str(uuid4().hex) - full_path = self.target.path.join(self.group_path, group_name) - self._create_directory(full_path) - - delegated_pids = _read_lines( - target=self.target, - path="{path}/cgroup.procs".format(path=self.group_path), - ) - - # Move PIDs to leaf CGroup. - for pid in delegated_pids: - self.target.write_value( - path=self.target.path.join(full_path, "cgroup.procs"), - value=pid, - verify=False, - ) - - # Write to Subtree - for controller in self.subtree_controllers: - self._set_controller_attribute( - "cgroup", - "subtree_control", - "+{cont}".format(cont=controller), - ) - - -class _CGroupV1(_CGroupBase): - """ - A Class representing a CGroup folder within a CGroup V1 hierarchy. - - :param name: The name assigned to the CGroup. Used to identify the CGroup and define the CGroup folder name. - :type name: str - - :param parent_path: The path to the parent CGroup this CGroup is a child of. - :type parent_path: str - - :param active_controllers: A dictionary of controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between a specific 'attribute' of the aforementioned - controller and a value for which that controller interface should be set to. - - :type active_controllers: Dict[str, Dict[str, Union[str,int]]] - - :param target: Interface to target device. - :type target: Target - """ - - def __enter__(self): - """ - Determines what happens when we enter the context of the CGroup, - in this case creating the required CGroup directory and calling the :meth:`_init_cgroup` method. - If an exception occurs during this phase, the context will be exited and the exception raised. - - :raises TargetStableError: If an exception occurs within the :meth:`_init_cgroup` method call. - - :return: An object reference to itself. - :rtype: :class:`_CGroupV1` - """ - - self._create_directory(self.group_path) - try: - self._init_cgroup() - except TargetStableError as err: - self.__exit__(err, type(err), err.__traceback__) - raise - else: - return self - - def __exit__(self, *exc): - self._delete_directory(self.group_path) - - def _init_cgroup(self): - """ - Performs the required steps in order to initialize the CGroup to the user specified configuration: - - * Write the values to be written to the specified controller interfaces files. - """ - - # Attributes to controller {controller : {attr : val, attr : val}} - - for controller, configuration in self.active_controllers.items(): - for attr, val in configuration.items(): - self._set_controller_attribute( - controller=controller, attribute=attr, value=val, verify=True - ) - - def _add_thread(self, tid: int, threaded_domain): - """ - Adds the thread associated with ``tid`` to the CGroup. - While thread level management suffers from no restrictions within a V1 hierarchy, - semantic equivalence with CGroup V2 is required. Therefore, adding a thread - to a CGroup within a V1 hierarchy still abides by the restrictions set within - a V2 hierarchy. In this case, the process that the thread associated with ``tid`` - is a part of must reside at the root of the threaded subtree, enabling thread level - granularity across the entire of the threaded subtree. - - :param tid: The TID of the thread to be added to the CGroup - :type tid: int - - :param threaded_domain: The :class:`ResponseTree` object representing the threaded domain - of the threaded CGroup subtree. The process will be added to all the CGroups - that the :class:`ResponseTree` represents. - :type threaded_domain: :class:`ResponseTree` - """ - - pid_of_tid = self._get_pid_from_tid(tid=tid) - - for low_level in threaded_domain.low_levels.values(): - low_level._add_process(pid_of_tid) - - self._set_controller_attribute("", "tasks", tid) - - -class _CGroupV1Root(_CGroupV1): - """ - A subclass of the :class:`_CGroupV1` class representing a root V1 CGroup directory. - Contains the necessary functionality that enables the setting-up / mounting of a V1 - CGroup hierarchy. - - :param mount_point: The path to which the root of the CGroup V1 controller hierarchy is mounted on. - :type mount_point: str - - :param target: Interface to target device. - :type target: Target - """ - - @classmethod - def _get_delegated_paths( - cls, - controllers: Dict[str, Dict[str, Union[str, int]]], - delegated_pid: int, - target: LinuxTarget, - ): - """ - Returns the relative sub-paths the delegated roots of the V1 hierarchies, via the parsing - of the /proc//cgroup file of the delegated PID. - - :param controllers: A dictionary of currently mounted CGroup controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type controllers: Dict[str, Dict[str, Union[str,int]]] - - :param delegated_pid: The Main PID of the transient service unit we request delegation for. - :type delegated_pid: int - - :param target: Interface to target device. - :type target: Target - - :raises TargetStableError: Occurs in the case where no V1 controllers have been delegated. - - :return: A dictionary mapping CGroup controllers to their respective delegated root paths. - :rtype: Dict[str, str] - """ - - delegated_mount_paths = _read_lines( - target=target, path="/proc/{pid}/cgroup".format(pid=delegated_pid) - ) - - # A snippet of the /proc//cgroup is shown below. - # - # 10:misc:/ - # 9:memory:/system.slice/xyz.service - # - # The regex is structured to only match V1 controller hierarchies. - - REL_PATH_REGEX = re.compile( - r"\d+:(?P.+):\/(?P.*)" - ) - - delegated_controllers = {} - - for mount_path in delegated_mount_paths: - regex_match = REL_PATH_REGEX.match(mount_path) - if regex_match: - con = regex_match.group("controllers") - path = regex_match.group("path_to_delegated_service_root") - # Multiple v1 controllers can be co-mounted on a single folder hierarchy. - co_mounted_controllers = con.strip().split(",") - for controller in co_mounted_controllers: - try: - configuration = controllers[controller] - except KeyError: - pass - else: - delegated_controllers[controller] = target.path.join( - configuration["mount_point"], path - ) - - if not delegated_controllers: - raise TargetStableError( - "No V1 CGroup controllers have been delegated on the target." - ) - - return delegated_controllers - - @classmethod - @contextmanager - def _systemd_offline_mount( - cls, - requested_controllers: Set[str], - all_controllers: Dict[str, Dict[str, Union[str, int]]], - target: LinuxTarget, - ): - """ - Manually mounts the V1 split hierarchy on the target device. Occurs in the absence of systemd. - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :param all_controllers: A Dictionary of currently mounted controller name keys to Dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type all_controllers: Dict[str, Dict[str, Union[str,int]]] - - :param target: Interface to target device. - :type target: Target - - :yield: A dictionary mapping CGroup controller names to their respective mount points. - :rtype: Dict[str,str] - """ - - available_controllers = _CGroupV1Root._get_available_v1_controllers( - controllers=all_controllers - ) - _validate_requested_hierarchy( - requested_controllers=requested_controllers, - available_controllers=available_controllers, - ) - - cm = _mount_v1_controllers(target=target, controllers=requested_controllers) - with cm as mounted: - yield mounted - - @classmethod - def _get_available_v1_controllers( - cls, controllers: Dict[str, Dict[str, Union[int, str]]] - ): - - unused_controllers = { - controller: configuration - for controller, configuration in controllers.items() - if configuration.get("version") is None - } - - if not unused_controllers: - raise TargetStableError("No V1 CGroup controllers available on target.") - - return unused_controllers - - @classmethod - @contextmanager - def _systemd_online_setup( - cls, - target: LinuxTarget, - requested_controllers: Set[str], - all_controllers: Dict[str, Dict[str, str]], - ): - """ - Sets up the required V1 hierarchy on the target device. Occurs in the presence of systemd. - - :param target: Interface to target device. - :type target: Target - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :param all_controllers: A Dictionary of currently mounted controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between various CGroup controller configuration keys - and their respectively obtained values for the respective CGroup controllers. - :type all_controllers: Dict[str, Dict[str, Union[str,int]]] - - :yield: A Dict[str, str] consisting of controller name keys mapped to their respective mount points. - :rtype: Dict[str, str] - """ - - with _request_delegation(target) as pid: - delegated_controllers = _CGroupV1Root._get_delegated_paths( - controllers=all_controllers, - delegated_pid=pid, - target=target, - ) - _validate_requested_hierarchy( - requested_controllers=requested_controllers, - available_controllers=delegated_controllers, - ) - - yield (delegated_controllers) - - @classmethod - @contextmanager - def _mount_filesystem(cls, target: LinuxTarget, requested_controllers: Set[str]): - """ - A context manager which Mounts/Sets-up a V1 split hierarchy on the target device, covering contexts where - systemd is both present and absent. This context manager Mounts/Sets-up a split V1 hierarchy (if possible) - and performs the clean up (process depends on whether systemd is present or not) necessary afterwards returning - the target device to the state before the mount/set-up occurred. - - :param target: Interface to target device. - :type target: Target - - :param requested_controllers: The set of controllers required to mount the requested hierarchy. - :type requested_controllers: Set[str] - - :yield: A dictionary mapping controller name to the paths where the controllers are mounted on, used to build the user requested V1 hierarchy. - :rtype: dict[str,str] - """ - - systemd_online = _is_systemd_online(target=target) - controllers = _get_cgroup_controllers(target=target) - - if systemd_online: - cm = _CGroupV1Root._systemd_online_setup( - target=target, - requested_controllers=requested_controllers, - all_controllers=controllers, - ) - with cm as controllers: - yield controllers - - else: - cm = _CGroupV1Root._systemd_offline_mount( - target=target, - requested_controllers=requested_controllers, - all_controllers=controllers, - ) - with cm as controllers: - yield controllers - - def __init__(self, mount_point: str, target: LinuxTarget): - - super().__init__( - # Root name is null. Isn't required. - name="", - parent_path=mount_point, - # Root can not have active controllers. - active_controllers={}, - target=target, - ) - - def __enter__(self): - """ - Determines what happens when we enter the context of the CGroup, - in this case no set-up/resource-management occurs. - - :return: An object reference to itself. - :rtype: :class:`_CGroupV1Root` - """ - - return self - - def __exit__(self, *exc): - pass - - -class _TreeBase(ABC): - """ - The abstract base class that all tree class types' subclass. - - :param name: The name assigned to the tree node. - :type name: str - - :param is_threaded: Whether the node is threaded or not. - :type is_threaded: bool - """ - - def __init__(self, name: str, is_threaded: bool): - self.name = name - self.is_threaded = is_threaded - self.threaded_domain = self - - # Propagates Threaded Property to - # sub-tree. - def make_threaded(grp): - grp.is_threaded = True - for child in grp._children_list: - make_threaded(child) - - # Propagates the Threaded domain - # to sub-tree. - def set_domain(grp): - grp.threaded_domain = domain - for child in grp._children_list: - set_domain(child) - - if is_threaded: - make_threaded(self) - else: - domain = self - if any([child.is_threaded for child in self._children_list]): - for child in self._children_list: - make_threaded(child) - set_domain(child) - - @property - def is_threaded_domain(self): - return ( - True - if any([child.is_threaded for child in self._children_list]) - and self.threaded_domain is self - else False - ) - - @property - @memoized - def group_type(self): - if self.is_threaded_domain: - return "threaded domain" - elif self.is_threaded: - return "threaded" - else: - return "domain" - - def __str__(self, level=0): - """ - Returns a string representation of the tree hierarchy, used for visualization and debugging. - - :param level: The current depth of the tree, defaults to 0. - :type level: int, optional - - :return: String formatted output, displaying the hierarchical structure of the tree. - :rtype: str - """ - - TAB = "\t" - ELBOW = "└──" - children = "\n".join( - child.__str__(level=level + 1) for child in (self._children_list) - ) - children = "\n" + children if children else children - - return "{tab}{elbow}{name} {node_info} {children}".format( - tab=TAB * level, - elbow=ELBOW, - name=self.name, - node_info=self._node_information, - children=children, - ) - - @property - @abstractmethod - def _node_information(self): - """ - Returns a formatted string displaying the information the :class:`_TreeBase` object represents. - """ - pass - - @property - @abstractmethod - def _children_list(self): - """ - Returns List[:class:`_TreeBase`]. - """ - pass - - -class RequestTree(_TreeBase): - """ - A class used to represent a unified, tree-like user-defined CGroup hierarchy. - Modelled as a V2 CGroup hierarchy, but able to represent both hierarchy versions (1 & 2) on the target device as - required by ensuring V2 semantic equivalence is maintained within the context of setting up a V1 hierarchy. - - :param name: Name assigned to the user defined :class:`RequestTree` object. - :type name: str - - :param children: A list of :class:`RequestTree` objects representing the children the object is - a hierarchical parent to, defaults to ``None``. - :type children: List[:class:`RequestTree`], optional - - :param controllers: A Dictionary of controller name keys to dictionary value mappings, - where the secondary dictionary contains a mapping between controller specific attributes and - their respective to be assigned values, , defaults to ``None``. - :type controllers: Dict[str, Dict[str, Union[str,int]]], optional - - :param threaded: defines whether the object will represent a CGroup capable of managing threads, defaults to ``False``. - :type threaded: bool, optional - """ - - def __init__( - self, - name: str, - children: Union[list, None] = None, - controllers: Union[Dict[str, Dict[str, Any]], None] = None, - threaded=False, - ): - self.children = children or [] - self.controllers = controllers or {} - super().__init__(name=name, is_threaded=threaded) - - @property - def _node_information(self): - # Returns Requests Tree Node Information. - active_controllers = [ - "({controller}) {config}".format( - controller=controller, config=configuration - ) - for controller, configuration in sorted(self.controllers.items()) - ] - return "{active_controllers} [{group_type}]".format( - active_controllers=", ".join(active_controllers), - group_type=self.group_type, - ) - - @property - @memoized - def _all_controllers(self): - # Returns a set of all the controllers that are active in that subtree, including its own. - return set( - itertools.chain( - self.controllers.keys(), - itertools.chain.from_iterable( - map(lambda child: child._all_controllers, self.children), - ), - ) - ) - - @property - def _subtree_controllers(self): - # Returns a set of all the controllers that are active in that subtree, excluding its own. - return set( - itertools.chain.from_iterable( - map(lambda child: child._all_controllers, self.children) - ) - ) - - @property - def _children_list(self): - return list(self.children) - - @contextmanager - def setup_hierarchy(self, version: int, target: LinuxTarget): - """ - A context manager which processes the user defined hierarchy and sets-up said hierarchy on the ``target`` device. - Uses an internal exit stack to the handle the entering and safe exiting of the lower level - contexts of the :class:`_CGroupBase` subclasses, - restoring the target device to the state it was in before the hierarchy was set up. - If the set-up was successful, it will yield an instance of the :class:`ResponseTree` class (representing the root of the tree) - which the user will interact with and can inspect. - - :param version: The version of the CGroup hierarchy to be set up on the Target device. - :type version: int - - :param target: Interface to target device. - :type target: Target - - :raises TargetStableError: Occurs when the version argument is neither ``1`` or ``2``; - the only two versions of CGroups currently available. - - :yield: An instance of the :class:`ResponseTree` class, representing the root of the CGroup hierarchy. - """ - - with ExitStack() as exit_stack: - if version == 1: - # Returns a {controller_name: controller_mount_point} dict - controller_paths = exit_stack.enter_context( - _CGroupV1Root._mount_filesystem( - target=target, requested_controllers=self._all_controllers - ) - ) - # Mounts the Roots Controller Parents. - root_parents = { - controller: _CGroupV1Root( - mount_point=mount_path, - target=target, - ) - for controller, mount_path in controller_paths.items() - if controller in self._all_controllers - } - - def make_groups(request: RequestTree, parents: Dict[str, _CGroupBase]): - """ - Defines and instantiates the low-level :class:`_CGroupV1` objects as per defined by the - configuration of the ``request`` :class:`RequestTree` object. - The parents of said :class:`_CGroupV1` objects will be determined via the - ``parents`` dictionary, ensuring the newly created :class:`_CGroupV1` objects are - created 'under' the suitable parent CGroup directory. - - :param request: The :class:`RequestTree` object that'll define the required :class:`_CGroupV1` objects it represents. - :type request: :class:`RequestTree` - - :param parents: The Dictionary mapping that maps CGroup controller names to their leaf CGroup directory. - :type parents: Dict[str, :class:`_CGroupBase`] - - :return: A tuple ``(request_defined_cgroups, all_cgroups, parents)`` where the first element defines the - dictionary mapping the controller names to the :class:`_CGroupV1` objects created directly - due to the :class:`RequestTree` object user configuration and the second and third elements - defining the dictionary mapping controller names to the leaf CGroup for said controller. - Duplication is required not only to maintain compatibility with the function - defined under the same name and signature under the context of setting up a CGroup V2 hierarchy, - but since we want to maintain a semantic equivalence to a V2 hierarchy, - the low-level :class:`_CGroupV1` objects a particular :class:`RequestTree` - instance indirectly defines given its parents and the :class:`_CGroupV1` objects it passes to it children - as potential suitable parents are the same. - :rtype: tuple(Dict[str,:class:`_CGroupV1`], Dict[str,:class:`_CGroupV1`], Dict[str,:class:`_CGroupV1`]) - """ - - request_defined_cgroups = { - controller: _CGroupV1( - name=request.name, - parent_path=parents[controller].group_path, - active_controllers={controller: attributes}, - target=target, - ) - for controller, attributes in request.controllers.items() - } - - # Parent dict updated to include the newly created leaf CGroups. - parents = {**parents, **request_defined_cgroups} - all_cgroups = parents - return (request_defined_cgroups, all_cgroups, parents) - - elif version == 2: - - # Returns a string representing the root of the V2 hierarchy - unified_mount_point = exit_stack.enter_context( - _CGroupV2Root._mount_filesystem( - target=target, requested_controllers=self._all_controllers - ) - ) - - root_parents = _CGroupV2Root( - mount_point=unified_mount_point, - subtree_controllers=self._all_controllers, - target=target, - ) - - # We require to enter the context of the root of the V2 hierarchy at this stage in order to perform the required - # root CGroup setup defined within the __enter__ method. - exit_stack.enter_context(root_parents) - - def make_groups(request: RequestTree, parent: _CGroupV2): - """ - Defines and instantiates the low-level :class:`_CGroupV2` object as per defined by the - configuration of the ``request`` :class:`RequestTree` object. The parents of said :class:`_CGroupV2` object - will be determined via the ``parents`` :class:`_CGroupV2` object, ensuring the newly created - :class:`_CGroupV2` object is created 'under' the suitable parent CGroup directory. - - :param request: The :class:`RequestTree` object that'll define the required :class:`_CGroupV2` object. - :type request: :class:`RequestTree` - - :param parents: The CGroup that'll be the parent of the :class:`_CGroupV2` object being defined. - :type parents: :class:`_CGroupV2` - - :return: A tuple ``(controllers_to_cgroup, controllers_to_cgroup, parent)`` where the first and second elements - define a dictionary mapping of controller names as per defined by the :class:`RequestTree` object - and the solitary :class:`_CGroupV2` object that has been instantiated, - (All enabled controllers map to a single V2 directory under a V2 hierarchy). - The Last element within the tuple defines the newly instantiated :class:`_CGroupV2` object set to be the - hierarchical parent of the subsequent V2 CGroups to be created. Duplication is required in this case since both the paths - the user defined V2 controllers are enabled at and the actual paths - of the low-level implementation are the same as per the structure of the unified V2 hierarchy. - :rtype: tuple(Dict[str,:class:`_CGroupV2`],Dict[str,:class:`_CGroupV2`],:class:`_CGroupV2`) - """ - - request_group = _CGroupV2( - name=request.name, - parent_path=parent.group_path, - active_controllers=request.controllers, - subtree_controllers=request._subtree_controllers, - is_threaded=request.is_threaded, - target=target, - ) - - # Creates a mapping between the enabled controllers within this CGroup to the low-level - # _CGroupV2 object - controllers_to_cgroup = dict.fromkeys( - request.controllers, request_group - ) - # Creating 'parent' variable for readability’s sake. - parent = request_group - return (controllers_to_cgroup, controllers_to_cgroup, parent) - - else: - raise TargetStableError( - "A {version} version hierarchy cannot be mounted. Ensure requested hierarchy version is 1 or 2.".format( - version=version - ) - ) - - # Create the Response Tree from the Request Tree. - response = self._create_response(root_parents, make_groups=make_groups) - # Returns a list of all the Low-level _CGroupBase objects the response object represents in the right order - groups = response._all_nodes - # Remove duplicates while preserving order. - groups = sorted(set(groups), key=groups.index) - # Enter the context for each object - for group in groups: - exit_stack.enter_context(group) - - yield response - - def _create_response(self, low_level_parent, make_groups): - """ - Creates the :class:`ResponseTree` object tree, using the appropriately defined :meth:`make_group` callable (defined as a local function - internally within :meth:`setup_hierarchy`) alongside the ``low_level_parent`` object to create the low-level CGroups a particular :class:`RequestTree` object represents. - This function is then recursively called on the children of the :class:`RequestTree` object in order to create subsequent child - :class:`ResponseTree` objects to create a Tree-like object that mirrors the Tree structure of the :class:`RequestTree` object. - - :param low_level_parent: The parent/s to the CGroups to be created. In the context of setting up a V1 hierarchy, this will be a - dictionary mapping controller names to :class:`_CGroupV1` objects; while in the case of V2, it'll be a solitary :class:`_CGroupV2` object. - :type low_level_parent: Dict[str,:class:`_CGroupV1`] | :class:`_CGroupV2` - - :param make_groups: The callable function definition used to create the low-level CGroup required. This callable is defined appropriately - depending on the CGroup hierarchy version we require to set-up/mount. - :type make_groups: callable - - :return: The root of the :class:`ResponseTree` object tree. - :rtype: :class:`ResponseTree` - """ - - user_visible_low_level_groups, low_level_groups, low_level_parent = make_groups( - self, low_level_parent - ) - return ResponseTree( - name="{name}/".format(name=self.name), - children={ - child.name: child._create_response( - low_level_parent=low_level_parent, - make_groups=make_groups, - ) - for child in self.children - }, - low_levels=low_level_groups, - # We dont want to show the user all the CGroups directories it represents within the context of a V1 hierarchy, - # since it contains directories not directly defined by the user (The V1 hierarchy we define resembles the unified V2 hierarchy, - # therefore there'll be some low-level _CGroupV1 objects it'll represent that haven't been directly defined by the its corresponding - # RequestTree object but instead inherited from its parents. - user_low_levels=user_visible_low_level_groups, - is_threaded=self.is_threaded, - ) - - -class ResponseTree(_TreeBase, collections.abc.Mapping): - """ - A class used to represent a collection of CGroup directories created on the target system, - abstracting the lower level complexities and allowing to user to interact with the CGroups. - The structure of the tree mirrors the structure of the :class:`RequestTree` object tree used to create it, where - each :class:`ResponseTree` object represents and abstracts the low-level CGroups its respective :class:`RequestTree` object defines. - - :param name: Name assigned to the :class:`ResponseTree` object, mirrors the name defined to its respective :class:`RequestTree` Object. - :type name: str - - :param children: A dictionary that maps children names that this :class:`ResponseTree` object is a parent to - and the respective :class:`ResponseTree` object the names represent. - :type children: dict[str,:class:`ResponseTree`] - - :param low_levels: A dictionary that maps CGroup controller names to the suitable low level CGroup this :class:`ResponseTree` abstracts. - :type low_levels: Dict[str, :class:`_CGroupBase`] - - :param user_low_levels: A dictionary that maps CGroup controller names to the suitable low level CGroup the - :class:`RequestTree` object this class mirrors has specified. This is used within the context of a V1 user - defined hierarchy in order to abstract the additional CGroups this class represents when trying to ensure V2 semantic - equivalence. Done purely for cosmetic reasons. - :type user_low_levels: Dict[str, :class:`_CGroupBase`] - - :param is_threaded: Boolean flag representing whether or not this ResponseTree object represents a single threaded V2 CGroup - or a collection of pseudo-threaded V1 CGroups. - :type is_threaded: bool - """ - - def __init__( - self, - name: str, - children: Dict[str, _TreeBase], - low_levels: Dict[str, _CGroupBase], - user_low_levels: Dict[str, _CGroupBase], - is_threaded: bool, - ): - self.children = children - self.low_levels = low_levels - self.user_low_levels = user_low_levels - super().__init__(name=name, is_threaded=is_threaded) - - @property - def _node_information(self): - # Returns a formatted string, displaying the enabled user-defined controllers and their paths - # (alongside the type of CGroup the controller resides in). - return ", ".join( - "{controller}@{path} [{cgroup_type}]".format( - controller=controller, - path=low_level.group_path, - cgroup_type=self.group_type, - ) - for controller, low_level in self.user_low_levels.items() - ) - - @property - def _children_list(self): - # Children Objects are the values in our self.children dict. - return list(self.children.values()) - - @property - def _all_nodes(self): - return list( - itertools.chain( - self.low_levels.values(), - itertools.chain.from_iterable( - map(lambda child: child._all_nodes, self.children.values()), - ), - ) - ) - - def add_process(self, pid: int): - """ - Adds the process associated with ``pid`` to the low level CGroups this :class:`ResponseTree` object represents. - - :param pid: the PID of the process to be added to the low-level CGroups. - :type pid: int - - :raises TargetStableError: Occurs in the case where this object is a parent to non-threaded children. - Ensures V2 hierarchy compatibility. - """ - - if self.is_threaded_domain or self.is_threaded or not self.children: - for low_level in self.low_levels.values(): - low_level._add_process(pid=pid) - else: - raise TargetStableError( - "Cannot add Process ID: {pid} to {name}. The ResponseTree object is a parent to a non-threaded ResponseTree.".format( - pid=pid, name=self.name - ) - ) - - def add_thread(self, tid: int): - """ - Adds the thread associated with the ``tid`` to the low level CGroups this :class:`ResponseTree` object represents. - - :param tid: the TID of the thread to be added to the low-level CGroups. - :type tid: int - - :raises TargetStableError: Occurs in the case where this object is not threaded. - Ensures V2 hierarchy compatibility. - """ - - if self.is_threaded: - for lower_level in set(self.low_levels.values()): - lower_level._add_thread(tid, self.threaded_domain) - else: - raise TargetStableError( - "Cannot add Thread ID: {tid} to {name}. The ResponseTree object is not threaded.".format( - tid=tid, name=self.name - ) - ) - - def __getitem__(self, child_name: str): - return self.children[child_name] - - def __iter__(self): - return iter(self.children) - - def __len__(self): - return len(self.children) diff --git a/external/devlib/devlib/module/vexpress.py b/external/devlib/devlib/module/vexpress.py index c597747be67f549bc2267efa44b31bcf86e6b65e..05e41467e725fd7cb20c42e1c204ae7462e7f9dd 100644 --- a/external/devlib/devlib/module/vexpress.py +++ b/external/devlib/devlib/module/vexpress.py @@ -21,7 +21,6 @@ from subprocess import CalledProcessError from devlib.module import HardRestModule, BootModule, FlashModule from devlib.exception import TargetError, TargetStableError, HostError -from devlib.utils.misc import safe_extract from devlib.utils.serial_port import open_serial_connection, pulse_dtr, write_characters from devlib.utils.uefi import UefiMenu, UefiConfig from devlib.utils.uboot import UbootMenu @@ -355,7 +354,7 @@ class VersatileExpressFlashModule(FlashModule): validate_image_bundle(bundle) self.logger.debug('Extracting {} into {}...'.format(bundle, self.vemsd_mount)) with tarfile.open(bundle) as tar: - safe_extract(tar, self.vemsd_mount) + tar.extractall(self.vemsd_mount) def _overlay_images(self, images): for dest, src in images.items(): diff --git a/external/devlib/devlib/platform/arm.py b/external/devlib/devlib/platform/arm.py index fbe81af8ccf0ed2393db1cccd533038e70347a05..eb5dbb5c052723a9b62b0a3359b1e141dd955fba 100644 --- a/external/devlib/devlib/platform/arm.py +++ b/external/devlib/devlib/platform/arm.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import division import os import sys import tempfile diff --git a/external/devlib/devlib/platform/gem5.py b/external/devlib/devlib/platform/gem5.py index 4bae58806b28b33327c08982fe6a57c8229a5bcb..9fa82547f84cdc61fb2b46c631eab7122be489e6 100644 --- a/external/devlib/devlib/platform/gem5.py +++ b/external/devlib/devlib/platform/gem5.py @@ -19,7 +19,7 @@ import shutil import time import types import shlex -from shlex import quote +from pipes import quote from devlib.exception import TargetStableError from devlib.host import PACKAGE_BIN_DIRECTORY diff --git a/external/devlib/devlib/target.py b/external/devlib/devlib/target.py index eaccd747dc7c9aa28c2894fdc0d0bcf9634ed69f..780008aa9c8bceb7efcc525efc6f1af16b11f600 100644 --- a/external/devlib/devlib/target.py +++ b/external/devlib/devlib/target.py @@ -36,10 +36,10 @@ import inspect import itertools from collections import namedtuple, defaultdict from contextlib import contextmanager +from pipes import quote from past.builtins import long from past.types import basestring from numbers import Number -from shlex import quote try: from collections.abc import Mapping except ImportError: @@ -61,7 +61,7 @@ from devlib.utils.misc import memoized, isiterable, convert_new_lines, groupby_v from devlib.utils.misc import commonprefix, merge_lists from devlib.utils.misc import ABI_MAP, get_cpu_name, ranges_to_list from devlib.utils.misc import batch_contextmanager, tls_property, _BoundTLSProperty, nullcontext -from devlib.utils.misc import strip_bash_colors, safe_extract +from devlib.utils.misc import strip_bash_colors from devlib.utils.types import integer, boolean, bitmask, identifier, caseless_string, bytes_regex import devlib.utils.asyn as asyn @@ -344,7 +344,6 @@ class Target(object): self._cache = {} self._shutils = None self._file_transfer_cache = None - self._max_async = max_async self.busybox = None if load_default_modules: @@ -388,7 +387,7 @@ class Target(object): # connection and initialization @asyn.asyncf - async def connect(self, timeout=None, check_boot_completed=True, max_async=None): + async def connect(self, timeout=None, check_boot_completed=True, max_async=50): self.platform.init_target_connection(self) # Forcefully set the thread-local value for the connection, with the # timeout we want @@ -401,7 +400,7 @@ class Target(object): self.execute('mkdir -p {}'.format(quote(self.executables_directory))) self.busybox = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, self.abi, 'busybox'), timeout=30) self.conn.busybox = self.busybox - self._detect_max_async(max_async or self._max_async) + self._detect_max_async(max_async) self.platform.update_from_target(self) self._update_modules('connected') if self.platform.big_core and self.load_default_modules: @@ -828,7 +827,7 @@ class Target(object): await self.pull.asyn(tar_file_name, tmpfile) # Decompress with tarfile.open(tmpfile, 'r') as f: - safe_extract(f, outdir) + f.extractall(outdir) os.remove(tmpfile) # execution @@ -1825,7 +1824,7 @@ class AndroidTarget(Target): raise TargetStableError('Connected but Android did not fully boot.') @asyn.asyncf - async def connect(self, timeout=30, check_boot_completed=True, max_async=None): # pylint: disable=arguments-differ + async def connect(self, timeout=30, check_boot_completed=True, max_async=50): # pylint: disable=arguments-differ device = self.connection_settings.get('device') await super(AndroidTarget, self).connect.asyn( timeout=timeout, @@ -2999,7 +2998,7 @@ class ChromeOsTarget(LinuxTarget): else: raise - def connect(self, timeout=30, check_boot_completed=True, max_async=None): + def connect(self, timeout=30, check_boot_completed=True, max_async=50): super(ChromeOsTarget, self).connect( timeout=timeout, check_boot_completed=check_boot_completed, diff --git a/external/devlib/devlib/utils/android.py b/external/devlib/devlib/utils/android.py index 1cecd06913e732827a049c3f81e6f56bda27c814..ecca40245e50e455b75ae292d0862ff7cdc53d8a 100755 --- a/external/devlib/devlib/utils/android.py +++ b/external/devlib/devlib/utils/android.py @@ -35,7 +35,11 @@ import threading from collections import defaultdict from io import StringIO from lxml import etree -from shlex import quote + +try: + from shlex import quote +except ImportError: + from pipes import quote from devlib.exception import TargetTransientError, TargetStableError, HostError, TargetTransientCalledProcessError, TargetStableCalledProcessError, AdbRootError from devlib.utils.misc import check_output, which, ABI_MAP, redirect_streams, get_subprocess diff --git a/external/devlib/devlib/utils/misc.py b/external/devlib/devlib/utils/misc.py index 47348927ff045e4696d32eeb0bb6d74b465bcabc..c73a9c601e6a730cf6f1c0c9cd5568ef283948ad 100644 --- a/external/devlib/devlib/utils/misc.py +++ b/external/devlib/devlib/utils/misc.py @@ -18,6 +18,7 @@ Miscellaneous functions that don't fit anywhere else. """ +from __future__ import division from contextlib import contextmanager from functools import partial, reduce, wraps from itertools import groupby @@ -46,7 +47,11 @@ try: except AttributeError: from contextlib2 import ExitStack -from shlex import quote +try: + from shlex import quote +except ImportError: + from pipes import quote + from past.builtins import basestring # pylint: disable=redefined-builtin @@ -457,7 +462,7 @@ def escape_quotes(text): """ Escape quotes, and escaped quotes, in the specified text. - .. note:: :func:`shlex.quote` should be favored where possible. + .. note:: :func:`pipes.quote` should be favored where possible. """ return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\'', '\\\'').replace('\"', '\\\"') @@ -466,7 +471,7 @@ def escape_single_quotes(text): """ Escape single quotes, and escaped single quotes, in the specified text. - .. note:: :func:`shlex.quote` should be favored where possible. + .. note:: :func:`pipes.quote` should be favored where possible. """ return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\'', '\'\\\'\'') @@ -475,7 +480,7 @@ def escape_double_quotes(text): """ Escape double quotes, and escaped double quotes, in the specified text. - .. note:: :func:`shlex.quote` should be favored where possible. + .. note:: :func:`pipes.quote` should be favored where possible. """ return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\"', '\\\"') @@ -484,7 +489,7 @@ def escape_spaces(text): """ Escape spaces in the specified text - .. note:: :func:`shlex.quote` should be favored where possible. + .. note:: :func:`pipes.quote` should be favored where possible. """ return text.replace(' ', '\\ ') @@ -986,26 +991,3 @@ def groupby_value(dct): tuple(map(itemgetter(0), _items)): v for v, _items in groupby(items, key=key) } - - -def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - """ - A wrapper around TarFile.extract all to mitigate CVE-2007-4995 - (see https://www.trellix.com/en-us/about/newsroom/stories/research/tarfile-exploiting-the-world.html) - """ - - for member in tar.getmembers(): - member_path = os.path.join(path, member.name) - if not _is_within_directory(path, member_path): - raise Exception("Attempted Path Traversal in Tar File") - - tar.extractall(path, members, numeric_owner=numeric_owner) - -def _is_within_directory(directory, target): - - abs_directory = os.path.abspath(directory) - abs_target = os.path.abspath(target) - - prefix = os.path.commonprefix([abs_directory, abs_target]) - - return prefix == abs_directory diff --git a/external/devlib/devlib/utils/rendering.py b/external/devlib/devlib/utils/rendering.py index e66dd8c98a24da293c74272f9a7103c80a9d36b2..1e98115422baf69fe30ed5ed508624d450c6c8f0 100644 --- a/external/devlib/devlib/utils/rendering.py +++ b/external/devlib/devlib/utils/rendering.py @@ -21,7 +21,7 @@ import tempfile import threading import time from collections import namedtuple -from shlex import quote +from pipes import quote # pylint: disable=redefined-builtin from devlib.exception import WorkerThreadError, TargetNotRespondingError, TimeoutError diff --git a/external/devlib/devlib/utils/ssh.py b/external/devlib/devlib/utils/ssh.py index 39d704645397f8e013987c77ef78e67975a8b887..451792bae92e47dba6a8bdbd7a3933b9368dfa09 100644 --- a/external/devlib/devlib/utils/ssh.py +++ b/external/devlib/devlib/utils/ssh.py @@ -32,7 +32,8 @@ import weakref import select import copy import functools -from shlex import quote +from pipes import quote +from future.utils import raise_from from paramiko.client import SSHClient, AutoAddPolicy, RejectPolicy import paramiko.ssh_exception @@ -847,8 +848,8 @@ class TelnetConnection(SshConnectionBase): try: check_output(command, timeout=timeout, shell=True) except subprocess.CalledProcessError as e: - msg = f"Failed to copy file with '{command_redacted}'. Output:\n{e.output}" - raise HostError(msg) from None + raise_from(HostError("Failed to copy file with '{}'. Output:\n{}".format( + command_redacted, e.output)), None) except TimeoutError as e: raise TimeoutError(command_redacted, e.output) diff --git a/external/devlib/setup.py b/external/devlib/setup.py index bf1b408ff5f3b25c08c485877bdce17b13088dbe..1bb7bd104048332248d38cbb1ba12fb6b690676d 100644 --- a/external/devlib/setup.py +++ b/external/devlib/setup.py @@ -90,6 +90,7 @@ params = dict( 'paramiko', # SSH connection 'scp', # SSH connection file transfers 'wrapt', # Basic for construction of decorator functions + 'future', # Python 2-3 compatibility 'numpy', 'pandas', 'lxml', # More robust xml parsing diff --git a/external/workload-automation/doc/source/developer_information/how_tos/adding_plugins.rst b/external/workload-automation/doc/source/developer_information/how_tos/adding_plugins.rst index 05921bb90ad548f542b464b59e2c96ff0e2daec1..cb3d7c0b6d90955229c1b809043d6160a5a504fd 100644 --- a/external/workload-automation/doc/source/developer_information/how_tos/adding_plugins.rst +++ b/external/workload-automation/doc/source/developer_information/how_tos/adding_plugins.rst @@ -492,10 +492,9 @@ Adding an Instrument ==================== This is an example of how we would create a instrument which will trace device errors using a custom "trace" binary file. For more detailed information please see the -:ref:`Instrument Reference `. The first thing to do is to create -a new file under ``$WA_USER_DIRECTORY/plugins/`` and subclass -:class:`Instrument`. Make sure to overwrite the variable name with what we want our instrument -to be called and then locate our binary for the instrument. +:ref:`Instrument Reference `. The first thing to do is to subclass +:class:`Instrument`, overwrite the variable name with what we want our instrument +to be called and locate our binary for our instrument. :: @@ -503,8 +502,8 @@ to be called and then locate our binary for the instrument. name = 'trace-errors' - def __init__(self, target, **kwargs): - super(TraceErrorsInstrument, self).__init__(target, **kwargs) + def __init__(self, target): + super(TraceErrorsInstrument, self).__init__(target) self.binary_name = 'trace' self.binary_file = os.path.join(os.path.dirname(__file__), self.binary_name) self.trace_on_target = None @@ -551,9 +550,8 @@ workload. The method can be passed 4 params, which are the metric `key`, def update_output(self, context): # pull the trace file from the target self.result = os.path.join(self.target.working_directory, 'trace.txt') - self.outfile = os.path.join(context.output_directory, 'trace.txt') - self.target.pull(self.result, self.outfile) - context.add_artifact('error_trace', self.outfile, kind='export') + self.target.pull(self.result, context.working_directory) + context.add_artifact('error_trace', self.result, kind='export') # parse the file if needs to be parsed, or add result directly to # context. @@ -574,14 +572,12 @@ At the very end of the run we would want to uninstall the binary we deployed ear So the full example would look something like:: - from wa import Instrument - class TraceErrorsInstrument(Instrument): name = 'trace-errors' - def __init__(self, target, **kwargs): - super(TraceErrorsInstrument, self).__init__(target, **kwargs) + def __init__(self, target): + super(TraceErrorsInstrument, self).__init__(target) self.binary_name = 'trace' self.binary_file = os.path.join(os.path.dirname(__file__), self.binary_name) self.trace_on_target = None @@ -599,9 +595,8 @@ So the full example would look something like:: def update_output(self, context): self.result = os.path.join(self.target.working_directory, 'trace.txt') - self.outfile = os.path.join(context.output_directory, 'trace.txt') - self.target.pull(self.result, self.outfile) - context.add_artifact('error_trace', self.outfile, kind='export') + self.target.pull(self.result, context.working_directory) + context.add_artifact('error_trace', self.result, kind='export') metric = # .. context.add_metric('number_of_errors', metric, lower_is_better=True @@ -618,9 +613,8 @@ Adding an Output Processor ========================== This is an example of how we would create an output processor which will format -the run metrics as a column-aligned table. The first thing to do is to create -a new file under ``$WA_USER_DIRECTORY/plugins/`` and subclass -:class:`OutputProcessor`. Make sure to overwrite the variable name with what we want our +the run metrics as a column-aligned table. The first thing to do is to subclass +:class:`OutputProcessor` and overwrite the variable name with what we want our processor to be called and provide a short description. Next we need to implement any relevant methods, (please see diff --git a/external/workload-automation/requirements.txt b/external/workload-automation/requirements.txt index ba33aff2e45478e56f048efee6ab3f05eb54c0d9..4df7265abd9231140535653fb55996def3a1e3dc 100644 --- a/external/workload-automation/requirements.txt +++ b/external/workload-automation/requirements.txt @@ -1,5 +1,5 @@ bcrypt==3.2.0 -certifi==2022.12.7 +certifi==2020.12.5 cffi==1.14.4 chardet==3.0.4 colorama==0.4.4 diff --git a/external/workload-automation/wa/framework/target/runtime_parameter_manager.py b/external/workload-automation/wa/framework/target/runtime_parameter_manager.py index 77365dd280b564a10ce63b5ba3d45767d833b782..c46235507ec3c4e04190fca6ffe7e94c9d2f9c80 100644 --- a/external/workload-automation/wa/framework/target/runtime_parameter_manager.py +++ b/external/workload-automation/wa/framework/target/runtime_parameter_manager.py @@ -22,7 +22,6 @@ from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig, CpuidleRuntimeConfig, AndroidRuntimeConfig) from wa.utils.types import obj_dict, caseless_string -from wa.framework import pluginloader class RuntimeParameterManager(object): @@ -38,15 +37,8 @@ class RuntimeParameterManager(object): def __init__(self, target): self.target = target - self.runtime_params = {} - - try: - for rt_cls in pluginloader.list_plugins(kind='runtime-config'): - if rt_cls not in self.runtime_config_cls: - self.runtime_config_cls.append(rt_cls) - except ValueError: - pass self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] + self.runtime_params = {} runtime_parameter = namedtuple('RuntimeParameter', 'cfg_point, rt_config') for cfg in self.runtime_configs: diff --git a/external/workload-automation/wa/workloads/geekbench/__init__.py b/external/workload-automation/wa/workloads/geekbench/__init__.py index 0aa7b342eab3dd0c4aa5dc92f2c60ade2d3b14ca..d7baf1d2e4ea09ce51beaa2bf5ae102be824045a 100644 --- a/external/workload-automation/wa/workloads/geekbench/__init__.py +++ b/external/workload-automation/wa/workloads/geekbench/__init__.py @@ -53,8 +53,8 @@ class Geekbench(ApkUiautoWorkload): """ summary_metrics = ['score', 'multicore_score'] - supported_versions = ['6', '5', '4.4.2', '4.4.0', '4.3.4', '4.3.2', '4.3.1', '4.2.0', '4.0.1', '3.4.1', '3.0.0', '2'] - package_names = ['com.primatelabs.geekbench6', 'com.primatelabs.geekbench5', 'com.primatelabs.geekbench', 'com.primatelabs.geekbench3', 'ca.primatelabs.geekbench2'] + supported_versions = ['5', '4.4.2', '4.4.0', '4.3.4', '4.3.2', '4.3.1', '4.2.0', '4.0.1', '3.4.1', '3.0.0', '2'] + package_names = ['com.primatelabs.geekbench5', 'com.primatelabs.geekbench', 'com.primatelabs.geekbench3', 'ca.primatelabs.geekbench2'] begin_regex = re.compile(r'^\s*D/WebViewClassic.loadDataWithBaseURL\(\s*\d+\s*\)' r'\s*:\s*(?P\<.*)\s*$') @@ -137,7 +137,7 @@ class Geekbench(ApkUiautoWorkload): context.add_metric(namemify(section['name'] + '_multicore_score', i), section['multicore_score']) - def update_result(self, context): + def update_result_4(self, context): outfile_glob = self.target.path.join(self.target.package_data_directory, self.apk.package, 'files', '*gb*') on_target_output_files = [f.strip() for f in self.target.execute('ls {}'.format(outfile_glob), as_root=True).split('\n') if f] @@ -151,7 +151,7 @@ class Geekbench(ApkUiautoWorkload): with open(host_output_file, 'w') as wfh: json.dump(data, wfh, indent=4) context.add_artifact('geekout', host_output_file, kind='data', - description='Geekbench output from target.') + description='Geekbench 4 output from target.') context.add_metric(namemify('score', i), data['score']) context.add_metric(namemify('multicore_score', i), data['multicore_score']) for section in data['sections']: @@ -161,9 +161,7 @@ class Geekbench(ApkUiautoWorkload): context.add_metric(namemify(section['name'] + '_' + workload_name + '_score', i), workloads['score']) - update_result_4 = update_result - update_result_5 = update_result - update_result_6 = update_result + update_result_5 = update_result_4 class GBWorkload(object): diff --git a/external/workload-automation/wa/workloads/geekbench/com.arm.wa.uiauto.geekbench.apk b/external/workload-automation/wa/workloads/geekbench/com.arm.wa.uiauto.geekbench.apk index cb232544b454f6c2a4f70b54c6e2b892c5403dee..5eb8bb080bd12351671c6931ec34f417324ac3b1 100644 Binary files a/external/workload-automation/wa/workloads/geekbench/com.arm.wa.uiauto.geekbench.apk and b/external/workload-automation/wa/workloads/geekbench/com.arm.wa.uiauto.geekbench.apk differ diff --git a/external/workload-automation/wa/workloads/geekbench/uiauto/app/src/main/java/com/arm/wa/uiauto/geekbench/UiAutomation.java b/external/workload-automation/wa/workloads/geekbench/uiauto/app/src/main/java/com/arm/wa/uiauto/geekbench/UiAutomation.java index fccfb373768f054eb215cdc2eb407bdf996c1cb5..cf31cab82482140a45ffd14d4163fde79c51d52b 100644 --- a/external/workload-automation/wa/workloads/geekbench/uiauto/app/src/main/java/com/arm/wa/uiauto/geekbench/UiAutomation.java +++ b/external/workload-automation/wa/workloads/geekbench/uiauto/app/src/main/java/com/arm/wa/uiauto/geekbench/UiAutomation.java @@ -96,7 +96,6 @@ public class UiAutomation extends BaseUiAutomation { break; case 4: case 5: - case 6: runCpuBenchmarks(isCorporate); waitForResultsv3onwards(); break;