From 357dd9c179646a293a652515b31ab798cc992020 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Fri, 1 Feb 2019 11:24:21 +0000 Subject: [PATCH 1/3] Squashed 'external/devlib/' changes from 17d4b22b9f0c..e01a76ef1bf5 e01a76ef1bf5 doc: Explain the 'tar' flag of target.read_tree_values 9fcca2503127 hwmon: Robustify hwmon scan a6b9542f0f8c target: Speed-up read_tree_values() 413e83f5d632 target/kernelconfig: Add alias for itteritems ac1987342328 target/TypedKernelConfig: Fix converting to string method git-subtree-dir: external/devlib git-subtree-split: e01a76ef1bf5a9f221136198f27233d2308a320b --- devlib/bin/scripts/shutils.in | 25 +++++++++++++++++++++++++ devlib/module/hwmon.py | 2 +- devlib/target.py | 35 +++++++++++++++++++++++++++++------ doc/target.rst | 11 ++++++++++- 4 files changed, 65 insertions(+), 8 deletions(-) diff --git a/devlib/bin/scripts/shutils.in b/devlib/bin/scripts/shutils.in index e230229e1..0cca5bba5 100755 --- a/devlib/bin/scripts/shutils.in +++ b/devlib/bin/scripts/shutils.in @@ -255,6 +255,28 @@ sched_get_kernel_attributes() { # Misc ################################################################################ +read_tree_values() { + BASEPATH=$1 + MAXDEPTH=$2 + + if [ ! -e $BASEPATH ]; then + echo "ERROR: $BASEPATH does not exist" + exit 1 + fi + + PATHS=$($BUSYBOX find $BASEPATH -follow -maxdepth $MAXDEPTH) + i=0 + for path in $PATHS; do + i=$(expr $i + 1) + if [ $i -gt 1 ]; then + break; + fi + done + if [ $i -gt 1 ]; then + $BUSYBOX grep -s '' $PATHS + fi +} + read_tree_tgz_b64() { BASEPATH=$1 MAXDEPTH=$2 @@ -353,6 +375,9 @@ ftrace_get_function_stats) hotplug_online_all) hotplug_online_all ;; +read_tree_values) + read_tree_values $* + ;; read_tree_tgz_b64) read_tree_tgz_b64 $* ;; diff --git a/devlib/module/hwmon.py b/devlib/module/hwmon.py index 7145ae541..3ecc55ca9 100644 --- a/devlib/module/hwmon.py +++ b/devlib/module/hwmon.py @@ -137,7 +137,7 @@ class HwmonModule(Module): self.scan() def scan(self): - values_tree = self.target.read_tree_values(self.root, depth=3) + values_tree = self.target.read_tree_values(self.root, depth=3, tar=True) for entry_id, fields in values_tree.items(): path = self.target.path.join(self.root, entry_id) name = fields.pop('name', None) diff --git a/devlib/target.py b/devlib/target.py index ad0b6d06c..18fb59ca6 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -695,7 +695,7 @@ class Target(object): timeout = duration + 10 self.execute('sleep {}'.format(duration), timeout=timeout) - def read_tree_values_flat(self, path, depth=1, check_exit_code=True, + def read_tree_tar_flat(self, path, depth=1, check_exit_code=True, decode_unicode=True, strip_null_chars=True): command = 'read_tree_tgz_b64 {} {} {}'.format(quote(path), depth, quote(self.working_directory)) @@ -732,8 +732,23 @@ class Target(object): return result + def read_tree_values_flat(self, path, depth=1, check_exit_code=True): + command = 'read_tree_values {} {}'.format(quote(path), depth) + output = self._execute_util(command, as_root=self.is_rooted, + check_exit_code=check_exit_code) + + accumulator = defaultdict(list) + for entry in output.strip().split('\n'): + if ':' not in entry: + continue + path, value = entry.strip().split(':', 1) + accumulator[path].append(value) + + result = {k: '\n'.join(v).strip() for k, v in accumulator.items()} + return result + def read_tree_values(self, path, depth=1, dictcls=dict, - check_exit_code=True, decode_unicode=True, + check_exit_code=True, tar=False, decode_unicode=True, strip_null_chars=True): """ Reads the content of all files under a given tree @@ -742,14 +757,20 @@ class Target(object): :depth: maximum tree depth to read :dictcls: type of the dict used to store the results :check_exit_code: raise an exception if the shutil command fails - :decode_unicode: decode the content of files as utf-8 + :tar: fetch the entire tree using tar rather than just the value (more + robust but slower in some use-cases) + :decode_unicode: decode the content of tar-ed files as utf-8 :strip_null_chars: remove '\x00' chars from the content of utf-8 decoded files :returns: a tree-like dict with the content of files as leafs """ - value_map = self.read_tree_values_flat(path, depth, check_exit_code, - decode_unicode, strip_null_chars) + if not tar: + value_map = self.read_tree_values_flat(path, depth, check_exit_code) + else: + value_map = self.read_tree_tar_flat(path, depth, check_exit_code, + decode_unicode, + strip_null_chars) return _build_path_tree(value_map, path, self.path.sep, dictcls) # internal methods @@ -1856,7 +1877,7 @@ class TypedKernelConfig(Mapping): elif isinstance(val, KernelConfigTristate): return val.value elif isinstance(val, basestring): - return '"{}"'.format(val) + return '"{}"'.format(val.strip('"')) else: return str(val) @@ -1999,6 +2020,8 @@ class KernelConfig(object): for k, v in self.typed_config.items(): yield (k, self.typed_config._val_to_str(v)) + items = iteritems + def get(self, name, strict=False): if strict: val = self.typed_config[name] diff --git a/doc/target.rst b/doc/target.rst index b63ba76b6..ea1dc780f 100644 --- a/doc/target.rst +++ b/doc/target.rst @@ -346,7 +346,7 @@ Target some sysfs entries silently failing to set the written value without returning an error code. -.. method:: Target.read_tree_values(path, depth=1, dictcls=dict): +.. method:: Target.read_tree_values(path, depth=1, dictcls=dict, [, tar [, decode_unicode [, strip_null_char ]]]): Read values of all sysfs (or similar) file nodes under ``path``, traversing up to the maximum depth ``depth``. @@ -358,9 +358,18 @@ Target value is a dict-line object with a key for every entry under ``path`` mapping onto its value or further dict-like objects as appropriate. + Although the default behaviour should suit most users, it is possible to + encounter issues when reading binary files, or files with colons in their + name for example. In such cases, the ``tar`` parameter can be set to force a + full archive of the tree using tar, hence providing a more robust behaviour. + This can, however, slow down the read process significantly. + :param path: sysfs path to scan :param depth: maximum depth to descend :param dictcls: a dict-like type to be used for each level of the hierarchy. + :param tar: the files will be read using tar rather than grep + :param decode_unicode: decode the content of tar-ed files as utf-8 + :param strip_null_char: remove null chars from utf-8 decoded files .. method:: Target.read_tree_values_flat(path, depth=1): -- GitLab From 47fccac071728e867d985a0da485abed9aeaf269 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Fri, 1 Feb 2019 11:24:27 +0000 Subject: [PATCH 2/3] Squashed 'external/workload-automation/' changes from 3f202205a5b0..4a9a2ad105b3 4a9a2ad105b3 fw/target/info: Fix for KernelConfig refactor 9f88459f56a7 fw/workload: Fix Typo a2087ea46714 workloads/manual: Fix incorrect attribute used to access target 31a5a95803a4 output_processors/postgresql: Ensure screen resolution is a list git-subtree-dir: external/workload-automation git-subtree-split: 4a9a2ad105b385c8ff7c7ae7eaf96d923ec65cd5 --- wa/framework/target/info.py | 14 +++++++++++--- wa/framework/workload.py | 2 +- wa/output_processors/postgresql.py | 2 +- wa/workloads/manual/__init__.py | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/wa/framework/target/info.py b/wa/framework/target/info.py index 5b11f7d83..52b214c83 100644 --- a/wa/framework/target/info.py +++ b/wa/framework/target/info.py @@ -53,9 +53,9 @@ def kernel_version_from_pod(pod): def kernel_config_from_pod(pod): config = KernelConfig('') - config._config = pod['kernel_config'] + config.typed_config._config = pod['kernel_config'] lines = [] - for key, value in config._config.items(): + for key, value in config.items(): if value == 'n': lines.append('# {} is not set'.format(key)) else: @@ -313,7 +313,7 @@ def cache_target_info(target_info, overwrite=False): class TargetInfo(Podable): - _pod_serialization_version = 2 + _pod_serialization_version = 3 @staticmethod def from_pod(pod): @@ -401,3 +401,11 @@ class TargetInfo(Podable): pod['page_size_kb'] = pod.get('page_size_kb') pod['_pod_version'] = pod.get('format_version', 0) return pod + + @staticmethod + def _pod_upgrade_v3(pod): + config = {} + for key, value in pod['kernel_config'].items(): + config[key.upper()] = value + pod['kernel_config'] = config + return pod diff --git a/wa/framework/workload.py b/wa/framework/workload.py index f5d0e1354..62f980ce1 100644 --- a/wa/framework/workload.py +++ b/wa/framework/workload.py @@ -73,7 +73,7 @@ class Workload(TargetedPlugin): supported_platforms = getattr(self, 'supported_platforms', []) if supported_platforms and self.target.os not in supported_platforms: - msg = 'Supported platforms for "{}" are "{}", attemping to run on "{}"' + msg = 'Supported platforms for "{}" are "{}", attempting to run on "{}"' raise WorkloadError(msg.format(self.name, ' '.join(self.supported_platforms), self.target.os)) diff --git a/wa/output_processors/postgresql.py b/wa/output_processors/postgresql.py index a2fa27b86..ba1ef3799 100644 --- a/wa/output_processors/postgresql.py +++ b/wa/output_processors/postgresql.py @@ -206,7 +206,7 @@ class PostgresqlResultProcessor(OutputProcessor): target_pod['sched_features'], target_pod['page_size_kb'], # Android Specific - target_pod.get('screen_resolution'), + list(target_pod.get('screen_resolution')), target_pod.get('prop'), target_pod.get('android_id'), target_pod.get('pod_version'), diff --git a/wa/workloads/manual/__init__.py b/wa/workloads/manual/__init__.py index 0f2257d24..289169801 100644 --- a/wa/workloads/manual/__init__.py +++ b/wa/workloads/manual/__init__.py @@ -85,7 +85,7 @@ class ManualWorkload(Workload): def run(self, context): self.logger.info('START NOW!') if self.duration: - self.device.sleep(self.duration) + self.target.sleep(self.duration) elif self.user_triggered: self.logger.info('') self.logger.info('hit any key to end your workload execution...') -- GitLab From 8b442ce63180e973e674ae2b898f08d33d6edcd5 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Fri, 1 Feb 2019 11:26:10 +0000 Subject: [PATCH 3/3] energy_model: Fix the read_tree_value call for debugfs Devlib's API for target.read_tree_value() just changed. In order to get a robust (but slow) implementation of it, the 'tar' parameter needs to be set. Do this for debugfs since the standard read_tree_values() fails on files having a colon in their name. Signed-off-by: Quentin Perret --- lisa/energy_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/energy_model.py b/lisa/energy_model.py index 29da91fef..2e8d02d79 100644 --- a/lisa/energy_model.py +++ b/lisa/energy_model.py @@ -873,7 +873,7 @@ class EnergyModel(Serializable, Loggable): pd_attr = defaultdict(dict) cpu_to_pd = {} - debugfs_em = target.read_tree_values(directory, depth=3) + debugfs_em = target.read_tree_values(directory, depth=3, tar=True) if not debugfs_em: raise TargetStableError('Energy Model not exposed at {} in sysfs.'.format(directory)) -- GitLab