From d9abab13f2096f241051dd6b8632d0373683aad1 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Tue, 28 Jun 2016 16:05:58 +0100 Subject: [PATCH 1/4] libs/utils/android: add wrapper for workload-automation Workload-automation (WA) already provides many workloads for Android targets. This is a wrapper for LISA to use WA. In order to use one of the WA workloads (`wa list workloads` tells you what is available) you need to implement a class in a .py file stored under libs/utils/android/workloads. inherits Wlauto and has to implement the methods from Workload and (eventually) Wlauto. WA agendas shall be stored under workloads/agendas. This way, the workload is visible by the Workload class that will list it as available and will eventually check that the related package is installed on the Android target. Signed-off-by: Michele Di Giorgio --- .travis.yml | 2 +- libs/utils/android/__init__.py | 1 + libs/utils/android/wlauto.py | 186 +++++++++++++++++++++++++++++++++ libs/utils/android/workload.py | 9 +- 4 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 libs/utils/android/wlauto.py diff --git a/.travis.yml b/.travis.yml index e771c37a1..b08919fa1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ sudo: required language: python install: # Use IPython 5.x because 6.0+ only supports Python 3.3 - - pip install --upgrade "ipython<6.0.0" Cython trappy bart-py devlib psutil wrapt jupyter matplotlib + - pip install --upgrade "ipython<6.0.0" Cython trappy bart-py devlib psutil wrapt jupyter matplotlib pyyaml script: - cd $TRAVIS_BUILD_DIR - source init_env && lisa-test tests/lisa/ diff --git a/libs/utils/android/__init__.py b/libs/utils/android/__init__.py index 896997045..91b5a2ccd 100644 --- a/libs/utils/android/__init__.py +++ b/libs/utils/android/__init__.py @@ -20,6 +20,7 @@ from screen import Screen from system import System from workload import Workload +from wlauto import Wlauto from viewer import ViewerWorkload from benchmark import LisaBenchmark diff --git a/libs/utils/android/wlauto.py b/libs/utils/android/wlauto.py new file mode 100644 index 000000000..bf705ca6f --- /dev/null +++ b/libs/utils/android/wlauto.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import subprocess as sp +import tempfile +import yaml +import glob + +from . import Workload +from distutils.spawn import find_executable +from shutil import copyfile, rmtree + +import logging + +WA_RUN_CMD = 'wa run -f {} -d {}' +WA_REPO = 'https://github.com/ARM-software/workload-automation.git' + +# Workloads directory path +workloads_dir = os.path.dirname(os.path.abspath(__file__)) +workloads_dir = os.path.join(workloads_dir, 'workloads') +agendas_dir = os.path.join(workloads_dir, 'agendas') + +class Wlauto(Workload): + """ + Wrapper class for workload-automation + + Workload-automation (WA) already provides many workloads for Android + targets. + + In order to use one of the WA workloads (`wa list workloads` tells you what + is available) you need to implement a class in a + .py file stored under libs/utils/android/workloads. + + inherits Wlauto and has to implement the methods from + Workload and (eventually) Wlauto. + + This way, the workload is visible by the Workload class that will list it + as available and will eventually check that the related package is + installed on the Android target. + + :param test_env: Test Environment object + :type test_env: TestEnv object + """ + + te = None + + def __init__(self, test_env): + if not find_executable('wa'): + raise RuntimeError( + 'wa not installed. Please, install it from %s', WA_REPO + ) + + Wlauto.te = test_env + super(Wlauto, self).__init__(test_env) + self._log = logging.getLogger('Wlauto') + self._log.debug('WA wrapper created') + + + @staticmethod + def _wload_is_available(workload_name): + """ + Check if the specified workload is available in workload-automation. + + :param workload_name: Name of the workload + :type workload_name: str + """ + _log = logging.getLogger('Wlauto') + _log.debug('Checking for %s', workload_name) + + if sp.check_output(['wa', 'list', '-n', workload_name, 'workloads']): + return True + + _log.error('%s not available in WA', workload_name) + return False + + @staticmethod + def wa_run(exp_dir, agenda, workload_name, collect=''): + """ + Run workload automation using the specified agenda. + + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Path to the YAML file to be passed to + workload-automation. You can pass either an absolute path or a + filename. In the latter case the file must be under + android/workloads. + :type agenda: str + + :param workload_name: Name of the workload + :type workload_name: str + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + _log = logging.getLogger('Wlauto') + + # Check if the specified path is absolute + if agenda.startswith('/'): + _agenda = agenda + else: + _agenda = os.path.join(agendas_dir, agenda) + + _log.debug('Using agenda %s', _agenda) + + # Create temprary output directory for WA + res_dir = tempfile.mkdtemp(prefix='lisa_wa_') + _log.debug('WA output dir is %s', res_dir) + + # Initialize energy meter results + nrg_report = None + # Start energy collection + if 'energy' in collect and Wlauto.te.emeter: + Wlauto.te.emeter.reset() + + # Execute workload-automation + p = sp.Popen(WA_RUN_CMD.format(_agenda, res_dir).split(), + stdout=sp.PIPE, + stderr=sp.PIPE) + out, err = p.communicate() + + # Stop energy collection + if 'energy' in collect and Wlauto.te.emeter: + nrg_report = Wlauto.te.emeter.report(exp_dir) + + _log.debug(out) + _log.debug(err) + + # Copy results to exp_dir + # Take into account results from multiple iterations + wload_dirs = glob.glob('{}/{}*'.format(res_dir, workload_name)) + + # Store relevant results under `exp_dir` + for res_dir in wload_dirs: + dst_res_dir = os.path.join(exp_dir, os.path.basename(res_dir)) + if os.path.exists(dst_res_dir): + # Remove possible outdated results + rmtree(dst_res_dir) + os.mkdir(dst_res_dir) + + for f in os.listdir(res_dir): + src_path = os.path.join(res_dir, f) + if os.path.isfile(src_path): + dst_path = os.path.join(dst_res_dir, f) + copyfile(src_path, dst_path) + + return nrg_report + + def generate_yaml_agenda(self, agenda): + """ + Converts a dictionary into a YAML file which should be used as the + agenda for running workload-automation benchmarks. + + The YAML file is put under the results directory specified in the test + environment. + + :param agenda: Dictionary to be converted into YAML file + :type agenda: dict + """ + if not isinstance(agenda, dict): + raise ValueError('Specified agenda must be a dictionary') + + _agenda = os.path.join(self.te.res_dir, 'agenda.yaml') + + with open(_agenda, 'w') as f: + f.write(yaml.dump(agenda, default_flow_style=True)) + + return _agenda + +# vim :set tabstop=4 shiftwidth=4 expandtab diff --git a/libs/utils/android/workload.py b/libs/utils/android/workload.py index 7387f5055..fa1bb1f7f 100644 --- a/libs/utils/android/workload.py +++ b/libs/utils/android/workload.py @@ -70,7 +70,14 @@ class Workload(object): _log.debug('Building list of available workloads...') for sc in Workload._subclasses(): _log.debug('Checking workload [%s]...', sc.__name__) - if sc.package in cls._packages or sc.package == '': + # Check if it is a WA-wrapped workload + if sc.__name__ == 'Wlauto': + cls._availables[sc.__name__.lower()] = sc + elif 'Wa' in sc.__name__: + if sc._is_available(test_env.target): + cls._availables[sc.__name__.lower()] = sc + # Check if non-WA-wrapped workload is available + elif sc.package in cls._packages or sc.package == '': cls._availables[sc.__name__.lower()] = sc _log.info('Supported workloads available on target:') -- GitLab From 6d3e1571a3523073d7709be94ffd4a8876793d4d Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Tue, 28 Jun 2016 16:07:49 +0100 Subject: [PATCH 2/4] libs/utils/android/workloads: add WA-wrapped RTApp workload Signed-off-by: Michele Di Giorgio --- .../android/workloads/agendas/rtapp.yaml | 8 +++ libs/utils/android/workloads/wa_rtapp.py | 67 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 libs/utils/android/workloads/agendas/rtapp.yaml create mode 100644 libs/utils/android/workloads/wa_rtapp.py diff --git a/libs/utils/android/workloads/agendas/rtapp.yaml b/libs/utils/android/workloads/agendas/rtapp.yaml new file mode 100644 index 000000000..3d2746ad2 --- /dev/null +++ b/libs/utils/android/workloads/agendas/rtapp.yaml @@ -0,0 +1,8 @@ +config: + device: Nexus5 +workloads: + - id: lisa_rtapp_mp3 + name: rt-app + workload_parameters: + duration: 5 + config: mp3-short diff --git a/libs/utils/android/workloads/wa_rtapp.py b/libs/utils/android/workloads/wa_rtapp.py new file mode 100644 index 000000000..df2074fa3 --- /dev/null +++ b/libs/utils/android/workloads/wa_rtapp.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto +from android import Workload + +import logging + +class WaRTApp(Wlauto): + """ + Android rt-app workload + """ + + # WA name of the workload + workload_name = 'rt-app' + + def __init__(self, test_env): + super(WaRTApp, self).__init__(test_env) + self._log = logging.getLogger('rt-app') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + _agenda = agenda + if isinstance(agenda, dict): + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, _agenda, + WaRTApp.workload_name, collect=collect) + + # RTApp has no db_file so return None as first value + return None, nrg_report + + @staticmethod + def _is_available(target): + if target.get_installed('rt-app'): + return Wlauto._wload_is_available(WaRTApp.workload_name) + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From e912ff458925479ba733561dc0a8129b8972bafc Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Tue, 28 Jun 2016 16:08:16 +0100 Subject: [PATCH 3/4] libs/utils/android/workloads: add WA-wrapped camera record workload Signed-off-by: Michele Di Giorgio --- .../workloads/agendas/camerarecord.yaml | 7 ++ .../android/workloads/wa_camerarecord.py | 78 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 libs/utils/android/workloads/agendas/camerarecord.yaml create mode 100644 libs/utils/android/workloads/wa_camerarecord.py diff --git a/libs/utils/android/workloads/agendas/camerarecord.yaml b/libs/utils/android/workloads/agendas/camerarecord.yaml new file mode 100644 index 000000000..f1e2e488f --- /dev/null +++ b/libs/utils/android/workloads/agendas/camerarecord.yaml @@ -0,0 +1,7 @@ +config: + device: generic_android +workloads: + - id: lisa_camerarecord_10 + name: camerarecord + workload_parameters: + recording_time: 10 diff --git a/libs/utils/android/workloads/wa_camerarecord.py b/libs/utils/android/workloads/wa_camerarecord.py new file mode 100644 index 000000000..98c747821 --- /dev/null +++ b/libs/utils/android/workloads/wa_camerarecord.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto, Workload + +import os +import glob +import logging + +from shutil import copyfile, rmtree + +class WaCameraRecord(Wlauto): + """ + Camera Record workload + """ + + # Package required by this workload + package = 'com.google.android.GoogleCamera' + + # WA name of the workload + workload_name = 'camerarecord' + + + def __init__(self, test_env): + super(WaCameraRecord, self).__init__(test_env) + self._log = logging.getLogger('Camera Record') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + # Setting agenda_yaml properly + _agenda = agenda + if isinstance(agenda, dict): + # Convert dictionary to YAML file + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, _agenda, + WaCameraRecord.workload_name, collect=collect) + + # Return frame stats file + db_file = os.path.join(exp_dir, "framestats.txt") + + return db_file, nrg_report + + @staticmethod + def _is_available(target): + if Wlauto._wload_is_available(WaCameraRecord.workload_name): + return True + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From 68abf77f40b68e44f23bd218de941f5e2f2c5bd1 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Wed, 6 Jul 2016 10:24:15 +0100 Subject: [PATCH 4/4] libs/utils/android/workloads: add WA-wrapped geekbench workload Signed-off-by: Michele Di Giorgio --- .../android/workloads/agendas/geekbench.yaml | 5 ++ libs/utils/android/workloads/wa_geekbench.py | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 libs/utils/android/workloads/agendas/geekbench.yaml create mode 100644 libs/utils/android/workloads/wa_geekbench.py diff --git a/libs/utils/android/workloads/agendas/geekbench.yaml b/libs/utils/android/workloads/agendas/geekbench.yaml new file mode 100644 index 000000000..7e031d2ff --- /dev/null +++ b/libs/utils/android/workloads/agendas/geekbench.yaml @@ -0,0 +1,5 @@ +config: + device: generic_android +workloads: + - id: lisa_geekbench + name: geekbench diff --git a/libs/utils/android/workloads/wa_geekbench.py b/libs/utils/android/workloads/wa_geekbench.py new file mode 100644 index 000000000..011ef8f27 --- /dev/null +++ b/libs/utils/android/workloads/wa_geekbench.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto +from android import Workload + +import glob +import logging + +class WaGeekbench(Wlauto): + """ + Geekbench workload + """ + + # Package required by this workload + package = 'com.primatelabs.geekbench' + + # WA name of the workload + workload_name = 'geekbench' + + def __init__(self, test_env): + super(WaGeekbench, self).__init__(test_env) + self._log = logging.getLogger('Geekbench') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + # Setting agenda_yaml properly + _agenda = agenda + if isinstance(agenda, dict): + # Convert dictionary to YAML file + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, agenda, + WaGeekbench.workload_name, collect=collect) + + return None, nrg_report + + @staticmethod + def _is_available(target): + if Wlauto._wload_is_available(WaGeekbench.workload_name): + return True + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab