diff --git a/.travis.yml b/.travis.yml index e771c37a12a11c304b414b39d7a5246ab69fcbad..b08919fa1ae5436c44e1dfe2da3b58a726ae068e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ sudo: required language: python install: # Use IPython 5.x because 6.0+ only supports Python 3.3 - - pip install --upgrade "ipython<6.0.0" Cython trappy bart-py devlib psutil wrapt jupyter matplotlib + - pip install --upgrade "ipython<6.0.0" Cython trappy bart-py devlib psutil wrapt jupyter matplotlib pyyaml script: - cd $TRAVIS_BUILD_DIR - source init_env && lisa-test tests/lisa/ diff --git a/libs/utils/android/__init__.py b/libs/utils/android/__init__.py index 896997045fd5a8fd1bfa01b34ccbe9b15a6cd86a..91b5a2ccd4d4835ea6974062b56b4cce5fbb608c 100644 --- a/libs/utils/android/__init__.py +++ b/libs/utils/android/__init__.py @@ -20,6 +20,7 @@ from screen import Screen from system import System from workload import Workload +from wlauto import Wlauto from viewer import ViewerWorkload from benchmark import LisaBenchmark diff --git a/libs/utils/android/wlauto.py b/libs/utils/android/wlauto.py new file mode 100644 index 0000000000000000000000000000000000000000..bf705ca6ffc87131d9bd159d991084828c6a6fdc --- /dev/null +++ b/libs/utils/android/wlauto.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import subprocess as sp +import tempfile +import yaml +import glob + +from . import Workload +from distutils.spawn import find_executable +from shutil import copyfile, rmtree + +import logging + +WA_RUN_CMD = 'wa run -f {} -d {}' +WA_REPO = 'https://github.com/ARM-software/workload-automation.git' + +# Workloads directory path +workloads_dir = os.path.dirname(os.path.abspath(__file__)) +workloads_dir = os.path.join(workloads_dir, 'workloads') +agendas_dir = os.path.join(workloads_dir, 'agendas') + +class Wlauto(Workload): + """ + Wrapper class for workload-automation + + Workload-automation (WA) already provides many workloads for Android + targets. + + In order to use one of the WA workloads (`wa list workloads` tells you what + is available) you need to implement a class in a + .py file stored under libs/utils/android/workloads. + + inherits Wlauto and has to implement the methods from + Workload and (eventually) Wlauto. + + This way, the workload is visible by the Workload class that will list it + as available and will eventually check that the related package is + installed on the Android target. + + :param test_env: Test Environment object + :type test_env: TestEnv object + """ + + te = None + + def __init__(self, test_env): + if not find_executable('wa'): + raise RuntimeError( + 'wa not installed. Please, install it from %s', WA_REPO + ) + + Wlauto.te = test_env + super(Wlauto, self).__init__(test_env) + self._log = logging.getLogger('Wlauto') + self._log.debug('WA wrapper created') + + + @staticmethod + def _wload_is_available(workload_name): + """ + Check if the specified workload is available in workload-automation. + + :param workload_name: Name of the workload + :type workload_name: str + """ + _log = logging.getLogger('Wlauto') + _log.debug('Checking for %s', workload_name) + + if sp.check_output(['wa', 'list', '-n', workload_name, 'workloads']): + return True + + _log.error('%s not available in WA', workload_name) + return False + + @staticmethod + def wa_run(exp_dir, agenda, workload_name, collect=''): + """ + Run workload automation using the specified agenda. + + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Path to the YAML file to be passed to + workload-automation. You can pass either an absolute path or a + filename. In the latter case the file must be under + android/workloads. + :type agenda: str + + :param workload_name: Name of the workload + :type workload_name: str + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + _log = logging.getLogger('Wlauto') + + # Check if the specified path is absolute + if agenda.startswith('/'): + _agenda = agenda + else: + _agenda = os.path.join(agendas_dir, agenda) + + _log.debug('Using agenda %s', _agenda) + + # Create temprary output directory for WA + res_dir = tempfile.mkdtemp(prefix='lisa_wa_') + _log.debug('WA output dir is %s', res_dir) + + # Initialize energy meter results + nrg_report = None + # Start energy collection + if 'energy' in collect and Wlauto.te.emeter: + Wlauto.te.emeter.reset() + + # Execute workload-automation + p = sp.Popen(WA_RUN_CMD.format(_agenda, res_dir).split(), + stdout=sp.PIPE, + stderr=sp.PIPE) + out, err = p.communicate() + + # Stop energy collection + if 'energy' in collect and Wlauto.te.emeter: + nrg_report = Wlauto.te.emeter.report(exp_dir) + + _log.debug(out) + _log.debug(err) + + # Copy results to exp_dir + # Take into account results from multiple iterations + wload_dirs = glob.glob('{}/{}*'.format(res_dir, workload_name)) + + # Store relevant results under `exp_dir` + for res_dir in wload_dirs: + dst_res_dir = os.path.join(exp_dir, os.path.basename(res_dir)) + if os.path.exists(dst_res_dir): + # Remove possible outdated results + rmtree(dst_res_dir) + os.mkdir(dst_res_dir) + + for f in os.listdir(res_dir): + src_path = os.path.join(res_dir, f) + if os.path.isfile(src_path): + dst_path = os.path.join(dst_res_dir, f) + copyfile(src_path, dst_path) + + return nrg_report + + def generate_yaml_agenda(self, agenda): + """ + Converts a dictionary into a YAML file which should be used as the + agenda for running workload-automation benchmarks. + + The YAML file is put under the results directory specified in the test + environment. + + :param agenda: Dictionary to be converted into YAML file + :type agenda: dict + """ + if not isinstance(agenda, dict): + raise ValueError('Specified agenda must be a dictionary') + + _agenda = os.path.join(self.te.res_dir, 'agenda.yaml') + + with open(_agenda, 'w') as f: + f.write(yaml.dump(agenda, default_flow_style=True)) + + return _agenda + +# vim :set tabstop=4 shiftwidth=4 expandtab diff --git a/libs/utils/android/workload.py b/libs/utils/android/workload.py index 7387f505525c4bdf93405f02a0bf1fb016823dc9..fa1bb1f7fc7112c7bfb8e92308779a15590044c6 100644 --- a/libs/utils/android/workload.py +++ b/libs/utils/android/workload.py @@ -70,7 +70,14 @@ class Workload(object): _log.debug('Building list of available workloads...') for sc in Workload._subclasses(): _log.debug('Checking workload [%s]...', sc.__name__) - if sc.package in cls._packages or sc.package == '': + # Check if it is a WA-wrapped workload + if sc.__name__ == 'Wlauto': + cls._availables[sc.__name__.lower()] = sc + elif 'Wa' in sc.__name__: + if sc._is_available(test_env.target): + cls._availables[sc.__name__.lower()] = sc + # Check if non-WA-wrapped workload is available + elif sc.package in cls._packages or sc.package == '': cls._availables[sc.__name__.lower()] = sc _log.info('Supported workloads available on target:') diff --git a/libs/utils/android/workloads/agendas/camerarecord.yaml b/libs/utils/android/workloads/agendas/camerarecord.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1e2e488f205531fc576ae294615775e3933ec5c --- /dev/null +++ b/libs/utils/android/workloads/agendas/camerarecord.yaml @@ -0,0 +1,7 @@ +config: + device: generic_android +workloads: + - id: lisa_camerarecord_10 + name: camerarecord + workload_parameters: + recording_time: 10 diff --git a/libs/utils/android/workloads/agendas/geekbench.yaml b/libs/utils/android/workloads/agendas/geekbench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e031d2ff713d190f79e262250e8a87628d6b719 --- /dev/null +++ b/libs/utils/android/workloads/agendas/geekbench.yaml @@ -0,0 +1,5 @@ +config: + device: generic_android +workloads: + - id: lisa_geekbench + name: geekbench diff --git a/libs/utils/android/workloads/agendas/rtapp.yaml b/libs/utils/android/workloads/agendas/rtapp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d2746ad2121e079ed7119b3a29ccb7cef5267d3 --- /dev/null +++ b/libs/utils/android/workloads/agendas/rtapp.yaml @@ -0,0 +1,8 @@ +config: + device: Nexus5 +workloads: + - id: lisa_rtapp_mp3 + name: rt-app + workload_parameters: + duration: 5 + config: mp3-short diff --git a/libs/utils/android/workloads/wa_camerarecord.py b/libs/utils/android/workloads/wa_camerarecord.py new file mode 100644 index 0000000000000000000000000000000000000000..98c747821fc6632ced7305f8564c4a5a62f17441 --- /dev/null +++ b/libs/utils/android/workloads/wa_camerarecord.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto, Workload + +import os +import glob +import logging + +from shutil import copyfile, rmtree + +class WaCameraRecord(Wlauto): + """ + Camera Record workload + """ + + # Package required by this workload + package = 'com.google.android.GoogleCamera' + + # WA name of the workload + workload_name = 'camerarecord' + + + def __init__(self, test_env): + super(WaCameraRecord, self).__init__(test_env) + self._log = logging.getLogger('Camera Record') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + # Setting agenda_yaml properly + _agenda = agenda + if isinstance(agenda, dict): + # Convert dictionary to YAML file + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, _agenda, + WaCameraRecord.workload_name, collect=collect) + + # Return frame stats file + db_file = os.path.join(exp_dir, "framestats.txt") + + return db_file, nrg_report + + @staticmethod + def _is_available(target): + if Wlauto._wload_is_available(WaCameraRecord.workload_name): + return True + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab diff --git a/libs/utils/android/workloads/wa_geekbench.py b/libs/utils/android/workloads/wa_geekbench.py new file mode 100644 index 0000000000000000000000000000000000000000..011ef8f271f2b2072177af702ac6b05a0790f825 --- /dev/null +++ b/libs/utils/android/workloads/wa_geekbench.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto +from android import Workload + +import glob +import logging + +class WaGeekbench(Wlauto): + """ + Geekbench workload + """ + + # Package required by this workload + package = 'com.primatelabs.geekbench' + + # WA name of the workload + workload_name = 'geekbench' + + def __init__(self, test_env): + super(WaGeekbench, self).__init__(test_env) + self._log = logging.getLogger('Geekbench') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + # Setting agenda_yaml properly + _agenda = agenda + if isinstance(agenda, dict): + # Convert dictionary to YAML file + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, agenda, + WaGeekbench.workload_name, collect=collect) + + return None, nrg_report + + @staticmethod + def _is_available(target): + if Wlauto._wload_is_available(WaGeekbench.workload_name): + return True + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab diff --git a/libs/utils/android/workloads/wa_rtapp.py b/libs/utils/android/workloads/wa_rtapp.py new file mode 100644 index 0000000000000000000000000000000000000000..df2074fa3fd1d5d2bc050a2c83eaf53484016931 --- /dev/null +++ b/libs/utils/android/workloads/wa_rtapp.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from android import Wlauto +from android import Workload + +import logging + +class WaRTApp(Wlauto): + """ + Android rt-app workload + """ + + # WA name of the workload + workload_name = 'rt-app' + + def __init__(self, test_env): + super(WaRTApp, self).__init__(test_env) + self._log = logging.getLogger('rt-app') + self._log.debug('Workload created') + + def run(self, exp_dir, agenda, collect=''): + """ + :param exp_dir: Path to experiment directory where to store results. + :type exp_dir: str + + :param agenda: Agenda to be passed to workload-automation. Can be a + YAML file or a dictionary. + :type agenda: str or dict + + :param collect: Specifies what to collect. Possible values: + - 'energy' + :type collect: list(str) + """ + _agenda = agenda + if isinstance(agenda, dict): + _agenda = self.generate_yaml_agenda(agenda) + + self._log.debug('Running') + + nrg_report = Wlauto.wa_run(exp_dir, _agenda, + WaRTApp.workload_name, collect=collect) + + # RTApp has no db_file so return None as first value + return None, nrg_report + + @staticmethod + def _is_available(target): + if target.get_installed('rt-app'): + return Wlauto._wload_is_available(WaRTApp.workload_name) + return False + +# vim :set tabstop=4 shiftwidth=4 expandtab