From 19477157cbabd28f6c132a28b1c7716658445589 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 10 May 2016 17:38:38 +0100 Subject: [PATCH 1/5] libs/utils/android: Add Screen control class When working on an Android target, some experiments requires to properly setup some components, especially to reduce variations on energy consumptions. The screen is one of the most energy consuming componets. This patch adds a Screen module which exposes some of the most common operations to tune the Screen settings of an Android target. Signed-off-by: Patrick Bellasi --- libs/utils/__init__.py | 2 + libs/utils/android/__init__.py | 21 ++++++ libs/utils/android/screen.py | 126 +++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 libs/utils/android/__init__.py create mode 100644 libs/utils/android/screen.py diff --git a/libs/utils/__init__.py b/libs/utils/__init__.py index 489f265a5..cdcfbe38f 100644 --- a/libs/utils/__init__.py +++ b/libs/utils/__init__.py @@ -30,3 +30,5 @@ from perf_analysis import PerfAnalysis from filters import Filters from report import Report + +import android diff --git a/libs/utils/android/__init__.py b/libs/utils/android/__init__.py new file mode 100644 index 000000000..008bfbf77 --- /dev/null +++ b/libs/utils/android/__init__.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Initialization for Android module""" + +from screen import Screen + diff --git a/libs/utils/android/screen.py b/libs/utils/android/screen.py new file mode 100644 index 000000000..0b9be30ec --- /dev/null +++ b/libs/utils/android/screen.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Screen(object): + """ + Set of utility functions to control and Android Screen + """ + + @staticmethod + def set_orientation(target, auto=True, portrait=None): + """ + Set screen orientation mode + """ + acc_mode = 1 if auto else 0 + # Force manual orientation of portrait specified + if portrait is not None: + acc_mode = 0 + target.logger.info("Force manual orientation") + if acc_mode == 0: + usr_mode = 0 if portrait else 1 + usr_mode_str = 'PORTRAIT' if portrait else 'LANDSCAPE' + target.logger.info('Set orientation: %s', usr_mode_str) + else: + usr_mode = 0 + target.logger.info('Set orientation: AUTO') + + if acc_mode == 0: + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:accelerometer_rotation '\ + '--bind value:i:{}'.format(acc_mode)) + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:user_rotation '\ + '--bind value:i:{}'.format(usr_mode)) + else: + # Force PORTRAIT mode when activation AUTO rotation + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:user_rotation '\ + '--bind value:i:{}'.format(usr_mode)) + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:accelerometer_rotation '\ + '--bind value:i:{}'.format(acc_mode)) + + @staticmethod + def set_brightness(target, auto=True, percent=None): + """ + Set screen brightness percentage + """ + bri_mode = 1 if auto else 0 + # Force manual brightness if a percent specified + if percent: + bri_mode = 0 + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:screen_brightness_mode '\ + '--bind value:i:{}'.format(bri_mode)) + if bri_mode == 0: + if percent<0 or percent>100: + msg = "Screen brightness {} out of range (0,100)"\ + .format(percent) + raise ValueError(msg) + value = 255 * percent / 100 + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:screen_brightness '\ + '--bind value:i:{}'.format(value)) + target.logger.info('Set brightness: %d%%', percent) + else: + target.logger.info('Set brightness: AUTO') + + @staticmethod + def set_dim(target, auto=True): + """ + Set screen dimming mode + """ + dim_mode = 1 if auto else 0 + dim_mode_str = 'ON' if auto else 'OFF' + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:dim_screen '\ + '--bind value:i:{}'.format(dim_mode)) + target.logger.info('Dim screen mode: %s', dim_mode_str) + + @staticmethod + def set_timeout(target, seconds=30): + """ + Set screen off timeout in seconds + """ + if seconds<0: + msg = "Screen timeout {}: cannot be negative".format(seconds) + raise ValueError(msg) + value = seconds * 1000 + target.execute('content insert '\ + '--uri content://settings/system '\ + '--bind name:s:screen_off_timeout '\ + '--bind value:i:{}'.format(value)) + target.logger.info('Screen timeout: %d [s]', seconds) + + @staticmethod + def set_defaults(target): + """ + Reset screen settings to a reasonable default + """ + Screen.set_orientation(target) + Screen.set_brightness(target) + Screen.set_dim(target) + Screen.set_timeout(target) + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From f1553b2a0b726922830ff93d5144b22d6619a876 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 10 May 2016 17:39:24 +0100 Subject: [PATCH 2/5] libs/utils/android: Add Workload base class Some workloads can be easily automated for an Android target by just issuing a proper set of ADB commands. This patch adds a base class to run workloads on Android. The goal of this base class is to make it simple to add new ADB-based automation to run specific workloads on Android. An Android workload is required to extend this class with a new one which is defined in a single Python source located under: libs/utils/android/workloads The base class is defined to list all the python files available under that folder and check if the (optionally) required package is installed in the target. The base class provides also a factory method which allows to get an instance of a specified workload, if available on the target. For example, a basic usage of this class is: wload = Workload.get(test_env, wload_name="YouTube") if wload: wload.run(test_env.res_dir, video_url="https://youtu.be/XSGBVzeBUbk?t=45s", video_duration_s=15) Signed-off-by: Patrick Bellasi --- libs/utils/android/__init__.py | 2 +- libs/utils/android/workload.py | 145 +++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 libs/utils/android/workload.py diff --git a/libs/utils/android/__init__.py b/libs/utils/android/__init__.py index 008bfbf77..c067fc8b1 100644 --- a/libs/utils/android/__init__.py +++ b/libs/utils/android/__init__.py @@ -18,4 +18,4 @@ """Initialization for Android module""" from screen import Screen - +from workload import Workload diff --git a/libs/utils/android/workload.py b/libs/utils/android/workload.py new file mode 100644 index 000000000..c82280153 --- /dev/null +++ b/libs/utils/android/workload.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys + +from glob import glob +from inspect import isclass +from importlib import import_module + +from collections import namedtuple + +import logging + +class Workload(object): + """ + Base class for Android related workloads + """ + _availables = None + + # Setup logger + logger = logging.getLogger('Workload') + logger.setLevel(logging.INFO) + + + _AW = namedtuple('AndroidWorkload', + ['module_name', 'module', 'class_name', 'ctor']) + + @staticmethod + def get(te, name='YouTube'): + """ + Get a reference to the specified Android workload + """ + if Workload._availables is None: + Workload.availables(te.target) + # Build list of case insensiteve workload names + if name not in Workload._availables: + logging.warning('Workload [%s] not available on target', name) + return None + return Workload._availables[name].ctor(te) + + @staticmethod + def availables(target): + """ + List the supported android workloads which are available on the target + """ + if Workload._availables: + return Workload._availables.keys() + + Workload._availables = {} + + # Add workloads dir to system path + workloads_dir = os.path.dirname(os.path.abspath(__file__)) + workloads_dir = os.path.join(workloads_dir, 'workloads') + logging.debug('%14s - Workdir: %s', 'Workload', workloads_dir) + + sys.path.insert(0, workloads_dir) + logging.debug('%14s - Syspath: %s', 'Workload', format(sys.path)) + + for filepath in glob(os.path.join(workloads_dir, '*.py')): + filename = os.path.splitext(os.path.basename(filepath))[0] + logging.debug('%14s - Filename: %s', 'Workload', filename) + + # Ignore __init__ files + if filename.startswith('__'): + continue + + # Import the module for inspection + module = import_module(filename) + for member in dir(module): + # Ignore the base class + if member == 'Workload': + continue + handler = getattr(module, member) + if handler and isclass(handler) and \ + issubclass(handler, Workload): + class_name = handler.__name__ + module_name = module.__name__ + # Check if a package is required and is available on target + aw = Workload._AW(module_name, module, class_name, handler) + if (Workload._is_available(target, aw)): + # Keep track of classes which are 'Android.Workload' + Workload._availables[class_name] = aw + + return Workload._availables.keys() + + @staticmethod + def _is_available(target, aw): + try: + package = getattr(aw.ctor, 'package') + except AttributeError: + # Assume workloads not requiring a package + # are always available + return True + + # Check for the package being available + count = target.execute('pm list packages | grep {} | wc -l'\ + .format(package)) + if int(count) >= 1: + return True + + logging.warning('%14s - Package [%s] not installed', + 'Workload', package) + logging.warning('%14s - Workload [%s] disabled', + 'Workload', aw.class_name) + return False + + def __init__(self, test_env): + """ + Initialized workloads available on the specified test environment + + test_env: target test environmen + """ + self.te = test_env + self.target = test_env.target + self.logger = self.target.logger + + logging.debug('%14s - Building list of available workloads...', 'Workload') + wloads = Workload.availables(self.target) + logging.info('%14s - Workloads available on target:', 'Workload') + logging.info('%14s - %s', 'Workload', wloads) + + def _adb(self, cmd): + return 'adb -s {} {}'.format(self.target.adb_name, cmd) + + + def run(self, exp_dir, **kwargs): + raise RuntimeError('Not implemeted') + + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From b5146c6c2d4a12299811b7e180974960d50f3b60 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 10 May 2016 17:39:48 +0100 Subject: [PATCH 3/5] libs/utils/android: Add Jankbench workload Jankbench is an Android Benchmark application to evaluate frame janks using many different kind of "scroll-based" workloads. This patch adds an example of class which inherit from the base android::Workload class to properly run a Junkbench workload on a target. The workload execution provides ADB support to properly start and stop energy measurements across the execution of the specified test. Energy samples are collected only if the "energy" tag is found in the "collect" input string and an energy meter (emeter) has been configured and properly initialized in the current test environment (test_env). Signed-off-by: Patrick Bellasi --- libs/utils/android/workloads/jankbench.py | 167 ++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 libs/utils/android/workloads/jankbench.py diff --git a/libs/utils/android/workloads/jankbench.py b/libs/utils/android/workloads/jankbench.py new file mode 100644 index 000000000..0146dad78 --- /dev/null +++ b/libs/utils/android/workloads/jankbench.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import os + +from subprocess import Popen, PIPE +from android import Screen, Workload + +import logging + +# Available test workloads +_jankbench = { + 'list_view' : 0, + 'image_list_view' : 1, + 'shadow_grid' : 2, + 'low_hitrate_text' : 3, + 'high_hitrate_text' : 4, + 'edit_text' : 5, +} + +# Regexps for benchmark synchronization +JANKBENCH_BENCHMARK_START_RE = re.compile( + r'ActivityManager: START.*' + '(cmp=com.android.benchmark/.app.RunLocalBenchmarksActivity)' +) +JANKBENCH_ITERATION_COUNT_RE = re.compile( + r'System.out: iteration: (?P[0-9]+)' +) +JANKBENCH_ITERATION_METRICS_RE = re.compile( + r'System.out: Mean: (?P[0-9\.]+)\s+JankP: (?P[0-9\.]+)\s+' + 'StdDev: (?P[0-9\.]+)\s+Count Bad: (?P[0-9]+)\s+' + 'Count Jank: (?P[0-9]+)' +) +JANKBENCH_BENCHMARK_DONE_RE = re.compile( + r'I BENCH\s+:\s+BenchmarkDone!' +) + +JANKBENCH_DB_PATH = '/data/data/com.android.benchmark/databases/' +JANKBENCH_DB_NAME = 'BenchmarkResults' + +class Jankbench(Workload): + """ + Android Jankbench workload + """ + + # Package required by this workload + package = 'com.android.benchmark' + + # Setup logger + logger = logging.getLogger('Jankbench') + logger.setLevel(logging.INFO) + + + def __init__(self, test_env): + super(Jankbench, self).__init__(test_env) + logging.debug('%14s - Workload created', 'Jankbench') + pass + + def run(self, exp_dir, test_name, iterations, collect=''): + # Setup test id + try: + test_id = _jankbench[test_name] + except KeyError: + raise ValueError('Jankbench test [%s] not supported', test_name) + + # Initialize energy meter results + nrg_data, nrg_file = None, None + + self.target.execute('input keyevent 82') + # Press Back button to be sure we run the video from the start + self.target.execute('input keyevent KEYCODE_BACK') + + # Close and clear application + self.target.execute('am force-stop com.android.benchmark') + self.target.execute('pm clear com.android.benchmark') + + # Force screen in PORTRAIT mode + Screen.set_orientation(self.target, portrait=True) + + # Clear logcat + os.system(self._adb('logcat -c')); + + self.logger.debug('Start Jank Benchmark [%d:%s]', test_id, test_name) + test_cmd = 'am start -n "com.android.benchmark/.app.RunLocalBenchmarksActivity" '\ + '--eia "com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS" {0} '\ + '--ei "com.android.benchmark.EXTRA_RUN_COUNT" {1}'\ + .format(test_id, iterations) + self.logger.info(test_cmd) + self.target.execute(test_cmd); + + # Parse logcat output lines + logcat_cmd = self._adb( + 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ + .format(self.target.adb_name)) + self.logger.info("%s", logcat_cmd) + + self.logger.debug("Iterations:") + logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) + while True: + + # read next logcat line (up to max 1024 chars) + message = logcat.stdout.readline(1024) + + # Benchmark start trigger + match = JANKBENCH_BENCHMARK_START_RE.search(message) + if match: + if 'energy' in collect and self.te.emeter: + self.te.emeter.reset() + self.logger.debug("Benchmark started!") + + # Benchmark completed trigger + match = JANKBENCH_BENCHMARK_DONE_RE.search(message) + if match: + if 'energy' in collect and self.te.emeter: + nrg_data, nrg_file = self.te.emeter.report(exp_dir) + self.logger.info("Estimated energy: %7.3f", + float(nrg_data['BAT'])) + self.logger.debug("Benchmark done!") + break + + # Iteration completd + match = JANKBENCH_ITERATION_COUNT_RE.search(message) + if match: + self.logger.debug("Iteration %2d:", + int(match.group('iteration'))+1) + # Iteration metrics + match = JANKBENCH_ITERATION_METRICS_RE.search(message) + if match: + self.logger.info(" Mean: %7.3f JankP: %7.3f StdDev: %7.3f Count Bad: %4d Count Jank: %4d", + float(match.group('mean')), + float(match.group('junk_p')), + float(match.group('std_dev')), + int(match.group('count_bad')), + int(match.group('count_junk'))) + + # get results + db_file = os.path.join(exp_dir, JANKBENCH_DB_NAME) + self.target.pull(JANKBENCH_DB_PATH + JANKBENCH_DB_NAME, db_file) + + # Close and clear application + self.target.execute('am force-stop com.android.benchmark') + self.target.execute('pm clear com.android.benchmark') + + # Go back to home screen + self.target.execute('input keyevent KEYCODE_HOME') + + # Switch back to screen auto rotation + Screen.set_orientation(self.target, auto=True) + + return db_file, nrg_data, nrg_file + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From 22c726e3389226c79600f7f9020eb8bba5e02b8c Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 10 May 2016 18:14:23 +0100 Subject: [PATCH 4/5] libs/utils/android: Add YouTube workload YouTube is a standard Android application which is particularly useful to evaluate the energy efficiency of an Android target configuration by just playing a video of a fixed amount of time. This patch adds an example of class which inherit from the base android::Workload class to properly playback a YouTube video on a target. The workload execution provides ADB support to properly collect frame statistics after the completion of the video which can be used to evaluation the performance of the youtube playback. Energy samples are collected only if the "energy" tag is found in the "collect" input string and an energy meter (emeter) has been configured and properly initialized in the current test environment (test_env). Signed-off-by: Patrick Bellasi --- libs/utils/android/workloads/youtube.py | 99 +++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 libs/utils/android/workloads/youtube.py diff --git a/libs/utils/android/workloads/youtube.py b/libs/utils/android/workloads/youtube.py new file mode 100644 index 000000000..018bc0367 --- /dev/null +++ b/libs/utils/android/workloads/youtube.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import os + +from subprocess import Popen, PIPE +from android import Screen, Workload +from time import sleep + +import logging + +YOUTUBE_CMD = 'shell dumpsys gfxinfo com.google.android.youtube > {}' + +class YouTube(Workload): + """ + Android YouTube workload + """ + + # Package required by this workload + package = 'com.google.android.youtube' + + # Setup logger + logger = logging.getLogger('YouTube') + logger.setLevel(logging.INFO) + + + def __init__(self, test_env): + super(YouTube, self).__init__(test_env) + logging.debug('%14s - Workload created', 'YouTube') + + def run(self, exp_dir, video_url, video_duration_s, collect=''): + + # Initialize energy meter results + nrg_data, nrg_file = None, None + + # Unlock device screen (assume no password required) + self.target.execute('input keyevent 82') + # Press Back button to be sure we run the video from the start + self.target.execute('input keyevent KEYCODE_BACK') + + # Force screen in LANDSCAPE mode + Screen.set_orientation(self.target, portrait=False) + + # Start YouTube video on the target device + youtube_cmd = 'am start -a android.intent.action.VIEW "{}"'\ + .format(video_url) + logging.info(youtube_cmd) + self.target.execute(youtube_cmd) + # Allow the activity to start + sleep(3) + + # Reset framestats collection + self.target.execute('dumpsys gfxinfo --reset') + + # Start energy collection + if 'energy' in collect and self.te.emeter: + self.te.emeter.reset() + + # Wait until the end of the video + logging.info("Play video for %d [s]", video_duration_s) + sleep(video_duration_s) + + # Stop energy collection + if 'energy' in collect and self.te.emeter: + nrg_data, nrg_file = self.te.emeter.report(exp_dir) + logging.info("Estimated energy: %7.3f", float(nrg_data['BAT'])) + + # Get frame stats + db_file = os.path.join(exp_dir, "framestats.txt") + self._adb(YOUTUBE_CMD.format(db_file)) + + # Close and clear application + self.target.execute('am force-stop com.google.android.youtube') + self.target.execute('pm clear com.google.android.youtube') + + # Go back to home screen + self.target.execute('input keyevent KEYCODE_HOME') + + # Switch back to screen auto rotation + Screen.set_orientation(self.target, auto=True) + + return db_file, nrg_data, nrg_file + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From 280df119cd5a06d93857bf266cd83a221fe0797a Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 10 May 2016 18:15:17 +0100 Subject: [PATCH 5/5] libs/utils/android: Add example notebook with multiple workloads The android:Workload base class add an abstraction which makes easy to run a predefined set of workloads on an Android target. This patch adds a simple example of a notebook which allows to run YouTube and Janknbench (if available on the target) while collecting both performance and energy consumption data. Signed-off-by: Patrick Bellasi --- ipynb/android/Android_Workloads.ipynb | 643 ++++++++++++++++++++++++++ 1 file changed, 643 insertions(+) create mode 100644 ipynb/android/Android_Workloads.ipynb diff --git a/ipynb/android/Android_Workloads.ipynb b/ipynb/android/Android_Workloads.ipynb new file mode 100644 index 000000000..b6963eb9e --- /dev/null +++ b/ipynb/android/Android_Workloads.ipynb @@ -0,0 +1,643 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Android Workloads Experiments" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import logging\n", + "reload(logging)\n", + "log_fmt = '%(asctime)-9s %(levelname)-8s: %(message)s'\n", + "logging.basicConfig(format=log_fmt)\n", + "\n", + "# Change to info once the notebook runs ok\n", + "logging.getLogger().setLevel(logging.INFO)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab inline\n", + "\n", + "import collections\n", + "import copy\n", + "import json\n", + "import os\n", + "import pexpect as pe\n", + "from time import sleep\n", + "\n", + "# Support to access the remote target\n", + "import devlib\n", + "from env import TestEnv\n", + "\n", + "# from devlib.utils.android import adb_command\n", + "\n", + "# Import support for Android devices\n", + "from android import Screen, Workload\n", + "\n", + "# Support for trace events analysis\n", + "from trace import Trace\n", + "from trace_analysis import TraceAnalysis\n", + "\n", + "# Suport for FTrace events parsing and visualization\n", + "import trappy" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test Environment set up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Devlib requires the ANDROID_HOME environment variable configured to point to your local installation of the Android SDK. If you have not this variable configured in the shell used to start the notebook server, you need to run the next cell to define where your Android SDK is installed." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Setup Androd SDK\n", + "os.environ['ANDROID_HOME'] = '/home/eas/Work/Android/android-sdk-linux/'\n", + "\n", + "# Setup Catapult for Systrace usage\n", + "CATAPULT_HOME = \"/home/eas/Work/Android/catapult\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "adbd is already running as root\r\n" + ] + } + ], + "source": [ + "# Android device to target\n", + "DEVICE = 'GA0113TP0178'\n", + "\n", + "# Ensure ADB is running as root\n", + "!adb -s {DEVICE} root" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In case more than one Android device are conencted to the host, you must specify the ID of the device you want to target in `my_target_conf`. Run `adb devices` on your host to get the ID." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Setup target configuration\n", + "my_conf = {\n", + "\n", + " # Target platform and board\n", + " \"platform\" : 'android',\n", + " \"device\" : DEVICE,\n", + "\n", + "# \"emeter\" : {\n", + "# \"instrument\" : \"aep\",\n", + "# \"conf\" : {\n", + "# 'labels' : ['BAT'],\n", + "# 'resistor_values' : [0.099],\n", + "# 'device_entry' : '/dev/ttyACM1',\n", + "# }\n", + "# },\n", + "\n", + " # Folder where all the results will be collected\n", + " \"results_dir\" : \"Android_Workloads\",\n", + "\n", + " # Define devlib modules to load\n", + " \"modules\" : [\n", + " 'cpufreq' # enable CPUFreq support\n", + " ],\n", + "\n", + " # FTrace events to collect for all the tests configuration which have\n", + " # the \"ftrace\" flag enabled\n", + " \"ftrace\" : {\n", + " \"events\" : [\n", + " \"sched_switch\",\n", + " \"sched_overutilized\",\n", + " \"sched_contrib_scale_f\",\n", + " \"sched_load_avg_cpu\",\n", + " \"sched_load_avg_task\",\n", + " \"sched_tune_tasks_update\",\n", + " \"sched_boost_cpu\",\n", + " \"sched_boost_task\",\n", + " \"sched_energy_diff\",\n", + " \"cpu_frequency\",\n", + " \"cpu_idle\",\n", + " \"cpu_capacity\",\n", + " ],\n", + " \"buffsize\" : 10 * 1024,\n", + " },\n", + "\n", + " # Tools required by the experiments\n", + " \"tools\" : [ 'trace-cmd' ],\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# List of configurations to test (keys of 'confs' defined in cell #9)\n", + "test_confs = ['std']\n", + "\n", + "# List of workloads to run, each workload consists of a workload name\n", + "# followed by a list of workload specific parameters\n", + "test_wloads = [\n", + " \n", + "\n", + " # YouTube workload:\n", + "# Params:\n", + "# - video URL (with optional start time)\n", + "# - duration [s] to playback\n", + " 'YouTube https://youtu.be/XSGBVzeBUbk?t=45s 15',\n", + "\n", + "# Jankbench workload:\n", + "# Params:\n", + "# - id of the benchmakr to run\n", + " 'Jankbench list_view',\n", + "# 'Jankbench image_list_view',\n", + "# 'Jankbench shadow_grid',\n", + "# 'Jankbench low_hitrate_text',\n", + "# 'Jankbench high_hitrate_text',\n", + "# 'Jankbench edit_text',\n", + "\n", + "]\n", + "\n", + "# Iterations for each test\n", + "iterations = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Define what we want to collect as a list of strings.\n", + "# Supported values are\n", + "# energy - Use the my_conf's defined emeter to measure energy consumption across experiments\n", + "# ftrace - Collect an execution trace using trace-cmd\n", + "# systrace - Collect an execution trace using Systrace/Atrace\n", + "# NOTE: energy is automatically enabled in case an \"emeter\" configuration is defined in my_conf\n", + "collect = ''" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Support Functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This set of support functions will help us running the benchmark using different CPUFreq governors." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def set_performance():\n", + " target.cpufreq.set_all_governors('performance')\n", + "\n", + "def set_powersave():\n", + " target.cpufreq.set_all_governors('powersave')\n", + "\n", + "def set_interactive():\n", + " target.cpufreq.set_all_governors('interactive')\n", + "\n", + "def set_sched():\n", + " target.cpufreq.set_all_governors('sched')\n", + "\n", + "def set_ondemand():\n", + " target.cpufreq.set_all_governors('ondemand')\n", + " \n", + " for cpu in target.list_online_cpus():\n", + " tunables = target.cpufreq.get_governor_tunables(cpu)\n", + " target.cpufreq.set_governor_tunables(\n", + " cpu,\n", + " 'ondemand',\n", + " **{'sampling_rate' : tunables['sampling_rate_min']}\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Available test configurations\n", + "confs = {\n", + " 'std' : {\n", + " 'label' : 'int',\n", + " 'set' : set_interactive,\n", + " },\n", + " 'eas' : {\n", + " 'label' : 'sch',\n", + " 'set' : set_sched,\n", + " }\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Experiments Execution Function" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "SYSTRACE_CMD = CATAPULT_HOME + \"/systrace/systrace/systrace.py -e {} -o {} gfx view sched freq idle -t {}\"\n", + "\n", + "def experiment(wl, res_dir, conf_name, wload_name, iterations, collect=''):\n", + " \n", + " # Load workload params\n", + " wload_kind = wload_name.split()[0]\n", + " wload_tag = wload_name.split()[1]\\\n", + " .replace('https://youtu.be/', '')\\\n", + " .replace('?t=', '_')\n", + " \n", + " # Check for workload being available\n", + " wload = Workload.get(te, wload_kind)\n", + " if not wload:\n", + " return {}\n", + " \n", + " # Setup test results folder\n", + " exp_dir = os.path.join(res_dir, conf_name, \"{}_{}\".format(wload_kind, wload_tag))\n", + " os.system('mkdir -p {}'.format(exp_dir));\n", + "\n", + " # Configure governor\n", + " confs[conf_name]['set']()\n", + " \n", + " # Configure screen to max brightness and no dimming\n", + " Screen.set_brightness(target, percent=100)\n", + " Screen.set_dim(target, auto=False)\n", + " Screen.set_timeout(target, 60*60*10) # 10 hours should be enought for an experiment\n", + " \n", + " # Start the required tracing command\n", + " if 'ftrace' in collect:\n", + " # Start FTrace and Energy monitoring\n", + " te.ftrace.start()\n", + " elif 'systrace' in collect:\n", + " # Start systrace\n", + " trace_file = os.path.join(exp_dir, 'trace.html')\n", + " trace_cmd = SYSTRACE_CMD.format(DEVICE, trace_file, wload['duration'] * iterations)\n", + " logging.info('SysTrace: %s', trace_cmd)\n", + " systrace_output = pe.spawn(trace_cmd)\n", + " \n", + " ###########################\n", + " # Run the required workload\n", + " \n", + " # Jankbench\n", + " if 'Jankbench' in wload_name:\n", + " db_file, nrg_data, nrg_file = wload.run(exp_dir, wload_tag, iterations, collect)\n", + " \n", + " # YouTube\n", + " elif 'YouTube' in wload_name:\n", + " video_url = wload_name.split()[1]\n", + " video_duration_s = wload_name.split()[2]\n", + " db_file, nrg_data, nrg_file = wload.run(exp_dir, video_url, int(video_duration_s), collect)\n", + "\n", + " ###########################\n", + " \n", + " # Stop the required trace command\n", + " if 'ftrace' in collect:\n", + " te.ftrace.stop()\n", + " # Collect and keep track of the trace\n", + " trace_file = os.path.join(exp_dir, 'trace.dat')\n", + " te.ftrace.get_trace(trace_file)\n", + " elif 'systrace' in collect:\n", + " logging.info('Waiting systrace report [%s]...', trace_file)\n", + " systrace_output.wait()\n", + "\n", + " # Reset screen brightness and auto dimming\n", + " Screen.set_defaults(target, )\n", + " \n", + " # Dump platform descriptor\n", + " te.platform_dump(exp_dir)\n", + "\n", + " # return all the experiment data\n", + " if 'trace' in collect:\n", + " return {\n", + " 'dir' : exp_dir,\n", + " 'db_file' : db_file,\n", + " 'nrg_data' : nrg_data,\n", + " 'nrg_file' : nrg_file,\n", + " 'trace_file' : trace_file,\n", + " }\n", + " else:\n", + " return {\n", + " 'dir' : exp_dir,\n", + " 'db_file' : db_file,\n", + " 'nrg_data' : nrg_data,\n", + " 'nrg_file' : nrg_file,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Main" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Target Connection" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# # Cleanup Caiman energy meter temporary folders\n", + "# !rm -rf /tmp/eprobe-caiman-*\n", + "# # Ensure there are not other \"caiman\" instanced running for the specified device\n", + "# # my_conf['emeter']['conf']['device_entry']\n", + "# !killall caiman" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2016-05-10 18:11:10,176 INFO : Target - Using base path: /home/derkling/Code/lisa\n", + "2016-05-10 18:11:10,177 INFO : Target - Loading custom (inline) target configuration\n", + "2016-05-10 18:11:10,178 INFO : Target - Devlib modules to load: ['cpufreq']\n", + "2016-05-10 18:11:10,179 INFO : Target - Connecting Android target [GA0113TP0178]\n", + "2016-05-10 18:11:11,481 INFO : Target - Initializing target workdir:\n", + "2016-05-10 18:11:11,482 INFO : Target - /data/local/tmp/devlib-target\n", + "2016-05-10 18:11:14,396 INFO : Target - Topology:\n", + "2016-05-10 18:11:14,397 INFO : Target - [[0, 1], [2, 3]]\n", + "2016-05-10 18:11:15,643 WARNING : Event [sched_overutilized] not available for tracing\n", + "2016-05-10 18:11:15,645 WARNING : Event [sched_contrib_scale_f] not available for tracing\n", + "2016-05-10 18:11:15,647 WARNING : Event [sched_load_avg_cpu] not available for tracing\n", + "2016-05-10 18:11:15,648 WARNING : Event [sched_load_avg_task] not available for tracing\n", + "2016-05-10 18:11:15,649 WARNING : Event [sched_tune_tasks_update] not available for tracing\n", + "2016-05-10 18:11:15,651 WARNING : Event [sched_boost_cpu] not available for tracing\n", + "2016-05-10 18:11:15,652 WARNING : Event [sched_boost_task] not available for tracing\n", + "2016-05-10 18:11:15,654 WARNING : Event [sched_energy_diff] not available for tracing\n", + "2016-05-10 18:11:15,657 WARNING : Event [cpu_capacity] not available for tracing\n", + "2016-05-10 18:11:15,657 INFO : FTrace - Enabled tracepoints:\n", + "2016-05-10 18:11:15,658 INFO : FTrace - sched_switch\n", + "2016-05-10 18:11:15,658 INFO : FTrace - sched_overutilized\n", + "2016-05-10 18:11:15,659 INFO : FTrace - sched_contrib_scale_f\n", + "2016-05-10 18:11:15,660 INFO : FTrace - sched_load_avg_cpu\n", + "2016-05-10 18:11:15,660 INFO : FTrace - sched_load_avg_task\n", + "2016-05-10 18:11:15,661 INFO : FTrace - sched_tune_tasks_update\n", + "2016-05-10 18:11:15,661 INFO : FTrace - sched_boost_cpu\n", + "2016-05-10 18:11:15,662 INFO : FTrace - sched_boost_task\n", + "2016-05-10 18:11:15,662 INFO : FTrace - sched_energy_diff\n", + "2016-05-10 18:11:15,663 INFO : FTrace - cpu_frequency\n", + "2016-05-10 18:11:15,664 INFO : FTrace - cpu_idle\n", + "2016-05-10 18:11:15,664 INFO : FTrace - cpu_capacity\n", + "2016-05-10 18:11:15,665 INFO : TestEnv - Set results folder to:\n", + "2016-05-10 18:11:15,665 INFO : TestEnv - /home/derkling/Code/lisa/results/Android_Workloads\n", + "2016-05-10 18:11:15,666 INFO : TestEnv - Experiment results available also in:\n", + "2016-05-10 18:11:15,666 INFO : TestEnv - /home/derkling/Code/lisa/results_latest\n" + ] + } + ], + "source": [ + "# Initialize a test environment using:\n", + "te = TestEnv(my_conf, wipe=False)\n", + "target = te.target" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Workloads Execution and Data Collection" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2016-05-10 18:11:17,190 INFO : Workload - Workloads available on target:\n", + "2016-05-10 18:11:17,192 INFO : Workload - ['YouTube', 'Jankbench']\n", + "2016-05-10 18:11:17,193 INFO : ------------------------\n", + "2016-05-10 18:11:17,194 INFO : Test 1/2: YOUTUBE in STD configuration\n", + "2016-05-10 18:11:17,195 INFO : Workload - Workloads available on target:\n", + "2016-05-10 18:11:17,195 INFO : Workload - ['YouTube', 'Jankbench']\n", + "2016-05-10 18:11:18,282 INFO : Set brightness: 100%\n", + "2016-05-10 18:11:18,829 INFO : Dim screen mode: OFF\n", + "2016-05-10 18:11:19,440 INFO : Screen timeout: 36000 [s]\n", + "2016-05-10 18:11:20,631 INFO : Force manual orientation\n", + "2016-05-10 18:11:20,632 INFO : Set orientation: LANDSCAPE\n", + "2016-05-10 18:11:21,776 INFO : am start -a android.intent.action.VIEW \"https://youtu.be/XSGBVzeBUbk?t=45s\"\n", + "2016-05-10 18:11:25,586 INFO : Play video for 15 [s]\n", + "2016-05-10 18:11:42,883 INFO : Set orientation: AUTO\n", + "2016-05-10 18:11:43,971 INFO : Set orientation: AUTO\n", + "2016-05-10 18:11:45,581 INFO : Set brightness: AUTO\n", + "2016-05-10 18:11:46,153 INFO : Dim screen mode: ON\n", + "2016-05-10 18:11:46,705 INFO : Screen timeout: 30 [s]\n", + "2016-05-10 18:11:46,707 INFO : ------------------------\n", + "2016-05-10 18:11:46,708 INFO : Test 2/2: JANKBENCH in STD configuration\n", + "2016-05-10 18:11:46,708 INFO : Workload - Workloads available on target:\n", + "2016-05-10 18:11:46,709 INFO : Workload - ['YouTube', 'Jankbench']\n", + "2016-05-10 18:11:47,841 INFO : Set brightness: 100%\n", + "2016-05-10 18:11:48,361 INFO : Dim screen mode: OFF\n", + "2016-05-10 18:11:48,971 INFO : Screen timeout: 36000 [s]\n", + "2016-05-10 18:11:51,289 INFO : Force manual orientation\n", + "2016-05-10 18:11:51,290 INFO : Set orientation: PORTRAIT\n", + "2016-05-10 18:11:52,543 INFO : am start -n \"com.android.benchmark/.app.RunLocalBenchmarksActivity\" --eia \"com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS\" 0 --ei \"com.android.benchmark.EXTRA_RUN_COUNT\" 1\n", + "2016-05-10 18:11:53,186 INFO : adb -s GA0113TP0178 logcat ActivityManager:* System.out:I *:S BENCH:*\n", + "2016-05-10 18:12:28,308 INFO : Mean: 24.202 JankP: 0.061 StdDev: 21.430 Count Bad: 4 Count Jank: 1\n", + "2016-05-10 18:12:31,396 INFO : Set orientation: AUTO\n", + "2016-05-10 18:12:32,452 INFO : Set orientation: AUTO\n", + "2016-05-10 18:12:34,180 INFO : Set brightness: AUTO\n", + "2016-05-10 18:12:34,750 INFO : Dim screen mode: ON\n", + "2016-05-10 18:12:35,320 INFO : Screen timeout: 30 [s]\n" + ] + } + ], + "source": [ + "# Unlock device screen (assume no password required)\n", + "target.execute('input keyevent 82')\n", + "\n", + "# Intialize Workloads for this test environment\n", + "wl = Workload(te)\n", + "\n", + "# The set of results for each comparison test\n", + "results = collections.defaultdict(dict)\n", + "\n", + "# Enable energy collection if an emeter has been configured\n", + "if 'emeter' in my_conf and te.emeter:\n", + " logging.info('Enabling ENERGY collection')\n", + " collect += ' energy'\n", + "\n", + "# Run the benchmark in all the configured governors\n", + "for conf_name in test_confs:\n", + "\n", + " for idx,wload_name in enumerate(test_wloads):\n", + " \n", + " wload_kind = wload_name.split()[0]\n", + " logging.info('------------------------')\n", + " logging.info('Test %d/%d: %s in %s configuration',\n", + " idx+1, len(test_wloads), wload_kind.upper(), conf_name.upper())\n", + " res = experiment(wl, te.res_dir, conf_name, wload_name, iterations, collect)\n", + " results[conf_name][wload_name] = copy.deepcopy(res)\n", + "\n", + " # Save collected results\n", + " res_file = os.path.join(te.res_dir, conf_name, 'results.json')\n", + " with open(res_file, 'w') as fh:\n", + " json.dump(results[conf_name], fh, indent=4)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Energy Measurements Report" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Energy consumption STD, YOUTUBE HTTPS://YOUTU.BE/XSGBVZEBUBK?T=45S 15 : NaN\n", + "Energy consumption STD, JANKBENCH LIST_VIEW : NaN\n" + ] + } + ], + "source": [ + "for conf_name in test_confs:\n", + " for idx,wload_name in enumerate(test_wloads):\n", + " nrg = 'NaN'\n", + " if results[conf_name][wload_name]['nrg_data']:\n", + " nrg = '{:6.1f}'.format(float(results[conf_name][wload_name]['nrg_data']['BAT']))\n", + " print \"Energy consumption {}, {:52}: {}\".format(conf_name.upper(), wload_name.upper(), nrg)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.9" + }, + "toc": { + "toc_cell": false, + "toc_number_sections": true, + "toc_threshold": 6, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} -- GitLab