From be59312a860a2f168b0cfbd445ef0e317c992fc1 Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Wed, 1 Mar 2017 17:43:49 +0000 Subject: [PATCH 1/2] android/workloads/pcmark.py: add pcmark workload Add a new benchmark to the android workload set. This is tested to work on Pixel with PCMark version 2.0.3710. Signed-off-by: Ionela Voinescu --- libs/utils/android/workloads/pcmark.py | 182 +++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 libs/utils/android/workloads/pcmark.py diff --git a/libs/utils/android/workloads/pcmark.py b/libs/utils/android/workloads/pcmark.py new file mode 100644 index 000000000..0371006b2 --- /dev/null +++ b/libs/utils/android/workloads/pcmark.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import os +import logging +import time + +from subprocess import Popen, PIPE + +from android import Screen, System +from android.workload import Workload + +PCMARK_DB_FILE = 'pcmarkscores.txt' +PCMARK_LOG_FILE = 'pcmarklog.txt' +PCMARK_TESTS = ['PERFORMANCE', 'BATTERY_LIFE'] + +# Regexps for benchmark synchronization +PCMARK_BENCHMARK_START_RE = re.compile( + r'com.futuremark.pcmandroid.AbstractMainActivity: '\ + 'running benchmark' +) + +PCMARK_BENCHMARK_END_RE = re.compile( + r'com.futuremark.pcmandroid.AbstractMainActivity: '\ + 'onWebViewReady.*view_scoredetails.html' +) + +class PCMark(Workload): + """ + Android PCMark workload + """ + + # Package required by this workload + package = 'com.futuremark.pcmark.android.benchmark' + + def __init__(self, test_env): + super(PCMark, self).__init__(test_env) + self._log = logging.getLogger('PCMark') + self._log.debug('Workload created') + + def run(self, out_dir, test_name, collect=''): + """ + Run single PCMark workload. Returns a collection of results. + + :param out_dir: Path to experiment directory on the host + where to store results. + :type out_dir: str + + :param test_name: Name of the test to run + :type test_name: str + + :param collect: Specifies what to collect. Possible values: + - 'energy' + - 'systrace' + - 'ftrace' + - any combination of the above as a single space-separated string. + :type collect: list(str) + """ + + self.out_dir = out_dir + self.collect = collect + + # Unlock device screen (assume no password required) + System.menu(self._target) + # Make sure we exit the app if already open + System.back(self._target) + + # Close and NOT clear application (benchmark tests are downloaded from web) + System.force_stop(self._target, self.package, clear=False) + + # Set airplane mode + System.set_airplane_mode(self._target, on=True) + + # Set min brightness + Screen.set_brightness(self._target, auto=False, percent=0) + + # Force screen in PORTRAIT mode + Screen.set_orientation(self._target, portrait=True) + + # Clear logcat + self._target.clear_logcat() + + # Start benchmark + System.monkey(self._target, self.package) + # Wait a few seconds while application is being loaded + time.sleep(8) + + # Parse logcat ActivityManager output lines to detect start + logcat_cmd = self._adb( + 'logcat com.futuremark.pcmandroid.AbstractMainActivity:I *:S'\ + .format(self._target.adb_name)) + self._log.info("%s", logcat_cmd) + + logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) + + # Run performance workload + if test_name.upper() == 'PERFORMANCE': + self._target.execute('input tap 750 1450') + self._log.info('PCmark started!') + else: + raise ValueError('PCMark test [%s] not supported', test_name) + + # The PCMark performance test is composed of multiple benchmark + # that have the same marker for starting + # Use 'trace_started' to only start tracing once + trace_started = False + while True: + + # read next logcat line (up to max 1024 chars) + message = logcat.stdout.readline(1024) + + match = PCMARK_BENCHMARK_START_RE.search(message) + # Benchmark start trigger + if match: + # Start tracing + if not trace_started: + self.tracingStart() + self._log.debug("Benchmark started!") + trace_started = True + + match = PCMARK_BENCHMARK_END_RE.search(message) + if match: + # Stop tracing + if trace_started: + self.tracingStop() + self._log.debug("Benchmark done!") + break + + # Kill the app but not clean the downloaded tests + System.force_stop(self._target, self.package, clear=False) + logcat.kill() + + db_file = os.path.join(out_dir, PCMARK_DB_FILE) + log_file = os.path.join(out_dir, PCMARK_LOG_FILE) + self._log.info("Results can be found in {}.".format(db_file)) + + with open(log_file, 'w') as log: + logcat = Popen(self._adb( + 'logcat com.futuremark.pcmandroid.VirtualMachineState:* *:S'\ + .format(self._target.adb_name)), + stdout=log, + shell=True) + time.sleep(2) + logcat.kill() + + # Parse logcat file and create db with scores + os.popen('grep -o "PCMA_.*_SCORE .*" {} | sed "s/ = / /g" | sort -u > {}' + .format(log_file, db_file)) + + self._log.debug("Score file: {}, size {}B".format(db_file, os.path.getsize(db_file))) + + # Restore default configuration + System.home(self._target) + + # Set orientation back to auto + Screen.set_orientation(self._target, auto=True) + + # Go back to home screen + System.home(self._target) + + # Turn off airplane mode + System.set_airplane_mode(self._target, on=False) + + # Set brightness back to auto + Screen.set_brightness(self._target, auto=True) + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab From a3747f41e65f8787cd7690a5d32119c1b27d862b Mon Sep 17 00:00:00 2001 From: Ionela Voinescu Date: Wed, 1 Mar 2017 17:45:45 +0000 Subject: [PATCH 2/2] tests/benchmarks/android_pcmark.py: Initial commit This provides an example of how to customize and execute an android PCMark workload. Signed-off-by: Ionela Voinescu --- tests/benchmarks/android_pcmark.py | 183 +++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100755 tests/benchmarks/android_pcmark.py diff --git a/tests/benchmarks/android_pcmark.py b/tests/benchmarks/android_pcmark.py new file mode 100755 index 000000000..6255b8cbb --- /dev/null +++ b/tests/benchmarks/android_pcmark.py @@ -0,0 +1,183 @@ +#!/usr/bin/python + +import os +from time import sleep + +# The workload class MUST be loaded before the LisaBenchmark +from android import Workload +from android import LisaBenchmark + +from devlib.exception import TargetError + +class PCMarkTest(LisaBenchmark): + + bm_conf = { + + # Target platform and board + "platform" : 'android', + + # Define devlib modules to load + "modules" : [ + 'cpufreq', + ], + + # FTrace events to collect for all the tests configuration which have + # the "ftrace" flag enabled + "ftrace" : { + "events" : [ + "sched_switch", + "sched_overutilized", + "sched_contrib_scale_f", + "sched_load_avg_cpu", + "sched_load_avg_task", + "sched_tune_tasks_update", + "sched_boost_cpu", + "sched_boost_task", + "sched_energy_diff", + "cpu_frequency", + "cpu_idle", + "cpu_capacity", + ], + "buffsize" : 10 * 1024, + }, + + # Default EnergyMeter Configuration + "emeter" : { + "instrument" : "acme", + "channel_map" : { + "Device0" : 0, + } + }, + + # Tools required by the experiments + "tools" : [ 'trace-cmd' ], + + # Default results folder + "results_dir" : "AndroidPCMark", + + } + + # Android Workload to run + bm_name = 'PCMark' + + # Default products to be collected + bm_collect = 'ftrace energy' + + def benchmarkInit(self): + self.setupWorkload() + self.setupGovernor() + if self.reboot: + self.reboot_target() + + def benchmarkFinalize(self): + if self.delay_after_s: + self._log.info("Waiting %d[s] before to continue...", + self.delay_after_s) + sleep(self.delay_after_s) + + def __init__(self, governor, test, reboot=False, delay_after_s=0): + self.reboot = reboot + self.governor = governor + self.test = test + self.delay_after_s = delay_after_s + super(PCMarkTest, self).__init__() + + def setupWorkload(self): + # Create a results folder for each "governor/test" + self.out_dir = os.path.join(self.te.res_dir, governor, self.test) + try: + os.stat(self.out_dir) + except: + os.makedirs(self.out_dir) + # Setup workload parameters + self.bm_params = { + 'test_name' : self.test + } + + def setupGovernor(self): + try: + self.target.cpufreq.set_all_governors(self.governor); + except TargetError: + self._log.warning('Governor [%s] not available on target', + self.governor) + raise + + # Setup schedutil parameters + if self.governor == 'schedutil': + rate_limit_us = 2000 + # Different schedutil versions have different tunables + tunables = self.target.cpufreq.list_governor_tunables(0) + if 'rate_limit_us' in tunables: + tunables = {'rate_limit_us' : str(rate_limit_us)} + else: + assert ('up_rate_limit_us' in tunables and + 'down_rate_limit_us' in tunables) + tunables = { + 'up_rate_limit_us' : str(rate_limit_us), + 'down_rate_limit_us' : str(rate_limit_us) + } + + try: + for cpu_id in range(self.te.platform['cpus_count']): + self.target.cpufreq.set_governor_tunables( + cpu_id, 'schedutil', **tunables) + except TargetError as e: + self._log.warning('Failed to set schedutils parameters: {}'\ + .format(e)) + raise + self._log.info('Set schedutil.rate_limit_us=%d', rate_limit_us) + + # Setup ondemand parameters + if self.governor == 'ondemand': + try: + for cpu_id in range(self.te.platform['cpus_count']): + tunables = self.target.cpufreq.get_governor_tunables(cpu_id) + self.target.cpufreq.set_governor_tunables( + cpu_id, 'ondemand', + **{'sampling_rate' : tunables['sampling_rate_min']}) + except TargetError as e: + self._log.warning('Failed to set ondemand parameters: {}'\ + .format(e)) + raise + self._log.info('Set ondemand.sampling_rate to minimum supported') + + # Report configured governor + governors = self.target.cpufreq.get_all_governors() + self._log.info('Using governors: %s', governors) + + +# Run the benchmark in each of the supported governors + +governors = [ + 'performance', + 'ondemand', + 'interactive', + 'sched', + 'schedutil', + 'powersave' +] + +tests = [ + 'PERFORMANCE' +] + +# Reboot device only the first time +do_reboot = True +tests_remaining = len(governors) * len(tests) +tests_completed = 0 +for governor in governors: + for test in tests: + tests_remaining -= 1 + delay_after_s = 30 if tests_remaining else 0 + try: + PCMarkTest(governor, test, do_reboot, delay_after_s) + tests_completed += 1 + except: + # A test configuraion failed, continue with other tests + pass + do_reboot = False + +# We want to collect data from at least one governor +assert(tests_completed >= 1) + +# vim :set tabstop=4 shiftwidth=4 expandtab -- GitLab