diff --git a/libs/utils/android/benchmark.py b/libs/utils/android/benchmark.py index 0f804ef405a990482147bae095dde45db7c9f225..842d3cff1ccb353af88d9c7d052923c741c54a84 100644 --- a/libs/utils/android/benchmark.py +++ b/libs/utils/android/benchmark.py @@ -59,6 +59,24 @@ class LisaBenchmark(object): bm_collect = None """Override this with the set of data to collect during test exeution""" + bm_reboot = False + """Override this with True if a boot image was passed as command line parameter""" + + bm_iterations = 1 + """Override this with the desired number of iterations of the test""" + + bm_iterations_pause = 30 + """ + Override this with the desired amount of time (in seconds) to pause + for before each iteration + """ + + bm_iterations_reboot = False + """ + Override this with the desired behaviour: reboot or not reboot before + each iteration + """ + def benchmarkInit(self): """ Code executed before running the benchmark @@ -85,8 +103,8 @@ class LisaBenchmark(object): default=None, help='Path of the Android boot.img to be used') parser.add_argument('--boot-timeout', type=int, - default=20, - help='Timeout in [s] to wait after a reboot (default 20)') + default=60, + help='Timeout in [s] to wait after a reboot (default 60)') # Android settings parser.add_argument('--android-device', type=str, @@ -106,6 +124,14 @@ class LisaBenchmark(object): help='Set of metrics to collect, ' 'e.g. "energy systrace_30" to sample energy and collect a 30s systrace, ' 'if specified overrides test defaults') + parser.add_argument('--iterations', type=int, + default=1, + help='Number of iterations the same test has to be repeated for (default 1)') + parser.add_argument('--iterations-pause', type=int, + default=30, + help='Amount of time (in seconds) to pause for before each iteration (default 30s)') + parser.add_argument('--iterations-reboot', action="store_true", + help='Reboot before each iteration (default False)') # Measurements settings parser.add_argument('--iio-channel-map', type=str, @@ -124,6 +150,8 @@ class LisaBenchmark(object): raise NotImplementedError(msg) # Override default configuration with command line parameters + if self.args.boot_image: + self.bm_reboot = True if self.args.android_device: self.bm_conf['device'] = self.args.android_device if self.args.android_home: @@ -132,6 +160,12 @@ class LisaBenchmark(object): self.bm_conf['results_dir'] = self.args.results_dir if self.args.collect: self.bm_collect = self.args.collect + if self.args.iterations: + self.bm_iterations = self.args.iterations + if self.args.iterations_pause: + self.bm_iterations_pause = self.args.iterations_pause + if self.args.iterations_reboot: + self.bm_iterations_reboot = True # Override energy meter configuration if self.args.iio_channel_map: @@ -178,6 +212,33 @@ class LisaBenchmark(object): return '' return self.bm_collect + def _preInit(self): + """ + Code executed before running the benchmark + """ + # If iterations_reboot is True we are going to reboot before the + # first iteration anyway. + if self.bm_reboot and not self.bm_iterations_reboot: + self.reboot_target() + + self.iterations_count = 1 + + def _preRun(self): + """ + Code executed before every iteration of the benchmark + """ + rebooted = False + + if self.bm_reboot and self.bm_iterations_reboot: + rebooted = self.reboot_target() + + if not rebooted and self.iterations_count > 1: + self._log.info('Waiting {}[s] before executing iteration {}...'\ + .format(self.bm_iterations_pause, self.iterations_count)) + sleep(self.bm_iterations_pause) + + self.iterations_count += 1 + def __init__(self): """ Set up logging and trigger running experiments @@ -197,15 +258,25 @@ class LisaBenchmark(object): self.wl = self._getWorkload() self.out_dir=self.te.res_dir try: + self._preInit() self.benchmarkInit() except: self._log.warning('Benchmark initialization failed: execution aborted') raise self._log.info('=== Execution...') - self.wl.run(out_dir=self.out_dir, - collect=self._getBmCollect(), - **self.bm_params) + for iter_id in range(1, self.bm_iterations+1): + self._log.info('=== Iteration {}/{}...'.format(iter_id, self.bm_iterations)) + out_dir = os.path.join(self.out_dir, "{:03d}".format(iter_id)) + try: + os.makedirs(out_dir) + except: pass + + self._preRun() + + self.wl.run(out_dir=out_dir, + collect=self._getBmCollect(), + **self.bm_params) self._log.info('=== Finalization...') self.benchmarkFinalize() @@ -247,6 +318,7 @@ class LisaBenchmark(object): method will reboot the target with the specified kernel and wait for the target to be up and running. """ + rebooted = False # Reboot the device, if a boot_image has been specified if self.args.boot_image: @@ -263,6 +335,7 @@ class LisaBenchmark(object): self._log.debug('Waiting {}[s] for boot to start...'\ .format(self.args.boot_timeout)) sleep(self.args.boot_timeout) + rebooted = True else: self._log.warning('Device NOT rebooted, using current image') @@ -286,4 +359,6 @@ class LisaBenchmark(object): # Wait for the system to complete the boot self._wait_for_logcat_idle() + return rebooted + # vim :set tabstop=4 shiftwidth=4 expandtab diff --git a/tests/benchmarks/android_geekbench.py b/tests/benchmarks/android_geekbench.py index 6cd1df5b4a0535d4493eeff6b853e84a178c3a69..d8b1af391b99f1b3c27e70283ffff8c0a156f52b 100755 --- a/tests/benchmarks/android_geekbench.py +++ b/tests/benchmarks/android_geekbench.py @@ -66,20 +66,10 @@ class GeekbenchTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, test, reboot=False, delay_after_s=0): - self.reboot = reboot + def __init__(self, governor, test): self.governor = governor self.test = test - self.delay_after_s = delay_after_s super(GeekbenchTest, self).__init__() def setupWorkload(self): @@ -161,21 +151,17 @@ tests = [ 'COMPUTE' ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(tests) tests_completed = 0 for governor in governors: for test in tests: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - GeekbenchTest(governor, test, do_reboot, delay_after_s) + GeekbenchTest(governor, test) tests_completed += 1 except: # A test configuraion failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1) diff --git a/tests/benchmarks/android_gmaps.py b/tests/benchmarks/android_gmaps.py index 77afa139f9ba5d35321ff608af4984ce385428dc..fc2b9295022e55e7cf4c920590a9f79eba1ab4e2 100755 --- a/tests/benchmarks/android_gmaps.py +++ b/tests/benchmarks/android_gmaps.py @@ -66,22 +66,11 @@ class GMapsTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, location_search, swipe_count, reboot=False, - delay_after_s=0): - self.reboot = reboot + + def __init__(self, governor, location_search, swipe_count): self.governor = governor self.location_search = location_search self.swipe_count = swipe_count - self.delay_after_s = delay_after_s super(GMapsTest, self).__init__() def setupWorkload(self): @@ -167,22 +156,17 @@ locations = [ "London British Museum" ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(locations) tests_completed = 0 for governor in governors: for location in locations: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - GMapsTest(governor, location, swipe_count, - do_reboot, delay_after_s) + GMapsTest(governor, location, swipe_count) tests_completed += 1 except: # A test configuration failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1) diff --git a/tests/benchmarks/android_jankbench.py b/tests/benchmarks/android_jankbench.py index 160373035319002236673fc76b4cfdbc3f3a5fe9..883cc3fcc7ddd995fc667366b0282fa0223e4ca2 100755 --- a/tests/benchmarks/android_jankbench.py +++ b/tests/benchmarks/android_jankbench.py @@ -67,22 +67,11 @@ class JankbenchTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, test, iterations, - reboot=False, delay_after_s=0): - self.reboot = reboot + + def __init__(self, governor, test, iterations): self.governor = governor self.test = test self.iterations = iterations - self.delay_after_s = delay_after_s super(JankbenchTest, self).__init__() def setupWorkload(self): @@ -172,22 +161,17 @@ tests = [ 'edit_text' ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(tests) tests_completed = 0 for governor in governors: for test in tests: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - JankbenchTest(governor, test, iterations, - do_reboot, delay_after_s) + JankbenchTest(governor, test, iterations) tests_completed += 1 except: # A test configuraion failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1) diff --git a/tests/benchmarks/android_uibench.py b/tests/benchmarks/android_uibench.py index c59ae6d0405cef65183fee6f5a7c7f9df8c84149..5058d16dc60a7c9d47a2cd6efd93edc72af81ffa 100755 --- a/tests/benchmarks/android_uibench.py +++ b/tests/benchmarks/android_uibench.py @@ -66,22 +66,11 @@ class UiBenchTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, test, duration_s, reboot=False, - delay_after_s=0): - self.reboot = reboot + + def __init__(self, governor, test, duration_s): self.governor = governor self.test = test self.duration_s = duration_s - self.delay_after_s = delay_after_s super(UiBenchTest, self).__init__() def setupWorkload(self): @@ -184,22 +173,17 @@ tests = [ 'ActivityTransitionDetails', ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(tests) tests_completed = 0 for governor in governors: for test in tests: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - UiBenchTest(governor, test, duration_s, - do_reboot, delay_after_s) + UiBenchTest(governor, test, duration_s) tests_completed += 1 except: # A test configuraion failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1) diff --git a/tests/benchmarks/android_vellamo.py b/tests/benchmarks/android_vellamo.py index 8afea22467e645568022fb38d9e9321ed05a0b12..64509316431ff11f6b6d167ac7600082494eac07 100755 --- a/tests/benchmarks/android_vellamo.py +++ b/tests/benchmarks/android_vellamo.py @@ -66,20 +66,10 @@ class VellamoTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, test, reboot=False, delay_after_s=0): - self.reboot = reboot + def __init__(self, governor, test): self.governor = governor self.test = test - self.delay_after_s = delay_after_s super(VellamoTest, self).__init__() def setupWorkload(self): @@ -164,21 +154,17 @@ tests = [ ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(tests) tests_completed = 0 for governor in governors: for test in tests: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - VellamoTest(governor, test, do_reboot, delay_after_s) + VellamoTest(governor, test) tests_completed += 1 except: # A test configuraion failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1) diff --git a/tests/benchmarks/android_youtube.py b/tests/benchmarks/android_youtube.py index e09ca7f140a18017050c4204f377d12cc39dc9ef..382296585d285d25d39c03b8dbc0adb567f682e0 100755 --- a/tests/benchmarks/android_youtube.py +++ b/tests/benchmarks/android_youtube.py @@ -66,22 +66,11 @@ class YouTubeTest(LisaBenchmark): def benchmarkInit(self): self.setupWorkload() self.setupGovernor() - if self.reboot: - self.reboot_target() - - def benchmarkFinalize(self): - if self.delay_after_s: - self._log.info("Waiting %d[s] before to continue...", - self.delay_after_s) - sleep(self.delay_after_s) - - def __init__(self, governor, video_url, video_duration_s, reboot=False, - delay_after_s=0): - self.reboot = reboot + + def __init__(self, governor, video_url, video_duration_s): self.governor = governor self.video_url = video_url self.video_duration_s = video_duration_s - self.delay_after_s = delay_after_s super(YouTubeTest, self).__init__() def setupWorkload(self): @@ -166,22 +155,17 @@ video_urls = [ 'https://youtu.be/XSGBVzeBUbk?t=45s', ] -# Reboot device only the first time -do_reboot = True tests_remaining = len(governors) * len(video_urls) tests_completed = 0 for governor in governors: for url in video_urls: tests_remaining -= 1 - delay_after_s = 30 if tests_remaining else 0 try: - YouTubeTest(governor, url, video_duration_s, - do_reboot, delay_after_s) + YouTubeTest(governor, url, video_duration_s) tests_completed += 1 except: # A test configuraion failed, continue with other tests pass - do_reboot = False # We want to collect data from at least one governor assert(tests_completed >= 1)