From 31be17dc4ee8402e755bae7cbcb533aaabb5c242 Mon Sep 17 00:00:00 2001
From: Michael Platings <michael.platings@arm.com>
Date: Mon, 8 Jul 2024 09:25:19 +0100
Subject: [PATCH] Make run_gtest_adb.py more permissive about errors

The output of OpenCV's benchmarks is not always consistent.
If the output is not as expected then print the error message but
continue executing the remaining benchmarks. Typically when running many
benchmarks the intention is to get a broad overview of performance so
the occasional missing result is acceptable.
---
 scripts/benchmark/run_gtest_adb.py | 78 ++++++++++++++++++------------
 1 file changed, 46 insertions(+), 32 deletions(-)

diff --git a/scripts/benchmark/run_gtest_adb.py b/scripts/benchmark/run_gtest_adb.py
index 83be85466..5d8e19b75 100755
--- a/scripts/benchmark/run_gtest_adb.py
+++ b/scripts/benchmark/run_gtest_adb.py
@@ -41,6 +41,7 @@ import json
 import os
 import shlex
 import subprocess
+import sys
 import tempfile
 import time
 
@@ -135,9 +136,13 @@ class ADBRunner:
     def check_output(self, args):
         command = self._make_adb_command() + ["shell"] + args
         self._print_command(command)
-        return subprocess.check_output(
-            command, stderr=subprocess.STDOUT
-        ).decode()
+        try:
+            return subprocess.check_output(
+                command, stderr=subprocess.STDOUT
+            ).decode()
+        except subprocess.CalledProcessError as e:
+            print(e.stdout.decode(), file=sys.stderr)
+            raise
 
     def push(self, filenames, dst):
         command = self._make_adb_command() + ["push", *filenames, dst]
@@ -207,16 +212,19 @@ def run_executable_tests(runner, args, host_executable, cpu, thermal_zone):
 
     for test_name in test_list:
         wait_for_cooldown(runner, thermal_zone)
-        runner.check_output(
-            [
-                "taskset",
-                taskset_mask,
-                executable,
-                f"--gtest_output=json:{output_file}",
-                f"--gtest_filter={test_name}",
-                f"--perf_min_samples={args.perf_min_samples}",
-            ]
-        )
+        try:
+            runner.check_output(
+                [
+                    "taskset",
+                    taskset_mask,
+                    executable,
+                    f"--gtest_output=json:{output_file}",
+                    f"--gtest_filter={test_name}",
+                    f"--perf_min_samples={args.perf_min_samples}",
+                ]
+            )
+        except subprocess.CalledProcessError:
+            continue
         test_result = runner.read_json(output_file)
 
         if not results:
@@ -233,13 +241,16 @@ def run_executable_tests(runner, args, host_executable, cpu, thermal_zone):
 
         test_result = testsuite["testsuite"][0]
 
-        output = (
-            f"{executable}-{cpu}\t{test_name}"
-            f"\t{test_result['value_param']}"
-        )
-        for key in args.tsv_columns:
-            output += f"\t{test_result[key]}"
-        print(output)
+        try:
+            output = (
+                f"{executable}-{cpu}\t{test_name}"
+                f"\t{test_result['value_param']}"
+            )
+            for key in args.tsv_columns:
+                output += f"\t{test_result[key]}"
+            print(output)
+        except KeyError:
+            pass
 
     results["testsuites"] = list(testsuites.values())
 
@@ -293,18 +304,21 @@ def get_results_table(args, results):
             value_param = test["value_param"]
             row = [f"{testsuite_name}.{test_name}", value_param]
 
-            for cpu in args.cpus:
-                for key in args.tsv_columns:
-                    for executable in args.executables:
-                        for rep in range(args.repetitions):
-                            result = results[
-                                get_run_name(rep, executable, cpu)
-                            ]
-                            exe_test = result["testsuites"][testsuite_index][
-                                "testsuite"
-                            ][test_index]
-                            row.append(exe_test[key])
-            rows.append(row)
+            try:
+                for cpu in args.cpus:
+                    for key in args.tsv_columns:
+                        for executable in args.executables:
+                            for rep in range(args.repetitions):
+                                result = results[
+                                    get_run_name(rep, executable, cpu)
+                                ]
+                                exe_test = result["testsuites"][testsuite_index][
+                                    "testsuite"
+                                ][test_index]
+                                row.append(exe_test[key])
+                rows.append(row)
+            except KeyError:
+                pass
 
     return rows
 
-- 
GitLab