From fd8fbbcc6ec00f5e2c1f55b5d3b4fdf38c99db63 Mon Sep 17 00:00:00 2001 From: Denes Tarjan Date: Tue, 8 Apr 2025 07:37:42 +0000 Subject: [PATCH] Implement benchmark list runner using the python script --- scripts/benchmark/benchmarks.yaml | 137 ++++++++++++++++++++ scripts/benchmark/requirements.txt | 5 + scripts/benchmark/run_benchmarks.py | 194 ++++++++++++++++++++++++++++ scripts/benchmark/run_gtest_adb.py | 37 +++--- 4 files changed, 358 insertions(+), 15 deletions(-) create mode 100644 scripts/benchmark/benchmarks.yaml create mode 100644 scripts/benchmark/requirements.txt create mode 100644 scripts/benchmark/run_benchmarks.py diff --git a/scripts/benchmark/benchmarks.yaml b/scripts/benchmark/benchmarks.yaml new file mode 100644 index 000000000..88f59ae05 --- /dev/null +++ b/scripts/benchmark/benchmarks.yaml @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2025 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 + +operations: +# name, executable, gtest_filter, gtest_param_filter + - [GRAY2BGR, opencv_perf_imgproc, "*cvtColor8u/*", '({pixel_format}, COLOR_GRAY2BGR)'] + - [GRAY2BGRA, opencv_perf_imgproc, "*cvtColor8u/*", '({pixel_format}, COLOR_GRAY2BGRA)'] + - [BGR2RGB, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGR2RGB)'] + - [BGRA2RGBA, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGRA2RGBA)'] + - [BGR2RGBA, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGR2RGBA)'] + - [BGR2BGRA, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGR2BGRA)'] + - [RGBA2BGR, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_RGBA2BGR)'] + - [BGRA2BGR, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGRA2BGR)'] + + - [YUVSP2BGR, opencv_perf_imgproc, '*cvtColorYUV420/*', '({pixel_format}, COLOR_YUV2BGR_NV12)'] + - [YUVSP2BGRA, opencv_perf_imgproc, '*cvtColorYUV420/*', '({pixel_format}, COLOR_YUV2BGRA_NV12)'] + - [YUVSP2RGB, opencv_perf_imgproc, '*cvtColorYUV420/*', '({pixel_format}, COLOR_YUV2RGB_NV12)'] + - [YUVSP2RGBA, opencv_perf_imgproc, '*cvtColorYUV420/*', '({pixel_format}, COLOR_YUV2RGBA_NV12)'] + + - [RGB2YUV, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_RGB2YUV)'] + - [BGR2YUV, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_BGR2YUV)'] + - [RGBA2YUV, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, CX_RGBA2YUV)'] + - [BGRA2YUV, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, CX_BGRA2YUV)'] + + - [YUV2RGB, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_YUV2RGB)'] + - [YUV2BGR, opencv_perf_imgproc, '*cvtColor8u/*', '({pixel_format}, COLOR_YUV2BGR)'] + + - [BinaryThreshold, opencv_perf_imgproc, '*ThreshFixture_Threshold.Threshold/*', '({pixel_format}, 8UC1, THRESH_BINARY)'] + + - [SepFilter2D_5x5_U8, opencv_perf_imgproc, '*KleidiCV_SepFilter2D.SepFilter2D/*', '({pixel_format}, 8UC1, 5, BORDER_REPLICATE)'] + - [SepFilter2D_5x5_U16, opencv_perf_imgproc, '*KleidiCV_SepFilter2D.SepFilter2D/*', '({pixel_format}, 16UC1, 5, BORDER_REPLICATE)'] + - [SepFilter2D_5x5_S16, opencv_perf_imgproc, '*KleidiCV_SepFilter2D.SepFilter2D/*', '({pixel_format}, 16SC1, 5, BORDER_REPLICATE)'] + + - [GaussianBlur3x3, opencv_perf_imgproc, '*gaussianBlur3x3/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur5x5, opencv_perf_imgproc, '*gaussianBlur5x5/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur7x7, opencv_perf_imgproc, '*gaussianBlur7x7/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur15x15, opencv_perf_imgproc, '*gaussianBlur15x15/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + + - [GaussianBlur3x3_CustomSigma, opencv_perf_imgproc, '*gaussianBlur3x3_CustomSigma/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur5x5_CustomSigma, opencv_perf_imgproc, '*gaussianBlur5x5_CustomSigma/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur7x7_CustomSigma, opencv_perf_imgproc, '*gaussianBlur7x7_CustomSigma/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + - [GaussianBlur15x15_CustomSigma, opencv_perf_imgproc, '*gaussianBlur15x15_CustomSigma/*', '({pixel_format}, 8UC1, BORDER_REPLICATE)'] + + - [Sobel_Gx, opencv_perf_imgproc, '*Border3x3_sobelFilter.sobelFilter/*', '({pixel_format}, 16SC1, (1, 0), BORDER_REPLICATE)'] + - [Sobel_Gy, opencv_perf_imgproc, '*Border3x3_sobelFilter.sobelFilter/*', '({pixel_format}, 16SC1, (0, 1), BORDER_REPLICATE)'] + + - [Dilate3x3, opencv_perf_imgproc, '*Dilate_big.big/*', '({pixel_format}, 8UC1, 3)'] + - [Dilate5x5, opencv_perf_imgproc, '*Dilate_big.big/*', '({pixel_format}, 8UC1, 5)'] + - [Dilate17x17, opencv_perf_imgproc, '*Dilate_big.big/*', '({pixel_format}, 8UC1, 17)'] + - [Erode3x3, opencv_perf_imgproc, '*Erode_big.big/*', '({pixel_format}, 8UC1, 3)'] + - [Erode5x5, opencv_perf_imgproc, '*Erode_big.big/*', '({pixel_format}, 8UC1, 5)'] + - [Erode17x17, opencv_perf_imgproc, '*Erode_big.big/*', '({pixel_format}, 8UC1, 17)'] + + - [Resize_0.5_8b, opencv_perf_imgproc, '*ResizeAreaFast/*', '(8UC1, {pixel_format}, 2)'] + - [Resize2x2_8b, opencv_perf_imgproc, '*resizeUpLinearNonExact/*', '(8UC1, ({pixel_format_half}, {pixel_format}))'] + - [Resize2x2_float, opencv_perf_imgproc, '*resizeUpLinearNonExact/*', '(32FC1, ({pixel_format_half}, {pixel_format}))'] + - [Resize4x4_8b, opencv_perf_imgproc, '*resizeUpLinearNonExact/*', '(8UC1, ({pixel_format_quarter}, {pixel_format}))'] + - [Resize4x4_float, opencv_perf_imgproc, '*resizeUpLinearNonExact/*', '(32FC1, ({pixel_format_quarter}, {pixel_format}))'] + - [Resize8x8_float, opencv_perf_imgproc, '*resizeUpLinearNonExact/*', '(32FC1, ({pixel_format_eighth}, {pixel_format}))'] + + - [Scale, opencv_perf_core, '*convertTo/*', '({pixel_format}, 8UC1, 8UC1, 1, 1.234, 4.567)'] + - [Scale_float_1.0, opencv_perf_core, '*convertTo/*', '({pixel_format}, 32FC1, 32FC1, 1, 1, 4.567)'] + - [Scale_float, opencv_perf_core, '*convertTo/*', '({pixel_format}, 32FC1, 32FC1, 1, 1.234, 4.567)'] + + - [MinMax_S8, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 8SC1)'] + - [MinMax_U8, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 8UC1)'] + - [MinMax_S16, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 16SC1)'] + - [MinMax_U16, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 16UC1)'] + - [MinMax_S32, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 32SC1)'] + - [MinMax_F32, opencv_perf_core, '*minMaxVals/*', '({pixel_format}, 32FC1)'] + + - [MinMaxLoc_U8, opencv_perf_core, '*minMaxLoc/*', '({pixel_format}, 8UC1)'] + + - [Sum_F32, opencv_perf_core, '*sum/*', '({pixel_format}, 32FC1)'] + + - [FloatToInt, opencv_perf_core, '*convertTo/*', '({pixel_format}, 32FC1, 8SC1, 1, 1, 0)'] + - [FloatToUint, opencv_perf_core, '*convertTo/*', '({pixel_format}, 32FC1, 8UC1, 1, 1, 0)'] + - [IntToFloat, opencv_perf_core, '*convertTo/*', '({pixel_format}, 8SC1, 32FC1, 1, 1, 0)'] + - [UintToFloat, opencv_perf_core, '*convertTo/*', '({pixel_format}, 8UC1, 32FC1, 1, 1, 0)'] + + - [CompareGt, opencv_perf_core, '*compare/*', '({pixel_format}, 8UC1, CMP_GT)'] + + - [InRange_U8, opencv_perf_core, '*inRangeScalar/*', '({pixel_format}, 8UC1, 1, 2)'] + - [InRange_F32, opencv_perf_core, '*inRangeScalar/*', '({pixel_format}, 32FC1, 1, 2)'] + + - [Remap_S16_U8_Replicate, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 16SC2, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_S16_U8_Constant, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 16SC2, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_S16_U16_Replicate, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 16SC2, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_S16_U16_Constant, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 16SC2, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_S16Point5_U8_Replicate, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 16SC2, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_S16Point5_U8_Replicate_4ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC4, 16SC2, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_S16Point5_U8_Constant, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 16SC2, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_S16Point5_U8_Constant_4ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC4, 16SC2, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_S16Point5_U16_Replicate, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 16SC2, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_S16Point5_U16_Replicate_4ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC4, 16SC2, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_S16Point5_U16_Constant, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 16SC2, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_S16Point5_U16_Constant_4ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC4, 16SC2, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_F32_U8_Replicate_Nearest, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 32FC1, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_F32_U8_Constant_Nearest, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 32FC1, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_F32_U16_Replicate_Nearest, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 32FC1, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_F32_U16_Constant_Nearest, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 32FC1, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_F32_U8_Replicate_Nearest_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC2, 32FC1, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_F32_U8_Constant_Nearest_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC2, 32FC1, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_F32_U16_Replicate_Nearest_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC2, 32FC1, INTER_NEAREST, BORDER_REPLICATE)'] + - [Remap_F32_U16_Constant_Nearest_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC2, 32FC1, INTER_NEAREST, BORDER_CONSTANT)'] + - [Remap_F32_U8_Replicate_Linear, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 32FC1, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_F32_U8_Constant_Linear, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC1, 32FC1, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_F32_U16_Replicate_Linear, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 32FC1, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_F32_U16_Constant_Linear, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC1, 32FC1, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_F32_U8_Replicate_Linear_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC2, 32FC1, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_F32_U8_Constant_Linear_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 8UC2, 32FC1, INTER_LINEAR, BORDER_CONSTANT)'] + - [Remap_F32_U16_Replicate_Linear_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC2, 32FC1, INTER_LINEAR, BORDER_REPLICATE)'] + - [Remap_F32_U16_Constant_Linear_2ch, opencv_perf_imgproc, '*Remap/*', '({pixel_format}, 16UC2, 32FC1, INTER_LINEAR, BORDER_CONSTANT)'] + + - [WarpPerspective_Nearest, opencv_perf_imgproc, '*WarpPerspective/*', '({pixel_format}, INTER_NEAREST, BORDER_REPLICATE, 1)'] + - [WarpPerspective_Linear, opencv_perf_imgproc, '*WarpPerspective/*', '({pixel_format}, INTER_LINEAR, BORDER_REPLICATE, 1)'] + - [WarpPerspective_Nearest_Constant, opencv_perf_imgproc, '*WarpPerspective/*', '({pixel_format}, INTER_NEAREST, BORDER_CONSTANT, 1)'] + - [WarpPerspective_Linear_Constant, opencv_perf_imgproc, '*WarpPerspective/*', '({pixel_format}, INTER_LINEAR, BORDER_CONSTANT, 1)'] + - [WarpPerspectiveNear_Nearest, opencv_perf_imgproc, '*WarpPerspectiveNear/*', '({pixel_format}, INTER_NEAREST, BORDER_REPLICATE, 8UC1)'] + - [WarpPerspectiveNear_Linear, opencv_perf_imgproc, '*WarpPerspectiveNear/*', '({pixel_format}, INTER_LINEAR, BORDER_REPLICATE, 8UC1)'] + - [WarpPerspectiveNear_Nearest_Constant, opencv_perf_imgproc, '*WarpPerspectiveNear/*', '({pixel_format}, INTER_NEAREST, BORDER_CONSTANT, 8UC1)'] + - [WarpPerspectiveNear_Linear_Constant, opencv_perf_imgproc, '*WarpPerspectiveNear/*', '({pixel_format}, INTER_LINEAR, BORDER_CONSTANT, 8UC1)'] + + - [BlurAndDownsample, opencv_perf_imgproc, '*pyrDown/*', '({pixel_format}, 8UC1)'] + + - [ScharrInterleaved, opencv_perf_video, '*calcScharrDeriv/*', '{pixel_format}'] + + - [Rotate_90_CLOCKWISE_8UC1, opencv_perf_core, '*rotate/*', '({pixel_format}, 0, 8UC1)'] + - [Rotate_90_CLOCKWISE_16SC1, opencv_perf_core, '*rotate/*', '({pixel_format}, 0, 16SC1)'] + - [Rotate_90_CLOCKWISE_8UC4, opencv_perf_core, '*rotate/*', '({pixel_format}, 0, 8UC4)'] + - [Rotate_90_CLOCKWISE_16SC4, opencv_perf_core, '*rotate/*', '({pixel_format}, 0, 16SC4)'] + +# 720p, not FHD but it's non-trivial to create FHD tests for optical flow and +# the cost of creating a separate category for them outweighs the benefit. + - [OpticalFlowPyrLK_inc_build_pyramid_720p, opencv_perf_video, '*OpticalFlowPyrLK_full/*', '("cv/optflow/frames/720p_%02d.png", 1, 1, (15, 15), 11)'] + - [OpticalFlowPyrLK_exc_build_pyramid_720p, opencv_perf_video, '*OpticalFlowPyrLK_self/*', '("cv/optflow/frames/720p_%02d.png", 1, 1, (15, 15), 11, true)'] diff --git a/scripts/benchmark/requirements.txt b/scripts/benchmark/requirements.txt new file mode 100644 index 000000000..8bfbb37a9 --- /dev/null +++ b/scripts/benchmark/requirements.txt @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2025 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 + +pyyaml diff --git a/scripts/benchmark/run_benchmarks.py b/scripts/benchmark/run_benchmarks.py new file mode 100644 index 000000000..5537bfb03 --- /dev/null +++ b/scripts/benchmark/run_benchmarks.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2025 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +from dataclasses import dataclass, field, KW_ONLY +import hashlib +from pathlib import Path +from typing import Any +import sys +import subprocess +import tempfile + +import yaml + +import run_gtest_adb + +TARGET_WORKSPACE_DIR: str = "/data/local/tmp" + +THIS_FILE: Path = Path(__file__) +THIS_DIR: Path = THIS_FILE.parent + +@dataclass +class Variation: + name: str + build_dir: str + +@dataclass +class Operation: + name: str + executable: str + gtest_filter: str + gtest_param_filter: str + +@dataclass +class Resolution: + params: dict[str, str] + operations: list[Operation] + +@dataclass +class RunGTestAdbArgs: + _: KW_ONLY + executables: list[str] + no_push: bool = False + adb: str = "adb" + serial: str + taskset_masks: list[int] + thermal_zones: list[int] + threads: int = 2 + tsv: str = "gtest_adb.tsv" + json: str = "gtest_adb.json" + tmpdir: str = "/data/local/tmp" + tsv_columns: list[str] = field(default_factory=lambda: ["median"]) + repetitions: int = 1 + verbose: bool = False + gtest_filter: str + gtest_param_filter: str + perf_min_samples: int = 100 + +def main(): + parser = argparse.ArgumentParser(prog=THIS_FILE.name) + parser.add_argument( + "--benchmarks_yaml_path", + type=Path, + help="Path to the benchmarks YAML file to run", + ) + parser.add_argument( + "--resolution", + type=str, + help="Name of the resolution defined in the benchmarks file to run", + ) + parser.add_argument( + "--serial", + type=str, + help="Serial of the device to connect to. " + "Optional if only one device is connected" + ) + + args = parser.parse_args() + + with open(args.benchmarks_yaml_path) as stream: + try: + benchmarks = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + variations = [ + Variation("vanilla", "build/opencv-vanilla"), + Variation("kleidicv", "build/opencv-kleidicv"), + Variation("kleidicv_sve2", "build/opencv-kleidicv-sve2") + ] + + executables = { + "opencv_perf_imgproc": "bin/opencv_perf_imgproc", + "opencv_perf_core": "bin/opencv_perf_core", + "opencv_perf_video": "bin/opencv_perf_video", + } + + resolutions = { + "480": { + "pixel_format": "640x480", + "pixel_format_half": "320x240", + "pixel_format_quarter": "160x120", + "pixel_format_eighth": "80x60" + }, + "720": { + "pixel_format": "1280x720", + "pixel_format_half": "640x360", + "pixel_format_quarter": "320x180", + "pixel_format_eighth": "160x90" + }, + "FHD": { + "pixel_format": "1920x1080", + "pixel_format_half": "960x540", + "pixel_format_quarter": "480x270", + "pixel_format_eighth": "240x135" + }, + "4K": { + "pixel_format": "3840x2160", + "pixel_format_half": "1920x1080", + "pixel_format_quarter": "960x540", + "pixel_format_eighth": "480x270" + } + } + + # Parse and store referenced objects. + operations: list[Operation] = [] + referenced_executables = set() + for raw_operation in benchmarks["operations"]: + operation = Operation(*raw_operation) + operations.append(operation) + if operation.executable not in executables: + raise ValueError(f"Unknown executable referenced: {operation.executable}") + referenced_executables.add(operation.executable) + resolution = Resolution(params=resolutions[args.resolution], operations=operations) + + adb = run_gtest_adb.ADBRunner( + adb_command="adb", + serial_number=args.serial, + verbose=True, + ) + + # Push referenced executables. + for variation in variations: + for executable in sorted(referenced_executables): + host_path: str = f"{THIS_DIR}/../../{variation.build_dir}/{executables[executable]}" + target_file_name: str = f"{executable}_{variation.name}" + if not Path(host_path).is_file(): + raise FileNotFoundError(host_path) + + target_md5: str = "" + not_on_target: bool = False + try: + result = adb.check_output(f"md5sum '{TARGET_WORKSPACE_DIR}/{target_file_name}'") + target_md5 = result.split()[0] + except subprocess.CalledProcessError: + not_on_target = True + host_md5 = hashlib.md5(open(host_path, "rb").read()).hexdigest() + + if not_on_target or (target_md5 != host_md5): + adb.push([host_path], f"{TARGET_WORKSPACE_DIR}/{target_file_name}") + + with tempfile.TemporaryDirectory() as host_temp_dir: + for operation in resolution.operations: + params: dict[str, str] = { + **resolution.params, + "name": operation.name, + } + gtest_param_filter = operation.gtest_param_filter.format(**params) + for variation in variations: + target_file_name: str = f"{operation.executable}_{variation.name}" + json_output_path: Path = Path(f"{host_temp_dir}/{operation.name}_{variation.name}_{args.resolution}") + run_gtest_adb_args = RunGTestAdbArgs( + executables=[target_file_name], + serial=args.serial, + no_push=True, + taskset_masks=[0x30], + thermal_zones=[1], + threads=[2], + json=str(json_output_path), + gtest_filter=operation.gtest_filter, + gtest_param_filter=gtest_param_filter, + ) + print(run_gtest_adb_args) + json_output_path.write_text(f"{{'name': '{json_output_path}'}}") + run_gtest_adb.main(run_gtest_adb_args) + print(f"{operation.name} - {variation.name}:\n" + f" exec {variation.build_dir}/{operation.executable} '{operation.gtest_filter}' '{gtest_param_filter}'") + print(json_output_path.read_text()) + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark/run_gtest_adb.py b/scripts/benchmark/run_gtest_adb.py index f82830e5b..fbff39c99 100755 --- a/scripts/benchmark/run_gtest_adb.py +++ b/scripts/benchmark/run_gtest_adb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2024 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: 2024 - 2025 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 @@ -55,8 +55,13 @@ def parse_args(): parser.add_argument( "--executables", nargs="+", - help="Path to the gtest executables on the host. " - "These will be copied to the device.", + help="Path to the gtest executables on the host (or target if --no-push is used). " + "These are copied to the device, unless --no-push is used.", + ) + parser.add_argument( + "--no_push", + action="store_true", + help="Don't push --executables from host, use them as target path instead.", ) parser.add_argument("--adb", default="adb", help="Path to adb") parser.add_argument( @@ -134,15 +139,15 @@ class ADBRunner: result.extend(["-s", self.serial_number]) return result - def _print_command(self, command): + def _print_command(self, command: list[str]): if self.verbose: - print("+ " + shlex.join(command)) + print("+ ", shlex.join(command)) - def check_output(self, script): + def check_output(self, script: str): command = self._make_adb_command() + ["shell", "su"] self._print_command(command) if self.verbose: - print("+ " + script) + print("+ ", script) try: return subprocess.check_output( command, @@ -350,8 +355,8 @@ def get_results_table(args, results): return rows -def main(): - args = parse_args() +def main(args): + print(args) runner = ADBRunner( adb_command=args.adb, @@ -360,11 +365,12 @@ def main(): ) # Copy executables to device - for host_filename in args.executables: - runner.push( - [host_filename], - host_filename_to_device_filename(args, host_filename), - ) + if not args.no_push: + for host_filename in args.executables: + runner.push( + [host_filename], + host_filename_to_device_filename(args, host_filename), + ) results = {} @@ -388,4 +394,5 @@ def main(): if __name__ == "__main__": - main() + args = parse_args() + main(args) -- GitLab