From 9144e94f283c6ce262508aecf248206ae9d2763d Mon Sep 17 00:00:00 2001 From: Michael Platings Date: Wed, 13 Mar 2024 15:57:14 +0000 Subject: [PATCH] Add benchmarks Move -Wl,--wrap,malloc flags specifically to tests to avoid breaking Google Benchmark build. --- CMakeLists.txt | 1 + README.md | 8 ++++ SECURITY.md | 1 + benchmark/CMakeLists.txt | 74 +++++++++++++++++++++++++++++++++++ benchmark/benchmark.cpp | 61 +++++++++++++++++++++++++++++ benchmark/main.cpp | 66 +++++++++++++++++++++++++++++++ scripts/ci.sh | 13 ++++++ scripts/format.sh | 1 + test/CMakeLists.txt | 1 - test/api/CMakeLists.txt | 8 ++++ test/framework/CMakeLists.txt | 8 ++++ 11 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/benchmark.cpp create mode 100644 benchmark/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6624d8ec0..d5c36161a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,3 +8,4 @@ project("IntrinsicCV" CXX) add_subdirectory(intrinsiccv) add_subdirectory(test) +add_subdirectory(benchmark) diff --git a/README.md b/README.md index 17dc6026d..d3a241b19 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,14 @@ To target Android devices the following CMake flags are required here as well: -DANDROID_ABI=arm64-v8a ``` +# Benchmarking + +Benchmarks can be enabled with the `cmake` argument `-DINTRINSICCV_BENCHMARK=ON`. +The benchmarks are based on [Google Benchmark](https://github.com/google/benchmark). +All the standard Google Benchmark command line arguments can be used. In addition, +the image size on which the tests will be run can be set with the command line +options `--image_width=123` and `--image_height=123`. + # Build Prerequisites While the core functionality of the library does not rely on any third-party libraries, there are build prerequisites that are essential for compiling the source code and generating the executable. diff --git a/SECURITY.md b/SECURITY.md index d5b3ca7dc..58022ed3c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,3 +16,4 @@ Scripts within this project may download and patch third party sources. It is the responsibility of the users of such scripts to track such third party sources for security issues. The third party sources that may be downloaded are: * Google Test 1.12.1. +* Google Benchmark 1.8.3. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 000000000..8550be738 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.16) + +# The requirements to build Google Benchmark are higher than the +# rest of the project so disable benchmarks by default. +# Another reason to disable it by default is that the configure time +# for Google Benchmark is quite noticeable. +option(INTRINSICCV_BENCHMARK "Enable IntrinsicCV benchmarks" OFF) + +if(INTRINSICCV_BENCHMARK) + +include(FetchContent) + +# Please update SECURITY.md if adding, removing or changing the version of third party content. +FetchContent_Declare( + benchmark + URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.tar.gz + URL_HASH SHA256=6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce +) +FetchContent_MakeAvailable(benchmark) + + +set(INTRINSICCV_INCLUDE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../intrinsiccv/include + ${CMAKE_CURRENT_BINARY_DIR}/../intrinsiccv/include +) +set(INTRINSICCV_BENCHMARK_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +set(INTRINSICCV_BENCHMARK_CXX_FLAGS + "-Werror" + "-Wall" + "-Wextra" + "-Wold-style-cast" +) + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND INTRINSICCV_BENCHMARK_CXX_FLAGS "-O0" "-g") +else() + list(APPEND INTRINSICCV_BENCHMARK_CXX_FLAGS "-O2" "-g0") +endif() + +file(GLOB intrinsiccv_benchmark_sources CONFIGURE_DEPENDS "*.h" "*.cpp") + +set_source_files_properties( + ${intrinsiccv_benchmark_sources} + PROPERTIES COMPILE_OPTIONS "${INTRINSICCV_BENCHMARK_CXX_FLAGS}" +) + +add_executable( + intrinsiccv-benchmark + ${intrinsiccv_benchmark_sources} +) + +set_target_properties( + intrinsiccv-benchmark + PROPERTIES CXX_STANDARD 17 +) + +target_include_directories( + intrinsiccv-benchmark + PRIVATE ${INTRINSICCV_INCLUDE_DIR} + PRIVATE ${INTRINSICCV_BENCHMARK_INCLUDE_DIR} +) + +target_link_libraries( + intrinsiccv-benchmark + intrinsiccv + benchmark::benchmark +) + +endif(INTRINSICCV_BENCHMARK) diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp new file mode 100644 index 000000000..a706e1ce7 --- /dev/null +++ b/benchmark/benchmark.cpp @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2024 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include + +#include "intrinsiccv/intrinsiccv.h" + +extern size_t image_width, image_height; + +template +static void bench_binary_op(Function f, benchmark::State& state) { + // Setup + std::vector src_a, src_b, dst; + src_a.resize(image_width * image_height); + src_b.resize(image_width * image_height); + dst.resize(image_width * image_height); + + std::mt19937 generator; + std::generate(src_a.begin(), src_a.end(), generator); + std::generate(src_b.begin(), src_b.end(), generator); + + for (auto _ : state) { + // This code gets benchmarked + auto unused = f(src_a.data(), image_width, src_a.data(), image_width, + dst.data(), image_width, image_width, image_height); + (void)unused; + } +} + +#define BENCH_BINARY_OP(name, type) \ + static void name(benchmark::State& state) { \ + bench_binary_op(intrinsiccv_##name, state); \ + } \ + BENCHMARK(name) + +BENCH_BINARY_OP(saturating_add_s8, int8_t); +BENCH_BINARY_OP(saturating_sub_u16, uint16_t); +BENCH_BINARY_OP(saturating_absdiff_s32, int32_t); + +static void min_max_loc_u8(benchmark::State& state) { + // Setup + std::vector src; + src.resize(image_width * image_height); + std::mt19937 generator; + std::generate(src.begin(), src.end(), generator); + + size_t min_offset = 0, max_offset = 0; + + for (auto _ : state) { + // This code gets benchmarked + auto unused = + intrinsiccv_min_max_loc_u8(src.data(), image_width, image_width, + image_height, &min_offset, &max_offset); + (void)unused; + } +} +BENCHMARK(min_max_loc_u8); diff --git a/benchmark/main.cpp b/benchmark/main.cpp new file mode 100644 index 000000000..a91b8500e --- /dev/null +++ b/benchmark/main.cpp @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: 2024 Arm Limited and/or its affiliates +// +// SPDX-License-Identifier: Apache-2.0 + +/* +To learn how to use this program see the Google Benchmark user guide: +https://github.com/google/benchmark/blob/main/docs/user_guide.md + +In addition to the regular Google Benchmark command line options, you can also +set the image size on which the tests will be run with the command line options +--image_width=123 and --image_height=123. +*/ + +#include + +#include +#include +#include + +size_t image_width = 1280, image_height = 720; + +bool get_argument(std::string_view arg, std::string_view prefix, + size_t& value) { + if (arg.substr(0, prefix.size()) != prefix) { + return false; + } + + auto value_str = arg.substr(prefix.size()); + + auto result = std::from_chars(value_str.data(), + value_str.data() + value_str.size(), value); + + if (result.ec == std::errc::invalid_argument) { + std::cerr << "Invalid argument: " << arg << std::endl; + exit(1); + } + + return true; +} + +int main(int argc, char** argv) { + for (int i = 1; i < argc;) { + std::string_view arg{argv[i]}; + if (get_argument(arg, "--image_width=", image_width) || + get_argument(arg, "--image_height=", image_height)) { + // Avoid passing the argument to Google Benchmark. + --argc; + for (int j = i; j < argc; ++j) { + argv[j] = argv[j + 1]; + } + } else { + ++i; + } + } + + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; + + std::cout << "IntrinsicCV: Running benchmarks with:\n"; + std::cout << " image_width: " << image_width << "\n"; + std::cout << " image_height: " << image_height << std::endl; + + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + return 0; +} diff --git a/scripts/ci.sh b/scripts/ci.sh index df225ef7c..bb780957c 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -78,6 +78,19 @@ if [[ $(dpkg --print-architecture) = arm64 ]]; then build/sanitize/test/api/intrinsiccv-api-test fi +# Build benchmarks, just to prevent bitrot. +cmake -S . -B build/build-benchmark -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CROSSCOMPILING_EMULATOR=qemu-aarch64 \ + -DCMAKE_CXX_COMPILER_TARGET=aarch64-linux-gnu \ + -DCMAKE_EXE_LINKER_FLAGS="--rtlib=compiler-rt -static -fuse-ld=lld" \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + -DINTRINSICCV_BENCHMARK=ON \ + -DINTRINSICCV_ENABLE_SVE2=ON \ + -DINTRINSICCV_ENABLE_SVE2_SELECTIVELY=OFF +ninja -C build/build-benchmark intrinsiccv-benchmark + # TODO: Cross-build OpenCV if [[ $(dpkg --print-architecture) = arm64 ]]; then # Check OpenCV-IntrinsicCV integration diff --git a/scripts/format.sh b/scripts/format.sh index a4c539cb1..fba356fed 100755 --- a/scripts/format.sh +++ b/scripts/format.sh @@ -30,6 +30,7 @@ INTRINSICCV_ROOT_PATH="$(realpath "${SCRIPT_PATH}"/..)" SOURCES="$(find \ "${INTRINSICCV_ROOT_PATH}"/adapters \ + "${INTRINSICCV_ROOT_PATH}"/benchmark \ "${INTRINSICCV_ROOT_PATH}"/intrinsiccv \ "${INTRINSICCV_ROOT_PATH}"/test \ \( -name \*.cpp -o -name \*.h \) \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6de1e2a88..a361914d6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,7 +38,6 @@ set(INTRINSICCV_TEST_FRAMEWORK_SOURCES ) if (INTRINSICCV_ALLOCATION_TESTS) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--wrap,malloc") list(APPEND INTRINSICCV_TEST_FRAMEWORK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/framework/wrap_malloc.cpp ) diff --git a/test/api/CMakeLists.txt b/test/api/CMakeLists.txt index 483a4fad1..ab591e547 100644 --- a/test/api/CMakeLists.txt +++ b/test/api/CMakeLists.txt @@ -27,6 +27,14 @@ target_include_directories( PRIVATE ${INTRINSICCV_TEST_INCLUDE_DIR} ) +if (INTRINSICCV_ALLOCATION_TESTS) + target_link_options( + intrinsiccv-api-test + PRIVATE -Wl,--wrap,malloc + ) +endif() + + target_link_libraries( intrinsiccv-api-test intrinsiccv diff --git a/test/framework/CMakeLists.txt b/test/framework/CMakeLists.txt index ff60c11a6..f8da3531a 100644 --- a/test/framework/CMakeLists.txt +++ b/test/framework/CMakeLists.txt @@ -27,6 +27,14 @@ target_include_directories( PRIVATE ${INTRINSICCV_TEST_INCLUDE_DIR} ) +if (INTRINSICCV_ALLOCATION_TESTS) + target_link_options( + intrinsiccv-framework-test + PRIVATE -Wl,--wrap,malloc + ) +endif() + + target_link_libraries( intrinsiccv-framework-test intrinsiccv -- GitLab