From 9018129572f419d99e6fcd69783a6f88bc6a9696 Mon Sep 17 00:00:00 2001 From: Michael Platings Date: Fri, 26 Jan 2024 12:24:40 +0000 Subject: [PATCH] Track combined line and branch coverage Many files rightly have zero branches and therefore do not contribute to the branch coverage number. To take both line and branch coverage into account, multiply the two numbers together. Branch coverage of test files is unimportant. Although it would be nice to track test code with only line coverage, the resulting system would lead to too much confusion. Therefore simply exclude test code from the coverage report. --- .gitlab-ci.yml | 2 +- scripts/ci.sh | 2 +- scripts/generate_coverage_report.py | 90 +++++++++++++++++++++++++++++ scripts/generate_coverage_report.sh | 56 ------------------ 4 files changed, 92 insertions(+), 58 deletions(-) create mode 100755 scripts/generate_coverage_report.py delete mode 100755 scripts/generate_coverage_report.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 791a65b0a..99fbcc312 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ image: registry.gitlab.arm.com/intrinsiccv/intrinsiccv:4 path: build/cobertura-coverage.xml paths: - build/html - coverage: /^branches:\s+([\d\.]+\%)/ + coverage: /^line and branch coverage:\s+([\d\.]+\%)/ # On the main branch publish pages. pages: diff --git a/scripts/ci.sh b/scripts/ci.sh index 4e6c25b2b..684ecfe60 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -54,6 +54,6 @@ scripts/prefix_testsuite_names.py build/test-results/sve2/intrinsiccv-api-test.x scripts/prefix_testsuite_names.py build/test-results/sme/intrinsiccv-api-test.xml "SME." # Generate test coverage report -LLVM_COV=llvm-cov scripts/generate_coverage_report.sh +LLVM_COV=llvm-cov scripts/generate_coverage_report.py exit $TESTRESULT diff --git a/scripts/generate_coverage_report.py b/scripts/generate_coverage_report.py new file mode 100755 index 000000000..8ce5cf9a2 --- /dev/null +++ b/scripts/generate_coverage_report.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 + +# Collects coverage data and generates Cobertura & HTML coverage reports. +# +# Arguments +# 1: Path to build directory. Defaults to default build directory. + +import argparse +import os +import shlex +import subprocess +import sys + + +def percentage_from_line(line): + # Given a line like this: + # lines: 58.4% (2676 out of 4580) + # extract the number 58.4 + return float(line.split()[1][:-1]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("build_directory", nargs="?") + args = parser.parse_args() + + llvm_cov = os.environ.get("LLVM_COV") + if not llvm_cov: + print( + """\ +Required environment variable 'LLVM_COV' is not set. +Please set it to point to an llvm-cov instance which is compatible with +the toolchain with which the binaries were built.""" + ) + sys.exit(1) + + source_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + build_dir = os.path.abspath( + args.build_directory or os.path.join(source_dir, "build") + ) + + cobertura = os.path.join(build_dir, "cobertura-coverage.xml") + + coverage_dir = os.path.join(build_dir, "html", "coverage") + os.makedirs(coverage_dir, exist_ok=True) + html = os.path.join(coverage_dir, "coverage_report.html") + + gcovr_command = [ + "gcovr", + "-j", + f"--gcov-executable={llvm_cov} gcov", + f"--cobertura={cobertura}", + f"--html-details={html}", + "--html-title=IntrinsicCV Coverage Report", + "--html-syntax-highlighting", + "--html-tab-size=2", + "--decisions", + "--exclude-noncode-lines", + f"--exclude={build_dir}", + "--exclude=test", + "--print-summary", + build_dir, + ] + + print("+", shlex.join(gcovr_command)) + + gcovr_output = subprocess.check_output( + gcovr_command, cwd=source_dir, text=True + ) + + print(gcovr_output, end="") + + lines = None + branches = None + for line in gcovr_output.splitlines(): + if line.startswith("lines:"): + lines = percentage_from_line(line) + if line.startswith("branches:"): + branches = percentage_from_line(line) + combined_coverage = lines * branches / 100 + print(f"line and branch coverage: {combined_coverage:.1f}%") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_coverage_report.sh b/scripts/generate_coverage_report.sh deleted file mode 100755 index 25bc28fcf..000000000 --- a/scripts/generate_coverage_report.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# -# SPDX-FileCopyrightText: 2023 Arm Limited and/or its affiliates -# -# SPDX-License-Identifier: Apache-2.0 -# -# Collects coverage data and generates Cobertura & HTML coverage reports. -# -# Arguments -# 1: Path to build folder. Defaults to default build folder. - -set -eu - -# ------------------------------------------------------------------------------ -# Mandatory arguments check -# ------------------------------------------------------------------------------ - -if [[ -z "${LLVM_COV+x}" ]]; then - echo "Required variable 'LLVM_COV' is not set." \ - "Please set it to point to an llvm-cov instance which is compatible" \ - "with the toolchain the binaries were built with." - exit 1 -fi - -set -x - -# ------------------------------------------------------------------------------ -# Automatic configuration -# ------------------------------------------------------------------------------ - -SCRIPT_PATH="$(realpath "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" - -source_path="$(realpath "${SCRIPT_PATH}"/..)" -build_path="$(realpath "${1:-${source_path}/build}")" -coverage_path="${build_path}"/html/coverage - -# ------------------------------------------------------------------------------ - -rm -rf "${coverage_path}" -mkdir -p "${coverage_path}" - -cd "${source_path}" - -gcovr \ - -j \ - --gcov-executable "${LLVM_COV} gcov" \ - --cobertura "${build_path}/cobertura-coverage.xml" \ - --html-details "${coverage_path}"/coverage_report.html \ - --html-title "IntrinsicCV Coverage Report" \ - --html-details-syntax-highlighting \ - --html-tab-size 2 \ - --decisions \ - --exclude-noncode-lines \ - --exclude "${build_path}" \ - --print-summary \ - "${build_path}" -- GitLab