diff --git a/coverage-tool/coverage-reporting/intermediate_layer.py b/coverage-tool/coverage-reporting/intermediate_layer.py index 55474b91eaa9aa10809ee93f8c87c91be3408e5d..18507d511d25bca07a1cbc9c6e7296af4123f2cf 100644 --- a/coverage-tool/coverage-reporting/intermediate_layer.py +++ b/coverage-tool/coverage-reporting/intermediate_layer.py @@ -1,6 +1,6 @@ # !/usr/bin/env python ############################################################################### -# Copyright (c) 2020-2023, ARM Limited and Contributors. All rights reserved. +# Copyright (c) 2020-2025, ARM Limited and Contributors. All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause ############################################################################### @@ -28,7 +28,10 @@ from typing import List from typing import Generator from typing import Union from typing import Tuple +from typing import Iterable import logging +import sys +from collections import defaultdict __version__ = "7.0" @@ -193,61 +196,82 @@ def remove_workspace(path, workspace): return ret -def get_function_line_numbers(source_file: str) -> Dict[str, int]: +def get_function_line_numbers(source_file_list: Iterable[str]) -> Dict[str, Dict[str, int]]: """ Using ctags get all the function names with their line numbers - within the source_file + within each source file. - :return: Dictionary with function name as key and line number as value + :return: Dictionary with source file name as key. Value is a dictionary with + function name as key and line number as value. """ - command = "ctags -x --c-kinds=f {}".format(source_file) - fln = {} + fln = defaultdict(dict) try: - function_lines = os_command(command).split("\n") - for line in function_lines: + proc = subprocess.Popen( + ['ctags', '--c-kinds=f', '-x', '-L', '-'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + universal_newlines=True # aka 'text' keyword parameter + ) + + proc.stdin.write('\n'.join(source_file_list) + '\n') + proc.stdin.close() + + for line in proc.stdout: cols = line.split() - if len(cols) < 3: + if len(cols) < 4: continue + line_no = int(cols[2]) + source_file = cols[3] if cols[1] == "function": - fln[cols[0]] = int(cols[2]) + fln[source_file][cols[0]] = line_no elif cols[1] == "label": if cols[0] == "func": - fln[cols[-1]] = int(cols[2]) + fln[source_file][cols[-1]] = line_no elif cols[0] + ":" == cols[-1]: - fln[cols[0]] = int(cols[2]) + fln[source_file][cols[0]] = line_no + + if proc.wait() != 0: + raise Exception() + except BaseException: - logger.warning("Warning: Can't get all function line numbers from %s" % - source_file) - except Exception as ex: - logger.warning(f"Warning: Unknown error '{ex}' when executing command " - f"'{command}'") - return {} + logger.warning("Warning: Can't get all function line numbers") return fln class FunctionLineNumbers(object): - """Helper class used to get a function start line number within - a source code file""" + """Helper class used to get all the source code files and start line + numbers for a function""" - def __init__(self, workspace: str): + def __init__(self, local_workspace: str): """ Initialise dictionary to allocate source code files with the corresponding function start line numbers. - :param workspace: The folder where the source files are deployed + :param local_workspace: The folder where the source files are deployed """ - self.filenames = {} - self.workspace = workspace + self.filenames = {} # Keys are file paths in the local workspace, where we run ctags + self.local_workspace = local_workspace - def get_line_number(self, filename: str, function_name: str) -> int: + def get_line_numbers(self, filenames: List[str], function_name: str) -> Dict[str, int]: if not FUNCTION_LINES_ENABLED: - return 0 - if filename not in self.filenames: - source_file = os.path.join(self.workspace, filename) - # Get all functions with their lines in the source file - self.filenames[filename] = get_function_line_numbers(source_file) - return 0 if function_name not in self.filenames[filename] else \ - self.filenames[filename][function_name] + return {} + + # Input filenames are relative to the workspace and need adjusting for + # the local_workspace. + def localize(filename): + return os.path.join(self.local_workspace, filename) + + local_filenames = { localize(f) for f in filenames } + + missing = local_filenames - self.filenames.keys() + if missing: + self.filenames.update(get_function_line_numbers(missing)) + + # A dict with the input file name as key and the line number as value. + return { f : self.filenames[localize(f)][function_name] + for f in filenames + if function_name in self.filenames.get(localize(f), {}) } class BinaryParser(object): @@ -474,11 +498,12 @@ class BinaryParser(object): source_code_block.source_file, self.workspace) yield source_code_block - def get_function_block(self) -> Generator['BinaryParser.FunctionBlock', - None, None]: - """Generator method to obtain all the function blocks contained in + def get_function_blocks(self) -> List['BinaryParser.FunctionBlock']: + """Get list of all the function blocks contained in the binary dump file. """ + result = [] + filenames = [] for function_block in BinaryParser.FunctionBlock.get(self.dump): if function_block.source_file is None: logger.warning(f"Source file not found for function " @@ -487,10 +512,14 @@ class BinaryParser(object): if self.remove_workspace: function_block.source_file = remove_workspace( function_block.source_file, self.workspace) - function_block.function_line_number = \ - self.function_line_numbers.get_line_number( - function_block.source_file, function_block.name) - yield function_block + + filenames.append(function_block.source_file) + result.append(function_block) + + for function_block in result: + line_numbers_by_file = self.function_line_numbers.get_line_numbers(filenames, function_block.name) + function_block.function_line_number = line_numbers_by_file.get(function_block.source_file, 0) + return result class CoverageHandler(object): @@ -564,6 +593,23 @@ class CoverageHandler(object): return self._source_files +IDENTIFIER = re.compile(r'\b[A-Za-z_]\w*\b') + +def identifiers(filename): + with open(filename) as f: + for match in IDENTIFIER.finditer(f.read()): + yield match.group() + +def locate_identifiers(ids, filenames): + """Map each identifier to the set of files in which it appears.""" + result = defaultdict(set) + for filename in filenames: + for s in identifiers(filename): + if s in ids: + result[s].add(filename) + return result + + class IntermediateCodeCoverage(object): """Class used to process the trace data along with the dwarf signature files to produce an intermediate layer in json with @@ -595,6 +641,7 @@ class IntermediateCodeCoverage(object): self.elf_map = {} # For elf custom mappings self.elf_custom = None + self.found_source_files = None def process(self): """ @@ -657,7 +704,7 @@ class IntermediateCodeCoverage(object): self.local_workspace) total_number_functions = 0 functions_covered = 0 - for function_block in parser.get_function_block(): + for function_block in parser.get_function_blocks(): total_number_functions += 1 # Function contains source code self.coverage.add_function_coverage(function_block) @@ -690,6 +737,17 @@ class IntermediateCodeCoverage(object): self._process_fn_no_sources(parser) return parser + def find_source_files(self): + # Cache the list of .c, .s and .S files found in the local workspace. + if self.found_source_files is None: + self.found_source_files = [sys.intern(f) for f in subprocess.check_output([ + 'find', self.local_workspace, + '(', '-name', '*.c', '-o', '-name', '*.s', '-o', '-name', '*.S', ')', + '-type', 'f'], + universal_newlines=True, + stderr=subprocess.DEVNULL).splitlines() ] + return self.found_source_files + def _process_fn_no_sources(self, parser: BinaryParser): """ Checks function coverage for functions with no dwarf signature i.e. @@ -705,23 +763,17 @@ class IntermediateCodeCoverage(object): traces_address_pointer = 0 _functions = parser.no_source_functions functions_addresses = sorted(_functions.keys()) + function_names = [v['name'] for v in _functions.values()] + hits = locate_identifiers(function_names, self.find_source_files()) address_size = 4 for start_address in functions_addresses: function_covered = False function_name = _functions[start_address]['name'] # Get all files in the source code where the function is defined - source_files = os_command("grep --include '*.c' --include '*.s' " - "--include '*.S' -nrw '{}' {}" - "| cut -d: -f1". - format(function_name, - self.local_workspace)) - unique_files = set(source_files.split()) - sources_found = [] - for source_file in unique_files: - line_number = parser.function_line_numbers.get_line_number( - source_file, function_name) - if line_number > 0: - sources_found.append((source_file, line_number)) + unique_files = hits[function_name] + line_numbers_by_file = parser.function_line_numbers.get_line_numbers( + unique_files, function_name) + sources_found = list(line_numbers_by_file.items()) if len(sources_found) == 0: logger.debug(f"'{function_name}' not found in sources") elif len(sources_found) > 1: