From 53a30127918a7759127500b5e4a0306fb6a70282 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Mon, 19 Nov 2018 12:13:25 +0000 Subject: [PATCH 01/16] report: remove lisa/report.py Remove as it is not used anymore --- lisa/report.py | 375 ----------------------------------------------- shell/lisa_shell | 17 --- tools/report.py | 41 ------ 3 files changed, 433 deletions(-) delete mode 100644 lisa/report.py delete mode 100755 tools/report.py diff --git a/lisa/report.py b/lisa/report.py deleted file mode 100644 index 31cb56bb0..000000000 --- a/lisa/report.py +++ /dev/null @@ -1,375 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import argparse -import fnmatch as fnm -import json -import math -import numpy as np -import os -import re -import sys -import logging -from collections import defaultdict - -from lisa.colors import TestColors -from lisa.results import Results -from lisa.utils import Loggable - - -# By default compare all the possible combinations -DEFAULT_COMPARE = [(r'base_', r'test_')] - -class Report(Loggable): - - - def __init__(self, results_dir, compare=None, formats=['relative']): - logger = self.get_logger() - self.results_json = results_dir + '/results.json' - self.results = {} - - self.compare = [] - - # Parse results (if required) - if not os.path.isfile(self.results_json): - Results(results_dir) - - # Load results from file (if already parsed) - logger.info('Load results from [%s]...', - self.results_json) - with open(self.results_json) as infile: - self.results = json.load(infile) - - # Setup configuration comparisons - if compare is None: - compare = DEFAULT_COMPARE - logger.warning('Comparing all the possible combination') - for (base_rexp, test_rexp) in compare: - logger.info('Configured regexps for comparisions ' - '(bases , tests): (%s, %s)', - base_rexp, test_rexp) - base_rexp = re.compile(base_rexp, re.DOTALL) - test_rexp = re.compile(test_rexp, re.DOTALL) - self.compare.append((base_rexp, test_rexp)) - - # Report all supported workload classes - self.__rtapp_report(formats) - self.__default_report(formats) - - ############################### REPORT RTAPP ############################### - - def __rtapp_report(self, formats): - logger = self.get_logger() - - if 'rtapp' not in list(self.results.keys()): - logger.debug('No RTApp workloads to report') - return - - logger.debug('Reporting RTApp workloads') - - # Setup lables depending on requested report - if 'absolute' in formats: - nrg_lable = 'Energy Indexes (Absolute)' - prf_lable = 'Performance Indexes (Absolute)' - logger.info('') - logger.info('Absolute comparisions:') - print('') - else: - nrg_lable = 'Energy Indexes (Relative)' - prf_lable = 'Performance Indexes (Relative)' - logger.info('') - logger.info('Relative comparisions:') - print('') - - # Dump headers - print('{:13s} {:20s} |'\ - ' {:33s} | {:54s} |'\ - .format('Test Id', 'Comparision', - nrg_lable, prf_lable)) - print('{:13s} {:20s} |'\ - ' {:>10s} {:>10s} {:>10s} |'\ - ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ - .format('', '', - 'LITTLE', 'big', 'Total', - 'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3')) - - # For each test - _results = self.results['rtapp'] - for tid in sorted(_results.keys()): - new_test = True - # For each configuration... - for base_idx in sorted(_results[tid].keys()): - # Which matches at least on base regexp - for (base_rexp, test_rexp) in self.compare: - if not base_rexp.match(base_idx): - continue - # Look for a configuration which matches the test regexp - for test_idx in sorted(_results[tid].keys()): - if test_idx == base_idx: - continue - if new_test: - print('{:-<37s}+{:-<35s}+{:-<56s}+'\ - .format('','', '')) - self.__rtapp_reference(tid, base_idx) - new_test = False - if test_rexp.match(test_idx) == None: - continue - self.__rtapp_compare(tid, base_idx, test_idx, formats) - - print('') - - def __rtapp_reference(self, tid, base_idx): - logger = self.get_logger() - _results = self.results['rtapp'] - - logger.debug('Test %s: compare against [%s] base', - tid, base_idx) - res_line = '{0:12s}: {1:22s} | '.format(tid, base_idx) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - # Dump absolute values - res_line += ' {0:10.3f}'.format(res_base) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s', pidx, res_base) - - if pidx in ['perf_avg']: - res_line += ' {0:s}'.format(TestColors.rate(res_base)) - continue - if pidx in ['slack_pct']: - res_line += ' {0:s}'.format( - TestColors.rate(res_base, positive_is_good = False)) - continue - if 'edp' in pidx: - res_line += ' {0:10.2e}'.format(res_base) - continue - res_line += ' |' - print(res_line) - - def __rtapp_compare(self, tid, base_idx, test_idx, formats): - logger = self.get_logger() - _results = self.results['rtapp'] - - logger.debug('Test %s: compare %s with %s', - tid, base_idx, test_idx) - res_line = '{0:12s}: {1:20s} | '.format(tid, test_idx) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - res_test = _results[tid][test_idx]['energy'][cpus]['avg'] - speedup_cnt = res_test - res_base - if 'absolute' in formats: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - speedup_pct = 0 - if res_base != 0: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'\ - .format(TestColors.rate( - speedup_pct, - positive_is_good = False)) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - res_test = _results[tid][test_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s, test: %s', - pidx, res_base, res_test) - - if pidx in ['perf_avg']: - res_line += ' {0:s}'.format(TestColors.rate(res_test)) - continue - - if pidx in ['slack_pct']: - res_line += ' {0:s}'.format( - TestColors.rate(res_test, positive_is_good = False)) - continue - - # Compute difference base-vs-test - if 'edp' in pidx: - speedup_cnt = res_base - res_test - if 'absolute': - res_line += ' {0:10.2e}'.format(speedup_cnt) - else: - res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) - - res_line += ' |' - print(res_line) - - ############################### REPORT DEFAULT ############################# - - def __default_report(self, formats): - logger = self.get_logger() - - # Build list of workload types which can be rendered using the default parser - wtypes = [] - for supported_wtype in DEFAULT_WTYPES: - if supported_wtype in list(self.results.keys()): - wtypes.append(supported_wtype) - - if len(wtypes) == 0: - logger.debug('No Default workloads to report') - return - - logger.debug('Reporting Default workloads') - - # Setup lables depending on requested report - if 'absolute' in formats: - nrg_lable = 'Energy Indexes (Absolute)' - prf_lable = 'Performance Indexes (Absolute)' - logger.info('') - logger.info('Absolute comparisions:') - print('') - else: - nrg_lable = 'Energy Indexes (Relative)' - prf_lable = 'Performance Indexes (Relative)' - logger.info('') - logger.info('Relative comparisions:') - print('') - - # Dump headers - print('{:9s} {:20s} |'\ - ' {:33s} | {:54s} |'\ - .format('Test Id', 'Comparision', - nrg_lable, prf_lable)) - print('{:9s} {:20s} |'\ - ' {:>10s} {:>10s} {:>10s} |'\ - ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ - .format('', '', - 'LITTLE', 'big', 'Total', - 'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3')) - - # For each default test - for wtype in wtypes: - _results = self.results[wtype] - for tid in sorted(_results.keys()): - new_test = True - # For each configuration... - for base_idx in sorted(_results[tid].keys()): - # Which matches at least on base regexp - for (base_rexp, test_rexp) in self.compare: - if not base_rexp.match(base_idx): - continue - # Look for a configuration which matches the test regexp - for test_idx in sorted(_results[tid].keys()): - if test_idx == base_idx: - continue - if new_test: - print('{:-<37s}+{:-<35s}+{:-<56s}+'\ - .format('','', '')) - new_test = False - if not test_rexp.match(test_idx): - continue - self.__default_compare(wtype, tid, base_idx, test_idx, formats) - - print('') - - def __default_compare(self, wtype, tid, base_idx, test_idx, formats): - logger = self.get_logger() - _results = self.results[wtype] - - logger.debug('Test %s: compare %s with %s', - tid, base_idx, test_idx) - res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx) - res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - - # If either base of test have a 0 MAX energy, this measn that - # energy has not been collected - base_max = _results[tid][base_idx]['energy'][cpus]['max'] - test_max = _results[tid][test_idx]['energy'][cpus]['max'] - if base_max == 0 or test_max == 0: - res_line += ' {0:10s}'.format('NA') - continue - - # Otherwise, report energy values - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - res_test = _results[tid][test_idx]['energy'][cpus]['avg'] - - speedup_cnt = res_test - res_base - if 'absolute' in formats: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'\ - .format(TestColors.rate( - speedup_pct, - positive_is_good = False)) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - res_test = _results[tid][test_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s, test: %s', - pidx, res_base, res_test) - - # Compute difference base-vs-test - speedup_cnt = 0 - if res_base != 0: - if pidx in ['perf_avg']: - speedup_cnt = res_test - res_base - else: - speedup_cnt = res_base - res_test - - # Compute speedup if required - speedup_pct = 0 - if 'absolute' in formats: - if 'edp' in pidx: - res_line += ' {0:10.2e}'.format(speedup_cnt) - else: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - if res_base != 0: - if pidx in ['perf_avg']: - # speedup_pct = 100.0 * speedup_cnt / res_base - speedup_pct = speedup_cnt - else: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) - res_line += ' |' - print(res_line) - -# List of workload types which can be parsed using the default test parser -DEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe'] - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/shell/lisa_shell b/shell/lisa_shell index 259421a9f..c6b49c939 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -365,23 +365,6 @@ function lisa-test { exekall "$@" } -function lisa-report { - CMD=$1 - [[ "$CMD" != "HELP" ]] && CMD=report - echo - case "$CMD" in - report) - ./tools/report.py $* - ;; - help|*) - ./tools/report.py --help - ;; - esac - echo - echo -} - - ################################################################################ # LISA Workloads utility functions ################################################################################ diff --git a/tools/report.py b/tools/report.py deleted file mode 100755 index 0a0756f31..000000000 --- a/tools/report.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -sys.path.insert(1, "./libs") -from utils.report import Report - -import argparse - -parser = argparse.ArgumentParser( - description='EAS RFC Configuration Comparator.') -parser.add_argument('--bases', type=str, - default='bases_', - help='Regexp of BASE configurations to compare with') -parser.add_argument('--tests', type=str, - default='tests_', - help='Regexp of TEST configurations to compare against') -parser.add_argument('--results', type=str, - default='./results_latest', - help='Folder containing experimental results') - -if __name__ == "__main__": - args = parser.parse_args() - Report(args.results, compare=[(args.bases, args.tests)]) - -- GitLab From 212136b8029f2ea89d109755446d63f6a953d845 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Mon, 19 Nov 2018 12:18:15 +0000 Subject: [PATCH 02/16] git: cleanup git.py Avoid a class with just a single staticmethod, a simple top-level function is perfectly acceptable in Python --- lisa/git.py | 67 +++++++++++++++++------------------- lisa/wa_results_collector.py | 4 +-- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/lisa/git.py b/lisa/git.py index 717e0d30d..99019c750 100644 --- a/lisa/git.py +++ b/lisa/git.py @@ -19,40 +19,37 @@ import logging import os import subprocess -class Git(object): - - @staticmethod - def find_shortest_symref(repo_path, sha1): - """ - Find the shortest symbolic reference (branch/tag) to a Git SHA1 - - :param repo_path: the path of a valid git repository - :type repo_path: str - - :param sha1: the SAH1 of a commit to lookup the reference for - :type sha1: str - - Returns None if nothing points to the requested SHA1 - """ - repo_path = os.path.expanduser(repo_path) - possibles = [] - # Can't use git for-each-ref --points-at because it only came in in Git 2.7 - # which is not in Ubuntu 14.04 - check by hand instead. - branches = subprocess.check_output( - "git for-each-ref --sort=-committerdate " - "--format='%(objectname:short) %(refname:short)' " - "refs/heads/ refs/remotes/ refs/tags", - cwd=repo_path, shell=True) - for line in branches.splitlines(): - try: - sha1_out, name = line.strip().split() - except ValueError: - continue - if sha1_out[:7] == sha1[:7]: - possibles.append(name) - if not possibles: - return None - - return min(possibles, key=len) +def find_shortest_symref(repo_path, sha1): + """ + Find the shortest symbolic reference (branch/tag) to a Git SHA1 + + :param repo_path: the path of a valid git repository + :type repo_path: str + + :param sha1: the SAH1 of a commit to lookup the reference for + :type sha1: str + + Returns None if nothing points to the requested SHA1 + """ + repo_path = os.path.expanduser(repo_path) + possibles = [] + # Can't use git for-each-ref --points-at because it only came in in Git 2.7 + # which is not in Ubuntu 14.04 - check by hand instead. + branches = subprocess.check_output( + "git for-each-ref --sort=-committerdate " + "--format='%(objectname:short) %(refname:short)' " + "refs/heads/ refs/remotes/ refs/tags", + cwd=repo_path, shell=True) + for line in branches.splitlines(): + try: + sha1_out, name = line.strip().split() + except ValueError: + continue + if sha1_out[:7] == sha1[:7]: + possibles.append(name) + if not possibles: + return None + + return min(possibles, key=len) # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/lisa/wa_results_collector.py b/lisa/wa_results_collector.py index 0afbd4b90..ccc5521cc 100644 --- a/lisa/wa_results_collector.py +++ b/lisa/wa_results_collector.py @@ -41,7 +41,7 @@ from IPython.display import display from lisa.platforms.platinfo import PlatformInfo from lisa.trace import Trace -from lisa.git import Git +from lisa.git import find_shortest_symref from lisa.utils import Loggable, memoized class WaResultsCollector(Loggable): @@ -164,7 +164,7 @@ class WaResultsCollector(Loggable): kernel_refs = {} if kernel_repo_path: for sha1 in df['kernel_sha1'].unique(): - ref = Git.find_shortest_symref(kernel_repo_path, sha1) + ref = find_shortest_symref(kernel_repo_path, sha1) if ref: kernel_refs[sha1] = ref -- GitLab From 40a7314333ca9afe0c32a79b77c3c0e5f19cacaf Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 14 Nov 2018 17:47:06 +0000 Subject: [PATCH 03/16] exekall: cleanup in main() --- tools/exekall/exekall/main.py | 47 ++++------------------------------ tools/exekall/exekall/utils.py | 39 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/tools/exekall/exekall/main.py b/tools/exekall/exekall/main.py index 742a302c2..0ef2c1bc9 100755 --- a/tools/exekall/exekall/main.py +++ b/tools/exekall/exekall/main.py @@ -160,47 +160,12 @@ the name of the parameter, the start value, stop value and step size.""") global show_traceback show_traceback = args.debug - # Look for a customization submodule in one of the toplevel packages - # of the modules we specified on the command line. - module_list = [utils.import_file(path) for path in args.python_files] - package_names_list = [ - module.__name__.split('.') - for module in reversed(module_list) - ] - - def build_full_names(l_l): - """Explode list of lists, and build full package names.""" - for l in l_l: - for i, _ in enumerate(l): - i += 1 - yield '.'.join(l[:i]) - - package_name_list = list(build_full_names(package_names_list)) + # Import all modules, before selecting the adaptor + module_set = {utils.import_file(path) for path in args.python_files} - adaptor_cls = AdaptorBase - module_set = set() - - try: - import_excep = ModuleNotFoundError - # Python < 3.6 - except NameError: - import_excep = AttributeError - - for name in reversed(package_name_list): - customize_name = name + '.exekall_customize' - # Only hide ModuleNotFoundError exceptions when looking up that - # specific module, we don't want to hide issues inside the module - # itself. - module_exists = False - with contextlib.suppress(import_excep): - module_exists = importlib.util.find_spec(customize_name) - - if module_exists: - # Importing that module is enough to make the adaptor visible - # to the Adaptor base class - customize_module = importlib.import_module(customize_name) - module_set.add(customize_module) - break + # Look for a customization submodule in one of the parent packages of the + # modules we specified on the command line. + module_set.update(utils.find_customization_module_set(module_set)) adaptor_name = args.adaptor adaptor_cls = AdaptorBase.get_adaptor_cls(adaptor_name) @@ -267,8 +232,6 @@ the name of the parameter, the start value, stop value and step size.""") utils.setup_logging(args.log_level, debug_log, info_log, verbose=verbose) - module_set.update(utils.import_file(path) for path in args.python_files) - non_reusable_type_set = adaptor.get_non_reusable_type_set() # Get the prebuilt operators from the adaptor diff --git a/tools/exekall/exekall/utils.py b/tools/exekall/exekall/utils.py index 30bfe6e72..69548a685 100644 --- a/tools/exekall/exekall/utils.py +++ b/tools/exekall/exekall/utils.py @@ -286,6 +286,45 @@ def resolve_annotations(annotations, module_vars): for param, cls in annotations.items() } +def find_customization_module_set(module_set): + def build_full_names(l_l): + """Explode list of lists, and build full package names.""" + for l in l_l: + for i, _ in enumerate(l): + i += 1 + yield '.'.join(l[:i]) + + try: + import_excep = ModuleNotFoundError + # Python < 3.6 + except NameError: + import_excep = AttributeError + + package_names_list = [ + module.__name__.split('.') + for module in module_set + ] + package_name_set = set(build_full_names(package_names_list)) + + customization_module_set = set() + + for name in package_name_set: + customize_name = name + '.exekall_customize' + # Only hide ModuleNotFoundError exceptions when looking up that + # specific module, we don't want to hide issues inside the module + # itself. + module_exists = False + with contextlib.suppress(import_excep): + module_exists = importlib.util.find_spec(customize_name) + + if module_exists: + # Importing that module is enough to make the adaptor visible + # to the Adaptor base class + customize_module = importlib.import_module(customize_name) + customization_module_set.add(customize_module) + + return customization_module_set + def import_file(python_src, module_name=None, is_package=False): python_src = pathlib.Path(python_src) if python_src.is_dir(): -- GitLab From de62a79b9e107f5410aad4cac4a088aea86ee4dc Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Mon, 19 Nov 2018 15:28:55 +0000 Subject: [PATCH 04/16] exekall: add setup.py Let setuptools create the wrapper shim that calls exekall.main.main() --- devmode_requirements.txt | 2 + modules_root/exekall | 1 - requirements.txt | 1 + shell/exekall | 6 -- shell/lisa_shell | 15 +-- tools/exekall/LICENSE.txt | 202 ++++++++++++++++++++++++++++++++++++++ tools/exekall/README.md | 0 tools/exekall/setup.py | 57 +++++++++++ 8 files changed, 263 insertions(+), 21 deletions(-) delete mode 120000 modules_root/exekall delete mode 100755 shell/exekall create mode 100644 tools/exekall/LICENSE.txt create mode 100644 tools/exekall/README.md create mode 100644 tools/exekall/setup.py diff --git a/devmode_requirements.txt b/devmode_requirements.txt index cfc57519b..693e81a5e 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -13,3 +13,5 @@ # WA before LISA -e ./external/workload-automation/ -e ./[notebook,doc,test] + +-e ./tools/exekall diff --git a/modules_root/exekall b/modules_root/exekall deleted file mode 120000 index 1be809308..000000000 --- a/modules_root/exekall +++ /dev/null @@ -1 +0,0 @@ -../tools/exekall/exekall \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9643a91e6..24622f6ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ ./[notebook,doc,test] +./tools/exekall diff --git a/shell/exekall b/shell/exekall deleted file mode 100755 index 69916958f..000000000 --- a/shell/exekall +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -export EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} -python3 -m exekall.main "$@" - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/shell/lisa_shell b/shell/lisa_shell index c6b49c939..8f610dcbb 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -29,15 +29,6 @@ export LISA_DEVMODE=${LISA_DEVMODE:-1} # Setup colors source "$LISA_HOME/shell/lisa_colors" -# TODO: Stop modifying the PYTHONPATH when all Python packages have a setup.py. -# It will be installed in edit mode using pip in requirements.txt - -# In devmode, we provide the lisa package and some of its dependencies within -# the git tree -if [[ "$LISA_DEVMODE" == 1 ]]; then - export PYTHONPATH="$LISA_HOME/modules_root/:$PYTHONPATH" -fi - # Add some shell utilities to the PATH, with lower priority than system's one # in case the user needs a different version of them export LISA_HOST_ABI=${LISA_HOST_ABI:-x86_64} @@ -355,11 +346,7 @@ echo ################################################################################ export LISA_RESULT_ROOT=$LISA_HOME/results - -function exekall { - EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} \ - python3 -m exekall.main "$@" -} +export EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} function lisa-test { exekall "$@" diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/tools/exekall/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/exekall/README.md b/tools/exekall/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py new file mode 100644 index 000000000..0ce7da5d4 --- /dev/null +++ b/tools/exekall/setup.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from setuptools import setup + +with open('README.md', 'r') as fh: + long_description = fh.read() + +setup( + name='exekall', + version='1.0', + author='Arm Ltd', + # TODO: figure out which email to put here + # author_email= + packages=['exekall'], + # url='http://pypi.python.org/pypi/TowelStuff/', + license='LICENSE.txt', + description='Python expression execution engine', + long_description=long_description, + entry_points={ + 'console_scripts': ['exekall=exekall.main:main'], + }, + python_requires='>= 3.5', + install_requires=[ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + ], + + classifiers=[ + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Intended Audience :: Developers", + ], +) + +# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab -- GitLab From 7a8a78bc705464943509bc94f5cb5c8b1c710820 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Mon, 19 Nov 2018 16:03:23 +0000 Subject: [PATCH 05/16] bisector: add setup.py --- devmode_requirements.txt | 1 + requirements.txt | 1 + tools/bisector/LICENSE.txt | 202 ++++++++++++++++++ tools/bisector/README.md | 0 .../bisector/bisector.py} | 63 +++--- .../bisector/xunit2json.py} | 12 +- tools/bisector/setup.py | 63 ++++++ 7 files changed, 311 insertions(+), 31 deletions(-) create mode 100644 tools/bisector/LICENSE.txt create mode 100644 tools/bisector/README.md rename tools/{bisector => bisector/bisector/bisector.py} (99%) rename tools/{xunit2json-analyze => bisector/bisector/xunit2json.py} (98%) create mode 100644 tools/bisector/setup.py diff --git a/devmode_requirements.txt b/devmode_requirements.txt index 693e81a5e..1a29b7be8 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -15,3 +15,4 @@ -e ./[notebook,doc,test] -e ./tools/exekall +-e ./tools/bisector diff --git a/requirements.txt b/requirements.txt index 24622f6ce..bd2b4136a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ./[notebook,doc,test] ./tools/exekall +./tools/bisector diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/tools/bisector/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/bisector/README.md b/tools/bisector/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/tools/bisector b/tools/bisector/bisector/bisector.py similarity index 99% rename from tools/bisector rename to tools/bisector/bisector/bisector.py index 8273d3cce..152a9af93 100755 --- a/tools/bisector +++ b/tools/bisector/bisector/bisector.py @@ -580,19 +580,19 @@ class ChoiceOrBoolParam(Param): def info(msg): """Write a log message at the INFO level.""" - logger.info(msg) + BISECTOR_LOGGER.info(msg) def debug(msg): """Write a log message at the DEBUG level.""" - logger.debug(msg) + BISECTOR_LOGGER.debug(msg) def warn(msg): """Write a log message at the WARNING level.""" - logger.warning(msg) + BISECTOR_LOGGER.warning(msg) def error(msg): """Write a log message at the ERROR level.""" - logger.error(msg) + BISECTOR_LOGGER.error(msg) class _DefaultType: """ @@ -886,8 +886,8 @@ def read_stdout(p, timeout=None, kill_timeout=3): if stdout: sys.stdout.buffer.write(stdout) sys.stdout.buffer.flush() - log_file.buffer.write(stdout) - log_file.buffer.flush() + LOG_FILE.buffer.write(stdout) + LOG_FILE.buffer.flush() stdout_list.append(stdout) ret = p.wait() @@ -1771,7 +1771,7 @@ class ExekallLISATestStep(ShellStep): testcase = CommaListParam('show only the test cases matching one of the patterns in the comma-separated list. * can be used to match any part of the name.'), ignore_non_issue = BoolParam('consider only tests that failed'), ignore_excep = CommaListParam('ignore the given comma-separated list of exceptions that caused tests failure or error. * can be used to match any part of the name'), - dump_artifact_dirs = BoolOrStrParam('write the list of exkeall artifact directories to a file. Useful to implement garbage collection of unreferenced artifact archives'), + dump_artifact_dirs = BoolOrStrParam('write the list of exekall artifact directories to a file. Useful to implement garbage collection of unreferenced artifact archives'), xunit2json = BoolOrStrParam('append consolidated xUnit information to a JSON file'), export_logs = BoolOrStrParam('export the logs, xUnit file and artifact directory symlink to the given directory'), download = BoolParam('Download the exekall artifact archives if necessary'), @@ -4964,9 +4964,9 @@ class YAMLCLIOptionsAction(argparse.Action): names = ', '.join('"'+name+'"' for name in overriden_names) )) -def main(argv): - global log_file - global show_traceback +def _main(argv): + global LOG_FILE + global SHOW_TRACEBACK parser = argparse.ArgumentParser(description=""" Git-bisect-compatible command driver. @@ -5262,7 +5262,7 @@ command line""") args = args, ) - show_traceback = args.debug + SHOW_TRACEBACK = args.debug service_hub = ServiceHub() try: service_hub.register_service('upload', ArtifactorialService()) @@ -5377,13 +5377,13 @@ command line""") # This log file is available for any function in the global scope ensure_dir(log_path) - log_file = open(log_path, log_file_mode, encoding='utf-8') + LOG_FILE = open(log_path, log_file_mode, encoding='utf-8') # Log to the main log file as well as on the console. - file_handler = logging.StreamHandler(log_file) + file_handler = logging.StreamHandler(LOG_FILE) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(formatter) - logger.addHandler(file_handler) + BISECTOR_LOGGER.addHandler(file_handler) info('Description: {desc}'.format(desc=desc)) @@ -5463,28 +5463,34 @@ command line""") return ret -if __name__ == '__main__': - # Might be changed by main() - show_traceback = True - # Log file opened for the duration of the execution. We initialize it as - # the standard error, and it will be overidden by a call to open() when - # we know which file needs to actually be opened (CLI parameter). - log_file = sys.stderr +# TODO: avoid the global variable by redirecting stdout to a tee (stdout and +# file) inheriting from io.TextIOWrapper and contextlib.redirect_stdout() + +# Log file opened for the duration of the execution. We initialize it as +# the standard error, and it will be overidden by a call to open() when +# we know which file needs to actually be opened (CLI parameter). +LOG_FILE = sys.stderr + +# Might be changed by _main() +SHOW_TRACEBACK = True + +BISECTOR_LOGGER = logging.getLogger('BISECTOR') + +def main(argv=sys.argv[1:]): formatter = logging.Formatter('[%(name)s][%(asctime)s] %(levelname)s %(message)s') - logger = logging.getLogger('BISECTOR') - logger.setLevel(logging.DEBUG) + BISECTOR_LOGGER.setLevel(logging.DEBUG) console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(formatter) - logger.addHandler(console_handler) + BISECTOR_LOGGER.addHandler(console_handler) return_code = None try: - return_code = main(argv=sys.argv[1:]) + return_code = _main(argv=argv) # Quietly exit for these exceptions except SILENT_EXCEPTIONS: pass @@ -5492,7 +5498,7 @@ if __name__ == '__main__': return_code = e.code # Catch-all except Exception as e: - if show_traceback: + if SHOW_TRACEBACK: error( 'Exception traceback:\n' + ''.join( @@ -5506,11 +5512,14 @@ if __name__ == '__main__': # We only flush without closing in case it is stderr, to avoid hidding # exception traceback. It will be closed when the process ends in any # case. - log_file.flush() + LOG_FILE.flush() if return_code is None: return_code = 0 sys.exit(return_code) +if __name__ == '__main__': + main() + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/tools/xunit2json-analyze b/tools/bisector/bisector/xunit2json.py similarity index 98% rename from tools/xunit2json-analyze rename to tools/bisector/bisector/xunit2json.py index 3f3ed99b8..dde94b60a 100755 --- a/tools/xunit2json-analyze +++ b/tools/bisector/bisector/xunit2json.py @@ -6,9 +6,10 @@ import functools import itertools import json import operator +import sys + import pandas as pd import scipy.stats -import sys def error(msg): print('Error: ' + str(msg), file=sys.stderr) @@ -101,7 +102,7 @@ def compare_df(json_df1, json_df2, alpha, alternative='two-sided', non_significa return regression_df -def main(argv): +def _main(argv): parser = argparse.ArgumentParser(description=""" Compare tests failure rate using Fisher exact test. """, @@ -226,13 +227,16 @@ def main(argv): failure_df= json_df['failure']/(json_df['passed'] + json_df['failure']) print(failure_df.to_csv(), end='') -if __name__ == '__main__': +def main(argv=sys.argv[1:]): try: - return_code = main(argv=sys.argv[1:]) + return_code = _main(argv=argv) except FileNotFoundError as e: error(e) return_code = 1 sys.exit(return_code) +if __name__ == '__main__': + main() + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py new file mode 100644 index 000000000..4ec33ecc5 --- /dev/null +++ b/tools/bisector/setup.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from setuptools import setup + +with open('README.md', 'r') as fh: + long_description = fh.read() + +setup( + name='bisector', + version='1.0', + author='Arm Ltd', + # TODO: figure out which email to put here + # author_email= + packages=['bisector'], + # url='http://pypi.python.org/pypi/TowelStuff/', + license='LICENSE.txt', + description='Command execution sequencer', + long_description=long_description, + entry_points={ + 'console_scripts': [ + 'bisector=bisector.bisector:main', + 'xunit2json-analyze=bisector.xunit2json:main' + ], + }, + python_requires='>= 3.5', + install_requires=[ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + "pandas", + "scipy", + ], + + classifiers=[ + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Topic :: Software Development :: Testing", + "Intended Audience :: Developers", + ], +) + +# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab -- GitLab From 8bdef3aa9d919eaec5a21a02562eddfddc11f1b0 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 11:59:49 +0000 Subject: [PATCH 06/16] lisa: improve setup.py * Add lisa.__version__ * Add some setup() parameters --- lisa/__init__.py | 1 + lisa/version.py | 18 ++++++++++++++++++ setup.py | 14 ++++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 lisa/version.py diff --git a/lisa/__init__.py b/lisa/__init__.py index e98d8036c..c2959c337 100644 --- a/lisa/__init__.py +++ b/lisa/__init__.py @@ -1,2 +1,3 @@ +from lisa.version import __version__ # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/lisa/version.py b/lisa/version.py new file mode 100644 index 000000000..10c23556e --- /dev/null +++ b/lisa/version.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = 2.0 diff --git a/setup.py b/setup.py index 864f44aa4..f723be382 100644 --- a/setup.py +++ b/setup.py @@ -20,14 +20,24 @@ from setuptools import setup with open('README.md', 'r') as fh: long_description = fh.read() +with open("lisa/version.py") as f: + version_globals = dict() + exec(f.read(), version_globals) + lisa_version = version_globals['__version__'] + setup( name='LISA', - version='2.0', + version=lisa_version, author='Arm Ltd', # TODO: figure out which email to put here # author_email= packages=['lisa'], - # url='http://pypi.python.org/pypi/TowelStuff/', + url='https://github.com/ARM-software/lisa', + project_urls={ + "Bug Tracker": "https://github.com/ARM-software/lisa/issues", + "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", + "Source Code": "https://github.com/ARM-software/lisa", + }, license='LICENSE.txt', description='A stick to probe the kernel with', long_description=long_description, -- GitLab From e2e91ff92dc456a01a7e58d4970ab08ce8d785fd Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 15:21:09 +0000 Subject: [PATCH 07/16] bisector: fix formatter setup --- tools/bisector/bisector/bisector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/bisector/bisector.py index 152a9af93..97de85c90 100755 --- a/tools/bisector/bisector/bisector.py +++ b/tools/bisector/bisector/bisector.py @@ -5382,7 +5382,7 @@ command line""") # Log to the main log file as well as on the console. file_handler = logging.StreamHandler(LOG_FILE) file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(formatter) + file_handler.setFormatter(BISECTOR_FORMATTER) BISECTOR_LOGGER.addHandler(file_handler) info('Description: {desc}'.format(desc=desc)) @@ -5476,16 +5476,16 @@ LOG_FILE = sys.stderr SHOW_TRACEBACK = True BISECTOR_LOGGER = logging.getLogger('BISECTOR') +BISECTOR_FORMATTER = logging.Formatter('[%(name)s][%(asctime)s] %(levelname)s %(message)s') def main(argv=sys.argv[1:]): - formatter = logging.Formatter('[%(name)s][%(asctime)s] %(levelname)s %(message)s') BISECTOR_LOGGER.setLevel(logging.DEBUG) console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) - console_handler.setFormatter(formatter) + console_handler.setFormatter(BISECTOR_FORMATTER) BISECTOR_LOGGER.addHandler(console_handler) return_code = None -- GitLab From f029c8a9af4076a155adff0a78b9fb8babb852df Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 16:05:06 +0000 Subject: [PATCH 08/16] buildroot: Avoid ARCH autodetection It appears to be broken for ARM, and CROSS_COMPILE also needs to be set properly anyway (otherwise make olddefconfig modifies other options), so full autodetection will never be possible. The user just have to set the right env var when calling the command. --- shell/lisa_buildroot_update_kernel_config | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/shell/lisa_buildroot_update_kernel_config b/shell/lisa_buildroot_update_kernel_config index 145a7d316..c5914a75e 100755 --- a/shell/lisa_buildroot_update_kernel_config +++ b/shell/lisa_buildroot_update_kernel_config @@ -25,7 +25,7 @@ KERNEL_DIR="$2" KERNEL_CONFIG="$2/.config" function print_usage { - echo "Usage: $0 " + echo "Usage: $0 " echo " options:" echo " -h: print this help message" } @@ -51,15 +51,8 @@ if [[ -z "$1" || -z "$2" ]]; then exit -1 fi -# Automatically detect the arm arch we are building for -ARCH="`grep -owE "(arm|arm64)" $KERNEL_CONFIG`" -if [ -z "$ARCH" ]; then - echo "ERROR: unknown arch, arm and arm64 are assumed" - exit -1 -fi - set -u # Update kernel .config to use the provided rootfs as builtin initramfs sed -i "s#[.]*CONFIG_INITRAMFS_SOURCE.*#CONFIG_INITRAMFS_SOURCE=\"$ROOTFS\"#" "$KERNEL_CONFIG" -make -C "$KERNEL_DIR" ARCH=$ARCH olddefconfig +make -C "$KERNEL_DIR" olddefconfig -- GitLab From f3c2fbc13a9b4f4c7090c8ae5acc04639727cd2a Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 17:14:38 +0000 Subject: [PATCH 09/16] bisector: add dbus dependencies --- devmode_requirements.txt | 2 +- install_base_ubuntu.sh | 2 +- requirements.txt | 2 +- tools/bisector/setup.py | 9 +++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/devmode_requirements.txt b/devmode_requirements.txt index 1a29b7be8..722b259d2 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -15,4 +15,4 @@ -e ./[notebook,doc,test] -e ./tools/exekall --e ./tools/bisector +-e ./tools/bisector[dbus] diff --git a/install_base_ubuntu.sh b/install_base_ubuntu.sh index 8aff27b0d..1a9a9eae8 100755 --- a/install_base_ubuntu.sh +++ b/install_base_ubuntu.sh @@ -65,7 +65,7 @@ apt-get update # venv is not installed by default on Ubuntu, even though it is part of the # Python standard library apt-get -y install build-essential git wget expect kernelshark \ - python3 python3-pip python3-venv python3-tk + python3 python3-pip python3-venv python3-tk gobject-introspection if [ "$install_android_sdk" == y ]; then install_sdk diff --git a/requirements.txt b/requirements.txt index bd2b4136a..aa56fac3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ ./[notebook,doc,test] ./tools/exekall -./tools/bisector +./tools/bisector[dbus] diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py index 4ec33ecc5..75edcdf15 100644 --- a/tools/bisector/setup.py +++ b/tools/bisector/setup.py @@ -46,6 +46,15 @@ setup( "scipy", ], + extras_require={ + 'dbus': [ + 'pydbus', + 'pygobject', + # You will also need gobject-introspection package from your + # distribution + ] + }, + classifiers=[ "Programming Language :: Python :: 3 :: Only", # This is not a standard classifier, as there is nothing defined for -- GitLab From 15d23d0a9bd4c538a7b9a35c581bdadfc0259a11 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 17:44:23 +0000 Subject: [PATCH 10/16] lisa: align tests IDs in xunit to exekall default ID format --- lisa/exekall_customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lisa/exekall_customize.py b/lisa/exekall_customize.py index 1d7a36e89..fbdcc8557 100644 --- a/lisa/exekall_customize.py +++ b/lisa/exekall_customize.py @@ -241,6 +241,7 @@ class LISAAdaptor(AdaptorBase): et_testcase = ET.SubElement(et_testsuite, 'testcase', dict( name = expr_val.get_id( full_qual=False, + qual=False, with_tags=True, hidden_callable_set=hidden_callable_set, ))) -- GitLab From 5c5ad03b6be2f4d21fe1ff280b1953a97963cf47 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 17:56:24 +0000 Subject: [PATCH 11/16] bisector: fix monitor subcommand --- tools/bisector/bisector/bisector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/bisector/bisector.py index 97de85c90..709d94c6f 100755 --- a/tools/bisector/bisector/bisector.py +++ b/tools/bisector/bisector/bisector.py @@ -4703,7 +4703,7 @@ def do_monitor(slave_id, args): report_path = props['ReportPath'] argv = ['report', report_path] argv.extend(args.report) - main(argv) + _main(argv) print('\n' + '#' * 80) if args.pause: -- GitLab From 65013cd3280e09c7643e55861b90c3e504f94878 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 18:18:24 +0000 Subject: [PATCH 12/16] load_tracking: Avoid testing load on CpuInvariance Since it does not make since, only test that for FreqInvariance(Item) --- lisa/tests/kernel/scheduler/load_tracking.py | 39 ++++++++++---------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/lisa/tests/kernel/scheduler/load_tracking.py b/lisa/tests/kernel/scheduler/load_tracking.py index 988da5a39..9e1eb350f 100644 --- a/lisa/tests/kernel/scheduler/load_tracking.py +++ b/lisa/tests/kernel/scheduler/load_tracking.py @@ -273,24 +273,6 @@ class InvarianceBase(LoadTrackingBase): """ return self._test_signal('util_avg', allowed_error_pct) - def test_task_load_avg(self, allowed_error_pct=15) -> ResultBundle: - """ - Test that the mean of the load_avg signal matched the expected value. - - Assuming that the system was under little stress (so the task was - RUNNING whenever it was RUNNABLE) and that the task was run with a - 'nice' value of 0, the load_avg should be similar to the util_avg. So, - this test does the same as test_task_util_avg but for load_avg. - - For asymmetric systems, this is only true for tasks run on the - biggest CPUs. - - :param allowed_error_pct: How much the real signal can stray from the - expected values - :type allowed_error_pct: float - """ - return self._test_signal('load_avg', allowed_error_pct) - class CpuInvariance(InvarianceBase): """ Basic check for CPU invariant load and utilization tracking @@ -398,6 +380,25 @@ class FreqInvarianceItem(InvarianceBase): capacity_scale=capacity_scale ) + def test_task_load_avg(self, allowed_error_pct=15) -> ResultBundle: + """ + Test that the mean of the load_avg signal matched the expected value. + + Assuming that the system was under little stress (so the task was + RUNNING whenever it was RUNNABLE) and that the task was run with a + 'nice' value of 0, the load_avg should be similar to the util_avg. So, + this test does the same as test_task_util_avg but for load_avg. + + For asymmetric systems, this is only true for tasks run on the + biggest CPUs. + + :param allowed_error_pct: How much the real signal can stray from the + expected values + :type allowed_error_pct: float + """ + return self._test_signal('load_avg', allowed_error_pct) + + class FreqInvariance(TestBundle, LoadTrackingHelpers): """ Basic check for frequency invariant load and utilization tracking @@ -480,7 +481,7 @@ class FreqInvariance(TestBundle, LoadTrackingHelpers): def test_task_load_avg(self, allowed_error_pct=15): """ - Aggregated version of :meth:`InvarianceBase.test_task_load_avg` + Aggregated version of :meth:`FreqInvarianceItem.test_task_load_avg` """ def item_test(test_item): return test_item.test_task_load_avg(allowed_error_pct=allowed_error_pct) -- GitLab From e6196e60718c53001588cd0531fbe7c30a81be67 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 18:34:02 +0000 Subject: [PATCH 13/16] tests: eas_behaviour: disable idle states --- lisa/tests/kernel/scheduler/eas_behaviour.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lisa/tests/kernel/scheduler/eas_behaviour.py b/lisa/tests/kernel/scheduler/eas_behaviour.py index f88557afa..f5e6ecb48 100644 --- a/lisa/tests/kernel/scheduler/eas_behaviour.py +++ b/lisa/tests/kernel/scheduler/eas_behaviour.py @@ -81,8 +81,9 @@ class EASBehaviour(RTATestBundle, abc.ABC): # EAS doesn't make a lot of sense without schedutil, # so make sure this is what's being used - with te.target.cpufreq.use_governor("schedutil"): - cls._run_rtapp(te, res_dir, rtapp_profile) + with te.disable_idle_states(): + with te.target.cpufreq.use_governor("schedutil"): + cls._run_rtapp(te, res_dir, rtapp_profile) return cls(res_dir, te.plat_info, rtapp_profile) -- GitLab From 928a6a01a94a5aac907ba415760db8674db3ea1b Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Tue, 20 Nov 2018 18:42:39 +0000 Subject: [PATCH 14/16] vagrant: fix Vagrantfile --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 8f7bb1fd8..6ef3dd794 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,7 +35,7 @@ Vagrant.configure(2) do |config| echo 'source init_env' >> /home/vagrant/.bashrc # Trigger the creation of a venv - su vagrant bash ./init_env + su vagrant bash -c 'source ./init_env' # We're all done! echo "Virtual Machine Installation completed successfully! " -- GitLab From fb0b62295a5459504f5c9ae482ca310bc6fc3edb Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 12:23:18 +0000 Subject: [PATCH 15/16] bisector: various fix in exekall support * Fix "passed" total count * Desynchronize the iteration numbers for each step to simplify the logic. That should not be a big issue in practice and allows supporting nested macrosteps as a bonus. * Support multiple trials --- tools/bisector/bisector/bisector.py | 125 +++++++++++++--------------- 1 file changed, 59 insertions(+), 66 deletions(-) diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/bisector/bisector.py index 709d94c6f..183dadc11 100755 --- a/tools/bisector/bisector/bisector.py +++ b/tools/bisector/bisector/bisector.py @@ -1796,10 +1796,10 @@ class ExekallLISATestStep(ShellStep): self.compress_artifact = compress_artifact def run(self, i_stack, service_hub): - # Set the path to tests artifact + # Add a level of UUID under the root, so we can handle multiple trials artifact_path = pathlib.Path( os.getenv('EXEKALL_ARTIFACT_ROOT', 'exekall_artifact'), - datetime.datetime.now().strftime('%Y%m%d_%H%M%S_') + uuid.uuid4().hex, + uuid.uuid4().hex, ) # This also strips the trailing /, which is needed later on when @@ -1809,12 +1809,11 @@ class ExekallLISATestStep(ShellStep): env = { # exekall will use that folder directly, so it has to be empty and # cannot be reused for another invocation - 'EXEKALL_ARTIFACT_DIR': str(artifact_path), + 'EXEKALL_ARTIFACT_ROOT': str(artifact_path), } - #TODO: revisit if self.trials > 1: - warn('More than one trials requested for exekall LISA test, the xUnit XML file will only be recorded for the last trial.') + warn("More than one trials requested for exekall LISA test, only the last trial's xUnit XML file will be used.") res_list = self._run_cmd(i_stack, env=env) ret = res_list[-1][0] @@ -1824,29 +1823,34 @@ class ExekallLISATestStep(ShellStep): else: bisect_ret = BisectRet.GOOD - xunit_path_list = list(artifact_path.glob('**/xunit.xml')) - if not xunit_path_list: - warn('No xUnit file found under: {}'.format(artifact_path)) - elif len(xunit_path_list) > 1: - warn('Found more than one xUnit file, only taking the first one: '+ - ', '.join(xunit_path_list) - ) - + # First item is the oldest created file + xunit_path_list = sorted( + artifact_path.glob('**/xunit.xml'), + key=lambda x: os.path.getmtime(x) + ) xunit_report = '' if xunit_path_list: - xunit_path = xunit_path_list[0] + # Take the most recent file + xunit_path = xunit_path_list[-1] + if len(xunit_path_list) > 1: + warn('Found more than one xUnit file, only taking the most recent one ({}): {}'.format( + xunit_path, + ', '.join(xunit_path_list) + )) try: with open(xunit_path, 'r') as xunit_file: xunit_report = xunit_file.read() - except OSError: - warn('Could not open xUnit XML at: {}'.format(xunit_path)) - xunit_report = '' + except OSError as e: + warn('Could not open xUnit XML {}: {}'.format(xunit_path, e)) + + else: + warn('No xUnit file found under: {}'.format(artifact_path)) # Compress artifact directory if self.compress_artifact: try: - artifact_dir = artifact_path + orig_artifact_path = artifact_path # Create a compressed tar archive info('Compressing exekall artifact directory {} ...'.format(artifact_path)) archive_name = shutil.make_archive( @@ -1855,16 +1859,19 @@ class ExekallLISATestStep(ShellStep): root_dir = os.path.join(artifact_path, '..'), base_dir = os.path.split(artifact_path)[-1], ) - info('exekall artifact directory {artifact_path} compressed as {archive_name}'.format(**locals())) + info('exekall artifact directory {artifact_path} compressed as {archive_name}'.format( + artifact_path=artifact_path, + archive_name=archive_name + )) # From now on, the artifact_path is the path to the archive. artifact_path = os.path.abspath(archive_name) # Delete the original artifact directory since we archived it # successfully. - info('Deleting exekall artifact directory {} ...'.format(artifact_dir)) - shutil.rmtree(artifact_dir) - info('exekall artifact directory {} deleted.'.format(artifact_dir)) + info('Deleting exekall artifact root directory {} ...'.format(orig_artifact_path)) + shutil.rmtree(orig_artifact_path) + info('exekall artifact directory {} deleted.'.format(orig_artifact_path)) except Exception as e: warn('Failed to compress exekall artifact: {e}'.format(e=e)) @@ -1939,16 +1946,6 @@ class ExekallLISATestStep(ShellStep): # Parse the xUnit XML report from LISA to know the failing tests testcase_map = collections.defaultdict(list) filtered_step_res_seq = list() - # Highest iteration number this step was ran during. We assume it ran - # from the start to the end, or that the iteration number at the leaf - # level of macro steps is the iteration number (in case of nested macro - # steps). We use the highest number we find. - # TODO: This should be replace by a function mapping the i_stack to a - # global iteration number, and we should take the maximum of that. - i_max = max( - len(step_res_seq), - max(res[0][-1] for res in step_res_seq) - ) for step_res_item in step_res_seq: i_stack, step_res = step_res_item @@ -2116,8 +2113,6 @@ class ExekallLISATestStep(ShellStep): if show_basic: out(basic_report) - # Display a summary of failed tests - counts = collections.defaultdict(int) # Contains the percentage of skipped, failed, crashed and passed # iterations for every testcase. testcase_stats = dict() @@ -2130,9 +2125,9 @@ class ExekallLISATestStep(ShellStep): testcase_stats[testcase_id] = stats stats['total'] = iteration_n - stats['events'] = dict() - stats['iterations_summary'] = dict() + stats['iterations_summary'] = [entry['result'] for entry in entry_list] stats['counters'] = dict() + stats['events'] = dict() for issue, pretty_issue in ( ('passed', 'passed'), ('skipped', 'skipped'), @@ -2142,27 +2137,16 @@ class ExekallLISATestStep(ShellStep): ): filtered_entry_list = [entry for entry in entry_list if entry['result'] == issue] - # Only works when no nested macro steps are used. That is the - # only way of getting a number that is guaranteed to be - # synchronised with other steps. - i_set = {entry['i_stack'][-1] for entry in filtered_entry_list} - stats['iterations_summary'].update({ - i: issue for i in i_set - }) - issue_n = len(filtered_entry_list) issue_pc = (100 * issue_n) / iteration_n stats['counters'][issue] = issue_n # The event value for a given issue at a given iteration will # be True if the issue appeared, False otherwise. stats['events'][issue] = [ - (i in i_set) - for i in range(1, i_max + 1) + summ_issue == issue + for summ_issue in stats['iterations_summary'] ] - if i_set: - counts[issue] += 1 - if show_rates and ( (issue == 'passed' and show_pass_rate) or @@ -2210,20 +2194,15 @@ class ExekallLISATestStep(ShellStep): 'undecided': 'u', 'error': '#', } - iterations_summary = [ - issue_letter[iterations_summary.get(i, 'passed')] - for i in range(1, i_max + 1) - ] + dist = str( + issue_letter[issue] + for issue in iterations_summary + ) dist_out('{testcase_id}:\n\t{dist}\n'.format( testcase_id = testcase_id, - dist = ''.join(iterations_summary) + dist = dist, )) - issues_total_n = sum( - count for issue, count in stats['counters'].items() - if issue != 'passed' - ) - if show_details: out(table_out) else: @@ -2232,12 +2211,26 @@ class ExekallLISATestStep(ShellStep): out() out(dist_out) - # This shows the number of tests that failed or crashed at least on - # one iteration. Therefore, the total might exceed the number of - # tests if one test failed at one iteration and crashed at another - # one. Only tests passing at all iterations are accounted for in - # the "Passed" count. - total = len(testcase_map) + + counts = { + issue: sum( + # If there is a non-zero count for that issue for that test, we + # account it + bool(stats['counters'][issue]) + for stats in testcase_stats.values() + ) + for issue in ('error', 'failure', 'undecided', 'skipped') + } + # Only account for tests that only passed and had no other issues + counts['passed'] = sum( + all( + not count + for issue, count in stats['counters'].items() + if issue != 'passed' + ) + for stats in testcase_stats.values() + ) + out( 'Crashed: {counts[error]}/{total}, ' 'Failed: {counts[failure]}/{total}, ' @@ -2245,7 +2238,7 @@ class ExekallLISATestStep(ShellStep): 'Skipped: {counts[skipped]}/{total}, ' 'Passed: {counts[passed]}/{total}'.format( counts=counts, - total=total, + total=len(testcase_map), ) ) -- GitLab From 08a60752637aeaf83f0eddc339a25e235aa758e2 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 14:58:17 +0000 Subject: [PATCH 16/16] lisa: Minor improvement in exekall glue --- lisa/exekall_customize.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lisa/exekall_customize.py b/lisa/exekall_customize.py index fbdcc8557..a6fb54775 100644 --- a/lisa/exekall_customize.py +++ b/lisa/exekall_customize.py @@ -234,6 +234,8 @@ class LISAAdaptor(AdaptorBase): testsuite_counters = dict(failures=0, errors=0, tests=0, skipped=0) for testcase in testcase_list: + artifact_path = testcase.data.get('artifact_dir', None) + # If there is more than one value for a given expression, we # assume that they testcase will have unique names using tags expr_val_list = result_map[testcase] @@ -244,7 +246,11 @@ class LISAAdaptor(AdaptorBase): qual=False, with_tags=True, hidden_callable_set=hidden_callable_set, - ))) + ), + # This may help locating the artifacts, even though it + # will only be valid on the machine it was produced on + artifact_path = str(artifact_path) + )) testsuite_counters['tests'] += 1 for failed_expr_val in expr_val.get_failed_values(): -- GitLab