diff --git a/Vagrantfile b/Vagrantfile index 8f7bb1fd8d93ec457e28f7c323d439201cd0068d..6ef3dd7944623f1d80035dd5cb2d900f566b8735 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,7 +35,7 @@ Vagrant.configure(2) do |config| echo 'source init_env' >> /home/vagrant/.bashrc # Trigger the creation of a venv - su vagrant bash ./init_env + su vagrant bash -c 'source ./init_env' # We're all done! echo "Virtual Machine Installation completed successfully! " diff --git a/devmode_requirements.txt b/devmode_requirements.txt index cfc57519bff713640f0b4f24381bbe80b1f24852..722b259d202d21f43c01ef26ca259eeb8735eb25 100644 --- a/devmode_requirements.txt +++ b/devmode_requirements.txt @@ -13,3 +13,6 @@ # WA before LISA -e ./external/workload-automation/ -e ./[notebook,doc,test] + +-e ./tools/exekall +-e ./tools/bisector[dbus] diff --git a/install_base_ubuntu.sh b/install_base_ubuntu.sh index 8aff27b0d5004bfd0c1119e9836f1fe23f859e82..1a9a9eae8e48a696f7d284f733da615731aa9e17 100755 --- a/install_base_ubuntu.sh +++ b/install_base_ubuntu.sh @@ -65,7 +65,7 @@ apt-get update # venv is not installed by default on Ubuntu, even though it is part of the # Python standard library apt-get -y install build-essential git wget expect kernelshark \ - python3 python3-pip python3-venv python3-tk + python3 python3-pip python3-venv python3-tk gobject-introspection if [ "$install_android_sdk" == y ]; then install_sdk diff --git a/lisa/__init__.py b/lisa/__init__.py index e98d8036c1a1b1ae0660f76c54fd56a57cc841ff..c2959c337414d3a208812e5921896272851dff53 100644 --- a/lisa/__init__.py +++ b/lisa/__init__.py @@ -1,2 +1,3 @@ +from lisa.version import __version__ # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/lisa/exekall_customize.py b/lisa/exekall_customize.py index 1d7a36e89588b6b44a0fb776b64eee32c55de759..a6fb54775a459514a54be3a4e4acd9986eac8739 100644 --- a/lisa/exekall_customize.py +++ b/lisa/exekall_customize.py @@ -234,6 +234,8 @@ class LISAAdaptor(AdaptorBase): testsuite_counters = dict(failures=0, errors=0, tests=0, skipped=0) for testcase in testcase_list: + artifact_path = testcase.data.get('artifact_dir', None) + # If there is more than one value for a given expression, we # assume that they testcase will have unique names using tags expr_val_list = result_map[testcase] @@ -241,9 +243,14 @@ class LISAAdaptor(AdaptorBase): et_testcase = ET.SubElement(et_testsuite, 'testcase', dict( name = expr_val.get_id( full_qual=False, + qual=False, with_tags=True, hidden_callable_set=hidden_callable_set, - ))) + ), + # This may help locating the artifacts, even though it + # will only be valid on the machine it was produced on + artifact_path = str(artifact_path) + )) testsuite_counters['tests'] += 1 for failed_expr_val in expr_val.get_failed_values(): diff --git a/lisa/git.py b/lisa/git.py index 717e0d30da68976eb5e7b083cf65177d3ac68aa6..99019c750e4890e1fcf526f825d4e516e76b955a 100644 --- a/lisa/git.py +++ b/lisa/git.py @@ -19,40 +19,37 @@ import logging import os import subprocess -class Git(object): - - @staticmethod - def find_shortest_symref(repo_path, sha1): - """ - Find the shortest symbolic reference (branch/tag) to a Git SHA1 - - :param repo_path: the path of a valid git repository - :type repo_path: str - - :param sha1: the SAH1 of a commit to lookup the reference for - :type sha1: str - - Returns None if nothing points to the requested SHA1 - """ - repo_path = os.path.expanduser(repo_path) - possibles = [] - # Can't use git for-each-ref --points-at because it only came in in Git 2.7 - # which is not in Ubuntu 14.04 - check by hand instead. - branches = subprocess.check_output( - "git for-each-ref --sort=-committerdate " - "--format='%(objectname:short) %(refname:short)' " - "refs/heads/ refs/remotes/ refs/tags", - cwd=repo_path, shell=True) - for line in branches.splitlines(): - try: - sha1_out, name = line.strip().split() - except ValueError: - continue - if sha1_out[:7] == sha1[:7]: - possibles.append(name) - if not possibles: - return None - - return min(possibles, key=len) +def find_shortest_symref(repo_path, sha1): + """ + Find the shortest symbolic reference (branch/tag) to a Git SHA1 + + :param repo_path: the path of a valid git repository + :type repo_path: str + + :param sha1: the SAH1 of a commit to lookup the reference for + :type sha1: str + + Returns None if nothing points to the requested SHA1 + """ + repo_path = os.path.expanduser(repo_path) + possibles = [] + # Can't use git for-each-ref --points-at because it only came in in Git 2.7 + # which is not in Ubuntu 14.04 - check by hand instead. + branches = subprocess.check_output( + "git for-each-ref --sort=-committerdate " + "--format='%(objectname:short) %(refname:short)' " + "refs/heads/ refs/remotes/ refs/tags", + cwd=repo_path, shell=True) + for line in branches.splitlines(): + try: + sha1_out, name = line.strip().split() + except ValueError: + continue + if sha1_out[:7] == sha1[:7]: + possibles.append(name) + if not possibles: + return None + + return min(possibles, key=len) # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/lisa/report.py b/lisa/report.py deleted file mode 100644 index 31cb56bb0381edb5b12565f1308e0ce6ab9be966..0000000000000000000000000000000000000000 --- a/lisa/report.py +++ /dev/null @@ -1,375 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import argparse -import fnmatch as fnm -import json -import math -import numpy as np -import os -import re -import sys -import logging -from collections import defaultdict - -from lisa.colors import TestColors -from lisa.results import Results -from lisa.utils import Loggable - - -# By default compare all the possible combinations -DEFAULT_COMPARE = [(r'base_', r'test_')] - -class Report(Loggable): - - - def __init__(self, results_dir, compare=None, formats=['relative']): - logger = self.get_logger() - self.results_json = results_dir + '/results.json' - self.results = {} - - self.compare = [] - - # Parse results (if required) - if not os.path.isfile(self.results_json): - Results(results_dir) - - # Load results from file (if already parsed) - logger.info('Load results from [%s]...', - self.results_json) - with open(self.results_json) as infile: - self.results = json.load(infile) - - # Setup configuration comparisons - if compare is None: - compare = DEFAULT_COMPARE - logger.warning('Comparing all the possible combination') - for (base_rexp, test_rexp) in compare: - logger.info('Configured regexps for comparisions ' - '(bases , tests): (%s, %s)', - base_rexp, test_rexp) - base_rexp = re.compile(base_rexp, re.DOTALL) - test_rexp = re.compile(test_rexp, re.DOTALL) - self.compare.append((base_rexp, test_rexp)) - - # Report all supported workload classes - self.__rtapp_report(formats) - self.__default_report(formats) - - ############################### REPORT RTAPP ############################### - - def __rtapp_report(self, formats): - logger = self.get_logger() - - if 'rtapp' not in list(self.results.keys()): - logger.debug('No RTApp workloads to report') - return - - logger.debug('Reporting RTApp workloads') - - # Setup lables depending on requested report - if 'absolute' in formats: - nrg_lable = 'Energy Indexes (Absolute)' - prf_lable = 'Performance Indexes (Absolute)' - logger.info('') - logger.info('Absolute comparisions:') - print('') - else: - nrg_lable = 'Energy Indexes (Relative)' - prf_lable = 'Performance Indexes (Relative)' - logger.info('') - logger.info('Relative comparisions:') - print('') - - # Dump headers - print('{:13s} {:20s} |'\ - ' {:33s} | {:54s} |'\ - .format('Test Id', 'Comparision', - nrg_lable, prf_lable)) - print('{:13s} {:20s} |'\ - ' {:>10s} {:>10s} {:>10s} |'\ - ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ - .format('', '', - 'LITTLE', 'big', 'Total', - 'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3')) - - # For each test - _results = self.results['rtapp'] - for tid in sorted(_results.keys()): - new_test = True - # For each configuration... - for base_idx in sorted(_results[tid].keys()): - # Which matches at least on base regexp - for (base_rexp, test_rexp) in self.compare: - if not base_rexp.match(base_idx): - continue - # Look for a configuration which matches the test regexp - for test_idx in sorted(_results[tid].keys()): - if test_idx == base_idx: - continue - if new_test: - print('{:-<37s}+{:-<35s}+{:-<56s}+'\ - .format('','', '')) - self.__rtapp_reference(tid, base_idx) - new_test = False - if test_rexp.match(test_idx) == None: - continue - self.__rtapp_compare(tid, base_idx, test_idx, formats) - - print('') - - def __rtapp_reference(self, tid, base_idx): - logger = self.get_logger() - _results = self.results['rtapp'] - - logger.debug('Test %s: compare against [%s] base', - tid, base_idx) - res_line = '{0:12s}: {1:22s} | '.format(tid, base_idx) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - # Dump absolute values - res_line += ' {0:10.3f}'.format(res_base) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s', pidx, res_base) - - if pidx in ['perf_avg']: - res_line += ' {0:s}'.format(TestColors.rate(res_base)) - continue - if pidx in ['slack_pct']: - res_line += ' {0:s}'.format( - TestColors.rate(res_base, positive_is_good = False)) - continue - if 'edp' in pidx: - res_line += ' {0:10.2e}'.format(res_base) - continue - res_line += ' |' - print(res_line) - - def __rtapp_compare(self, tid, base_idx, test_idx, formats): - logger = self.get_logger() - _results = self.results['rtapp'] - - logger.debug('Test %s: compare %s with %s', - tid, base_idx, test_idx) - res_line = '{0:12s}: {1:20s} | '.format(tid, test_idx) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - res_test = _results[tid][test_idx]['energy'][cpus]['avg'] - speedup_cnt = res_test - res_base - if 'absolute' in formats: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - speedup_pct = 0 - if res_base != 0: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'\ - .format(TestColors.rate( - speedup_pct, - positive_is_good = False)) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - res_test = _results[tid][test_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s, test: %s', - pidx, res_base, res_test) - - if pidx in ['perf_avg']: - res_line += ' {0:s}'.format(TestColors.rate(res_test)) - continue - - if pidx in ['slack_pct']: - res_line += ' {0:s}'.format( - TestColors.rate(res_test, positive_is_good = False)) - continue - - # Compute difference base-vs-test - if 'edp' in pidx: - speedup_cnt = res_base - res_test - if 'absolute': - res_line += ' {0:10.2e}'.format(speedup_cnt) - else: - res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) - - res_line += ' |' - print(res_line) - - ############################### REPORT DEFAULT ############################# - - def __default_report(self, formats): - logger = self.get_logger() - - # Build list of workload types which can be rendered using the default parser - wtypes = [] - for supported_wtype in DEFAULT_WTYPES: - if supported_wtype in list(self.results.keys()): - wtypes.append(supported_wtype) - - if len(wtypes) == 0: - logger.debug('No Default workloads to report') - return - - logger.debug('Reporting Default workloads') - - # Setup lables depending on requested report - if 'absolute' in formats: - nrg_lable = 'Energy Indexes (Absolute)' - prf_lable = 'Performance Indexes (Absolute)' - logger.info('') - logger.info('Absolute comparisions:') - print('') - else: - nrg_lable = 'Energy Indexes (Relative)' - prf_lable = 'Performance Indexes (Relative)' - logger.info('') - logger.info('Relative comparisions:') - print('') - - # Dump headers - print('{:9s} {:20s} |'\ - ' {:33s} | {:54s} |'\ - .format('Test Id', 'Comparision', - nrg_lable, prf_lable)) - print('{:9s} {:20s} |'\ - ' {:>10s} {:>10s} {:>10s} |'\ - ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ - .format('', '', - 'LITTLE', 'big', 'Total', - 'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3')) - - # For each default test - for wtype in wtypes: - _results = self.results[wtype] - for tid in sorted(_results.keys()): - new_test = True - # For each configuration... - for base_idx in sorted(_results[tid].keys()): - # Which matches at least on base regexp - for (base_rexp, test_rexp) in self.compare: - if not base_rexp.match(base_idx): - continue - # Look for a configuration which matches the test regexp - for test_idx in sorted(_results[tid].keys()): - if test_idx == base_idx: - continue - if new_test: - print('{:-<37s}+{:-<35s}+{:-<56s}+'\ - .format('','', '')) - new_test = False - if not test_rexp.match(test_idx): - continue - self.__default_compare(wtype, tid, base_idx, test_idx, formats) - - print('') - - def __default_compare(self, wtype, tid, base_idx, test_idx, formats): - logger = self.get_logger() - _results = self.results[wtype] - - logger.debug('Test %s: compare %s with %s', - tid, base_idx, test_idx) - res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx) - res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp) - - # Dump all energy metrics - for cpus in ['LITTLE', 'big', 'Total']: - - # If either base of test have a 0 MAX energy, this measn that - # energy has not been collected - base_max = _results[tid][base_idx]['energy'][cpus]['max'] - test_max = _results[tid][test_idx]['energy'][cpus]['max'] - if base_max == 0 or test_max == 0: - res_line += ' {0:10s}'.format('NA') - continue - - # Otherwise, report energy values - res_base = _results[tid][base_idx]['energy'][cpus]['avg'] - res_test = _results[tid][test_idx]['energy'][cpus]['avg'] - - speedup_cnt = res_test - res_base - if 'absolute' in formats: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'\ - .format(TestColors.rate( - speedup_pct, - positive_is_good = False)) - res_line += ' |' - - # If available, dump also performance results - if 'performance' not in list(_results[tid][base_idx].keys()): - print(res_line) - return - - for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']: - res_base = _results[tid][base_idx]['performance'][pidx]['avg'] - res_test = _results[tid][test_idx]['performance'][pidx]['avg'] - - logger.debug('idx: %s, base: %s, test: %s', - pidx, res_base, res_test) - - # Compute difference base-vs-test - speedup_cnt = 0 - if res_base != 0: - if pidx in ['perf_avg']: - speedup_cnt = res_test - res_base - else: - speedup_cnt = res_base - res_test - - # Compute speedup if required - speedup_pct = 0 - if 'absolute' in formats: - if 'edp' in pidx: - res_line += ' {0:10.2e}'.format(speedup_cnt) - else: - res_line += ' {0:10.2f}'.format(speedup_cnt) - else: - if res_base != 0: - if pidx in ['perf_avg']: - # speedup_pct = 100.0 * speedup_cnt / res_base - speedup_pct = speedup_cnt - else: - speedup_pct = 100.0 * speedup_cnt / res_base - res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) - res_line += ' |' - print(res_line) - -# List of workload types which can be parsed using the default test parser -DEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe'] - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/lisa/tests/kernel/scheduler/eas_behaviour.py b/lisa/tests/kernel/scheduler/eas_behaviour.py index f88557afa0d0db50f851ffaf77d21777f3f268a9..f5e6ecb4813feafaa95b37e0b198293def1e7938 100644 --- a/lisa/tests/kernel/scheduler/eas_behaviour.py +++ b/lisa/tests/kernel/scheduler/eas_behaviour.py @@ -81,8 +81,9 @@ class EASBehaviour(RTATestBundle, abc.ABC): # EAS doesn't make a lot of sense without schedutil, # so make sure this is what's being used - with te.target.cpufreq.use_governor("schedutil"): - cls._run_rtapp(te, res_dir, rtapp_profile) + with te.disable_idle_states(): + with te.target.cpufreq.use_governor("schedutil"): + cls._run_rtapp(te, res_dir, rtapp_profile) return cls(res_dir, te.plat_info, rtapp_profile) diff --git a/lisa/tests/kernel/scheduler/load_tracking.py b/lisa/tests/kernel/scheduler/load_tracking.py index 988da5a3923ed94c09fddd6c4d86a60dfd6cc341..9e1eb350f6e2fa478884d3424834604a7274a779 100644 --- a/lisa/tests/kernel/scheduler/load_tracking.py +++ b/lisa/tests/kernel/scheduler/load_tracking.py @@ -273,24 +273,6 @@ class InvarianceBase(LoadTrackingBase): """ return self._test_signal('util_avg', allowed_error_pct) - def test_task_load_avg(self, allowed_error_pct=15) -> ResultBundle: - """ - Test that the mean of the load_avg signal matched the expected value. - - Assuming that the system was under little stress (so the task was - RUNNING whenever it was RUNNABLE) and that the task was run with a - 'nice' value of 0, the load_avg should be similar to the util_avg. So, - this test does the same as test_task_util_avg but for load_avg. - - For asymmetric systems, this is only true for tasks run on the - biggest CPUs. - - :param allowed_error_pct: How much the real signal can stray from the - expected values - :type allowed_error_pct: float - """ - return self._test_signal('load_avg', allowed_error_pct) - class CpuInvariance(InvarianceBase): """ Basic check for CPU invariant load and utilization tracking @@ -398,6 +380,25 @@ class FreqInvarianceItem(InvarianceBase): capacity_scale=capacity_scale ) + def test_task_load_avg(self, allowed_error_pct=15) -> ResultBundle: + """ + Test that the mean of the load_avg signal matched the expected value. + + Assuming that the system was under little stress (so the task was + RUNNING whenever it was RUNNABLE) and that the task was run with a + 'nice' value of 0, the load_avg should be similar to the util_avg. So, + this test does the same as test_task_util_avg but for load_avg. + + For asymmetric systems, this is only true for tasks run on the + biggest CPUs. + + :param allowed_error_pct: How much the real signal can stray from the + expected values + :type allowed_error_pct: float + """ + return self._test_signal('load_avg', allowed_error_pct) + + class FreqInvariance(TestBundle, LoadTrackingHelpers): """ Basic check for frequency invariant load and utilization tracking @@ -480,7 +481,7 @@ class FreqInvariance(TestBundle, LoadTrackingHelpers): def test_task_load_avg(self, allowed_error_pct=15): """ - Aggregated version of :meth:`InvarianceBase.test_task_load_avg` + Aggregated version of :meth:`FreqInvarianceItem.test_task_load_avg` """ def item_test(test_item): return test_item.test_task_load_avg(allowed_error_pct=allowed_error_pct) diff --git a/lisa/version.py b/lisa/version.py new file mode 100644 index 0000000000000000000000000000000000000000..10c23556edf16b27226a188f14c48c7d01e45a84 --- /dev/null +++ b/lisa/version.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +__version__ = 2.0 diff --git a/lisa/wa_results_collector.py b/lisa/wa_results_collector.py index 0afbd4b909689957ec6b60aeed8d10070acce1fb..ccc5521cc1b7b9dd60fc632c4b2a507d7bdefc0d 100644 --- a/lisa/wa_results_collector.py +++ b/lisa/wa_results_collector.py @@ -41,7 +41,7 @@ from IPython.display import display from lisa.platforms.platinfo import PlatformInfo from lisa.trace import Trace -from lisa.git import Git +from lisa.git import find_shortest_symref from lisa.utils import Loggable, memoized class WaResultsCollector(Loggable): @@ -164,7 +164,7 @@ class WaResultsCollector(Loggable): kernel_refs = {} if kernel_repo_path: for sha1 in df['kernel_sha1'].unique(): - ref = Git.find_shortest_symref(kernel_repo_path, sha1) + ref = find_shortest_symref(kernel_repo_path, sha1) if ref: kernel_refs[sha1] = ref diff --git a/modules_root/exekall b/modules_root/exekall deleted file mode 120000 index 1be809308d1972ee81be2b3e6cc6e5851632748b..0000000000000000000000000000000000000000 --- a/modules_root/exekall +++ /dev/null @@ -1 +0,0 @@ -../tools/exekall/exekall \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9643a91e66940c6ccd6f07a2e77878d86060ee62..aa56fac3cfa8c83da36da6b6a26add509f777fdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ ./[notebook,doc,test] +./tools/exekall +./tools/bisector[dbus] diff --git a/setup.py b/setup.py index 864f44aa44bf144ee47e8a7b40351cb559dd4861..f723be382a157639d22f3077cf3e067e4882e6ed 100644 --- a/setup.py +++ b/setup.py @@ -20,14 +20,24 @@ from setuptools import setup with open('README.md', 'r') as fh: long_description = fh.read() +with open("lisa/version.py") as f: + version_globals = dict() + exec(f.read(), version_globals) + lisa_version = version_globals['__version__'] + setup( name='LISA', - version='2.0', + version=lisa_version, author='Arm Ltd', # TODO: figure out which email to put here # author_email= packages=['lisa'], - # url='http://pypi.python.org/pypi/TowelStuff/', + url='https://github.com/ARM-software/lisa', + project_urls={ + "Bug Tracker": "https://github.com/ARM-software/lisa/issues", + "Documentation": "https://lisa-linux-integrated-system-analysis.readthedocs.io/", + "Source Code": "https://github.com/ARM-software/lisa", + }, license='LICENSE.txt', description='A stick to probe the kernel with', long_description=long_description, diff --git a/shell/exekall b/shell/exekall deleted file mode 100755 index 69916958f9bd19ac10e63106774000b8e39a93f3..0000000000000000000000000000000000000000 --- a/shell/exekall +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -export EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} -python3 -m exekall.main "$@" - -# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/shell/lisa_buildroot_update_kernel_config b/shell/lisa_buildroot_update_kernel_config index 145a7d31683c617563c2da89902c3d32f39d9a00..c5914a75eda5d1d48f874f6fd8ea3ad66428684e 100755 --- a/shell/lisa_buildroot_update_kernel_config +++ b/shell/lisa_buildroot_update_kernel_config @@ -25,7 +25,7 @@ KERNEL_DIR="$2" KERNEL_CONFIG="$2/.config" function print_usage { - echo "Usage: $0 " + echo "Usage: $0 " echo " options:" echo " -h: print this help message" } @@ -51,15 +51,8 @@ if [[ -z "$1" || -z "$2" ]]; then exit -1 fi -# Automatically detect the arm arch we are building for -ARCH="`grep -owE "(arm|arm64)" $KERNEL_CONFIG`" -if [ -z "$ARCH" ]; then - echo "ERROR: unknown arch, arm and arm64 are assumed" - exit -1 -fi - set -u # Update kernel .config to use the provided rootfs as builtin initramfs sed -i "s#[.]*CONFIG_INITRAMFS_SOURCE.*#CONFIG_INITRAMFS_SOURCE=\"$ROOTFS\"#" "$KERNEL_CONFIG" -make -C "$KERNEL_DIR" ARCH=$ARCH olddefconfig +make -C "$KERNEL_DIR" olddefconfig diff --git a/shell/lisa_shell b/shell/lisa_shell index 259421a9f200addc1f9b68bfb92060d328e4490b..8f610dcbb005197646e4019000a5d651980ac27c 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -29,15 +29,6 @@ export LISA_DEVMODE=${LISA_DEVMODE:-1} # Setup colors source "$LISA_HOME/shell/lisa_colors" -# TODO: Stop modifying the PYTHONPATH when all Python packages have a setup.py. -# It will be installed in edit mode using pip in requirements.txt - -# In devmode, we provide the lisa package and some of its dependencies within -# the git tree -if [[ "$LISA_DEVMODE" == 1 ]]; then - export PYTHONPATH="$LISA_HOME/modules_root/:$PYTHONPATH" -fi - # Add some shell utilities to the PATH, with lower priority than system's one # in case the user needs a different version of them export LISA_HOST_ABI=${LISA_HOST_ABI:-x86_64} @@ -355,33 +346,12 @@ echo ################################################################################ export LISA_RESULT_ROOT=$LISA_HOME/results - -function exekall { - EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} \ - python3 -m exekall.main "$@" -} +export EXEKALL_ARTIFACT_ROOT=${EXEKALL_ARTIFACT_ROOT:-$LISA_RESULT_ROOT} function lisa-test { exekall "$@" } -function lisa-report { - CMD=$1 - [[ "$CMD" != "HELP" ]] && CMD=report - echo - case "$CMD" in - report) - ./tools/report.py $* - ;; - help|*) - ./tools/report.py --help - ;; - esac - echo - echo -} - - ################################################################################ # LISA Workloads utility functions ################################################################################ diff --git a/tools/bisector/LICENSE.txt b/tools/bisector/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/tools/bisector/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/bisector/README.md b/tools/bisector/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/bisector b/tools/bisector/bisector/bisector.py similarity index 97% rename from tools/bisector rename to tools/bisector/bisector/bisector.py index 8273d3cce83751819d832ebfa8ea791e6687b9ee..183dadc11f91a925177f7414ebc306e8499af2e7 100755 --- a/tools/bisector +++ b/tools/bisector/bisector/bisector.py @@ -580,19 +580,19 @@ class ChoiceOrBoolParam(Param): def info(msg): """Write a log message at the INFO level.""" - logger.info(msg) + BISECTOR_LOGGER.info(msg) def debug(msg): """Write a log message at the DEBUG level.""" - logger.debug(msg) + BISECTOR_LOGGER.debug(msg) def warn(msg): """Write a log message at the WARNING level.""" - logger.warning(msg) + BISECTOR_LOGGER.warning(msg) def error(msg): """Write a log message at the ERROR level.""" - logger.error(msg) + BISECTOR_LOGGER.error(msg) class _DefaultType: """ @@ -886,8 +886,8 @@ def read_stdout(p, timeout=None, kill_timeout=3): if stdout: sys.stdout.buffer.write(stdout) sys.stdout.buffer.flush() - log_file.buffer.write(stdout) - log_file.buffer.flush() + LOG_FILE.buffer.write(stdout) + LOG_FILE.buffer.flush() stdout_list.append(stdout) ret = p.wait() @@ -1771,7 +1771,7 @@ class ExekallLISATestStep(ShellStep): testcase = CommaListParam('show only the test cases matching one of the patterns in the comma-separated list. * can be used to match any part of the name.'), ignore_non_issue = BoolParam('consider only tests that failed'), ignore_excep = CommaListParam('ignore the given comma-separated list of exceptions that caused tests failure or error. * can be used to match any part of the name'), - dump_artifact_dirs = BoolOrStrParam('write the list of exkeall artifact directories to a file. Useful to implement garbage collection of unreferenced artifact archives'), + dump_artifact_dirs = BoolOrStrParam('write the list of exekall artifact directories to a file. Useful to implement garbage collection of unreferenced artifact archives'), xunit2json = BoolOrStrParam('append consolidated xUnit information to a JSON file'), export_logs = BoolOrStrParam('export the logs, xUnit file and artifact directory symlink to the given directory'), download = BoolParam('Download the exekall artifact archives if necessary'), @@ -1796,10 +1796,10 @@ class ExekallLISATestStep(ShellStep): self.compress_artifact = compress_artifact def run(self, i_stack, service_hub): - # Set the path to tests artifact + # Add a level of UUID under the root, so we can handle multiple trials artifact_path = pathlib.Path( os.getenv('EXEKALL_ARTIFACT_ROOT', 'exekall_artifact'), - datetime.datetime.now().strftime('%Y%m%d_%H%M%S_') + uuid.uuid4().hex, + uuid.uuid4().hex, ) # This also strips the trailing /, which is needed later on when @@ -1809,12 +1809,11 @@ class ExekallLISATestStep(ShellStep): env = { # exekall will use that folder directly, so it has to be empty and # cannot be reused for another invocation - 'EXEKALL_ARTIFACT_DIR': str(artifact_path), + 'EXEKALL_ARTIFACT_ROOT': str(artifact_path), } - #TODO: revisit if self.trials > 1: - warn('More than one trials requested for exekall LISA test, the xUnit XML file will only be recorded for the last trial.') + warn("More than one trials requested for exekall LISA test, only the last trial's xUnit XML file will be used.") res_list = self._run_cmd(i_stack, env=env) ret = res_list[-1][0] @@ -1824,29 +1823,34 @@ class ExekallLISATestStep(ShellStep): else: bisect_ret = BisectRet.GOOD - xunit_path_list = list(artifact_path.glob('**/xunit.xml')) - if not xunit_path_list: - warn('No xUnit file found under: {}'.format(artifact_path)) - elif len(xunit_path_list) > 1: - warn('Found more than one xUnit file, only taking the first one: '+ - ', '.join(xunit_path_list) - ) - + # First item is the oldest created file + xunit_path_list = sorted( + artifact_path.glob('**/xunit.xml'), + key=lambda x: os.path.getmtime(x) + ) xunit_report = '' if xunit_path_list: - xunit_path = xunit_path_list[0] + # Take the most recent file + xunit_path = xunit_path_list[-1] + if len(xunit_path_list) > 1: + warn('Found more than one xUnit file, only taking the most recent one ({}): {}'.format( + xunit_path, + ', '.join(xunit_path_list) + )) try: with open(xunit_path, 'r') as xunit_file: xunit_report = xunit_file.read() - except OSError: - warn('Could not open xUnit XML at: {}'.format(xunit_path)) - xunit_report = '' + except OSError as e: + warn('Could not open xUnit XML {}: {}'.format(xunit_path, e)) + + else: + warn('No xUnit file found under: {}'.format(artifact_path)) # Compress artifact directory if self.compress_artifact: try: - artifact_dir = artifact_path + orig_artifact_path = artifact_path # Create a compressed tar archive info('Compressing exekall artifact directory {} ...'.format(artifact_path)) archive_name = shutil.make_archive( @@ -1855,16 +1859,19 @@ class ExekallLISATestStep(ShellStep): root_dir = os.path.join(artifact_path, '..'), base_dir = os.path.split(artifact_path)[-1], ) - info('exekall artifact directory {artifact_path} compressed as {archive_name}'.format(**locals())) + info('exekall artifact directory {artifact_path} compressed as {archive_name}'.format( + artifact_path=artifact_path, + archive_name=archive_name + )) # From now on, the artifact_path is the path to the archive. artifact_path = os.path.abspath(archive_name) # Delete the original artifact directory since we archived it # successfully. - info('Deleting exekall artifact directory {} ...'.format(artifact_dir)) - shutil.rmtree(artifact_dir) - info('exekall artifact directory {} deleted.'.format(artifact_dir)) + info('Deleting exekall artifact root directory {} ...'.format(orig_artifact_path)) + shutil.rmtree(orig_artifact_path) + info('exekall artifact directory {} deleted.'.format(orig_artifact_path)) except Exception as e: warn('Failed to compress exekall artifact: {e}'.format(e=e)) @@ -1939,16 +1946,6 @@ class ExekallLISATestStep(ShellStep): # Parse the xUnit XML report from LISA to know the failing tests testcase_map = collections.defaultdict(list) filtered_step_res_seq = list() - # Highest iteration number this step was ran during. We assume it ran - # from the start to the end, or that the iteration number at the leaf - # level of macro steps is the iteration number (in case of nested macro - # steps). We use the highest number we find. - # TODO: This should be replace by a function mapping the i_stack to a - # global iteration number, and we should take the maximum of that. - i_max = max( - len(step_res_seq), - max(res[0][-1] for res in step_res_seq) - ) for step_res_item in step_res_seq: i_stack, step_res = step_res_item @@ -2116,8 +2113,6 @@ class ExekallLISATestStep(ShellStep): if show_basic: out(basic_report) - # Display a summary of failed tests - counts = collections.defaultdict(int) # Contains the percentage of skipped, failed, crashed and passed # iterations for every testcase. testcase_stats = dict() @@ -2130,9 +2125,9 @@ class ExekallLISATestStep(ShellStep): testcase_stats[testcase_id] = stats stats['total'] = iteration_n - stats['events'] = dict() - stats['iterations_summary'] = dict() + stats['iterations_summary'] = [entry['result'] for entry in entry_list] stats['counters'] = dict() + stats['events'] = dict() for issue, pretty_issue in ( ('passed', 'passed'), ('skipped', 'skipped'), @@ -2142,27 +2137,16 @@ class ExekallLISATestStep(ShellStep): ): filtered_entry_list = [entry for entry in entry_list if entry['result'] == issue] - # Only works when no nested macro steps are used. That is the - # only way of getting a number that is guaranteed to be - # synchronised with other steps. - i_set = {entry['i_stack'][-1] for entry in filtered_entry_list} - stats['iterations_summary'].update({ - i: issue for i in i_set - }) - issue_n = len(filtered_entry_list) issue_pc = (100 * issue_n) / iteration_n stats['counters'][issue] = issue_n # The event value for a given issue at a given iteration will # be True if the issue appeared, False otherwise. stats['events'][issue] = [ - (i in i_set) - for i in range(1, i_max + 1) + summ_issue == issue + for summ_issue in stats['iterations_summary'] ] - if i_set: - counts[issue] += 1 - if show_rates and ( (issue == 'passed' and show_pass_rate) or @@ -2210,20 +2194,15 @@ class ExekallLISATestStep(ShellStep): 'undecided': 'u', 'error': '#', } - iterations_summary = [ - issue_letter[iterations_summary.get(i, 'passed')] - for i in range(1, i_max + 1) - ] + dist = str( + issue_letter[issue] + for issue in iterations_summary + ) dist_out('{testcase_id}:\n\t{dist}\n'.format( testcase_id = testcase_id, - dist = ''.join(iterations_summary) + dist = dist, )) - issues_total_n = sum( - count for issue, count in stats['counters'].items() - if issue != 'passed' - ) - if show_details: out(table_out) else: @@ -2232,12 +2211,26 @@ class ExekallLISATestStep(ShellStep): out() out(dist_out) - # This shows the number of tests that failed or crashed at least on - # one iteration. Therefore, the total might exceed the number of - # tests if one test failed at one iteration and crashed at another - # one. Only tests passing at all iterations are accounted for in - # the "Passed" count. - total = len(testcase_map) + + counts = { + issue: sum( + # If there is a non-zero count for that issue for that test, we + # account it + bool(stats['counters'][issue]) + for stats in testcase_stats.values() + ) + for issue in ('error', 'failure', 'undecided', 'skipped') + } + # Only account for tests that only passed and had no other issues + counts['passed'] = sum( + all( + not count + for issue, count in stats['counters'].items() + if issue != 'passed' + ) + for stats in testcase_stats.values() + ) + out( 'Crashed: {counts[error]}/{total}, ' 'Failed: {counts[failure]}/{total}, ' @@ -2245,7 +2238,7 @@ class ExekallLISATestStep(ShellStep): 'Skipped: {counts[skipped]}/{total}, ' 'Passed: {counts[passed]}/{total}'.format( counts=counts, - total=total, + total=len(testcase_map), ) ) @@ -4703,7 +4696,7 @@ def do_monitor(slave_id, args): report_path = props['ReportPath'] argv = ['report', report_path] argv.extend(args.report) - main(argv) + _main(argv) print('\n' + '#' * 80) if args.pause: @@ -4964,9 +4957,9 @@ class YAMLCLIOptionsAction(argparse.Action): names = ', '.join('"'+name+'"' for name in overriden_names) )) -def main(argv): - global log_file - global show_traceback +def _main(argv): + global LOG_FILE + global SHOW_TRACEBACK parser = argparse.ArgumentParser(description=""" Git-bisect-compatible command driver. @@ -5262,7 +5255,7 @@ command line""") args = args, ) - show_traceback = args.debug + SHOW_TRACEBACK = args.debug service_hub = ServiceHub() try: service_hub.register_service('upload', ArtifactorialService()) @@ -5377,13 +5370,13 @@ command line""") # This log file is available for any function in the global scope ensure_dir(log_path) - log_file = open(log_path, log_file_mode, encoding='utf-8') + LOG_FILE = open(log_path, log_file_mode, encoding='utf-8') # Log to the main log file as well as on the console. - file_handler = logging.StreamHandler(log_file) + file_handler = logging.StreamHandler(LOG_FILE) file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) + file_handler.setFormatter(BISECTOR_FORMATTER) + BISECTOR_LOGGER.addHandler(file_handler) info('Description: {desc}'.format(desc=desc)) @@ -5463,28 +5456,34 @@ command line""") return ret -if __name__ == '__main__': - # Might be changed by main() - show_traceback = True - # Log file opened for the duration of the execution. We initialize it as - # the standard error, and it will be overidden by a call to open() when - # we know which file needs to actually be opened (CLI parameter). - log_file = sys.stderr +# TODO: avoid the global variable by redirecting stdout to a tee (stdout and +# file) inheriting from io.TextIOWrapper and contextlib.redirect_stdout() + +# Log file opened for the duration of the execution. We initialize it as +# the standard error, and it will be overidden by a call to open() when +# we know which file needs to actually be opened (CLI parameter). +LOG_FILE = sys.stderr + +# Might be changed by _main() +SHOW_TRACEBACK = True + +BISECTOR_LOGGER = logging.getLogger('BISECTOR') +BISECTOR_FORMATTER = logging.Formatter('[%(name)s][%(asctime)s] %(levelname)s %(message)s') - formatter = logging.Formatter('[%(name)s][%(asctime)s] %(levelname)s %(message)s') +def main(argv=sys.argv[1:]): - logger = logging.getLogger('BISECTOR') - logger.setLevel(logging.DEBUG) + + BISECTOR_LOGGER.setLevel(logging.DEBUG) console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) - console_handler.setFormatter(formatter) - logger.addHandler(console_handler) + console_handler.setFormatter(BISECTOR_FORMATTER) + BISECTOR_LOGGER.addHandler(console_handler) return_code = None try: - return_code = main(argv=sys.argv[1:]) + return_code = _main(argv=argv) # Quietly exit for these exceptions except SILENT_EXCEPTIONS: pass @@ -5492,7 +5491,7 @@ if __name__ == '__main__': return_code = e.code # Catch-all except Exception as e: - if show_traceback: + if SHOW_TRACEBACK: error( 'Exception traceback:\n' + ''.join( @@ -5506,11 +5505,14 @@ if __name__ == '__main__': # We only flush without closing in case it is stderr, to avoid hidding # exception traceback. It will be closed when the process ends in any # case. - log_file.flush() + LOG_FILE.flush() if return_code is None: return_code = 0 sys.exit(return_code) +if __name__ == '__main__': + main() + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/tools/xunit2json-analyze b/tools/bisector/bisector/xunit2json.py similarity index 98% rename from tools/xunit2json-analyze rename to tools/bisector/bisector/xunit2json.py index 3f3ed99b86585b15b3d62acc5f819c20da70f1e8..dde94b60a73401003fe19086eaf80765dbe3b7c3 100755 --- a/tools/xunit2json-analyze +++ b/tools/bisector/bisector/xunit2json.py @@ -6,9 +6,10 @@ import functools import itertools import json import operator +import sys + import pandas as pd import scipy.stats -import sys def error(msg): print('Error: ' + str(msg), file=sys.stderr) @@ -101,7 +102,7 @@ def compare_df(json_df1, json_df2, alpha, alternative='two-sided', non_significa return regression_df -def main(argv): +def _main(argv): parser = argparse.ArgumentParser(description=""" Compare tests failure rate using Fisher exact test. """, @@ -226,13 +227,16 @@ def main(argv): failure_df= json_df['failure']/(json_df['passed'] + json_df['failure']) print(failure_df.to_csv(), end='') -if __name__ == '__main__': +def main(argv=sys.argv[1:]): try: - return_code = main(argv=sys.argv[1:]) + return_code = _main(argv=argv) except FileNotFoundError as e: error(e) return_code = 1 sys.exit(return_code) +if __name__ == '__main__': + main() + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..75edcdf155332cac38f4ca69c9d2bdc14e867b55 --- /dev/null +++ b/tools/bisector/setup.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from setuptools import setup + +with open('README.md', 'r') as fh: + long_description = fh.read() + +setup( + name='bisector', + version='1.0', + author='Arm Ltd', + # TODO: figure out which email to put here + # author_email= + packages=['bisector'], + # url='http://pypi.python.org/pypi/TowelStuff/', + license='LICENSE.txt', + description='Command execution sequencer', + long_description=long_description, + entry_points={ + 'console_scripts': [ + 'bisector=bisector.bisector:main', + 'xunit2json-analyze=bisector.xunit2json:main' + ], + }, + python_requires='>= 3.5', + install_requires=[ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + "pandas", + "scipy", + ], + + extras_require={ + 'dbus': [ + 'pydbus', + 'pygobject', + # You will also need gobject-introspection package from your + # distribution + ] + }, + + classifiers=[ + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Topic :: Software Development :: Testing", + "Intended Audience :: Developers", + ], +) + +# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/exekall/LICENSE.txt b/tools/exekall/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/tools/exekall/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/exekall/README.md b/tools/exekall/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/exekall/exekall/main.py b/tools/exekall/exekall/main.py index 742a302c25646d9faea5c9f7df8e9ecfe2536f7e..0ef2c1bc955936c2e29e082f45c1cb028b5aff2c 100755 --- a/tools/exekall/exekall/main.py +++ b/tools/exekall/exekall/main.py @@ -160,47 +160,12 @@ the name of the parameter, the start value, stop value and step size.""") global show_traceback show_traceback = args.debug - # Look for a customization submodule in one of the toplevel packages - # of the modules we specified on the command line. - module_list = [utils.import_file(path) for path in args.python_files] - package_names_list = [ - module.__name__.split('.') - for module in reversed(module_list) - ] - - def build_full_names(l_l): - """Explode list of lists, and build full package names.""" - for l in l_l: - for i, _ in enumerate(l): - i += 1 - yield '.'.join(l[:i]) - - package_name_list = list(build_full_names(package_names_list)) + # Import all modules, before selecting the adaptor + module_set = {utils.import_file(path) for path in args.python_files} - adaptor_cls = AdaptorBase - module_set = set() - - try: - import_excep = ModuleNotFoundError - # Python < 3.6 - except NameError: - import_excep = AttributeError - - for name in reversed(package_name_list): - customize_name = name + '.exekall_customize' - # Only hide ModuleNotFoundError exceptions when looking up that - # specific module, we don't want to hide issues inside the module - # itself. - module_exists = False - with contextlib.suppress(import_excep): - module_exists = importlib.util.find_spec(customize_name) - - if module_exists: - # Importing that module is enough to make the adaptor visible - # to the Adaptor base class - customize_module = importlib.import_module(customize_name) - module_set.add(customize_module) - break + # Look for a customization submodule in one of the parent packages of the + # modules we specified on the command line. + module_set.update(utils.find_customization_module_set(module_set)) adaptor_name = args.adaptor adaptor_cls = AdaptorBase.get_adaptor_cls(adaptor_name) @@ -267,8 +232,6 @@ the name of the parameter, the start value, stop value and step size.""") utils.setup_logging(args.log_level, debug_log, info_log, verbose=verbose) - module_set.update(utils.import_file(path) for path in args.python_files) - non_reusable_type_set = adaptor.get_non_reusable_type_set() # Get the prebuilt operators from the adaptor diff --git a/tools/exekall/exekall/utils.py b/tools/exekall/exekall/utils.py index 30bfe6e72d28ea537e3f6bd7b0190f8e34b4deda..69548a685ca714fff5fd274a560af968384f5a46 100644 --- a/tools/exekall/exekall/utils.py +++ b/tools/exekall/exekall/utils.py @@ -286,6 +286,45 @@ def resolve_annotations(annotations, module_vars): for param, cls in annotations.items() } +def find_customization_module_set(module_set): + def build_full_names(l_l): + """Explode list of lists, and build full package names.""" + for l in l_l: + for i, _ in enumerate(l): + i += 1 + yield '.'.join(l[:i]) + + try: + import_excep = ModuleNotFoundError + # Python < 3.6 + except NameError: + import_excep = AttributeError + + package_names_list = [ + module.__name__.split('.') + for module in module_set + ] + package_name_set = set(build_full_names(package_names_list)) + + customization_module_set = set() + + for name in package_name_set: + customize_name = name + '.exekall_customize' + # Only hide ModuleNotFoundError exceptions when looking up that + # specific module, we don't want to hide issues inside the module + # itself. + module_exists = False + with contextlib.suppress(import_excep): + module_exists = importlib.util.find_spec(customize_name) + + if module_exists: + # Importing that module is enough to make the adaptor visible + # to the Adaptor base class + customize_module = importlib.import_module(customize_name) + customization_module_set.add(customize_module) + + return customization_module_set + def import_file(python_src, module_name=None, is_package=False): python_src = pathlib.Path(python_src) if python_src.is_dir(): diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce7da5d44e3e31f304964301371ca4abbb3bdd4 --- /dev/null +++ b/tools/exekall/setup.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from setuptools import setup + +with open('README.md', 'r') as fh: + long_description = fh.read() + +setup( + name='exekall', + version='1.0', + author='Arm Ltd', + # TODO: figure out which email to put here + # author_email= + packages=['exekall'], + # url='http://pypi.python.org/pypi/TowelStuff/', + license='LICENSE.txt', + description='Python expression execution engine', + long_description=long_description, + entry_points={ + 'console_scripts': ['exekall=exekall.main:main'], + }, + python_requires='>= 3.5', + install_requires=[ + # Older versions will have troubles with serializing complex nested + # objects hierarchy implementing custom __getstate__ and __setstate__ + "ruamel.yaml >= 0.15.72", + ], + + classifiers=[ + "Programming Language :: Python :: 3 :: Only", + # This is not a standard classifier, as there is nothing defined for + # Apache 2.0 yet: + # https://pypi.org/classifiers/ + "License :: OSI Approved :: Apache 2.0", + # It has not been tested under any other OS + "Operating System :: POSIX :: Linux", + + "Intended Audience :: Developers", + ], +) + +# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/report.py b/tools/report.py deleted file mode 100755 index 0a0756f3139c377c496862c3197a51d5eeaba150..0000000000000000000000000000000000000000 --- a/tools/report.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python -# -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -sys.path.insert(1, "./libs") -from utils.report import Report - -import argparse - -parser = argparse.ArgumentParser( - description='EAS RFC Configuration Comparator.') -parser.add_argument('--bases', type=str, - default='bases_', - help='Regexp of BASE configurations to compare with') -parser.add_argument('--tests', type=str, - default='tests_', - help='Regexp of TEST configurations to compare against') -parser.add_argument('--results', type=str, - default='./results_latest', - help='Folder containing experimental results') - -if __name__ == "__main__": - args = parser.parse_args() - Report(args.results, compare=[(args.bases, args.tests)]) -