From 6dffe65ff009bef17823e8e064c1c0e3b45dea72 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 20 Apr 2017 13:32:54 +0100 Subject: [PATCH 1/5] executor: Add :ivar: doc for experiments attribute --- libs/utils/executor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/utils/executor.py b/libs/utils/executor.py index 7c7060b22..84d8a9583 100644 --- a/libs/utils/executor.py +++ b/libs/utils/executor.py @@ -166,6 +166,9 @@ class Executor(): Number of iterations for each workload/conf combination. Default is 1. :type experiments_conf: dict + + :ivar experiments: After calling `meth`:run:, the list of + :class:`Experiment` s that were run """ critical_tasks = { -- GitLab From 36bddfca77e30ee884df009dd511eedf47bd3011 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 20 Apr 2017 13:33:13 +0100 Subject: [PATCH 2/5] executor: Make iterations attribute "public" --- libs/utils/executor.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/utils/executor.py b/libs/utils/executor.py index 84d8a9583..883c07470 100644 --- a/libs/utils/executor.py +++ b/libs/utils/executor.py @@ -169,6 +169,10 @@ class Executor(): :ivar experiments: After calling `meth`:run:, the list of :class:`Experiment` s that were run + + :ivar iterations: The number of iterations run for each wload/conf pair + (i.e. ``experiments_conf['iterations']``. + """ critical_tasks = { @@ -218,9 +222,9 @@ class Executor(): self.te = test_env self.target = self.te.target - self._iterations = self._experiments_conf.get('iterations', 1) + self.iterations = self._experiments_conf.get('iterations', 1) # Compute total number of experiments - self._exp_count = self._iterations \ + self._exp_count = self.iterations \ * len(self._experiments_conf['wloads']) \ * len(self._experiments_conf['confs']) @@ -236,7 +240,7 @@ class Executor(): self._log.info(' %3d workloads (%d iterations each)', len(self._experiments_conf['wloads']), - self._iterations) + self.iterations) wload_confs = ', '.join(self._experiments_conf['wloads']) self._log.info(' %s', wload_confs) @@ -264,7 +268,7 @@ class Executor(): for wl_idx in self._experiments_conf['wloads']: # TEST: configuration wload, test_dir = self._wload_init(tc, wl_idx) - for itr_idx in range(1, self._iterations + 1): + for itr_idx in range(1, self.iterations + 1): exp = Experiment( wload_name=wl_idx, wload=wload, @@ -666,7 +670,7 @@ class Executor(): self._print_title('Experiment {}/{}, [{}:{}] {}/{}'\ .format(exp_idx, self._exp_count, tc_idx, experiment.wload_name, - experiment.iteration, self._iterations)) + experiment.iteration, self.iterations)) # Setup local results folder self._log.debug('out_dir set to [%s]', experiment.out_dir) -- GitLab From 3a01b131174b28d2d5409bd060a58b08be9bf6d6 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 20 Apr 2017 17:10:38 +0100 Subject: [PATCH 3/5] LisaTest: rename logger to _log For consistency with other LISA code --- libs/utils/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/utils/test.py b/libs/utils/test.py index b60621684..db5ba4439 100644 --- a/libs/utils/test.py +++ b/libs/utils/test.py @@ -81,9 +81,9 @@ class LisaTest(unittest.TestCase): """ Set up logging and trigger running experiments """ - cls.logger = logging.getLogger('LisaTest') + cls._log = logging.getLogger('LisaTest') - cls.logger.info('Setup tests execution engine...') + cls._log.info('Setup tests execution engine...') test_env = TestEnv(test_conf=cls._getTestConf()) experiments_conf = cls._getExperimentsConf(test_env) @@ -100,7 +100,7 @@ class LisaTest(unittest.TestCase): # Execute pre-experiments code defined by the test cls._experimentsInit() - cls.logger.info('Experiments execution...') + cls._log.info('Experiments execution...') cls.executor.run() cls.experiments = cls.executor.experiments -- GitLab From 8ba281d201a54bd0bd34cc0248830f87af030269 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Tue, 28 Mar 2017 17:34:04 +0100 Subject: [PATCH 4/5] LisaTest: Improve running tests with multiple iterations Currently when you run a LisaTest with experiments_conf['iterations']!=0, the test assertions are made for every experiment and the test is failed as soon as any experiment produces a failure. Only the first failure encountered is displayed. With this patch all the results for each wload/conf pair are aggregated. The problem is only solved across iterations: if multiple wloads or confs fail, only the first wload/conf pair that was found to have failed will be displayed. --- libs/utils/test.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libs/utils/test.py b/libs/utils/test.py index db5ba4439..6c3fe62b7 100644 --- a/libs/utils/test.py +++ b/libs/utils/test.py @@ -213,16 +213,22 @@ def experiment_test(wrapped_test, instance, args, kwargs): The method will be passed the experiment object and a list of the names of tasks that were run as the experiment's workload. """ + failures = {} for experiment in instance.executor.experiments: tasks = experiment.wload.tasks.keys() try: wrapped_test(experiment, tasks, *args, **kwargs) except AssertionError as e: trace_relpath = os.path.join(experiment.out_dir, "trace.dat") - add_msg = "\n\tCheck trace file: " + os.path.abspath(trace_relpath) - orig_msg = e.args[0] if len(e.args) else "" - e.args = (orig_msg + add_msg,) + e.args[1:] - raise + add_msg = "Check trace file: " + os.path.abspath(trace_relpath) + msg = str(e) + "\n\t" + add_msg + + test_key = (experiment.wload_name, experiment.conf['tag']) + failures[test_key] = failures.get(test_key, []) + [msg] + + for fails in failures.itervalues(): + raise AssertionError("{} failures from {} iteration(s):\n{}".format( + len(fails), instance.executor.iterations, '\n'.join(fails))) # Prevent nosetests from running experiment_test directly as a test case experiment_test.__test__ = False -- GitLab From 59ad067fd62b8554c9656e635cc1dae172fd96c3 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 20 Apr 2017 16:49:00 +0100 Subject: [PATCH 5/5] LisaTest: Add permitted_fail_pct attribute This allows specifying a percentage of iterations of a given test (tuple wload, conf, test method) can be permitted to fail. If some tests fail but not enough to cross this percentage threshold, a warning is printed. --- libs/utils/test.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/libs/utils/test.py b/libs/utils/test.py index 6c3fe62b7..d3b96ec45 100644 --- a/libs/utils/test.py +++ b/libs/utils/test.py @@ -56,6 +56,9 @@ class LisaTest(unittest.TestCase): experiments_conf = None """Override this with a dictionary or JSON path to configure the Executor""" + permitted_fail_pct = 0 + """The percentage of iterations of each test that may be permitted to fail""" + @classmethod def _getTestConf(cls): if cls.test_conf is None: @@ -227,8 +230,19 @@ def experiment_test(wrapped_test, instance, args, kwargs): failures[test_key] = failures.get(test_key, []) + [msg] for fails in failures.itervalues(): - raise AssertionError("{} failures from {} iteration(s):\n{}".format( - len(fails), instance.executor.iterations, '\n'.join(fails))) + iterations = instance.executor.iterations + fail_pct = 100. * len(fails) / iterations + + msg = "{} failures from {} iteration(s):\n{}".format( + len(fails), iterations, '\n'.join(fails)) + if fail_pct > instance.permitted_fail_pct: + raise AssertionError(msg) + else: + instance._log.warning(msg) + instance._log.warning( + 'ALLOWING due to permitted_fail_pct={}'.format( + instance.permitted_fail_pct)) + # Prevent nosetests from running experiment_test directly as a test case experiment_test.__test__ = False -- GitLab