diff --git a/lisa/energy_model.py b/lisa/energy_model.py index 2b44258e0a7e394062fccd646103c1959a27e46b..328885005599d86f9744b339710bb7f2b781ded9 100644 --- a/lisa/energy_model.py +++ b/lisa/energy_model.py @@ -16,12 +16,12 @@ # from collections import namedtuple, OrderedDict -from itertools import product, groupby +from itertools import product import logging import operator import re -from lisa.utils import Loggable, Serializable, memoized +from lisa.utils import Loggable, Serializable, memoized, groupby import pandas as pd import numpy as np @@ -437,7 +437,7 @@ class EnergyModel(Serializable, Loggable): List of lists of CPUs nodes who share the same active state values """ def key(node): - return node.active_states + return sorted(node.active_states.items()) return [ list(group) diff --git a/lisa/exekall_customize.py b/lisa/exekall_customize.py index a6fb54775a459514a54be3a4e4acd9986eac8739..cb6fdfd8408bf3a30b45ba28e7d99a1f7948edcf 100644 --- a/lisa/exekall_customize.py +++ b/lisa/exekall_customize.py @@ -29,7 +29,7 @@ import traceback from lisa.env import TestEnv, TargetConf from lisa.platforms.platinfo import PlatformInfo -from lisa.utils import HideExekallID, Loggable, ArtifactPath +from lisa.utils import HideExekallID, Loggable, ArtifactPath, get_subclasses, MultiSrcConf, groupby, Serializable from lisa.tests.kernel.test_bundle import TestBundle, Result, ResultBundle, CannotCreateError from lisa.tests.kernel.scheduler.load_tracking import FreqInvarianceItem @@ -75,19 +75,47 @@ class LISAAdaptor(AdaptorBase): def get_prebuilt_list(self): non_reusable_type_set = self.get_non_reusable_type_set() op_list = [] - if self.args.target_conf: + + # Try to build as many configurations instances from all the files we + # are given + conf_cls_set = set(get_subclasses(MultiSrcConf)) + conf_list = [] + for conf_path in self.args.conf: + for conf_cls in conf_cls_set: + try: + conf = conf_cls.from_yaml_map(conf_path) + except KeyError: + continue + else: + conf_list.append((conf, conf_path)) + + def keyfunc(conf_and_path): + cls = type(conf_and_path[0]) + # We use the ID since classes are not comparable + return id(cls), cls + + # Then aggregate all the conf from each type, so they just act as + # alternative sources. + for (_, conf_cls), conf_and_path_seq in groupby(conf_list, key=keyfunc): + conf_and_path_list = list(conf_and_path_seq) + # Since we use reversed order, we get the source override from the + # last one. + conf_and_path_list.reverse() + + conf = conf_and_path_list[0][0] + for conf_src, conf_path in conf_and_path_list[1:]: + conf.add_src(conf_path, conf_src, fallback=True) + op_list.append( - PrebuiltOperator(TargetConf, [ - TargetConf.from_yaml_map(self.args.target_conf) - ], + PrebuiltOperator(conf_cls, [conf], non_reusable_type_set=non_reusable_type_set )) - if self.args.platform_info: + # Inject serialized objects as root operators + for path in self.args.inject: + obj = Serializable.from_path(path) op_list.append( - PrebuiltOperator(PlatformInfo, [ - PlatformInfo.from_yaml_map(self.args.platform_info) - ], + PrebuiltOperator(type(obj), [obj], non_reusable_type_set=non_reusable_type_set )) @@ -104,11 +132,13 @@ class LISAAdaptor(AdaptorBase): @staticmethod def register_cli_param(parser): - parser.add_argument('--target-conf', - help="Target config file") + parser.add_argument('--conf', action='append', + default=[], + help="Configuration file") - parser.add_argument('--platform-info', - help="Platform info file") + parser.add_argument('--inject', action='append', + default=[], + help="Serialized object to inject when building expressions") @staticmethod def get_default_type_goal_pattern_set(): @@ -140,7 +170,7 @@ class LISAAdaptor(AdaptorBase): def finalize_expr(self, expr): testcase_artifact_dir = expr.data['testcase_artifact_dir'] artifact_dir = expr.data['artifact_dir'] - for expr_val in expr.get_all_values(): + for expr_val in expr.get_all_vals(): self._finalize_expr_val(expr_val, artifact_dir, testcase_artifact_dir) def _finalize_expr_val(self, expr_val, artifact_dir, testcase_artifact_dir): @@ -177,29 +207,25 @@ class LISAAdaptor(AdaptorBase): symlink.symlink_to(target, target_is_directory=True) - for param, param_expr_val in expr_val.param_value_map.items(): + for param, param_expr_val in expr_val.param_expr_val_map.items(): self._finalize_expr_val(param_expr_val, artifact_dir, testcase_artifact_dir) @classmethod - def get_tag_list(cls, value): - tags = [] + def get_tags(cls, value): + tags = {} if isinstance(value, TestEnv): - board_name = value.target_conf.get('name') - if board_name: - tags.append(board_name) + tags['board'] = value.target_conf.get('name') elif isinstance(value, PlatformInfo): - name = value.get('name') - if name: - tags.append(name) + tags['board'] = value.get('name') elif isinstance(value, TestBundle): - name = value.plat_info.get('name') - if name: - tags.append(name) + tags['board'] = value.plat_info.get('name') if isinstance(value, FreqInvarianceItem): - tag_str = 'cpu{cpu}@{freq}' if value.cpu is not None else '{}' - tags.append(tag_str.format(cpu=value.cpu, freq=value.freq)) + if value.cpu is not None: + tags['cpu'] = '{}@{}'.format(value.cpu, value.freq) else: - tags = super().get_tag_list(value) + tags = super().get_tags(value) + + tags = {k: v for k, v in tags.items() if v is not None} return tags @@ -226,7 +252,7 @@ class LISAAdaptor(AdaptorBase): return expr.op.mod_name # One testsuite per module where a root operator is defined - for mod_name, group in itertools.groupby(testcase_list, key=key): + for mod_name, group in groupby(testcase_list, key=key): testcase_list = list(group) et_testsuite = ET.SubElement(et_testsuites, 'testsuite', attrib=dict( name = mod_name @@ -240,6 +266,17 @@ class LISAAdaptor(AdaptorBase): # assume that they testcase will have unique names using tags expr_val_list = result_map[testcase] for expr_val in expr_val_list: + + # Get the set of UUIDs of all TestBundle instances that were + # involved in the testcase. + def bundle_predicate(expr_val, param): + return issubclass(expr_val.expr.op.value_type, TestBundle) + bundle_uuid_set = { + expr_val.value_uuid + for expr_val in expr_val.get_parent_expr_vals(bundle_predicate) + } + bundle_uuid_set.discard(None) + et_testcase = ET.SubElement(et_testsuite, 'testcase', dict( name = expr_val.get_id( full_qual=False, @@ -249,11 +286,12 @@ class LISAAdaptor(AdaptorBase): ), # This may help locating the artifacts, even though it # will only be valid on the machine it was produced on - artifact_path = str(artifact_path) + artifact_path=str(artifact_path), + bundle_uuids=','.join(sorted(bundle_uuid_set)), )) testsuite_counters['tests'] += 1 - for failed_expr_val in expr_val.get_failed_values(): + for failed_expr_val in expr_val.get_failed_expr_vals(): excep = failed_expr_val.excep # When one critical object cannot be created, we assume # the test was skipped. diff --git a/lisa/utils.py b/lisa/utils.py index 8d3fe52556889dde31b148b6953d1f765d9eb570..3b1f8fc9f062589af006e866714799224f51d5a0 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -33,6 +33,7 @@ import operator import numbers import difflib import threading +import itertools import ruamel.yaml from ruamel.yaml import YAML @@ -532,17 +533,17 @@ class KeyDesc(KeyDescBase): def validate_val(self, val): # Or if that key is supposed to hold a value classinfo = self.classinfo - key = self.name + key = self.qualname def get_excep(key, val, classinfo, cls, msg): classinfo = ' or '.join(self._get_cls_name(cls) for cls in classinfo) msg = ': ' + msg if msg else '' - return ValueError('Key "{key}" is an instance of {actual_cls}, but should be instance of {classinfo}{msg}. Help: {help}'.format( - key=self.qualname, + return TypeError('Key "{key}" is an instance of {actual_cls}, but should be instance of {classinfo}{msg}. Help: {help}'.format( + key=key, actual_cls=self._get_cls_name(type(val)), classinfo=classinfo, msg=msg, help=self.help, - )) + ), key) def checkinstance(key, val, classinfo): excep_list = [] @@ -558,7 +559,7 @@ class KeyDesc(KeyDescBase): elif hasattr(cls, 'instancecheck'): try: cls.instancecheck(val) - except ValueError as e: + except TypeError as e: excep_list.append( get_excep(key, val, classinfo, cls, str(e)) ) @@ -628,11 +629,12 @@ class LevelKeyDesc(KeyDescBase, Mapping): else: closest_match = ', maybe you meant "{}" ?'.format(closest_match) - raise ValueError('Key "{key}" is not allowed in {parent}{maybe}'.format( + parent = self.qualname + raise KeyError('Key "{key}" is not allowed in {parent}{maybe}'.format( key=key, - parent=self.qualname, + parent=parent, maybe=closest_match, - )) + ), parent, key) def validate_key(self, key, val): self[key].validate_val(val) @@ -640,9 +642,10 @@ class LevelKeyDesc(KeyDescBase, Mapping): def validate_val(self, conf): """Validate a mapping to be used as a source""" if not isinstance(conf, Mapping): - raise ValueError('Configuration of {key} must be a Mapping'.format( - key=self.qualname, - )) + key = self.qualname + raise TypeError('Configuration of {key} must be a Mapping'.format( + key=key, + ), key) for key, val in conf.items(): self.validate_key(key, val) @@ -825,10 +828,11 @@ class MultiSrcConf(SerializableConfABC, Loggable, Mapping, metaclass=MultiSrcCon def force_src(self, key, src_prio): key_desc = self._structure[key] if isinstance(key_desc, LevelKeyDesc): + key = key_desc.qualname raise ValueError('Cannot force source of the sub-level "{key}" in {cls}'.format( - key=key_desc.qualname, + key=key, cls=type(self).__qualname__ - )) + ), key) # None means removing the src override for that key if src_prio is None: @@ -883,9 +887,10 @@ class MultiSrcConf(SerializableConfABC, Loggable, Mapping, metaclass=MultiSrcCon key_desc = self._structure[key] if isinstance(key_desc, LevelKeyDesc): + key = key_desc.qualname raise ValueError('Key "{key}" is a nested configuration level, it does not have a source on its own.'.format( - key=key_desc.qualname, - )) + key=key, + ), key) # Get the priority list from the prio override list, or just the # default prio list @@ -893,9 +898,10 @@ class MultiSrcConf(SerializableConfABC, Loggable, Mapping, metaclass=MultiSrcCon if src_prio: return src_prio[0] else: + key = key_desc.qualname raise KeyError('Could not find any source for key "{key}"'.format( - key=key_desc.qualname, - )) + key=key, + ), key) def _eval_deferred_val(self, src, key): key_desc = self._structure[key] @@ -950,10 +956,11 @@ class MultiSrcConf(SerializableConfABC, Loggable, Mapping, metaclass=MultiSrcCon try: val = self._key_map[key][src] except KeyError: + key = key_desc.qualname raise KeyError('Key "{key}" is not available from source "{src}"'.format( - key=key_desc.qualname, + key=key, src=src, - )) + ), key) if eval_deferred: val = self._eval_deferred_val(src, key) @@ -980,10 +987,11 @@ class MultiSrcConf(SerializableConfABC, Loggable, Mapping, metaclass=MultiSrcCon def get_src_map(self, key): key_desc = self._structure[key] if isinstance(key_desc, LevelKeyDesc): + key = key_desc.qualname raise ValueError('Key "{key}" is a nested configuration level in {cls}, it does not have a source on its own.'.format( - key=key_desc.qualname, + key=key, cls=type(self).__qualname__, - )) + ), key) return OrderedDict( (src, self._eval_deferred_val(src, key)) @@ -1055,7 +1063,7 @@ class GenericContainerMetaBase(type): def __instancecheck__(cls, instance): try: cls.instancecheck(instance) - except ValueError: + except TypeError: return False else: return True @@ -1068,23 +1076,23 @@ class GenericContainerBase: class GenericMappingMeta(GenericContainerMetaBase, type(Mapping)): def instancecheck(cls, instance): if not isinstance(instance, Mapping): - raise ValueError('not a Mapping') + raise TypeError('not a Mapping') k_type, v_type = cls._type for k, v in instance.items(): if not isinstance(k, k_type): - raise ValueError('Key "{key}" of type {actual_cls} should be of type {k_type}'.format( + raise TypeError('Key "{key}" of type {actual_cls} should be of type {k_type}'.format( key=k, actual_cls=type(k).__qualname__, k_type=k_type.__qualname__, - )) + ), k) if not isinstance(v, v_type): - raise ValueError('Value of {actual_cls} key "{key}" should be of type {v_type}'.format( + raise TypeError('Value of {actual_cls} key "{key}" should be of type {v_type}'.format( key=k, actual_cls=type(v).__qualname__, v_type=v_type.__qualname__, - )) + ), k) class TypedDict(GenericContainerBase, dict, metaclass=GenericMappingMeta): pass @@ -1092,17 +1100,17 @@ class TypedDict(GenericContainerBase, dict, metaclass=GenericMappingMeta): class GenericSequenceMeta(GenericContainerMetaBase, type(Sequence)): def instancecheck(cls, instance): if not isinstance(instance, Sequence): - raise ValueError('not a Sequence') + raise TypeError('not a Sequence') type_ = cls._type for i, x in enumerate(instance): if not isinstance(x, type_): - raise ValueError('Item #{i} "{val}" of type {actual_cls} should be of type {type_}'.format( + raise TypeError('Item #{i} "{val}" of type {actual_cls} should be of type {type_}'.format( i=i, val=x, actual_cls=type(x).__qualname__, type_=type_.__qualname__ - )) + ), i) class TypedList(GenericContainerBase, list, metaclass=GenericSequenceMeta): pass @@ -1189,4 +1197,10 @@ class ArtifactPath(str, Loggable, HideExekallID): # Swap-in the new root and return a new instance return type(self)(root, relative) +def groupby(iterable, key=None): + # We need to sort before feeding to groupby, or it will fail to establish + # the groups as expected. + iterable = sorted(iterable, key=key) + return itertools.groupby(iterable, key=key) + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/bisector/bisector.py index 392e11ac55cfb5b4f6f04dd56d25a3f41f46a87c..665123b223ed9733c1a45cc8a81b1779567a709f 100755 --- a/tools/bisector/bisector/bisector.py +++ b/tools/bisector/bisector/bisector.py @@ -236,7 +236,9 @@ class BisectRet(enum.Enum): BAD = 1 # Bisect must abort due to non-recoverable error (the board can't boot # anymore for example) - ABORT = 254 + ABORT = 253 + # Yield to the caller of bisector, without taking any decision + YIELD = 252 @property def lower_name(self): @@ -262,7 +264,7 @@ BisectRet.yaml_tag = '!git-bisect' yaml.register_class(BisectRet) # This will map to an "abort" meaning for "git bisect run" -GENERIC_ERROR_CODE = 253 +GENERIC_ERROR_CODE = 254 assert GENERIC_ERROR_CODE not in (e.value for e in BisectRet) def parse_step_options(opts_seq): @@ -1500,10 +1502,13 @@ class StepBase(StepABC): class YieldStep(StepBase): """ - Abort current iteration. + Abort current iteration with the yield return code. If the specified command returns a non-zero return code, bisector will - abort the current iteration. + abort the current iteration with the yield return code. + + .. note:: This step can only be used under the main macrostep (it cannot be + used in nested macrosteps). """ yaml_tag = '!yield-step' @@ -1535,7 +1540,7 @@ class YieldStep(StepBase): # Return value is the one from the last trial ret = res_list[-1][0] - bisect_ret = BisectRet.ABORT if ret != 0 else BisectRet.NA + bisect_ret = BisectRet.YIELD if ret != 0 else BisectRet.NA step_res = StepResult( step = self, res_list = res_list, @@ -3349,6 +3354,12 @@ class MacroStep(StepBase): info('{step.cat} step ({step.name}) requested bisect abortion, aborting ...'.format(step=step)) break + # Yielding to the caller of bisector + if res.bisect_ret == BisectRet.YIELD: + info('{step.cat} step ({step.name}) requested yielding ...'.format(step=step)) + break + + # If the commit is not testable or bad, bail out early if self.bail_out_early and step.bail_out and ( (res.bisect_ret == BisectRet.UNTESTABLE) or @@ -3380,8 +3391,8 @@ class MacroStep(StepBase): res = self._run_steps(i_stack, service_hub) res_list.append(res) - # Propagate ABORT - if res.bisect_ret == BisectRet.ABORT: + # Propagate ABORT and YIELD + if res.bisect_ret in (BisectRet.ABORT, BisectRet.YIELD): break i_stack.pop() @@ -3450,6 +3461,12 @@ class MacroStep(StepBase): slave_manager.signal.State = 'aborted' break + # Propagate YIELD + if res.bisect_ret == BisectRet.YIELD: + if slave_manager: + slave_manager.signal.State = 'yielded' + break + if slave_manager: while slave_manager.pause_loop.is_set(): info('Iterations paused.') diff --git a/tools/exekall/exekall/customization.py b/tools/exekall/exekall/customization.py index d3df04ffc4ce282fe1ec9d28062715508e510eb4..a180606be1e3cc1c1500a7bbce22530d16049783 100644 --- a/tools/exekall/exekall/customization.py +++ b/tools/exekall/exekall/customization.py @@ -33,11 +33,11 @@ class AdaptorBase: return set() @staticmethod - def get_tag_list(value): + def get_tags(value): if isinstance(value, numbers.Number): - tags = [str(value)] + tags = {'': value} else: - tags = [] + tags = {} return tags load_db = None @@ -81,7 +81,7 @@ class AdaptorBase: def result_str(self, result): val = result.value if val is NoValue or val is None: - failed_parents = result.get_failed_values() + failed_parents = result.get_failed_expr_vals() for failed_parent in failed_parents: excep = failed_parent.excep return 'EXCEPTION ({type}): {msg}'.format( @@ -101,7 +101,7 @@ class AdaptorBase: hidden_callable_set=hidden_callable_set, full_qual=False, qual=False, - ) + ':' + ) for expr, result_list in result_map.items() for result in result_list } diff --git a/tools/exekall/exekall/engine.py b/tools/exekall/exekall/engine.py index 64c51fe1eb5a32969d8783c8953285557886efa9..29e5d5126945aacd70638f6296142fdbfc44da94 100644 --- a/tools/exekall/exekall/engine.py +++ b/tools/exekall/exekall/engine.py @@ -34,7 +34,7 @@ import exekall._utils as utils def take_first(iterable): for i in iterable: return i - return engine.NoValue + return NoValue class NoOperatorError(Exception): pass @@ -164,7 +164,7 @@ class ObjectStore: def _do_serial_val_dfs(cls, serial_val, callback): callback(serial_val) - for serial_val in serial_val.param_value_map.values(): + for serial_val in serial_val.param_expr_val_map.values(): cls._do_serial_val_dfs(serial_val, callback) def get_all(self): @@ -358,14 +358,13 @@ class Expression: self.discard_result() def validate_expr(self, op_map): - return True - expr_map, valid = self._dfs_visit() + type_map, valid = self._get_type_map() if not valid: return False # Check that the Expression does not involve 2 classes that are compatible cls_bags = [set(cls_list) for cls_list in op_map.values()] - cls_used = set(expr_map.keys()) + cls_used = set(type_map.keys()) for cls1, cls2 in itertools.product(cls_used, repeat=2): for cls_bag in cls_bags: if cls1 in cls_bag and cls2 in cls_bag: @@ -373,21 +372,21 @@ class Expression: return True - def _dfs_visit(self): - expr_map = dict() - return (expr_map, expr._dfs_visit(expr_map)) + def _get_type_map(self): + type_map = dict() + return (type_map, self._populate_type_map(type_map)) - def _do_dfs_visit(self, expr_map): + def _populate_type_map(self, type_map): value_type = self.op.value_type # If there was already an Expression producing that type, the Expression # is not valid - found_callable = expr_map.get(value_type) + found_callable = type_map.get(value_type) if found_callable is not None and found_callable is not self.op.callable_: return False - expr_map[value_type] = self.op.callable_ + type_map[value_type] = self.op.callable_ for param_expr in self.param_map.values(): - if not param_expr._do_dfs_visit(expr_map): + if not param_expr._populate_type_map(type_map): return False return True @@ -401,27 +400,27 @@ class Expression: if bool(param_expr.op.reusable) == reusable ) - def get_all_values(self): + def get_all_vals(self): for result in self.result_list: yield from result.value_list def find_result_list(self, param_expr_val_map): - def value_map(expr_value_map): + def value_map(expr_val_map): return OrderedDict( # Extract the actual value from ExprValue (param, expr_val.value) - for param, expr_val in expr_value_map.items() + for param, expr_val in expr_val_map.items() ) - param_value_map = value_map(param_expr_val_map) + param_expr_val_map = value_map(param_expr_val_map) # Find the results that are matching the param_expr_val_map return [ result for result in self.result_list - # Check if param_expr_val_map is a subset of the param_value_map + # Check if param_expr_val_map is a subset of the param_expr_val_map # of the ExprValue. That allows checking for reusable parameters # only. - if param_value_map.items() <= value_map(result.param_expr_val_map).items() + if param_expr_val_map.items() <= value_map(result.param_expr_val_map).items() ] def discard_result(self): @@ -456,11 +455,11 @@ class Expression: ) return out - def get_failed_values(self): - for expr_val in self.get_all_values(): - yield from expr_val.get_failed_values() + def get_failed_expr_vals(self): + for expr_val in self.get_all_vals(): + yield from expr_val.get_failed_expr_vals() - def get_id(self, *args, marked_value_set=None, mark_excep=False, hidden_callable_set=None, **kwargs): + def get_id(self, *args, marked_expr_val_set=None, mark_excep=False, hidden_callable_set=None, **kwargs): if hidden_callable_set is None: hidden_callable_set = set() @@ -471,18 +470,18 @@ class Expression: # Mark all the values that failed to be computed because of an # exception if mark_excep: - marked_value_set = set(self.get_failed_values()) + marked_expr_val_set = set(self.get_failed_expr_vals()) for id_, marker in self._get_id( - marked_value_set=marked_value_set, hidden_callable_set=hidden_callable_set, + marked_expr_val_set=marked_expr_val_set, hidden_callable_set=hidden_callable_set, *args, **kwargs ): - if marked_value_set: + if marked_expr_val_set: yield '\n'.join((id_, marker)) else: yield id_ - def _get_id(self, with_tags=True, full_qual=True, qual=True, expr_val=None, marked_value_set=None, hidden_callable_set=None): + def _get_id(self, with_tags=True, full_qual=True, qual=True, expr_val=None, marked_expr_val_set=None, hidden_callable_set=None): # When asked about NoValue, it means the caller did not have any value # computed for that parameter, but still wants an ID. Obviously, it # cannot have any tag since there is no ExprValue available to begin @@ -493,7 +492,7 @@ class Expression: # No specific value was asked for, so we will cover the IDs of all # values if expr_val is None or expr_val is NoValue: - def grouped_value_list(): + def grouped_expr_val_list(): # Make sure we yield at least once even if no computed value # is available, so _get_id() is called at least once if (not self.result_list) or (not with_tags): @@ -505,27 +504,27 @@ class Expression: # If we were asked about the ID of a specific value, make sure we # don't explore other paths that lead to different values else: - def grouped_value_list(): + def grouped_expr_val_list(): # Only yield the ExprValue we are interested in - yield (expr_val.param_value_map, [expr_val]) + yield (expr_val.param_expr_val_map, [expr_val]) - for param_value_map, value_list in grouped_value_list(): + for param_expr_val_map, value_list in grouped_expr_val_list(): yield from self._get_id_internal( - param_value_map=param_value_map, + param_expr_val_map=param_expr_val_map, value_list=value_list, with_tags=with_tags, - marked_value_set=marked_value_set, + marked_expr_val_set=marked_expr_val_set, hidden_callable_set=hidden_callable_set, full_qual=full_qual, qual=qual ) - def _get_id_internal(self, param_value_map, value_list, with_tags, marked_value_set, hidden_callable_set, full_qual, qual): + def _get_id_internal(self, param_expr_val_map, value_list, with_tags, marked_expr_val_set, hidden_callable_set, full_qual, qual): separator = ':' marker_char = '^' - if marked_value_set is None: - marked_value_set = set() + if marked_expr_val_set is None: + marked_expr_val_set = set() # We only get the ID's of the parameter ExprValue that lead to the # ExprValue we are interested in @@ -536,15 +535,15 @@ class Expression: qual = qual, # Pass a NoValue when there is no value available, since # None means all possible IDs (we just want one here). - expr_val = param_value_map.get(param, NoValue), - marked_value_set = marked_value_set, + expr_val = param_expr_val_map.get(param, NoValue), + marked_expr_val_set = marked_expr_val_set, hidden_callable_set = hidden_callable_set, ))) for param, param_expr in self.param_map.items() if ( param_expr.op.callable_ not in hidden_callable_set # If the value is marked, the ID will not be hidden - or param_value_map.get(param) in marked_value_set + or param_expr_val_map.get(param) in marked_expr_val_set ) ) @@ -552,8 +551,7 @@ class Expression: if value_list: for expr_val in value_list: if with_tags: - tag = expr_val.format_tag_list() - tag = '[{}]'.format(tag) if tag else '' + tag = expr_val.format_tags() else: tag = '' yield (expr_val, tag) @@ -563,7 +561,7 @@ class Expression: yield None, '' def get_marker_char(expr_val): - return marker_char if expr_val in marked_value_set else ' ' + return marker_char if expr_val in marked_expr_val_set else ' ' # No parameter to worry about if not param_id_map: @@ -632,7 +630,7 @@ class Expression: yield (id_, marker_str) @classmethod - def get_all_serializable_values(cls, expr_seq, *args, **kwargs): + def get_all_serializable_vals(cls, expr_seq, *args, **kwargs): serialized_map = dict() result_list = list() for expr in expr_seq: @@ -652,7 +650,7 @@ class Expression: assert expr_list if obj_store is None: - serial_list = Expression.get_all_serializable_values(expr_list) + serial_list = Expression.get_all_serializable_vals(expr_list) obj_store = ObjectStore(serial_list) db_var_name = obj_store.db_var_name @@ -680,7 +678,7 @@ class Expression: ) idt = IndentationManager(' '*4) - expr_val_set = set(expr.get_all_values()) + expr_val_set = set(expr.get_all_vals()) result_name, snippet = expr._get_script( reusable_outvar_map = reusable_outvar_map, prefix = prefix + str(i), @@ -835,7 +833,7 @@ class Expression: out = list() for param, expr_val in param_expr_val_map.items(): try: - value = format_expr_value(expr_val) + value = format_expr_val(expr_val) # Cannot be serialized, so we skip it except utils.NotSerializableError: continue @@ -845,7 +843,7 @@ class Expression: return '\n' + ',\n'.join(out) - def format_expr_value(expr_val, com=lambda x: ' # ' + x): + def format_expr_val(expr_val, com=lambda x: ' # ' + x): excep = expr_val.excep value = expr_val.value @@ -883,7 +881,7 @@ class Expression: assert expr_val_list[1:] == expr_val_list[:-1] expr_data = take_first(expr_val_set) - return (format_expr_value(expr_data, lambda x:''), '') + return (format_expr_val(expr_data, lambda x:''), '') # Prior to execution, we don't have an ExprValue yet else: is_user_defined = True @@ -929,7 +927,7 @@ class Expression: # When there is no value for that parameter, that means it # could not be computed and therefore we skip that result with contextlib.suppress(KeyError): - param_expr_val = expr_val.param_value_map[param] + param_expr_val = expr_val.param_expr_val_map[param] param_expr_val_set.add(param_expr_val) # Do a deep first search traversal of the expression. @@ -1055,7 +1053,7 @@ class Expression: try: if is_genfunc: serialized_list = '\n' + idt.style + ('\n' + idt.style).join( - format_expr_value(expr_val, lambda x: ', # ' + x) + format_expr_val(expr_val, lambda x: ', # ' + x) for expr_val in value_list ) + '\n' serialized_instance = 'for {outname} in ({values}):'.format( @@ -1066,7 +1064,7 @@ class Expression: elif value_list: serialized_instance = '{outname} = {value}'.format( outname = outname, - value = format_expr_value(value_list[0]) + value = format_expr_val(value_list[0]) ) # The values cannot be serialized so we hide them except utils.NotSerializableError: @@ -1186,7 +1184,7 @@ class Expression: # Consume all the reusable parameters, since they are generators for param_expr_val_map in consume_gen_map( - reusable_param_exec_map, product=ExprValue.expr_value_product + reusable_param_exec_map, product=ExprValue.expr_val_product ): # If some parameters could not be computed, we will not get all # values @@ -1206,7 +1204,7 @@ class Expression: # that was computed with a given param_expr_val_map assert len(result_list) == 1 expr_val_seq = result_list[0] - yield from expr_val_seq.iter_expr_value() + yield from expr_val_seq.iter_expr_val() continue # Only compute the non-reusable parameters if all the reusable one @@ -1250,7 +1248,7 @@ class Expression: # If no value has been found, compute it and save the results in # a list. - param_value_map = OrderedDict( + param_val_map = OrderedDict( # Extract the actual computed values wrapped in ExprValue (param, param_expr_val.value) for param, param_expr_val in param_expr_val_map.items() @@ -1271,7 +1269,7 @@ class Expression: # Otherwise, we just call the operators with its parameters else: - iterated = self.op.generator_wrapper(**param_value_map) + iterated = self.op.generator_wrapper(**param_val_map) iterator = iter(iterated) expr_val_seq = ExprValueSeq( @@ -1279,7 +1277,7 @@ class Expression: post_compute_cb ) self.result_list.append(expr_val_seq) - yield from expr_val_seq.iter_expr_value() + yield from expr_val_seq.iter_expr_val() def infinite_iter(generator, value_list, from_gen): """Exhaust the `generator` when `from_gen=True`, yield from `value_list` @@ -1314,13 +1312,13 @@ class AnnotationError(Exception): pass class Operator: - def __init__(self, callable_, non_reusable_type_set=None, tag_list_getter=None): + def __init__(self, callable_, non_reusable_type_set=None, tags_getter=None): if non_reusable_type_set is None: non_reusable_type_set = set() - if not tag_list_getter: - tag_list_getter = lambda v: [] - self.tag_list_getter = tag_list_getter + if not tags_getter: + tags_getter = lambda v: [] + self.tags_getter = tags_getter assert callable(callable_) self.callable_ = callable_ @@ -1367,7 +1365,7 @@ class Operator: def __repr__(self): return '' - def force_param(self, param_callable_map, tag_list_getter=None): + def force_param(self, param_callable_map, tags_getter=None): def define_type(param_type): class ForcedType(param_type): pass @@ -1390,7 +1388,7 @@ class Operator: self.annotations[param] = ForcedType prebuilt_op_set.add( PrebuiltOperator(ForcedType, value_list, - tag_list_getter=tag_list_getter + tags_getter=tags_getter )) # Make sure the parameter is not optional anymore @@ -1683,7 +1681,7 @@ class ExprValueSeq: ) return new - def iter_expr_value(self): + def iter_expr_val(self): callback = self.post_compute_cb if not callback: callback = lambda x, reused: None @@ -1756,13 +1754,13 @@ class SerializableExprValue: if type_ is not object ] - self.param_value_map = OrderedDict() - for param, param_expr_val in expr_val.param_value_map.items(): + self.param_expr_val_map = OrderedDict() + for param, param_expr_val in expr_val.param_expr_val_map.items(): param_serialzable = param_expr_val._get_serializable( serialized_map, hidden_callable_set=hidden_callable_set ) - self.param_value_map[param] = param_serialzable + self.param_expr_val_map[param] = param_serialzable def get_id(self, full_qual=True, qual=True, with_tags=True): args = (full_qual, qual, with_tags) @@ -1773,13 +1771,13 @@ class SerializableExprValue: if predicate(self): parent_set.add(self) - for parent in self.param_value_map.values(): + for parent in self.param_expr_val_map.values(): parent.get_parent_set(predicate, _parent_set=parent_set) return parent_set class ExprValue: - def __init__(self, expr, param_value_map, + def __init__(self, expr, param_expr_val_map, value=NoValue, value_uuid=None, excep=NoValue, excep_uuid=None, ): @@ -1788,12 +1786,15 @@ class ExprValue: self.excep = excep self.excep_uuid = excep_uuid self.expr = expr - self.param_value_map = param_value_map + self.param_expr_val_map = param_expr_val_map - def format_tag_list(self): - tag_list = self.expr.op.tag_list_getter(self.value) - if tag_list: - return '+'.join(str(v) for v in tag_list) + def format_tags(self): + tag_map = self.expr.op.tags_getter(self.value) + if tag_map: + return ''.join( + '[{}={}]'.format(k, v) if k else '[{}]'.format(val) + for k, v in sorted(tag_map.items()) + ) else: return '' @@ -1808,27 +1809,16 @@ class ExprValue: serialized_map[self] = serializable return serializable - def _dfs_visit(self): - expr_map = dict() - self._do_dfs_visit(expr_map) - return expr_map - - def _do_dfs_visit(self, expr_map): - expr_map[self.expr] = self - - for param_expr_val in self.param_value_map.values(): - param_expr_val._do_dfs_visit(expr_map) - @classmethod - def validate_expr_value_list(cls, expr_value_list): - if not expr_value_list: + def validate_expr_val_list(cls, expr_val_list): + if not expr_val_list: return True - expr_value_ref = expr_value_list[0] - expr_map_ref = expr_value_ref._dfs_visit() + expr_val_ref = expr_val_list[0] + expr_map_ref = expr_val_ref._get_expr_map() - for expr_val in expr_value_list[1:]: - expr_map = expr_val._dfs_visit() + for expr_val in expr_val_list[1:]: + expr_map = expr_val._get_expr_map() # For all Expression's that directly or indirectly lead to both the # reference ExprValue and the ExprValue, check that it had the same # value. That ensures that we are not making incompatible combinations. @@ -1843,23 +1833,23 @@ class ExprValue: ): return False - if not cls.validate_expr_value_list(expr_value_list[2:]): + if not cls.validate_expr_val_list(expr_val_list[2:]): return False return True @classmethod - def expr_value_product(cls, *gen_list): + def expr_val_product(cls, *gen_list): """Similar to the cartesian product provided by itertools.product, with special handling of NoValue and some checks on the yielded sequences. It will only yield the combinations of values that are validated by - :meth:`validate_expr_value_list`. + :meth:`validate_expr_val_list`. """ generator = gen_list[0] sub_generator_list = gen_list[1:] - sub_generator_list_iterator = cls.expr_value_product(*sub_generator_list) + sub_generator_list_iterator = cls.expr_val_product(*sub_generator_list) if sub_generator_list: from_gen = True value_list = list() @@ -1873,21 +1863,21 @@ class ExprValue: yield [expr_val] continue - for sub_value_list in infinite_iter( + for sub_expr_val_list in infinite_iter( sub_generator_list_iterator, value_list, from_gen ): - expr_value_list = [expr_val] + sub_value_list - if cls.validate_expr_value_list(expr_value_list): - yield expr_value_list + expr_val_list = [expr_val] + sub_expr_val_list + if cls.validate_expr_val_list(expr_val_list): + yield expr_val_list # After the first traversal of sub_generator_list_iterator, we # want to yield from the saved value_list from_gen = False else: for expr_val in generator: - expr_value_list = [expr_val] - if cls.validate_expr_value_list(expr_value_list): - yield expr_value_list + expr_val_list = [expr_val] + if cls.validate_expr_val_list(expr_val_list): + yield expr_val_list def get_id(self, *args, with_tags=True, **kwargs): @@ -1896,12 +1886,32 @@ class ExprValue: return take_first(self.expr.get_id(with_tags=with_tags, expr_val=self, *args, **kwargs)) - def get_failed_values(self): - if self.excep is not NoValue: + def get_parent_expr_vals(self, predicate): + yield from self._get_parent_expr_vals(predicate) + + def _get_parent_expr_vals(self, predicate, param=None): + if predicate(self, param): yield self - for param, expr_val in self.param_value_map.items(): - yield from expr_val.get_failed_values() + for param, expr_val in self.param_expr_val_map.items(): + yield from expr_val._get_parent_expr_vals(predicate, param) + + def get_failed_expr_vals(self): + def predicate(expr_val, param): + return expr_val.excep is not NoValue + + yield from self.get_parent_expr_vals(predicate) + + def _get_expr_map(self): + expr_map = {} + def callback(expr_val, param): + expr_map[expr_val.expr] = expr_val + + # Consume the generator + for _ in self.get_parent_expr_vals(callback): + pass + + return expr_map class Consumer: def __init__(self): diff --git a/tools/exekall/exekall/main.py b/tools/exekall/exekall/main.py index 0ef2c1bc955936c2e29e082f45c1cb028b5aff2c..5f72eb9d1272e2872806ab4a7c6d819ab6bc4622 100755 --- a/tools/exekall/exekall/main.py +++ b/tools/exekall/exekall/main.py @@ -270,7 +270,7 @@ the name of the parameter, the start value, stop value and step size.""") # Get all the UUIDs of its parameters param_uuid_list = [ param_serial.value_uuid - for param_serial in serial.param_value_map.values() + for param_serial in serial.param_expr_val_map.values() ] serial_res_set.update( @@ -329,7 +329,7 @@ the name of the parameter, the start value, stop value and step size.""") engine.PrebuiltOperator( type_, serial_list, id_=id_, non_reusable_type_set=non_reusable_type_set, - tag_list_getter=adaptor.get_tag_list, + tags_getter=adaptor.get_tags, )) # Pool of all callable considered @@ -338,7 +338,7 @@ the name of the parameter, the start value, stop value and step size.""") engine.Operator( callable_, non_reusable_type_set=non_reusable_type_set, - tag_list_getter=adaptor.get_tag_list + tags_getter=adaptor.get_tags ) for callable_ in callable_pool } @@ -372,7 +372,7 @@ the name of the parameter, the start value, stop value and step size.""") try: new_op_pool = op.force_param( param_patch_map, - tag_list_getter=adaptor.get_tag_list + tags_getter=adaptor.get_tags ) prebuilt_op_pool_list.extend(new_op_pool) except KeyError as e: @@ -686,7 +686,7 @@ the name of the parameter, the start value, stop value and step size.""") out('') for result in utils.iterate_cb(executor, pre_line, flush_std_streams): - for failed_val in result.get_failed_values(): + for failed_val in result.get_failed_expr_vals(): excep = failed_val.excep tb = utils.format_exception(excep) error('Error ({e_name}): {e}\nID: {id}\n{tb}'.format( @@ -742,7 +742,7 @@ the name of the parameter, the start value, stop value and step size.""") f.write(expr_val.excep_uuid + '\n') obj_store = engine.ObjectStore( - engine.Expression.get_all_serializable_values( + engine.Expression.get_all_serializable_vals( testcase_list, hidden_callable_set, ) )