From 9b13073e06e8f9cd12ad762c2f73af52249e8ce3 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 30 Mar 2022 20:23:58 +0100 Subject: [PATCH 01/11] lisa.utils: Add instancemethod decorator FEATURE --- lisa/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lisa/utils.py b/lisa/utils.py index 069317638..2480b6711 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -208,6 +208,33 @@ class bothmethod: return getattr(self.f, attr) +class instancemethod: + """ + Decorator providing a hybrid of a normal method and a classmethod: + + * Like a classmethod, it can be looked up on the class itself, and the + class is passed as first parameter. This allows selecting the class + "manually" before applying on an instance. + + * Like a normal method, it can be looked up on an instance. In that + case, the first parameter is the class of the instance and the second + parameter is the instance itself. + """ + def __init__(self, f): + self.__wrapped__ = classmethod(f) + + def __get__(self, instance, owner=None): + # Binding to a class + if instance is None: + return self.__wrapped__.__get__(instance, owner) + # Binding to an instance + else: + return functools.partial( + self.__wrapped__.__get__(instance, instance.__class__), + instance, + ) + + class _DummyLogger: def __getattr__(self, attr): x = getattr(logging, attr) @@ -3623,4 +3650,5 @@ class SerializeViaConstructor(metaclass=_SerializeViaConstructorMeta): } return (self._make_instance, (self._ctor, dct)) + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab -- GitLab From 153edc9efd6ed66ed2b74c091358fac31bde4fb6 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 23 Mar 2022 16:10:47 +0000 Subject: [PATCH 02/11] lisa.utils: Add foldr() Add right-associative version of functools.reduce(), so that: [a, b, c, d] is turned into f(a, f(b, f(c, f(d, init)))) instead of f(f(f(f(init, a), b), c), d) --- lisa/utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lisa/utils.py b/lisa/utils.py index 2480b6711..85fa4e491 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -1372,6 +1372,25 @@ def fold(f, xs, init=None): ) +def foldr(f, xs, init=None): + """ + Right-associative version of :func:`fold`. + + .. note:: This requires reversing `xs`. If reversing is not supported by + the iterator, it will be first converted to a tuple. + """ + try: + xs = reversed(xs) + except TypeError: + xs = reversed(tuple(xs)) + + return fold( + lambda x, y: f(y, x), + xs, + init, + ) + + def add(iterable): """ Same as :func:`sum` but works on any object that defines ``__add__`` -- GitLab From 35f9dca8a21c4744da05248bce9a1f5db8f6994c Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 17 Feb 2022 17:37:48 +0000 Subject: [PATCH 03/11] lisa.wlgen.rta: Fix with_props() examples FIX Fix examples: with_props(prop_X=...) should be with_props(X=...) --- lisa/wlgen/rta.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lisa/wlgen/rta.py b/lisa/wlgen/rta.py index 0d1dcb6cf..0dda68469 100644 --- a/lisa/wlgen/rta.py +++ b/lisa/wlgen/rta.py @@ -1606,8 +1606,8 @@ def leaf_precedence(val, **kwargs): """ Give precedence to the leaf values when combining with ``&``:: - phase = phase.with_props(prop_meta=({'hello': 'leaf'}) - phase = phase.with_props(prop_meta=leaf_precedence({'hello': 'root'}) + phase = phase.with_props(meta=({'hello': 'leaf'}) + phase = phase.with_props(meta=leaf_precedence({'hello': 'root'}) assert phase['meta'] == {'hello': 'leaf'} This allows setting a property with some kind of default value on a root @@ -1633,7 +1633,7 @@ def override(val, **kwargs): Override a property with the given value, rather than combining it with the property-specific ``&`` implementation:: - phase = phase.with_props(prop_cpus=override({1,2})) + phase = phase.with_props(cpus=override({1,2})) """ return _OverridingValue(val, **kwargs) @@ -1687,7 +1687,7 @@ def delete(): """ Remove the given property from the phase:: - phase = phase.with_props(prop_cpus=delete()) + phase = phase.with_props(cpus=delete()) """ return _DeletingValue() -- GitLab From 1b2740f5a4fd79122f5c3b4babcbbf7973bbe0ed Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 30 Mar 2022 11:58:47 +0100 Subject: [PATCH 04/11] lisa.fuzz: Give a default to Gen(name=...) Give the name of the function passed if name is None. --- lisa/fuzz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/fuzz.py b/lisa/fuzz.py index 4f91b61d8..aebbb3fd1 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -153,7 +153,7 @@ class Gen(StateMonad, Loggable): self.logger.log(log_level, f'Drawn {val}{trials}from {self}{info}') return x - self.name = name + self.name = name or f.__qualname__ super().__init__(wrapper) class _STATE: -- GitLab From 3b28ca12ef7ec9b077e25e15cdf8ac0e6f62cf6e Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 23 Feb 2022 12:57:48 +0000 Subject: [PATCH 05/11] lisa.fuzz: Catch RetryException in decorator Rather than catching RetryException in state-modifying functions, wrap directly the user-provided coroutine-functions. This is done in preparation for the lisa.monad.StateMonad change where user coroutine-functions will not be wrapped with Gen.from_f() anymore. --- lisa/fuzz.py | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/lisa/fuzz.py b/lisa/fuzz.py index aebbb3fd1..b82462887 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -126,43 +126,16 @@ class Gen(StateMonad, Loggable): Random generator monad inspired by Haskell's QuickCheck. """ def __init__(self, f, name=None): - log_level = logging.DEBUG - logger = self.logger - if logger.isEnabledFor(log_level): - caller_info = inspect.stack()[2] - else: - caller_info = None - - @functools.wraps(f) - def wrapper(state): - for i in itertools.count(1): - try: - x = f(state) - except RetryException: - continue - else: - trials = f'after {i} trials ' if i > 1 else '' - if caller_info: - info = f' ({caller_info.filename}:{caller_info.lineno})' - else: - info = '' - val, _ = x - val = str(val) - sep = '\n' + ' ' * 4 - val = sep + val.replace('\n', sep) + '\n' if '\n' in val else val + ' ' - self.logger.log(log_level, f'Drawn {val}{trials}from {self}{info}') - return x - self.name = name or f.__qualname__ - super().__init__(wrapper) + super().__init__(f) - class _STATE: + class _State: def __init__(self, rng): self.rng = rng @classmethod def make_state(cls, *, rng=None, seed=None): - return cls._STATE( + return cls._State( rng=rng or random.Random(seed), ) @@ -170,6 +143,25 @@ class Gen(StateMonad, Loggable): name = self.name or self._f.__qualname__ return f'{self.__class__.__qualname__}({name})' + @classmethod + def _wrap_coroutine_f(cls, f): + @functools.wraps(f) + async def wrapper(*args, **kwargs): + for i in itertools.count(1): + try: + x = await f(*args, **kwargs) + except RetryException: + continue + else: + trials = f'after {i} trials ' if i > 1 else '' + val = str(x) + sep = '\n' + ' ' * 4 + val = sep + val.replace('\n', sep) + '\n' if '\n' in val else val + ' ' + cls.get_logger().debug(f'Drawn {val}{trials}from {f.__qualname__}') + return x + + return wrapper + class Choices(Gen): """ -- GitLab From 4309ee5a086ce2d3f3a4a545830c5d7a5197c025 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Fri, 25 Feb 2022 11:11:02 +0000 Subject: [PATCH 06/11] lisa.fuzz: Update docstring FIX Remove the statement that monadic arguments are automatically awaited, as this is not the case anymore. --- lisa/fuzz.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lisa/fuzz.py b/lisa/fuzz.py index b82462887..96a0e98ad 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -18,6 +18,10 @@ """ Fuzzing API to build random constrained values. +.. note:: The following example shows a direct use of the :class:`Gen` monad, + but be aware that :mod:`lisa.wlgen.rta` API allows mixing both :class:`Gen` + and RTA DSL into the same coroutine function. + **Example**:: import operator @@ -28,9 +32,7 @@ Fuzzing API to build random constrained values. from lisa.fuzz import Gen, Choice, Int, Float, retry_until # The function must be decorated with Gen.lift() so that "await" gains its - # special meaning. In addition to that, parameters are automatically awaited if - # they are an instance of Gen, and the return value is automatically promoted - # to an instance of Gen if it is not already. + # special meaning. @Gen.lift async def make_task(duration=None): # Draw a value from an iterable. -- GitLab From 7a6a224be9993ebd9aaf76b7397fe1018978d85d Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Thu, 17 Feb 2022 16:55:12 +0000 Subject: [PATCH 07/11] lisa.monad: Replace the State monad implementation BREAKING CHANGE FEATURE Replace the original custom-made State monad implementation with a full featured monad transformer implementation. This will allow combining multiple independent state monads, as well as interop with event loops (e.g. asyncio), and allow new effects to be added along the way such as the short-circuiting Option. Breaking changes: * Remove the automatic awaiting of monadic arguments to user-defined coroutine functions. The reason is that it was a misfeature that worked for the lisa.fuzz.Gen monad, but is actually quite problematic for other state monads as the user looses control over the ordering of side effects. This can unfortunately be extremely important for some monads. * Re-purpose the "lift" method to act like a monad transformer lift. Existing use of "Monad.lift()" need to be converted to "Monad.do()" instead. * Rename StateMonad into State * StateMonad values can now only be called once. --- lisa/monad.py | 1017 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 913 insertions(+), 104 deletions(-) diff --git a/lisa/monad.py b/lisa/monad.py index 6e66ee11c..f84e2d6bd 100644 --- a/lisa/monad.py +++ b/lisa/monad.py @@ -58,171 +58,980 @@ This allow composing lifted functions easily """ import abc +import itertools import functools import inspect +import operator +import asyncio +from operator import attrgetter +from functools import partial +from weakref import WeakKeyDictionary +import nest_asyncio +nest_asyncio.apply() -class StateMonad(abc.ABC): - """ - The state monad. +from lisa.utils import memoized, foldr, instancemethod - :param f: Callable that takes the state as parameter and returns an - instance of the monad. - :type f: collections.abc.Callable +class _MonadBase: + """ + Abstract Base Class parametrized by a type T, + like a container of some sort. """ - def __init__(self, f): - self._f = f + @abc.abstractclassmethod + def pure(cls, x): + """ + Takes a value of type T and turns it into a "monadic value" of type Monad[T]. + """ + pass + + @abc.abstractmethod + def bind(self, continuation): + """ + Takes a monadic value Monad[A], a function that takes an A and returns Monad[B], and returns a Monad[B]. + + .. note:: It is allowed to return a :class:`_TailCall` instance. + """ + pass + + @abc.abstractmethod + def map(self, f): + """ + Takes a monadic value Monad[A], a function that takes an A and returns B, and returns a Monad[B]. + """ + pass + + @abc.abstractmethod + def join(self): + """ + Takes a monadic value Monad[Monad[A]], and returns a Monad[A]. + """ + pass def __await__(self): - # What happens here is executed as if the code was inlined in the - # coroutine's body ("await x" is actually equivalent to - # "yield from x"): - # 1. "yield self" allows to relinquish control back to the loop, - # providing the monadic value that was awaited on by the user code. - # 2. Returning the result of the yield allows the loop to inject any - # value it sees fit using coro.send(). return (yield self) + +class Monad(_MonadBase): + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + + def wrap_bind(f): + @functools.wraps(f) + def wrapper(self, continuation): + # Ensure that all the parameters are resolved before use, so + # that implementations don't have to be sprinkled with + # _TailCall.run() + run = _TailCall.run + return f( + run(self), + _TailCall._trampoline(run(continuation)) + ) + return wrapper + + cls.bind = wrap_bind(cls.bind) + + @instancemethod + def map(cls, self, f): + return cls.bind(self, lambda x: self.pure(f(x))) + + @instancemethod + def join(cls, self): + return cls.bind(self, lambda x: x) + + +# Inherit from _MonadBase directly to avoid the automatic trampoline on bind() +# that would lead to infinite recursion +class _Identity(_MonadBase): + """ + This monad is private as it is not "proper". + + It does not actually wrap the value, which means that the only way e.g. + bind() can be called is when referring to the class directly, since no + instance of :class:`_Identity` will be created. + + This does have the advantage of avoiding unnecessary boxing, which improves + performance and also avoids a layer of indirection when inspecting the + resulting values. + """ + def __new__(cls, x): + return x + + def bind(self, continuation): + return _TailCall(_TailCall.run(continuation), _TailCall.run(self)) + + @staticmethod + def pure(x): + return x + + def map(self, f): + return f(self) + + def join(self): + return self + + +class _TailCall: + """ + Represents a function call, to be returned so that an enclosing loop + can execute the call, rather than using code itself:: + + return f(x) + # becomes + return _TailCall(f, x) + """ + __slots__ = ('f', '__weakref__') + + def __init__(self, f, *args, **kwargs): + self.f = functools.partial(f, *args, **kwargs) + + def run(x): + """ + Evaluates a chain of _TailCall until it's all evaluated. + """ + # Do not use self.__class__ so that _TailCall.run() can be called on + # arbitrary objects. + while isinstance(x, _TailCall): + x = x.f() + + # Recursive implementation, sometimes useful for debugging + # if isinstance(x, _TailCall): + # x = _TailCall.run(x.f()) + return x + + @classmethod + def trampoline(cls, f): + """ + Decorator to allow a function to return :class:`_TailCall`. + """ + return functools.wraps(f)(cls._trampoline(f)) + + @classmethod + def _trampoline(cls, f): + run = cls.run + def wrapper(*args, **kwargs): + return run( + f(*args, **kwargs) + ) + return wrapper + + +class AlreadyCalledError(Exception): + """ + Exception raised by :class:`_CallOnce` when the wrapped function has already + been called once. + """ + pass + + +class _CallOnce: + """ + Only allow calling the wrapped function once. + + This prevents accidentally calling ``coro.send(...)`` multiple times, + wrongly expecting it to resume twice from the same point. + """ + __slots__ = ('f', '__weakref__') + + def __init__(self, f): + self.f = f + + @staticmethod + def _raise(*args, **kwargs): + raise AlreadyCalledError(f'This function cannot be called more than once') + def __call__(self, *args, **kwargs): - state = self.make_state(*args, **kwargs) - x, _ = self._f(state) + x = self.f(*args, **kwargs) + self.f = self._raise return x - def __init_subclass__(cls, **kwargs): - # The one inheriting directly from StateMonad is the base of the - # hierarchy - if StateMonad in cls.__bases__: - cls._MONAD_BASE = cls - super().__init_subclass__(**kwargs) + +def _build_stack(stack): + *stack, init = tuple(stack) + if stack: + return foldr( + lambda trans, base: trans._rebase_on(base), + stack, + init, + ) + else: + return init + + +class MonadTrans(Monad, abc.ABC): + """ + Base class for monad transformers. + + Heavily inspired by transformers as defined by: + https://hackage.haskell.org/package/transformers + + And stack manipulation inspired by: + https://hackage.haskell.org/package/mmorph + """ + _BASE = _Identity @classmethod - def from_f(cls, *args, **kwargs): + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + cls._UNAPPLIED = cls + """ + Unapplied monad transformer, i.e. applied to (a subclass of) :class:`_Identity`. + """ + + # Stacks of more than one transformer will override that + cls._TRANSFORMER_STACK = (cls,) """ - Build an instance of the monad from a state transformation function. - The callback takes the current state as parameter and returns - ``tuple(value, new_state)``. + Stack of transformers. + """ + + cls._TRANSFORM_CACHE = WeakKeyDictionary() + """ + Cache of the transformer applied to other base monads. + + .. note:: This cache is necessary for functional purposes, so that + rebasing a transformer on the same monad twice gives back the same + class, which can therefore be used as key in mappings. """ - return cls._MONAD_BASE(*args, **kwargs) @abc.abstractclassmethod - def make_state(cls, *args, **kwargs): + def lift(cls, m): """ - Create the state from user-defined parameters. This is used by - :meth:`lisa.monad.StateMonad.__call__` in order to initialize the - state. + Lift a monadic value ``m`` by one level in the stack, i.e.: + Given a stack for 3 transformers ``T1(T2(T3(Identity)))``, + a value ``m = T3(Identity).pure(42)``. we have + ``T2.lift(m) == T2(T3(Identity)).pure(42)``. + + .. seealso:: ``lift`` as defined in https://hackage.haskell.org/package/transformers + """ + pass + + @abc.abstractclassmethod + def hoist(cls, self, nat): + """ + Lift a monadic value ``m`` by one level in the stack, i.e.: + Given a stack for 3 transformers ``T1(T2(T3(Identity)))``, + a value ``m = T2(Identity).pure(42)``. we have + ``T2.hoist(m, T3.pure) == T2(T3(Identity)).pure(42)``. + + In other words, it allows adding a level "from below", whereas ``lift`` + adds a level "from above". It's similar to ``map``, except that instead + of traversing all the nested functor layers, it stops at the first one. + + :param self: Monadic value to hoist. + :type self: lisa.monad.Monad + + :param nat: Natural transform. i.e. a morphism from ``Monad1[A]`` to + ``Monad2[A]`` that obeys certain laws. + :type nat: collections.abc.Callable + + .. seealso:: ``hoist`` as defined in https://hackage.haskell.org/package/mmorph + + .. note:: Note for implementers: A monad transformers ``t m a`` (``t`` + is the transformer HKT, ``m`` is the base monad and ``a`` is the + "contained type) usually ends up containing an "m (f a)" (``f`` being + some kind of functor). For example, ``MaybeT`` in Haskell + (:class:`Option` here) is more or less defined as ``data MaybeT m a = + MaybeT (m (Maybe a))``. What the ``hoist`` implementation must do is to + "rebuild" a value with a call to ``nat()`` around the ``m (...)`` part. + For ``MaybeT``, this gives + ``hoist nat (MaybeT (m (Maybe a))) = MaybeT(nat(m (Maybe a)))``. """ pass @classmethod def pure(cls, x): """ - Lift a value in the state monad, i.e. create a monad instance with a - function that returns the value and the state unchanged. + Turn a regular value of type ``A`` into a monadic value of type ``Monad[A]``. """ - return cls.from_f(lambda state: (x, state)) + return cls.lift(cls._BASE.pure(x)) @classmethod - def lift(cls, f): + def _rebase_on(cls, monad): """ - Decorator used to lift a function into the monad, such that it can take - monadic parameters that will be evaluated in the current state, and - returns a monadic value as well. + Rebase the transformer onto a new base monad and returns it. """ + key = monad + # "unapply" the transformer, since we are rebuilding the stack from the + # bottom to the top + cls = cls._UNAPPLIED - cls = cls._MONAD_BASE + # This cache is necessary for functional purposes, it's not only an + # optimisation. It guarantees that rebasing the same transformer on the + # same monad will give the same class, so it can be used as key in + # dictionaries. + try: + return cls._TRANSFORM_CACHE[key] + except KeyError: + transformed = cls._do_rebase_on(monad) + cls._TRANSFORM_CACHE[key] = transformed + return transformed - def run(_f, args, kwargs): - call = lambda: _f(*args, **kwargs) - x = call() - if inspect.iscoroutine(x): - def body(state): - if inspect.getcoroutinestate(x) == inspect.CORO_CLOSED: - _x = call() - else: - _x = x + @classmethod + def _do_rebase_on(cls, base): + class _Monad(cls): + _BASE = base - next_ = lambda: _x.send(None) - while True: - try: - future = next_() - except StopIteration as e: - val = e.value - break - else: - assert isinstance(future, cls) - try: - val, state = future._f(state) - except Exception as e: - # We need an intermediate variable here, since - # "e" is not really bound in this scope. - excep = e - next_ = lambda: _x.throw(excep) - else: - next_ = lambda: _x.send(val) - - if isinstance(val, cls): - return val._f(state) - else: - return (val, state) + _Monad._UNAPPLIED = cls + _Monad.__name__ = cls.__name__ + _Monad.__qualname__ = cls.__qualname__ + return _Monad + + @staticmethod + def _decorate_coroutine_function(f): + """ + Called by :meth:`MonadTrans.do` to wrap the user-provided coroutine function, i.e. ``async def`` functions. + """ + return f + + @classmethod + def do(cls, f): + """ + Decorate a coroutine function so that ``awaits`` gains the powers of the monad. + + .. seealso:: This decorator is very similar to the do-notation in Haskell. + """ + if not inspect.iscoroutinefunction(f): + raise TypeError(f'{cls.__qualname__}.do() can only decorate generator functions, i.e. defined with "async def"') + + do_cls = cls + stack = cls._TRANSFORMER_STACK + + # The direct "await" at our level are expected to yield instances of + # the "bare" transformers (i.e. applied to _Identity), or an instance + # of a substack + substacks = { + cls._UNAPPLIED_TOP: cls + for cls in stack + if issubclass(cls, _MonadTransStack) + } + + transformers = { + cls._UNAPPLIED: cls + for cls in stack + } + + def resolve(cls): + try: + return substacks[cls] + except KeyError: + try: + return transformers[cls._UNAPPLIED] + except KeyError: + _stack = ', '.join( + cls.__qualname__ + for cls in stack + ) + raise TypeError(f'Could not find the transformer {cls.__qualname__} in the stack {do_cls.__name__}. Only the following transformers are allowed at that level: {_stack}') + + lifters = { + cls: tuple(reversed(tuple(itertools.takewhile( + lambda x: x is not cls, + stack + )))) + for cls in stack + } - val = cls.from_f(body, name=f.__qualname__) + def make_hoister(stack): + try: + cls1, cls2 = stack + except ValueError: + return lambda x: x else: - if isinstance(x, cls): - val = x - else: - val = cls.pure(x) + return lambda x: cls1.hoist(x, cls2.pure) - return val + hoisters = { + cls: make_hoister( + itertools.islice( + itertools.dropwhile( + lambda x: x is not cls, + stack + ), + None, + 2, + ) + ) + for cls in stack + } - @functools.wraps(f) + def convert(action): + if action.__class__ is stack[0]: + return action + else: + action_cls = resolve(action.__class__) + hoisted = hoisters[action_cls](action) + + lifted = hoisted + for _cls in lifters[action_cls]: + lifted = _cls.lift(lifted) + + return lifted + + def decorate(f): + for trans in reversed(stack): + f = trans._decorate_coroutine_function(f) + return f + + _f = decorate(f) + + @functools.wraps(_f) def wrapper(*args, **kwargs): - async def _f(*args, **kwargs): - args = [ - (await arg) if isinstance(arg, cls) else arg - for arg in args - ] - kwargs = { - k: (await v) if isinstance(v, cls) else v - for k, v in kwargs.items() - } - return run(f, args, kwargs) - return run(_f, args, kwargs) + coro = _f(*args, **kwargs) + def next_(x): + run = functools.partial(coro.send, x) + + while True: + try: + action = run() + except StopIteration as e: + x = e.value + if isinstance(x, Monad): + return x + else: + return cls.pure(x) + else: + try: + # Automatically lift actions based on their class, + # which indicates their position in the transformer + # stack. + action = convert(action) + except Exception as e: + run = functools.partial(coro.throw, e) + else: + return action.bind(_CallOnce(next_)) + + return _TailCall.run(next_(None)) + + wrapper.monad = cls return wrapper + +class _CloneIdentity(_Identity): @classmethod - def get_state(cls): + def clone(cls): + # Derive from _CloneIdentity to avoid long chains of inheritance, which + # would prevent garbage collection classes and also slow down attribute + # resolution. + class new(_CloneIdentity): + pass + return new + + +class _MonadTransStack(MonadTrans): + """ + Stack of monad transformers. + + * The ``_TOP`` attribute points at the top of the stack, which can the be + unfolded by accessing ``_BASE``. + + * ``_UNAPPLIED_TOP`` is the "original" ``_TOP``, i.e. before the stack is + rebased using :meth:`MonadTrans.rebase_on`. + + .. note:: Each stack is based on its own copy of :class:`_CloneIdentity`, + so that all the transformers composing the stack are independent from + any other transformer. This allows natural mixing of stacks, as there + is never any ambiguity on what stack a given monadic value belongs to. + """ + _BASE = _CloneIdentity + + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + + def get_stack(trans): + if issubclass(trans, MonadTrans): + yield trans + yield from get_stack(trans._BASE) + else: + return + + _base = cls._BASE + + # Update the base we will build _TOP on, but do not update _BASE + # itself. This allows subclasses of non-rebased classes to all have + # _BASE==_CloneIdentity, so they all get independent _TOP stack. + if _base is _CloneIdentity: + # Ensure each stack is independent by providing a separate + # _Identity base = _base.clone() if _base is _CloneIdentity else + # _base + base = _base.clone() + else: + base = _base + + # Rebuild the stack on top of the new base + top = _build_stack( + itertools.chain( + get_stack(cls._UNAPPLIED_TOP), + (base,) + ) + ) + + # We are defining a new stack, not just rebasing an existing stack. + if _base is _CloneIdentity: + cls._UNAPPLIED_TOP = top + + cls._TOP = top + cls._TRANSFORMER_STACK = tuple(get_stack(top)) + + def bind(self, continuation): + # This looks like a dummy implementation, but is actually useful since + # it can be called as T.bind(m, f), regardless of the precise type of + # m. + return self.bind(continuation) + + @classmethod + def lift(cls, m): + # Lift a monadic value on the whole original stack (i.e. disregarding + # any other stack this transformer might be rebased on). + def lift(monad, unapplied, x): + if issubclass(unapplied, MonadTrans): + return monad.lift(lift(monad._BASE, unapplied._BASE, x)) + else: + return x + + return lift(cls._TOP, cls._UNAPPLIED_TOP, m) + + @instancemethod + def hoist(cls, self, nat): + def hoist(monad, unapplied, x): + return monad.hoist( + x, + ( + functools.partial( + hoist, + monad._BASE, unapplied._BASE, + ) + if issubclass(unapplied._BASE, MonadTrans) else + nat + ) + ) + + # Traverse the structure using the _UNAPPLIED_TOP, but build values of the + # correct type that reflects the actual structure of the value. + return hoist(cls._TOP, cls._UNAPPLIED_TOP, self) + + +def TransformerStack(*stack): + """ + Allows stacking together multiple :class:`MonadTrans`, e.g.:: + + class Stack(TransformerStack(T1, T2, T3)): + pass + + @Stack.do + async def foo(): + + # Any monadic value from the stack's direct components can be used. + await T1.pure(42) + await T2.pure(42) + await T3.pure(42) + + """ + + if len(set(stack)) != len(stack): + raise ValueError(f'Monad transformers can only appear once in any given stack') + else: + class Stack(_MonadTransStack): + _UNAPPLIED_TOP = _build_stack(stack) + + return Stack + + +class _Optional: + """ + Instances of this class represents either the absence of a value, or a + value. + """ + pass + + +class Some(_Optional): + """ + Wraps an arbitrary value to indicate its presence. + """ + __slots__ = ('x',) + + def __init__(self, x): + self.x = x + + def __repr__(self): + return f'{self.__class__.__qualname__}({self.x})' + + __str__ = __repr__ + + def __eq__(self, other): + assert type(self) is type(other) + return self.x == other.x + + +class _Nothing(_Optional): + """ + Do not make your own instances, use the ``Nothing`` singleton. + """ + def __repr__(self): + return 'Nothing' + + __str__ = __repr__ + + def __eq__(self, other): + return self is other + + +Nothing = _Nothing() +""" +Similar to :class:`Some` but indicating the absence of value. +""" + + +# Do that in a base class so that Option itself gets __init_subclass__ ran on +class _AddPureNothing: + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + # Specialize the _OPTION_NOTHING to the correct right _BASE. + # Pre-computing it allows sharing a single instance for all the cases + # where it is needed. + cls._PURE_NOTHING = cls._BASE.pure(Nothing) + + +class Option(MonadTrans, _AddPureNothing): + """ + Monad transformer that manipulates :class:`Some` and :attr:`Nothing`. + + :meth:`Option.bind` will short-circuit if :attr:`Nothing` is passed, much + like the Rust or Javascript ``?`` operator, or the ``MaybeT`` monad + transformer in Haskell. + """ + __slots__ = ('_x',) + + def __init__(self, x): + self._x = x + + @property + def x(self): + """ + Wrapped value, of type ``Base[_Optional[A]]`` with ``Base`` the base + monad of the transformer. """ - Returns a monadic value making the current state available. - To be used inside a lifted function using:: + # Ensure we only run the _TailCall once, since it will run a + # continuation + x = _TailCall.run(self._x) + self._x = x + return x + + def __repr__(self): + return f'Option({self.x})' + + __str__ = __repr__ + + def bind(self, continuation): + cls = self.__class__ + base = self._BASE + + def k(v): + if v is Nothing: + # Use pre-computed object + return cls._PURE_NOTHING + else: + return continuation(v.x)._x + + return cls(base.bind(self.x, k)) + + @classmethod + def lift(cls, m): + base = cls._BASE + return cls(base.map(m, Some)) + + @instancemethod + def hoist(cls, self, nat): + return cls(nat(self.x)) + + +class State(MonadTrans): + """ + Monad transformer analogous to Haskell's ``StateT`` transformer. + + It manipulates state-transforming functions of type ``state -> (value, + new_state)``. This allows simulating a global state, without actually + requiring one. + """ + __slots__ = ('_f',) + + def __init__(self, f): + self._f = f - state = await StateMonad.get_state() + @property + def f(self): """ - return cls.from_f(lambda state: (state, state)) + State-transforming function of type ``state -> (value, new_state)`` + """ + return _TailCall._trampoline(self._f) @classmethod - def set_state(cls, new_state): + def make_state(cls, x): """ - Returns a monadic value to set the current state. - To be used inside a lifted function using:: + Create an initial state. All the parameters of :meth:`State.__call__` + are passed to :meth:`State.make_state`. + """ + return x - await StateMonad.set_state(new_state) + def __call__(self, *args, **kwargs): + """ + Allow calling monadic values to run the state-transforming function, + with the initial state provided by :meth:`State.make_state`. """ - return cls.from_f(lambda state: (state, new_state)) + init = self.make_state(*args, **kwargs) + return self.f(init) + + def bind(self, continuation): + base = self._BASE + def k(res): + x, state = res + return continuation(x)._f(state) + + return self.__class__( + lambda state: base.bind(self.f(state), k) + ) @classmethod - def modify_state(cls, f): + def lift(cls, m): + base = cls._BASE + return cls( + lambda state: base.bind( + m, + lambda a: base.pure((a, state)) + ) + ) + + @instancemethod + def hoist(cls, self, nat): + return cls(lambda state: nat(self.f(state))) + + @classmethod + def from_f(cls, f): + """ + Build a monadic value out of a state modifying function of type + ``state -> (value, new_state)``. """ - Returns a monadic value to modify the current state. - To be used inside a lifted function using:: + base = cls._BASE + return cls(lambda state: base.pure(f(state))) - await StateMonad.modify_state(lambda state: new_state) + @classmethod + def get_state(cls): + """ + Returns a monadic value returning the current state. + """ + return cls.from_f(lambda state: (state, state)) + + @classmethod + def set_state(cls, new): + """ + Returns a monadic value setting the current state and returning the old + one. + """ + return cls.from_f(lambda state: (state, new)) + + @classmethod + def modify_state(cls, f): + """ + Returns a monadic value applying ``f`` on the current state, setting + the new state and then returning it. """ def _f(state): new_state = f(state) return (new_state, new_state) return cls.from_f(_f) + +class StateDiscard(State): + """ + Same as :class:`State` except that calling monadic values will return the + computed value instead of a tuple ``(value, state)``. + + This is useful for APIs where the final state is of no interest to the + user. + """ + def __call__(self, *args, **kwargs): + # We are part of a monad transformer stack, so we need to apply + # the function at the right level. + # super().__call__() returns a "m (a, s)", so we need to map on that + # monadic value to turn it into "m a". + return self._BASE.map( + super().__call__(*args, **kwargs), + operator.itemgetter(0), + ) + + +def _wrap_in_coro(x): + """ + Wrap ``x`` in a coroutine that will simply return it. + """ + async def f(): + return x + return f() + + +class _AwaitWrapper: + """ + Dummy wrapper that allows transparently forwarding an "await" from a + wrapped coroutine + """ + __slots__ = ('x', '__weakref__') + + def __init__(self, x): + self.x = x + + def __await__(self): + return (yield self.x) + + +class _StopIteration(Exception): + """ + Custom :exc:`StopIteration` that is allowed to be raised by an async + generator. + """ + pass + + +class Async(MonadTrans): + """ + Monad transformer allowing the decorated coroutine function to await on + non-monadic values. This is useful to mix any monad transformer defined in + this module with other async APIs, such as :mod:`asyncio`. + """ + __slots__ = ('_coro',) + + def __init__(self, coro): + self._coro = coro + + @property + def coro(self): + """ + Coroutine that will only yield non-monadic values. All the monadic + values will be processed by the monad transformer stack as expected and + will stay hidden. + """ + async def _f(): + return _TailCall.run(await self._coro) + return _f() + + @property + @memoized + def x(self): + """ + Run the coroutine to completion in its event loop. + """ + return self._run(self.coro) + + @staticmethod + def _run(coro): + """ + Run a coroutine to completion, handling any non-monadic value along the + way. This is typically the entry point of an event loop, such as + :func:`asyncio.run`. + + .. note:: This function must be re-entrant. :func:`asyncio.run` is not + re-entrant by default, but can be made to with + https://pypi.org/project/nest-asyncio/ + """ + try: + action = coro.send(None) + except StopIteration as e: + return e.value + else: + raise RuntimeError(f'Did not expect coroutine to await on {action.__class__.__qualname__}, please subclass Async and override the _run() method with e.g. asyncio.run') + + @classmethod + def _decorate_coroutine_function(cls, f): + """ + Wrap the coroutine function into an async generator that will: + + 1. yield any monadic action + 2. await on everything else, e.g. asyncio actions + + A simpler design would simply ``cls._run()`` or await each action + individually, but it is unfortunately not enough for ``asyncio``. + Indeed, the coroutine itself must be run by ``asyncio.run()``, otherwise + creating the action itself (e.g. ``asyncio.sleep(1)``) will fail as it will + not find the global event loop. + """ + async def _genf(*args, **kwargs): + coro = f(*args, **kwargs) + run = functools.partial(coro.send, None) + while True: + try: + action = run() + # Convert to a custom type, as an async generator is not + # allowed to raise a StopIteration (if it does, it is punished + # by a RuntimeError). We also cannot simply "return e.value" + # since async generators are not allowed to return anything. + except StopIteration as e: + raise _StopIteration(e.value) + else: + is_monad = isinstance(action, Monad) + try: + if is_monad: + x = yield action + else: + x = await _AwaitWrapper(action) + except Exception as e: + run = functools.partial(coro.throw, e) + else: + run = functools.partial(coro.send, x) + + @functools.wraps(f) + async def wrapper(*args, **kwargs): + gen = _genf(*args, **kwargs) + + # All the action the generator yields is to be processed by our + # monadic stack, the others are already handled by the _run() call + x = None + while True: + next_ = gen.asend(x) + try: + action = cls._run(next_) + except _StopIteration as e: + return e.args[0] + else: + x = await action + + return wrapper + + @classmethod + def lift(cls, x): + return cls(_wrap_in_coro(x)) + + @instancemethod + def hoist(cls, self, nat): + async def f(): + # We could use "await self.coro" but expand it to save on a layer + # of coroutine + x = _TailCall.run(await self._coro) + return nat(x) + return cls(f()) + + def bind(self, continuation): + cls = self.__class__ + base = cls._BASE + + async def f(): + return base.bind( + await self._coro, + lambda x: _TailCall(cls._run, continuation(x)._coro) + ) + + return cls(f()) + + +class AsyncIO(Async): + """ + Specialization of :class:`lisa.monad.Async` to :mod:`asyncio` event loop. + """ + # Note that this only works properly with nest_asyncio package. Otherwise, + # asyncio.run() is not re-entrant. + _run = staticmethod(asyncio.run) + + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 -- GitLab From f14e3844bc82525e20bd852fceaa249b20e0df6a Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 30 Mar 2022 11:54:20 +0100 Subject: [PATCH 08/11] lisa.fuzz: Port lisa.fuzz.Gen over the new lisa.monad API BREAKING CHANGE Introduce GenMonad inheriting from lisa.monad.State and make Gen simply embed a GenMonad action. Breaking change: retry_until() now needs to be awaited, otherwise it's a no-op. This will ensure future-proofing of the API, should the implementation of retry_until need access to the internal state. --- lisa/fuzz.py | 94 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 20 deletions(-) diff --git a/lisa/fuzz.py b/lisa/fuzz.py index 96a0e98ad..fcd012800 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -29,11 +29,11 @@ Fuzzing API to build random constrained values. from lisa.platforms.platinfo import PlatformInfo from lisa.wlgen.rta import RTAPhase, RTAConf, PeriodicWload - from lisa.fuzz import Gen, Choice, Int, Float, retry_until + from lisa.fuzz import GenMonad, Choice, Int, Float, retry_until - # The function must be decorated with Gen.lift() so that "await" gains its + # The function must be decorated with GenMonad.do() so that "await" gains its # special meaning. - @Gen.lift + @GenMonad.do async def make_task(duration=None): # Draw a value from an iterable. period = await Choice([16e-3, 8e-3]) @@ -42,7 +42,7 @@ Fuzzing API to build random constrained values. # Arbitrary properties can be enforced. If they are not satisfied, the # function will run again until the condition is true. - retry_until(0 < nr <= 2) + await retry_until(0 < nr <= 2) phase = functools.reduce( operator.add, @@ -60,7 +60,7 @@ Fuzzing API to build random constrained values. return phase - @Gen.lift + @GenMonad.do async def make_profile(plat_info, **kwargs): nr_tasks = await Int(1, plat_info['cpus-count']) @@ -73,11 +73,11 @@ Fuzzing API to build random constrained values. def main(): plat_info = PlatformInfo.from_yaml_map('./doc/traces/plat_info.yml') - # When called, profile_gen() will create a random profiles - profile_gen = make_profile(plat_info, duration=1) - # Display a few randomly generated tasks for _ in range(2): + # When called, profile_gen() will create a random profiles + profile_gen = make_profile(plat_info, duration=1) + # seed (or rng) can be fixed for reproducible results # profile = profile_gen(seed=1) profile = profile_gen(seed=None) @@ -93,11 +93,11 @@ import functools import itertools import inspect import logging -from operator import attrgetter +from operator import attrgetter, itemgetter from collections.abc import Iterable, Mapping -from lisa.monad import StateMonad -from lisa.utils import Loggable +from lisa.monad import StateDiscard +from lisa.utils import Loggable, deprecate class RetryException(Exception): @@ -110,20 +110,33 @@ class RetryException(Exception): pass +class _Retrier: + def __init__(self, cond): + self.cond = cond + + def __await__(self): + if self.cond: + return + else: + raise RetryException() + # Ensures __await__ is a generator function + yield + + def retry_until(cond): """ - If ``cond`` is ``True``, signify to the :class:`lisa.fuzz.Gen` monad to - retry the computation. This is used to enforce constraints on the output. + Returns an awaitable that will signify to the :class:`lisa.fuzz.Gen` monad + to retry the computation until ``cond`` is ``True``. This is used to + enforce arbitrary constraints on generated data. .. note:: If possible, it's a better idea to generate the data in a way that satisfy the constraints, as retrying can happen an arbitrary number of time and thus become quite costly. """ - if not cond: - raise RetryException() + return _Retrier(cond) -class Gen(StateMonad, Loggable): +class GenMonad(StateDiscard, Loggable): """ Random generator monad inspired by Haskell's QuickCheck. """ @@ -137,6 +150,15 @@ class Gen(StateMonad, Loggable): @classmethod def make_state(cls, *, rng=None, seed=None): + """ + Initialize the RNG state with either an rng or a seed. + + :param seed: Seed to initialize the :class:`random.Random` instance. + :type seed: object + + :param rng: Instance of RNG. + :type rng: random.Random + """ return cls._State( rng=rng or random.Random(seed), ) @@ -146,12 +168,14 @@ class Gen(StateMonad, Loggable): return f'{self.__class__.__qualname__}({name})' @classmethod - def _wrap_coroutine_f(cls, f): - @functools.wraps(f) + def _decorate_coroutine_function(cls, f): + _f = super()._decorate_coroutine_function(f) + + @functools.wraps(_f) async def wrapper(*args, **kwargs): for i in itertools.count(1): try: - x = await f(*args, **kwargs) + x = await _f(*args, **kwargs) except RetryException: continue else: @@ -159,12 +183,41 @@ class Gen(StateMonad, Loggable): val = str(x) sep = '\n' + ' ' * 4 val = sep + val.replace('\n', sep) + '\n' if '\n' in val else val + ' ' - cls.get_logger().debug(f'Drawn {val}{trials}from {f.__qualname__}') + cls.get_logger().debug(f'Drawn {val}{trials}from {_f.__qualname__}') return x return wrapper +class Gen: + def __init__(self, *args, **kwargs): + self._action = GenMonad(*args, **kwargs) + + def __await__(self): + return (yield from self._action.__await__()) + + @classmethod + @deprecate(deprecated_in='2.0', removed_in='3.0', replaced_by=GenMonad.do, + msg='Note that GenMonad.do() will not automatically await on arguments if they are Gen instances, this must be done manually.', + ) + def lift(cls, f): + + @GenMonad.do + @functools.wraps(f) + async def wrapper(*args, **kwargs): + args = [ + (await arg) if isinstance(arg, cls) else arg + for arg in args + ] + kwargs = { + k: (await v) if isinstance(v, cls) else v + for k, v in kwargs.items() + } + return await f(*args, **kwargs) + + return wrapper + + class Choices(Gen): """ Randomly choose ``n`` values among ``xs``. @@ -195,6 +248,7 @@ class Choices(Gen): def __str__(self): return f'{self.__class__.__qualname__}({self._xs_str})' + class Set(Choices): """ Same as :class:`lisa.fuzz.Choices` but returns a set. -- GitLab From 799da68a4ced6c11ad4efda6e896efb4dcee9e06 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 23 Feb 2022 12:58:42 +0000 Subject: [PATCH 09/11] lisa.wlgen.rta: Add async API FEATURE Add an async API to build wlgen rta tasks. The new API is based on the previous one and its sole purpose is to provide a more intuitive front end by somewhat using Python as a DSL. The new API is not more or less powerful than the old one, and both can be used together. --- lisa/wlgen/rta.py | 279 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 269 insertions(+), 10 deletions(-) diff --git a/lisa/wlgen/rta.py b/lisa/wlgen/rta.py index 0dda68469..fdd21754c 100644 --- a/lisa/wlgen/rta.py +++ b/lisa/wlgen/rta.py @@ -49,7 +49,10 @@ The most important classes are: A typical workload would be created this way:: - from lisa.wlgen.rta import RTA, RTAPhase, PeriodicWload, SleepWload + from lisa.wlgen.rta import RTA, RTAPhase, PeriodicWload, RunWload, SleepWload, override, delete, WithProperties, task_factory, RTAConf + from lisa.fuzz import Float, Int + from lisa.platforms.platinfo import PlatformInfo + from lisa.target import Target task = ( # Phases can be added together so they will be executed in order @@ -73,6 +76,38 @@ A typical workload would be created this way:: ) ) + # Alternatively, an async API is available to define tasks. + # Each workload awaited will become a new phase, and WithProperties async + # context manager can be used to set properties for multiple such phases. + # + # Additionally, the random data generators from lisa.fuzz can be used as well. + + @task_factory + async def make_task(run_duration): + async with WithProperties(name='first', uclamp=(256, 512)): + await RunWload(run_duration) + + # Create a bounded random value + sleep_duration = await Float(min_=1, max_=5) + sleep_duration = round(sleep_duration, 2) + + async with WithProperties(name='second'): + # We could await once on each workload, but that would lead to 2 phases + # instead of one. Whether this is desired or an issue depends on the + # use-case. + await ( + SleepWload(sleep_duration) + + PeriodicWload( + duty_cycle_pct=20, + period=16e-3, + duration=2, + ) + ) + + # "seed" is used to initialize the random number generator. If not + # provided, the seed will be random. + task = make_task(run_duration=1)(seed=1) + # Important note: all the classes in this module are immutable. Modifying # attributes is not allowed, use the RTAPhase.with_props() if you want to # get a new object with updated properties. @@ -85,11 +120,16 @@ A typical workload would be created this way:: # If you want to set the clamp and override any existing value rather than # combining it, you can use the override() function - task = task.with_props(uclamp=override((300, 800))) + task = task.with_props(uclamp=override((400, 700))) # Similarly, you can delete any property that was already set with delete() task = task.with_props(uclamp=delete()) + + ################################################### + # Alternative 1: create a simple profile and run it + ################################################### + # Connect to a target target = Target.from_default_conf() @@ -103,6 +143,38 @@ A typical workload would be created this way:: with wload: wload.run() + + ################################################### + # Alternative 2: a more complex example generating multiple tasks and dumping a JSON. + ################################################### + + @task_factory + async def make_profile(plat_info, **kwargs): + nr_tasks = await Int(1, plat_info['cpus-count']) + + profile = {} + for i in range(nr_tasks): + async with WithProperties(name=f'task{i}'): + profile[f'task{i}'] = await make_task(**kwargs) + + # If we return anything else than None, the return value will not be + # replaced by the phase defined in the function. + return profile + + + plat_info = PlatformInfo.from_yaml_map('./doc/traces/plat_info.yml') + + # Display a few randomly generated tasks + for _ in range(2): + # When called, profile_gen() will create a random profiles + profile_gen = make_profile(plat_info, run_duration=1) + + # seed (or rng) can be fixed for reproducible results + # profile = profile_gen(seed=1) + profile = profile_gen(seed=None) + + conf = RTAConf.from_profile(profile, plat_info=plat_info) + print(conf.json) """ import abc @@ -117,7 +189,7 @@ import re import weakref from collections import OrderedDict from collections.abc import Iterable, Mapping, Callable -from itertools import chain, product, starmap +from itertools import chain, product, starmap, islice from operator import itemgetter from shlex import quote from statistics import mean @@ -153,6 +225,8 @@ from lisa.utils import ( ) from lisa.wlgen.workload import Workload from lisa.conf import DeferredValueComputationError +from lisa.monad import StateDiscard, TransformerStack +from lisa.fuzz import GenMonad def _to_us(x): @@ -2480,6 +2554,9 @@ class WloadPropertyBase(ConcretePropertyBase): def val(self): return self + def __await__(self): + return (yield from _wload_action(self).__await__()) + def __add__(self, other): """ Adding two workloads together concatenates them. @@ -3153,15 +3230,15 @@ class _RTAPhaseBase: docstring = inspect.getdoc(cls) if docstring: cls.__doc__ = docstring.format( - prop_kwargs=cls._get_rst_prop_kwargs_doc() + prop_kwargs=cls._get_rst_prop_kwargs_doc(param_prefix='prop_') ) super().__init_subclass__(**kwargs) @classmethod - def _get_rst_prop_kwargs_doc(cls): + def _get_rst_prop_kwargs_doc(cls, param_prefix=''): def make(key, cls): - param = f'prop_{key}' + param = f'{param_prefix}{key}' doc, type_ = cls._get_cls_doc() fst = f':param {param}: {doc}' snd = f':type {param}: {type_}' @@ -3175,6 +3252,9 @@ class _RTAPhaseBase: return '\n\n'.join(starmap(make, sorted(properties.items()))) + def __await__(self): + return (yield from _phase_action(self).__await__()) + class RTAPhaseBase(_RTAPhaseBase, SimpleHash, Mapping, abc.ABC): """ @@ -3216,11 +3296,15 @@ class RTAPhaseBase(_RTAPhaseBase, SimpleHash, Mapping, abc.ABC): """ Return a cloned instance with the properties combined with the given ``properties`` using :meth:`RTAPhaseProperties.__and__` (``&``). The - ``properties`` parameter is the left operand. + ``properties`` parameter is the left operand. If ``properties`` is + ``None``, just return the phase itself. """ - new = copy.copy(self) - new.properties = properties & new.properties - return new + if properties is None: + return self + else: + new = copy.copy(self) + new.properties = properties & new.properties + return new def with_properties_map(self, properties, **kwargs): """ @@ -4362,4 +4446,179 @@ class RunAndSync(_RTATask): **kwargs, ) + +class RTAMonad(StateDiscard): + """ + Monad from derived from :class:`lisa.monad.StateDiscard` used to define + :class:`RTAPhaseBase` in a more natural way. + + If the function returns ``None``, the return value will be replaced by the + :class:`RTAPhaseBase` created while executing the function, with all the + currently active properties applied to it. + + .. seealso:: See the :func:`task_factory` decorator to be able to mix these + actions with the ones from :class:`lisa.fuzz.Gen`. + """ + class _State: + def __init__(self): + # Each level is a tuple of: + # 1. RTAPhaseProperties applied at this level, or None + # 2. A list of phases + # + # To compute the final phase object, we add the phases together and + # then apply the properties on the resulting object. + self.levels = [(None, [])] + + @property + def curr_level(self): + return self.levels[-1] + + def add_phase(self, phase): + self.curr_level[1].append(phase) + + def begin_prop(self, props): + self.levels.append((props, [])) + + def end_prop(self): + phase = self.merge_level() + self.levels.pop() + self.curr_level[1].append(phase) + return phase + + def merge_level(self): + """ + Compute the :class:`RTAPhaseBase` instance for that level with the + phases added so far in the current ``async with WithProperties``. + """ + props, level = self.curr_level + + if level: + phase = functools.reduce( + operator.add, + level, + ) + else: + phase = RTAPhase() + + return phase.with_phase_properties(props) + + @classmethod + def make_state(cls): + """ + Create a fresh instance of :class:`RTAMonad._State` + """ + return cls._State() + + @classmethod + def _decorate_coroutine_function(cls, f): + _f = super()._decorate_coroutine_function(f) + + @functools.wraps(_f) + async def wrapper(*args, **kwargs): + state = await RTAMonad.get_state() + + # For each user function call, we introduce a new level so that we + # can track easily the phases that are created under it. + state.begin_prop(RTAPhaseProperties(None)) + + x = await _f(*args, **kwargs) + + phase = state.end_prop() + + # Replace None return value by the phase created during the call to + # this function + if x is None: + # Compute a phase with all the currently active properties + # applied. This is used to provide a value to user-defined + # functions returning None. + return functools.reduce( + lambda phase, prop: phase.with_phase_properties(prop), + reversed( + list(map( + itemgetter(0), + state.levels, + )) + ), + phase, + ) + else: + return x + + return wrapper + + +class _RTAMonadStack(TransformerStack(GenMonad, RTAMonad)): + pass + + +def _compute(self, rng=None, seed=None): + return RTAMonad.__call__( + GenMonad.__call__( + self, + rng=rng, + seed=seed, + ), + ) +_RTAMonadStack._TOP.__call__ = _compute + + +def task_factory(f): + """ + Coroutine function decorator allowing to create tasks using actions from both + :class:`RTAMonad` and :class:`lisa.fuzz.GenMonad`. + + Calling the decorated function will result in another callable that can be + called once with: + + * ``seed``: Seed to use to automatically initialize a :class:`random.Random`. + * ``rng``: Alternatively, an existing instance of + :class:`random.Random` to use. + + If the user-defined coroutine function returns ``None``, the return value + will be replaced by an :class:`RTAPhaseBase` representing all the phases + that got added while running the function, on which the current active + properties set with :class:`WithProperties` are applied. + + .. seealso:: :mod:`lisa.wlgen.rta` for an example. + """ + return _RTAMonadStack.do(f) + + +def _action_from_f(f): + @functools.wraps(f) + def wrapper(state): + f(state) + return (None, state) + return RTAMonad.from_f(wrapper) + + +def _phase_action(phase): + return _action_from_f(lambda state: state.add_phase(phase)) + + +def _wload_action(wload): + return _phase_action(RTAPhase(prop_wload=wload)) + + +class WithProperties: + """ +Asynchronous context manager used to set properties on the enclosed phases. + +{prop_kwargs} + """ + def __init__(self, **kwargs): + self.props = RTAPhaseProperties.from_polymorphic(kwargs) + + async def __aenter__(self): + await _action_from_f(lambda state: state.begin_prop(self.props)) + return + + async def __aexit__(self, exc_type, exc_value, traceback): + await _action_from_f(lambda state: state.end_prop()) + return + +WithProperties.__doc__ = WithProperties.__doc__.format( + prop_kwargs=_RTAPhaseBase._get_rst_prop_kwargs_doc(), +) + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab -- GitLab From 835df57ff2940e0038955c534570e17b2f176822 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 30 Mar 2022 15:17:47 +0100 Subject: [PATCH 10/11] lisa.fuzz: Rewrite docstring example Simplify the docstring example to remove mentions of lisa.wlgen.rta. This part is now covered by lisa.wlgen.rta itself so there is no need to duplicate it. --- lisa/fuzz.py | 57 +++++++++------------------------------------------- 1 file changed, 9 insertions(+), 48 deletions(-) diff --git a/lisa/fuzz.py b/lisa/fuzz.py index fcd012800..763ab5b6f 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -20,21 +20,18 @@ Fuzzing API to build random constrained values. .. note:: The following example shows a direct use of the :class:`Gen` monad, but be aware that :mod:`lisa.wlgen.rta` API allows mixing both :class:`Gen` - and RTA DSL into the same coroutine function. + and RTA DSL into the same coroutine function using + :func:`lisa.wlgen.rta.task_factory`. **Example**:: - import operator - import functools - from lisa.platforms.platinfo import PlatformInfo - from lisa.wlgen.rta import RTAPhase, RTAConf, PeriodicWload from lisa.fuzz import GenMonad, Choice, Int, Float, retry_until - # The function must be decorated with GenMonad.do() so that "await" gains its - # special meaning. + # The function must be decorated with GenMonad.do() so that "await" gains + # its special meaning. @GenMonad.do - async def make_task(duration=None): + async def make_data(duration=None): # Draw a value from an iterable. period = await Choice([16e-3, 8e-3]) nr = await Choice(range(1, 4)) @@ -44,48 +41,12 @@ Fuzzing API to build random constrained values. # function will run again until the condition is true. await retry_until(0 < nr <= 2) - phase = functools.reduce( - operator.add, - [ - RTAPhase( - prop_wload=PeriodicWload( - duty_cycle_pct=await Choice(range(100)), - period=period, - duration=duration, - ), - ) - for i in range(nr) - ] - ) - - return phase - - @GenMonad.do - async def make_profile(plat_info, **kwargs): - nr_tasks = await Int(1, plat_info['cpus-count']) - - profile = {} - for i in range(nr_tasks): - profile[f'task{i}'] = await make_task(**kwargs) - return profile - - - def main(): - plat_info = PlatformInfo.from_yaml_map('./doc/traces/plat_info.yml') - - # Display a few randomly generated tasks - for _ in range(2): - # When called, profile_gen() will create a random profiles - profile_gen = make_profile(plat_info, duration=1) - - # seed (or rng) can be fixed for reproducible results - # profile = profile_gen(seed=1) - profile = profile_gen(seed=None) + return (nr, duration, period) - conf = RTAConf.from_profile(profile, plat_info=plat_info) - print(conf.json) + # seed (or rng) can be fixed for reproducible results + data = make_data(duration=42)(seed=1) + print(data) - main() """ import random -- GitLab From d607fc8cbf71783b81af44e715342ed26627a175 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 30 Mar 2022 16:23:58 +0100 Subject: [PATCH 11/11] pytest.ini: Ignore nest_asyncio warning Ignore warning reported at: https://github.com/erdewit/nest_asyncio/issues/70 --- pytest.ini | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index d42174104..97071038d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,8 +5,10 @@ testpaths=tests filterwarnings = error ignore::DeprecationWarning:past.builtins.misc - # Avoid: - # .lisa-venv-3.9/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/version.py:111: in __init__ - # warnings.warn( - # DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release - ignore::DeprecationWarning:pkg_resources.*: + # Avoid: + # .lisa-venv-3.9/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/version.py:111: in __init__ + # warnings.warn( + # DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release + ignore::DeprecationWarning:pkg_resources.*: + # Ignore this warning: https://github.com/erdewit/nest_asyncio/issues/70 + ignore::DeprecationWarning:nest_asyncio.*: -- GitLab