diff --git a/lisa/fuzz.py b/lisa/fuzz.py index 4f91b61d8c5e7f670291fa874f185e6f4bc098a9..763ab5b6f76335b8bbf0aeea6a139e1f48b2a2b9 100644 --- a/lisa/fuzz.py +++ b/lisa/fuzz.py @@ -18,21 +18,20 @@ """ Fuzzing API to build random constrained values. -**Example**:: +.. note:: The following example shows a direct use of the :class:`Gen` monad, + but be aware that :mod:`lisa.wlgen.rta` API allows mixing both :class:`Gen` + and RTA DSL into the same coroutine function using + :func:`lisa.wlgen.rta.task_factory`. - import operator - import functools +**Example**:: from lisa.platforms.platinfo import PlatformInfo - from lisa.wlgen.rta import RTAPhase, RTAConf, PeriodicWload - from lisa.fuzz import Gen, Choice, Int, Float, retry_until - - # The function must be decorated with Gen.lift() so that "await" gains its - # special meaning. In addition to that, parameters are automatically awaited if - # they are an instance of Gen, and the return value is automatically promoted - # to an instance of Gen if it is not already. - @Gen.lift - async def make_task(duration=None): + from lisa.fuzz import GenMonad, Choice, Int, Float, retry_until + + # The function must be decorated with GenMonad.do() so that "await" gains + # its special meaning. + @GenMonad.do + async def make_data(duration=None): # Draw a value from an iterable. period = await Choice([16e-3, 8e-3]) nr = await Choice(range(1, 4)) @@ -40,50 +39,14 @@ Fuzzing API to build random constrained values. # Arbitrary properties can be enforced. If they are not satisfied, the # function will run again until the condition is true. - retry_until(0 < nr <= 2) - - phase = functools.reduce( - operator.add, - [ - RTAPhase( - prop_wload=PeriodicWload( - duty_cycle_pct=await Choice(range(100)), - period=period, - duration=duration, - ), - ) - for i in range(nr) - ] - ) - - return phase + await retry_until(0 < nr <= 2) - @Gen.lift - async def make_profile(plat_info, **kwargs): - nr_tasks = await Int(1, plat_info['cpus-count']) + return (nr, duration, period) - profile = {} - for i in range(nr_tasks): - profile[f'task{i}'] = await make_task(**kwargs) - return profile + # seed (or rng) can be fixed for reproducible results + data = make_data(duration=42)(seed=1) + print(data) - - def main(): - plat_info = PlatformInfo.from_yaml_map('./doc/traces/plat_info.yml') - - # When called, profile_gen() will create a random profiles - profile_gen = make_profile(plat_info, duration=1) - - # Display a few randomly generated tasks - for _ in range(2): - # seed (or rng) can be fixed for reproducible results - # profile = profile_gen(seed=1) - profile = profile_gen(seed=None) - - conf = RTAConf.from_profile(profile, plat_info=plat_info) - print(conf.json) - - main() """ import random @@ -91,11 +54,11 @@ import functools import itertools import inspect import logging -from operator import attrgetter +from operator import attrgetter, itemgetter from collections.abc import Iterable, Mapping -from lisa.monad import StateMonad -from lisa.utils import Loggable +from lisa.monad import StateDiscard +from lisa.utils import Loggable, deprecate class RetryException(Exception): @@ -108,67 +71,112 @@ class RetryException(Exception): pass +class _Retrier: + def __init__(self, cond): + self.cond = cond + + def __await__(self): + if self.cond: + return + else: + raise RetryException() + # Ensures __await__ is a generator function + yield + + def retry_until(cond): """ - If ``cond`` is ``True``, signify to the :class:`lisa.fuzz.Gen` monad to - retry the computation. This is used to enforce constraints on the output. + Returns an awaitable that will signify to the :class:`lisa.fuzz.Gen` monad + to retry the computation until ``cond`` is ``True``. This is used to + enforce arbitrary constraints on generated data. .. note:: If possible, it's a better idea to generate the data in a way that satisfy the constraints, as retrying can happen an arbitrary number of time and thus become quite costly. """ - if not cond: - raise RetryException() + return _Retrier(cond) -class Gen(StateMonad, Loggable): +class GenMonad(StateDiscard, Loggable): """ Random generator monad inspired by Haskell's QuickCheck. """ def __init__(self, f, name=None): - log_level = logging.DEBUG - logger = self.logger - if logger.isEnabledFor(log_level): - caller_info = inspect.stack()[2] - else: - caller_info = None + self.name = name or f.__qualname__ + super().__init__(f) - @functools.wraps(f) - def wrapper(state): + class _State: + def __init__(self, rng): + self.rng = rng + + @classmethod + def make_state(cls, *, rng=None, seed=None): + """ + Initialize the RNG state with either an rng or a seed. + + :param seed: Seed to initialize the :class:`random.Random` instance. + :type seed: object + + :param rng: Instance of RNG. + :type rng: random.Random + """ + return cls._State( + rng=rng or random.Random(seed), + ) + + def __str__(self): + name = self.name or self._f.__qualname__ + return f'{self.__class__.__qualname__}({name})' + + @classmethod + def _decorate_coroutine_function(cls, f): + _f = super()._decorate_coroutine_function(f) + + @functools.wraps(_f) + async def wrapper(*args, **kwargs): for i in itertools.count(1): try: - x = f(state) + x = await _f(*args, **kwargs) except RetryException: continue else: trials = f'after {i} trials ' if i > 1 else '' - if caller_info: - info = f' ({caller_info.filename}:{caller_info.lineno})' - else: - info = '' - val, _ = x - val = str(val) + val = str(x) sep = '\n' + ' ' * 4 val = sep + val.replace('\n', sep) + '\n' if '\n' in val else val + ' ' - self.logger.log(log_level, f'Drawn {val}{trials}from {self}{info}') + cls.get_logger().debug(f'Drawn {val}{trials}from {_f.__qualname__}') return x - self.name = name - super().__init__(wrapper) + return wrapper - class _STATE: - def __init__(self, rng): - self.rng = rng + +class Gen: + def __init__(self, *args, **kwargs): + self._action = GenMonad(*args, **kwargs) + + def __await__(self): + return (yield from self._action.__await__()) @classmethod - def make_state(cls, *, rng=None, seed=None): - return cls._STATE( - rng=rng or random.Random(seed), - ) + @deprecate(deprecated_in='2.0', removed_in='3.0', replaced_by=GenMonad.do, + msg='Note that GenMonad.do() will not automatically await on arguments if they are Gen instances, this must be done manually.', + ) + def lift(cls, f): - def __str__(self): - name = self.name or self._f.__qualname__ - return f'{self.__class__.__qualname__}({name})' + @GenMonad.do + @functools.wraps(f) + async def wrapper(*args, **kwargs): + args = [ + (await arg) if isinstance(arg, cls) else arg + for arg in args + ] + kwargs = { + k: (await v) if isinstance(v, cls) else v + for k, v in kwargs.items() + } + return await f(*args, **kwargs) + + return wrapper class Choices(Gen): @@ -201,6 +209,7 @@ class Choices(Gen): def __str__(self): return f'{self.__class__.__qualname__}({self._xs_str})' + class Set(Choices): """ Same as :class:`lisa.fuzz.Choices` but returns a set. diff --git a/lisa/monad.py b/lisa/monad.py index 6e66ee11c7fc883ba7020399ce735aebb3ee8c0f..f84e2d6bdb78fca90dd5e40a09e4b53b2b80be92 100644 --- a/lisa/monad.py +++ b/lisa/monad.py @@ -58,171 +58,980 @@ This allow composing lifted functions easily """ import abc +import itertools import functools import inspect +import operator +import asyncio +from operator import attrgetter +from functools import partial +from weakref import WeakKeyDictionary +import nest_asyncio +nest_asyncio.apply() -class StateMonad(abc.ABC): - """ - The state monad. +from lisa.utils import memoized, foldr, instancemethod - :param f: Callable that takes the state as parameter and returns an - instance of the monad. - :type f: collections.abc.Callable +class _MonadBase: + """ + Abstract Base Class parametrized by a type T, + like a container of some sort. """ - def __init__(self, f): - self._f = f + @abc.abstractclassmethod + def pure(cls, x): + """ + Takes a value of type T and turns it into a "monadic value" of type Monad[T]. + """ + pass + + @abc.abstractmethod + def bind(self, continuation): + """ + Takes a monadic value Monad[A], a function that takes an A and returns Monad[B], and returns a Monad[B]. + + .. note:: It is allowed to return a :class:`_TailCall` instance. + """ + pass + + @abc.abstractmethod + def map(self, f): + """ + Takes a monadic value Monad[A], a function that takes an A and returns B, and returns a Monad[B]. + """ + pass + + @abc.abstractmethod + def join(self): + """ + Takes a monadic value Monad[Monad[A]], and returns a Monad[A]. + """ + pass def __await__(self): - # What happens here is executed as if the code was inlined in the - # coroutine's body ("await x" is actually equivalent to - # "yield from x"): - # 1. "yield self" allows to relinquish control back to the loop, - # providing the monadic value that was awaited on by the user code. - # 2. Returning the result of the yield allows the loop to inject any - # value it sees fit using coro.send(). return (yield self) + +class Monad(_MonadBase): + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + + def wrap_bind(f): + @functools.wraps(f) + def wrapper(self, continuation): + # Ensure that all the parameters are resolved before use, so + # that implementations don't have to be sprinkled with + # _TailCall.run() + run = _TailCall.run + return f( + run(self), + _TailCall._trampoline(run(continuation)) + ) + return wrapper + + cls.bind = wrap_bind(cls.bind) + + @instancemethod + def map(cls, self, f): + return cls.bind(self, lambda x: self.pure(f(x))) + + @instancemethod + def join(cls, self): + return cls.bind(self, lambda x: x) + + +# Inherit from _MonadBase directly to avoid the automatic trampoline on bind() +# that would lead to infinite recursion +class _Identity(_MonadBase): + """ + This monad is private as it is not "proper". + + It does not actually wrap the value, which means that the only way e.g. + bind() can be called is when referring to the class directly, since no + instance of :class:`_Identity` will be created. + + This does have the advantage of avoiding unnecessary boxing, which improves + performance and also avoids a layer of indirection when inspecting the + resulting values. + """ + def __new__(cls, x): + return x + + def bind(self, continuation): + return _TailCall(_TailCall.run(continuation), _TailCall.run(self)) + + @staticmethod + def pure(x): + return x + + def map(self, f): + return f(self) + + def join(self): + return self + + +class _TailCall: + """ + Represents a function call, to be returned so that an enclosing loop + can execute the call, rather than using code itself:: + + return f(x) + # becomes + return _TailCall(f, x) + """ + __slots__ = ('f', '__weakref__') + + def __init__(self, f, *args, **kwargs): + self.f = functools.partial(f, *args, **kwargs) + + def run(x): + """ + Evaluates a chain of _TailCall until it's all evaluated. + """ + # Do not use self.__class__ so that _TailCall.run() can be called on + # arbitrary objects. + while isinstance(x, _TailCall): + x = x.f() + + # Recursive implementation, sometimes useful for debugging + # if isinstance(x, _TailCall): + # x = _TailCall.run(x.f()) + return x + + @classmethod + def trampoline(cls, f): + """ + Decorator to allow a function to return :class:`_TailCall`. + """ + return functools.wraps(f)(cls._trampoline(f)) + + @classmethod + def _trampoline(cls, f): + run = cls.run + def wrapper(*args, **kwargs): + return run( + f(*args, **kwargs) + ) + return wrapper + + +class AlreadyCalledError(Exception): + """ + Exception raised by :class:`_CallOnce` when the wrapped function has already + been called once. + """ + pass + + +class _CallOnce: + """ + Only allow calling the wrapped function once. + + This prevents accidentally calling ``coro.send(...)`` multiple times, + wrongly expecting it to resume twice from the same point. + """ + __slots__ = ('f', '__weakref__') + + def __init__(self, f): + self.f = f + + @staticmethod + def _raise(*args, **kwargs): + raise AlreadyCalledError(f'This function cannot be called more than once') + def __call__(self, *args, **kwargs): - state = self.make_state(*args, **kwargs) - x, _ = self._f(state) + x = self.f(*args, **kwargs) + self.f = self._raise return x - def __init_subclass__(cls, **kwargs): - # The one inheriting directly from StateMonad is the base of the - # hierarchy - if StateMonad in cls.__bases__: - cls._MONAD_BASE = cls - super().__init_subclass__(**kwargs) + +def _build_stack(stack): + *stack, init = tuple(stack) + if stack: + return foldr( + lambda trans, base: trans._rebase_on(base), + stack, + init, + ) + else: + return init + + +class MonadTrans(Monad, abc.ABC): + """ + Base class for monad transformers. + + Heavily inspired by transformers as defined by: + https://hackage.haskell.org/package/transformers + + And stack manipulation inspired by: + https://hackage.haskell.org/package/mmorph + """ + _BASE = _Identity @classmethod - def from_f(cls, *args, **kwargs): + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + cls._UNAPPLIED = cls + """ + Unapplied monad transformer, i.e. applied to (a subclass of) :class:`_Identity`. + """ + + # Stacks of more than one transformer will override that + cls._TRANSFORMER_STACK = (cls,) """ - Build an instance of the monad from a state transformation function. - The callback takes the current state as parameter and returns - ``tuple(value, new_state)``. + Stack of transformers. + """ + + cls._TRANSFORM_CACHE = WeakKeyDictionary() + """ + Cache of the transformer applied to other base monads. + + .. note:: This cache is necessary for functional purposes, so that + rebasing a transformer on the same monad twice gives back the same + class, which can therefore be used as key in mappings. """ - return cls._MONAD_BASE(*args, **kwargs) @abc.abstractclassmethod - def make_state(cls, *args, **kwargs): + def lift(cls, m): """ - Create the state from user-defined parameters. This is used by - :meth:`lisa.monad.StateMonad.__call__` in order to initialize the - state. + Lift a monadic value ``m`` by one level in the stack, i.e.: + Given a stack for 3 transformers ``T1(T2(T3(Identity)))``, + a value ``m = T3(Identity).pure(42)``. we have + ``T2.lift(m) == T2(T3(Identity)).pure(42)``. + + .. seealso:: ``lift`` as defined in https://hackage.haskell.org/package/transformers + """ + pass + + @abc.abstractclassmethod + def hoist(cls, self, nat): + """ + Lift a monadic value ``m`` by one level in the stack, i.e.: + Given a stack for 3 transformers ``T1(T2(T3(Identity)))``, + a value ``m = T2(Identity).pure(42)``. we have + ``T2.hoist(m, T3.pure) == T2(T3(Identity)).pure(42)``. + + In other words, it allows adding a level "from below", whereas ``lift`` + adds a level "from above". It's similar to ``map``, except that instead + of traversing all the nested functor layers, it stops at the first one. + + :param self: Monadic value to hoist. + :type self: lisa.monad.Monad + + :param nat: Natural transform. i.e. a morphism from ``Monad1[A]`` to + ``Monad2[A]`` that obeys certain laws. + :type nat: collections.abc.Callable + + .. seealso:: ``hoist`` as defined in https://hackage.haskell.org/package/mmorph + + .. note:: Note for implementers: A monad transformers ``t m a`` (``t`` + is the transformer HKT, ``m`` is the base monad and ``a`` is the + "contained type) usually ends up containing an "m (f a)" (``f`` being + some kind of functor). For example, ``MaybeT`` in Haskell + (:class:`Option` here) is more or less defined as ``data MaybeT m a = + MaybeT (m (Maybe a))``. What the ``hoist`` implementation must do is to + "rebuild" a value with a call to ``nat()`` around the ``m (...)`` part. + For ``MaybeT``, this gives + ``hoist nat (MaybeT (m (Maybe a))) = MaybeT(nat(m (Maybe a)))``. """ pass @classmethod def pure(cls, x): """ - Lift a value in the state monad, i.e. create a monad instance with a - function that returns the value and the state unchanged. + Turn a regular value of type ``A`` into a monadic value of type ``Monad[A]``. """ - return cls.from_f(lambda state: (x, state)) + return cls.lift(cls._BASE.pure(x)) @classmethod - def lift(cls, f): + def _rebase_on(cls, monad): """ - Decorator used to lift a function into the monad, such that it can take - monadic parameters that will be evaluated in the current state, and - returns a monadic value as well. + Rebase the transformer onto a new base monad and returns it. """ + key = monad + # "unapply" the transformer, since we are rebuilding the stack from the + # bottom to the top + cls = cls._UNAPPLIED - cls = cls._MONAD_BASE + # This cache is necessary for functional purposes, it's not only an + # optimisation. It guarantees that rebasing the same transformer on the + # same monad will give the same class, so it can be used as key in + # dictionaries. + try: + return cls._TRANSFORM_CACHE[key] + except KeyError: + transformed = cls._do_rebase_on(monad) + cls._TRANSFORM_CACHE[key] = transformed + return transformed - def run(_f, args, kwargs): - call = lambda: _f(*args, **kwargs) - x = call() - if inspect.iscoroutine(x): - def body(state): - if inspect.getcoroutinestate(x) == inspect.CORO_CLOSED: - _x = call() - else: - _x = x + @classmethod + def _do_rebase_on(cls, base): + class _Monad(cls): + _BASE = base - next_ = lambda: _x.send(None) - while True: - try: - future = next_() - except StopIteration as e: - val = e.value - break - else: - assert isinstance(future, cls) - try: - val, state = future._f(state) - except Exception as e: - # We need an intermediate variable here, since - # "e" is not really bound in this scope. - excep = e - next_ = lambda: _x.throw(excep) - else: - next_ = lambda: _x.send(val) - - if isinstance(val, cls): - return val._f(state) - else: - return (val, state) + _Monad._UNAPPLIED = cls + _Monad.__name__ = cls.__name__ + _Monad.__qualname__ = cls.__qualname__ + return _Monad + + @staticmethod + def _decorate_coroutine_function(f): + """ + Called by :meth:`MonadTrans.do` to wrap the user-provided coroutine function, i.e. ``async def`` functions. + """ + return f + + @classmethod + def do(cls, f): + """ + Decorate a coroutine function so that ``awaits`` gains the powers of the monad. + + .. seealso:: This decorator is very similar to the do-notation in Haskell. + """ + if not inspect.iscoroutinefunction(f): + raise TypeError(f'{cls.__qualname__}.do() can only decorate generator functions, i.e. defined with "async def"') + + do_cls = cls + stack = cls._TRANSFORMER_STACK + + # The direct "await" at our level are expected to yield instances of + # the "bare" transformers (i.e. applied to _Identity), or an instance + # of a substack + substacks = { + cls._UNAPPLIED_TOP: cls + for cls in stack + if issubclass(cls, _MonadTransStack) + } + + transformers = { + cls._UNAPPLIED: cls + for cls in stack + } + + def resolve(cls): + try: + return substacks[cls] + except KeyError: + try: + return transformers[cls._UNAPPLIED] + except KeyError: + _stack = ', '.join( + cls.__qualname__ + for cls in stack + ) + raise TypeError(f'Could not find the transformer {cls.__qualname__} in the stack {do_cls.__name__}. Only the following transformers are allowed at that level: {_stack}') + + lifters = { + cls: tuple(reversed(tuple(itertools.takewhile( + lambda x: x is not cls, + stack + )))) + for cls in stack + } - val = cls.from_f(body, name=f.__qualname__) + def make_hoister(stack): + try: + cls1, cls2 = stack + except ValueError: + return lambda x: x else: - if isinstance(x, cls): - val = x - else: - val = cls.pure(x) + return lambda x: cls1.hoist(x, cls2.pure) - return val + hoisters = { + cls: make_hoister( + itertools.islice( + itertools.dropwhile( + lambda x: x is not cls, + stack + ), + None, + 2, + ) + ) + for cls in stack + } - @functools.wraps(f) + def convert(action): + if action.__class__ is stack[0]: + return action + else: + action_cls = resolve(action.__class__) + hoisted = hoisters[action_cls](action) + + lifted = hoisted + for _cls in lifters[action_cls]: + lifted = _cls.lift(lifted) + + return lifted + + def decorate(f): + for trans in reversed(stack): + f = trans._decorate_coroutine_function(f) + return f + + _f = decorate(f) + + @functools.wraps(_f) def wrapper(*args, **kwargs): - async def _f(*args, **kwargs): - args = [ - (await arg) if isinstance(arg, cls) else arg - for arg in args - ] - kwargs = { - k: (await v) if isinstance(v, cls) else v - for k, v in kwargs.items() - } - return run(f, args, kwargs) - return run(_f, args, kwargs) + coro = _f(*args, **kwargs) + def next_(x): + run = functools.partial(coro.send, x) + + while True: + try: + action = run() + except StopIteration as e: + x = e.value + if isinstance(x, Monad): + return x + else: + return cls.pure(x) + else: + try: + # Automatically lift actions based on their class, + # which indicates their position in the transformer + # stack. + action = convert(action) + except Exception as e: + run = functools.partial(coro.throw, e) + else: + return action.bind(_CallOnce(next_)) + + return _TailCall.run(next_(None)) + + wrapper.monad = cls return wrapper + +class _CloneIdentity(_Identity): @classmethod - def get_state(cls): + def clone(cls): + # Derive from _CloneIdentity to avoid long chains of inheritance, which + # would prevent garbage collection classes and also slow down attribute + # resolution. + class new(_CloneIdentity): + pass + return new + + +class _MonadTransStack(MonadTrans): + """ + Stack of monad transformers. + + * The ``_TOP`` attribute points at the top of the stack, which can the be + unfolded by accessing ``_BASE``. + + * ``_UNAPPLIED_TOP`` is the "original" ``_TOP``, i.e. before the stack is + rebased using :meth:`MonadTrans.rebase_on`. + + .. note:: Each stack is based on its own copy of :class:`_CloneIdentity`, + so that all the transformers composing the stack are independent from + any other transformer. This allows natural mixing of stacks, as there + is never any ambiguity on what stack a given monadic value belongs to. + """ + _BASE = _CloneIdentity + + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + + def get_stack(trans): + if issubclass(trans, MonadTrans): + yield trans + yield from get_stack(trans._BASE) + else: + return + + _base = cls._BASE + + # Update the base we will build _TOP on, but do not update _BASE + # itself. This allows subclasses of non-rebased classes to all have + # _BASE==_CloneIdentity, so they all get independent _TOP stack. + if _base is _CloneIdentity: + # Ensure each stack is independent by providing a separate + # _Identity base = _base.clone() if _base is _CloneIdentity else + # _base + base = _base.clone() + else: + base = _base + + # Rebuild the stack on top of the new base + top = _build_stack( + itertools.chain( + get_stack(cls._UNAPPLIED_TOP), + (base,) + ) + ) + + # We are defining a new stack, not just rebasing an existing stack. + if _base is _CloneIdentity: + cls._UNAPPLIED_TOP = top + + cls._TOP = top + cls._TRANSFORMER_STACK = tuple(get_stack(top)) + + def bind(self, continuation): + # This looks like a dummy implementation, but is actually useful since + # it can be called as T.bind(m, f), regardless of the precise type of + # m. + return self.bind(continuation) + + @classmethod + def lift(cls, m): + # Lift a monadic value on the whole original stack (i.e. disregarding + # any other stack this transformer might be rebased on). + def lift(monad, unapplied, x): + if issubclass(unapplied, MonadTrans): + return monad.lift(lift(monad._BASE, unapplied._BASE, x)) + else: + return x + + return lift(cls._TOP, cls._UNAPPLIED_TOP, m) + + @instancemethod + def hoist(cls, self, nat): + def hoist(monad, unapplied, x): + return monad.hoist( + x, + ( + functools.partial( + hoist, + monad._BASE, unapplied._BASE, + ) + if issubclass(unapplied._BASE, MonadTrans) else + nat + ) + ) + + # Traverse the structure using the _UNAPPLIED_TOP, but build values of the + # correct type that reflects the actual structure of the value. + return hoist(cls._TOP, cls._UNAPPLIED_TOP, self) + + +def TransformerStack(*stack): + """ + Allows stacking together multiple :class:`MonadTrans`, e.g.:: + + class Stack(TransformerStack(T1, T2, T3)): + pass + + @Stack.do + async def foo(): + + # Any monadic value from the stack's direct components can be used. + await T1.pure(42) + await T2.pure(42) + await T3.pure(42) + + """ + + if len(set(stack)) != len(stack): + raise ValueError(f'Monad transformers can only appear once in any given stack') + else: + class Stack(_MonadTransStack): + _UNAPPLIED_TOP = _build_stack(stack) + + return Stack + + +class _Optional: + """ + Instances of this class represents either the absence of a value, or a + value. + """ + pass + + +class Some(_Optional): + """ + Wraps an arbitrary value to indicate its presence. + """ + __slots__ = ('x',) + + def __init__(self, x): + self.x = x + + def __repr__(self): + return f'{self.__class__.__qualname__}({self.x})' + + __str__ = __repr__ + + def __eq__(self, other): + assert type(self) is type(other) + return self.x == other.x + + +class _Nothing(_Optional): + """ + Do not make your own instances, use the ``Nothing`` singleton. + """ + def __repr__(self): + return 'Nothing' + + __str__ = __repr__ + + def __eq__(self, other): + return self is other + + +Nothing = _Nothing() +""" +Similar to :class:`Some` but indicating the absence of value. +""" + + +# Do that in a base class so that Option itself gets __init_subclass__ ran on +class _AddPureNothing: + @classmethod + def __init_subclass__(cls, *args, **kwargs): + super().__init_subclass__(*args, **kwargs) + # Specialize the _OPTION_NOTHING to the correct right _BASE. + # Pre-computing it allows sharing a single instance for all the cases + # where it is needed. + cls._PURE_NOTHING = cls._BASE.pure(Nothing) + + +class Option(MonadTrans, _AddPureNothing): + """ + Monad transformer that manipulates :class:`Some` and :attr:`Nothing`. + + :meth:`Option.bind` will short-circuit if :attr:`Nothing` is passed, much + like the Rust or Javascript ``?`` operator, or the ``MaybeT`` monad + transformer in Haskell. + """ + __slots__ = ('_x',) + + def __init__(self, x): + self._x = x + + @property + def x(self): + """ + Wrapped value, of type ``Base[_Optional[A]]`` with ``Base`` the base + monad of the transformer. """ - Returns a monadic value making the current state available. - To be used inside a lifted function using:: + # Ensure we only run the _TailCall once, since it will run a + # continuation + x = _TailCall.run(self._x) + self._x = x + return x + + def __repr__(self): + return f'Option({self.x})' + + __str__ = __repr__ + + def bind(self, continuation): + cls = self.__class__ + base = self._BASE + + def k(v): + if v is Nothing: + # Use pre-computed object + return cls._PURE_NOTHING + else: + return continuation(v.x)._x + + return cls(base.bind(self.x, k)) + + @classmethod + def lift(cls, m): + base = cls._BASE + return cls(base.map(m, Some)) + + @instancemethod + def hoist(cls, self, nat): + return cls(nat(self.x)) + + +class State(MonadTrans): + """ + Monad transformer analogous to Haskell's ``StateT`` transformer. + + It manipulates state-transforming functions of type ``state -> (value, + new_state)``. This allows simulating a global state, without actually + requiring one. + """ + __slots__ = ('_f',) + + def __init__(self, f): + self._f = f - state = await StateMonad.get_state() + @property + def f(self): """ - return cls.from_f(lambda state: (state, state)) + State-transforming function of type ``state -> (value, new_state)`` + """ + return _TailCall._trampoline(self._f) @classmethod - def set_state(cls, new_state): + def make_state(cls, x): """ - Returns a monadic value to set the current state. - To be used inside a lifted function using:: + Create an initial state. All the parameters of :meth:`State.__call__` + are passed to :meth:`State.make_state`. + """ + return x - await StateMonad.set_state(new_state) + def __call__(self, *args, **kwargs): + """ + Allow calling monadic values to run the state-transforming function, + with the initial state provided by :meth:`State.make_state`. """ - return cls.from_f(lambda state: (state, new_state)) + init = self.make_state(*args, **kwargs) + return self.f(init) + + def bind(self, continuation): + base = self._BASE + def k(res): + x, state = res + return continuation(x)._f(state) + + return self.__class__( + lambda state: base.bind(self.f(state), k) + ) @classmethod - def modify_state(cls, f): + def lift(cls, m): + base = cls._BASE + return cls( + lambda state: base.bind( + m, + lambda a: base.pure((a, state)) + ) + ) + + @instancemethod + def hoist(cls, self, nat): + return cls(lambda state: nat(self.f(state))) + + @classmethod + def from_f(cls, f): + """ + Build a monadic value out of a state modifying function of type + ``state -> (value, new_state)``. """ - Returns a monadic value to modify the current state. - To be used inside a lifted function using:: + base = cls._BASE + return cls(lambda state: base.pure(f(state))) - await StateMonad.modify_state(lambda state: new_state) + @classmethod + def get_state(cls): + """ + Returns a monadic value returning the current state. + """ + return cls.from_f(lambda state: (state, state)) + + @classmethod + def set_state(cls, new): + """ + Returns a monadic value setting the current state and returning the old + one. + """ + return cls.from_f(lambda state: (state, new)) + + @classmethod + def modify_state(cls, f): + """ + Returns a monadic value applying ``f`` on the current state, setting + the new state and then returning it. """ def _f(state): new_state = f(state) return (new_state, new_state) return cls.from_f(_f) + +class StateDiscard(State): + """ + Same as :class:`State` except that calling monadic values will return the + computed value instead of a tuple ``(value, state)``. + + This is useful for APIs where the final state is of no interest to the + user. + """ + def __call__(self, *args, **kwargs): + # We are part of a monad transformer stack, so we need to apply + # the function at the right level. + # super().__call__() returns a "m (a, s)", so we need to map on that + # monadic value to turn it into "m a". + return self._BASE.map( + super().__call__(*args, **kwargs), + operator.itemgetter(0), + ) + + +def _wrap_in_coro(x): + """ + Wrap ``x`` in a coroutine that will simply return it. + """ + async def f(): + return x + return f() + + +class _AwaitWrapper: + """ + Dummy wrapper that allows transparently forwarding an "await" from a + wrapped coroutine + """ + __slots__ = ('x', '__weakref__') + + def __init__(self, x): + self.x = x + + def __await__(self): + return (yield self.x) + + +class _StopIteration(Exception): + """ + Custom :exc:`StopIteration` that is allowed to be raised by an async + generator. + """ + pass + + +class Async(MonadTrans): + """ + Monad transformer allowing the decorated coroutine function to await on + non-monadic values. This is useful to mix any monad transformer defined in + this module with other async APIs, such as :mod:`asyncio`. + """ + __slots__ = ('_coro',) + + def __init__(self, coro): + self._coro = coro + + @property + def coro(self): + """ + Coroutine that will only yield non-monadic values. All the monadic + values will be processed by the monad transformer stack as expected and + will stay hidden. + """ + async def _f(): + return _TailCall.run(await self._coro) + return _f() + + @property + @memoized + def x(self): + """ + Run the coroutine to completion in its event loop. + """ + return self._run(self.coro) + + @staticmethod + def _run(coro): + """ + Run a coroutine to completion, handling any non-monadic value along the + way. This is typically the entry point of an event loop, such as + :func:`asyncio.run`. + + .. note:: This function must be re-entrant. :func:`asyncio.run` is not + re-entrant by default, but can be made to with + https://pypi.org/project/nest-asyncio/ + """ + try: + action = coro.send(None) + except StopIteration as e: + return e.value + else: + raise RuntimeError(f'Did not expect coroutine to await on {action.__class__.__qualname__}, please subclass Async and override the _run() method with e.g. asyncio.run') + + @classmethod + def _decorate_coroutine_function(cls, f): + """ + Wrap the coroutine function into an async generator that will: + + 1. yield any monadic action + 2. await on everything else, e.g. asyncio actions + + A simpler design would simply ``cls._run()`` or await each action + individually, but it is unfortunately not enough for ``asyncio``. + Indeed, the coroutine itself must be run by ``asyncio.run()``, otherwise + creating the action itself (e.g. ``asyncio.sleep(1)``) will fail as it will + not find the global event loop. + """ + async def _genf(*args, **kwargs): + coro = f(*args, **kwargs) + run = functools.partial(coro.send, None) + while True: + try: + action = run() + # Convert to a custom type, as an async generator is not + # allowed to raise a StopIteration (if it does, it is punished + # by a RuntimeError). We also cannot simply "return e.value" + # since async generators are not allowed to return anything. + except StopIteration as e: + raise _StopIteration(e.value) + else: + is_monad = isinstance(action, Monad) + try: + if is_monad: + x = yield action + else: + x = await _AwaitWrapper(action) + except Exception as e: + run = functools.partial(coro.throw, e) + else: + run = functools.partial(coro.send, x) + + @functools.wraps(f) + async def wrapper(*args, **kwargs): + gen = _genf(*args, **kwargs) + + # All the action the generator yields is to be processed by our + # monadic stack, the others are already handled by the _run() call + x = None + while True: + next_ = gen.asend(x) + try: + action = cls._run(next_) + except _StopIteration as e: + return e.args[0] + else: + x = await action + + return wrapper + + @classmethod + def lift(cls, x): + return cls(_wrap_in_coro(x)) + + @instancemethod + def hoist(cls, self, nat): + async def f(): + # We could use "await self.coro" but expand it to save on a layer + # of coroutine + x = _TailCall.run(await self._coro) + return nat(x) + return cls(f()) + + def bind(self, continuation): + cls = self.__class__ + base = cls._BASE + + async def f(): + return base.bind( + await self._coro, + lambda x: _TailCall(cls._run, continuation(x)._coro) + ) + + return cls(f()) + + +class AsyncIO(Async): + """ + Specialization of :class:`lisa.monad.Async` to :mod:`asyncio` event loop. + """ + # Note that this only works properly with nest_asyncio package. Otherwise, + # asyncio.run() is not re-entrant. + _run = staticmethod(asyncio.run) + + # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/lisa/utils.py b/lisa/utils.py index 069317638f99a18311a509d5b1aa4326ba10f7ba..85fa4e4915dbd896e1943d98a4215f30710688c0 100644 --- a/lisa/utils.py +++ b/lisa/utils.py @@ -208,6 +208,33 @@ class bothmethod: return getattr(self.f, attr) +class instancemethod: + """ + Decorator providing a hybrid of a normal method and a classmethod: + + * Like a classmethod, it can be looked up on the class itself, and the + class is passed as first parameter. This allows selecting the class + "manually" before applying on an instance. + + * Like a normal method, it can be looked up on an instance. In that + case, the first parameter is the class of the instance and the second + parameter is the instance itself. + """ + def __init__(self, f): + self.__wrapped__ = classmethod(f) + + def __get__(self, instance, owner=None): + # Binding to a class + if instance is None: + return self.__wrapped__.__get__(instance, owner) + # Binding to an instance + else: + return functools.partial( + self.__wrapped__.__get__(instance, instance.__class__), + instance, + ) + + class _DummyLogger: def __getattr__(self, attr): x = getattr(logging, attr) @@ -1345,6 +1372,25 @@ def fold(f, xs, init=None): ) +def foldr(f, xs, init=None): + """ + Right-associative version of :func:`fold`. + + .. note:: This requires reversing `xs`. If reversing is not supported by + the iterator, it will be first converted to a tuple. + """ + try: + xs = reversed(xs) + except TypeError: + xs = reversed(tuple(xs)) + + return fold( + lambda x, y: f(y, x), + xs, + init, + ) + + def add(iterable): """ Same as :func:`sum` but works on any object that defines ``__add__`` @@ -3623,4 +3669,5 @@ class SerializeViaConstructor(metaclass=_SerializeViaConstructorMeta): } return (self._make_instance, (self._ctor, dct)) + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/lisa/wlgen/rta.py b/lisa/wlgen/rta.py index 0d1dcb6cffdef9fa4e08bafff8a2fdf68dbd26c5..fdd21754c39cd93dc50ae3fef8ffea474993d653 100644 --- a/lisa/wlgen/rta.py +++ b/lisa/wlgen/rta.py @@ -49,7 +49,10 @@ The most important classes are: A typical workload would be created this way:: - from lisa.wlgen.rta import RTA, RTAPhase, PeriodicWload, SleepWload + from lisa.wlgen.rta import RTA, RTAPhase, PeriodicWload, RunWload, SleepWload, override, delete, WithProperties, task_factory, RTAConf + from lisa.fuzz import Float, Int + from lisa.platforms.platinfo import PlatformInfo + from lisa.target import Target task = ( # Phases can be added together so they will be executed in order @@ -73,6 +76,38 @@ A typical workload would be created this way:: ) ) + # Alternatively, an async API is available to define tasks. + # Each workload awaited will become a new phase, and WithProperties async + # context manager can be used to set properties for multiple such phases. + # + # Additionally, the random data generators from lisa.fuzz can be used as well. + + @task_factory + async def make_task(run_duration): + async with WithProperties(name='first', uclamp=(256, 512)): + await RunWload(run_duration) + + # Create a bounded random value + sleep_duration = await Float(min_=1, max_=5) + sleep_duration = round(sleep_duration, 2) + + async with WithProperties(name='second'): + # We could await once on each workload, but that would lead to 2 phases + # instead of one. Whether this is desired or an issue depends on the + # use-case. + await ( + SleepWload(sleep_duration) + + PeriodicWload( + duty_cycle_pct=20, + period=16e-3, + duration=2, + ) + ) + + # "seed" is used to initialize the random number generator. If not + # provided, the seed will be random. + task = make_task(run_duration=1)(seed=1) + # Important note: all the classes in this module are immutable. Modifying # attributes is not allowed, use the RTAPhase.with_props() if you want to # get a new object with updated properties. @@ -85,11 +120,16 @@ A typical workload would be created this way:: # If you want to set the clamp and override any existing value rather than # combining it, you can use the override() function - task = task.with_props(uclamp=override((300, 800))) + task = task.with_props(uclamp=override((400, 700))) # Similarly, you can delete any property that was already set with delete() task = task.with_props(uclamp=delete()) + + ################################################### + # Alternative 1: create a simple profile and run it + ################################################### + # Connect to a target target = Target.from_default_conf() @@ -103,6 +143,38 @@ A typical workload would be created this way:: with wload: wload.run() + + ################################################### + # Alternative 2: a more complex example generating multiple tasks and dumping a JSON. + ################################################### + + @task_factory + async def make_profile(plat_info, **kwargs): + nr_tasks = await Int(1, plat_info['cpus-count']) + + profile = {} + for i in range(nr_tasks): + async with WithProperties(name=f'task{i}'): + profile[f'task{i}'] = await make_task(**kwargs) + + # If we return anything else than None, the return value will not be + # replaced by the phase defined in the function. + return profile + + + plat_info = PlatformInfo.from_yaml_map('./doc/traces/plat_info.yml') + + # Display a few randomly generated tasks + for _ in range(2): + # When called, profile_gen() will create a random profiles + profile_gen = make_profile(plat_info, run_duration=1) + + # seed (or rng) can be fixed for reproducible results + # profile = profile_gen(seed=1) + profile = profile_gen(seed=None) + + conf = RTAConf.from_profile(profile, plat_info=plat_info) + print(conf.json) """ import abc @@ -117,7 +189,7 @@ import re import weakref from collections import OrderedDict from collections.abc import Iterable, Mapping, Callable -from itertools import chain, product, starmap +from itertools import chain, product, starmap, islice from operator import itemgetter from shlex import quote from statistics import mean @@ -153,6 +225,8 @@ from lisa.utils import ( ) from lisa.wlgen.workload import Workload from lisa.conf import DeferredValueComputationError +from lisa.monad import StateDiscard, TransformerStack +from lisa.fuzz import GenMonad def _to_us(x): @@ -1606,8 +1680,8 @@ def leaf_precedence(val, **kwargs): """ Give precedence to the leaf values when combining with ``&``:: - phase = phase.with_props(prop_meta=({'hello': 'leaf'}) - phase = phase.with_props(prop_meta=leaf_precedence({'hello': 'root'}) + phase = phase.with_props(meta=({'hello': 'leaf'}) + phase = phase.with_props(meta=leaf_precedence({'hello': 'root'}) assert phase['meta'] == {'hello': 'leaf'} This allows setting a property with some kind of default value on a root @@ -1633,7 +1707,7 @@ def override(val, **kwargs): Override a property with the given value, rather than combining it with the property-specific ``&`` implementation:: - phase = phase.with_props(prop_cpus=override({1,2})) + phase = phase.with_props(cpus=override({1,2})) """ return _OverridingValue(val, **kwargs) @@ -1687,7 +1761,7 @@ def delete(): """ Remove the given property from the phase:: - phase = phase.with_props(prop_cpus=delete()) + phase = phase.with_props(cpus=delete()) """ return _DeletingValue() @@ -2480,6 +2554,9 @@ class WloadPropertyBase(ConcretePropertyBase): def val(self): return self + def __await__(self): + return (yield from _wload_action(self).__await__()) + def __add__(self, other): """ Adding two workloads together concatenates them. @@ -3153,15 +3230,15 @@ class _RTAPhaseBase: docstring = inspect.getdoc(cls) if docstring: cls.__doc__ = docstring.format( - prop_kwargs=cls._get_rst_prop_kwargs_doc() + prop_kwargs=cls._get_rst_prop_kwargs_doc(param_prefix='prop_') ) super().__init_subclass__(**kwargs) @classmethod - def _get_rst_prop_kwargs_doc(cls): + def _get_rst_prop_kwargs_doc(cls, param_prefix=''): def make(key, cls): - param = f'prop_{key}' + param = f'{param_prefix}{key}' doc, type_ = cls._get_cls_doc() fst = f':param {param}: {doc}' snd = f':type {param}: {type_}' @@ -3175,6 +3252,9 @@ class _RTAPhaseBase: return '\n\n'.join(starmap(make, sorted(properties.items()))) + def __await__(self): + return (yield from _phase_action(self).__await__()) + class RTAPhaseBase(_RTAPhaseBase, SimpleHash, Mapping, abc.ABC): """ @@ -3216,11 +3296,15 @@ class RTAPhaseBase(_RTAPhaseBase, SimpleHash, Mapping, abc.ABC): """ Return a cloned instance with the properties combined with the given ``properties`` using :meth:`RTAPhaseProperties.__and__` (``&``). The - ``properties`` parameter is the left operand. + ``properties`` parameter is the left operand. If ``properties`` is + ``None``, just return the phase itself. """ - new = copy.copy(self) - new.properties = properties & new.properties - return new + if properties is None: + return self + else: + new = copy.copy(self) + new.properties = properties & new.properties + return new def with_properties_map(self, properties, **kwargs): """ @@ -4362,4 +4446,179 @@ class RunAndSync(_RTATask): **kwargs, ) + +class RTAMonad(StateDiscard): + """ + Monad from derived from :class:`lisa.monad.StateDiscard` used to define + :class:`RTAPhaseBase` in a more natural way. + + If the function returns ``None``, the return value will be replaced by the + :class:`RTAPhaseBase` created while executing the function, with all the + currently active properties applied to it. + + .. seealso:: See the :func:`task_factory` decorator to be able to mix these + actions with the ones from :class:`lisa.fuzz.Gen`. + """ + class _State: + def __init__(self): + # Each level is a tuple of: + # 1. RTAPhaseProperties applied at this level, or None + # 2. A list of phases + # + # To compute the final phase object, we add the phases together and + # then apply the properties on the resulting object. + self.levels = [(None, [])] + + @property + def curr_level(self): + return self.levels[-1] + + def add_phase(self, phase): + self.curr_level[1].append(phase) + + def begin_prop(self, props): + self.levels.append((props, [])) + + def end_prop(self): + phase = self.merge_level() + self.levels.pop() + self.curr_level[1].append(phase) + return phase + + def merge_level(self): + """ + Compute the :class:`RTAPhaseBase` instance for that level with the + phases added so far in the current ``async with WithProperties``. + """ + props, level = self.curr_level + + if level: + phase = functools.reduce( + operator.add, + level, + ) + else: + phase = RTAPhase() + + return phase.with_phase_properties(props) + + @classmethod + def make_state(cls): + """ + Create a fresh instance of :class:`RTAMonad._State` + """ + return cls._State() + + @classmethod + def _decorate_coroutine_function(cls, f): + _f = super()._decorate_coroutine_function(f) + + @functools.wraps(_f) + async def wrapper(*args, **kwargs): + state = await RTAMonad.get_state() + + # For each user function call, we introduce a new level so that we + # can track easily the phases that are created under it. + state.begin_prop(RTAPhaseProperties(None)) + + x = await _f(*args, **kwargs) + + phase = state.end_prop() + + # Replace None return value by the phase created during the call to + # this function + if x is None: + # Compute a phase with all the currently active properties + # applied. This is used to provide a value to user-defined + # functions returning None. + return functools.reduce( + lambda phase, prop: phase.with_phase_properties(prop), + reversed( + list(map( + itemgetter(0), + state.levels, + )) + ), + phase, + ) + else: + return x + + return wrapper + + +class _RTAMonadStack(TransformerStack(GenMonad, RTAMonad)): + pass + + +def _compute(self, rng=None, seed=None): + return RTAMonad.__call__( + GenMonad.__call__( + self, + rng=rng, + seed=seed, + ), + ) +_RTAMonadStack._TOP.__call__ = _compute + + +def task_factory(f): + """ + Coroutine function decorator allowing to create tasks using actions from both + :class:`RTAMonad` and :class:`lisa.fuzz.GenMonad`. + + Calling the decorated function will result in another callable that can be + called once with: + + * ``seed``: Seed to use to automatically initialize a :class:`random.Random`. + * ``rng``: Alternatively, an existing instance of + :class:`random.Random` to use. + + If the user-defined coroutine function returns ``None``, the return value + will be replaced by an :class:`RTAPhaseBase` representing all the phases + that got added while running the function, on which the current active + properties set with :class:`WithProperties` are applied. + + .. seealso:: :mod:`lisa.wlgen.rta` for an example. + """ + return _RTAMonadStack.do(f) + + +def _action_from_f(f): + @functools.wraps(f) + def wrapper(state): + f(state) + return (None, state) + return RTAMonad.from_f(wrapper) + + +def _phase_action(phase): + return _action_from_f(lambda state: state.add_phase(phase)) + + +def _wload_action(wload): + return _phase_action(RTAPhase(prop_wload=wload)) + + +class WithProperties: + """ +Asynchronous context manager used to set properties on the enclosed phases. + +{prop_kwargs} + """ + def __init__(self, **kwargs): + self.props = RTAPhaseProperties.from_polymorphic(kwargs) + + async def __aenter__(self): + await _action_from_f(lambda state: state.begin_prop(self.props)) + return + + async def __aexit__(self, exc_type, exc_value, traceback): + await _action_from_f(lambda state: state.end_prop()) + return + +WithProperties.__doc__ = WithProperties.__doc__.format( + prop_kwargs=_RTAPhaseBase._get_rst_prop_kwargs_doc(), +) + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/pytest.ini b/pytest.ini index d421741043c1207c828f0cc6b09706117c0a70ad..97071038d1c406ef747eaa3fc10f4a51f76140fc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,8 +5,10 @@ testpaths=tests filterwarnings = error ignore::DeprecationWarning:past.builtins.misc - # Avoid: - # .lisa-venv-3.9/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/version.py:111: in __init__ - # warnings.warn( - # DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release - ignore::DeprecationWarning:pkg_resources.*: + # Avoid: + # .lisa-venv-3.9/lib/python3.9/site-packages/pkg_resources/_vendor/packaging/version.py:111: in __init__ + # warnings.warn( + # DeprecationWarning: Creating a LegacyVersion has been deprecated and will be removed in the next major release + ignore::DeprecationWarning:pkg_resources.*: + # Ignore this warning: https://github.com/erdewit/nest_asyncio/issues/70 + ignore::DeprecationWarning:nest_asyncio.*: