diff --git a/.travis.yml b/.travis.yml index 7da785246b1193261d20a3166c584caa2b0b3581..41a97647aa44fea7f886a6966003cb33f18717d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ sudo: required language: python install: - pip install --upgrade trappy bart-py devlib psutil wrapt matplotlib - future jupyter sphinx + future jupyter sphinx ruamel.yaml script: - cd $TRAVIS_BUILD_DIR - 'echo backend : Agg > matplotlibrc' # Otherwise it tries to use tkinter diff --git a/doc/conf.py b/doc/conf.py index a5a97ed2fbaf2f23dab7d7d2d0b80551483c158b..bcf5b6db259dd039a69f3e434c563b88d66eec41 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -118,7 +118,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/index.rst b/doc/index.rst index dfe64133ce73e621427975c3f5cb603196572d4c..fc4a00e0d69f4d719213ca9400aac5cac283ffd7 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,9 +31,13 @@ Contents: .. TODO: due to our slightly weird package structure the index here is wildly nested where it needn't be. +.. TODO: Move wiki to here, wirte a proper module doc, proove Riemann's hypothesis + .. toctree:: + :maxdepth: 2 + + tests - modules Indices and tables ================== diff --git a/doc/libs.rst b/doc/libs.rst deleted file mode 100644 index 1af3a1992fedf0c35317a7226f9f5a0e9e2f7322..0000000000000000000000000000000000000000 --- a/doc/libs.rst +++ /dev/null @@ -1,17 +0,0 @@ -libs package -============ - -Subpackages ------------ - -.. toctree:: - - libs.utils - -Module contents ---------------- - -.. automodule:: libs - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/libs.utils.android.rst b/doc/libs.utils.android.rst deleted file mode 100644 index d49a1a747d9ecc5325e06a1199c6afb538b6a87d..0000000000000000000000000000000000000000 --- a/doc/libs.utils.android.rst +++ /dev/null @@ -1,38 +0,0 @@ -libs.utils.android package -========================== - -Submodules ----------- - -libs.utils.android.screen module --------------------------------- - -.. automodule:: libs.utils.android.screen - :members: - :undoc-members: - :show-inheritance: - -libs.utils.android.system module --------------------------------- - -.. automodule:: libs.utils.android.system - :members: - :undoc-members: - :show-inheritance: - -libs.utils.android.workload module ----------------------------------- - -.. automodule:: libs.utils.android.workload - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: libs.utils.android - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/libs.utils.rst b/doc/libs.utils.rst deleted file mode 100644 index 23da13c19f2cf8f8c09b0b46bef75f3e7b0f9ede..0000000000000000000000000000000000000000 --- a/doc/libs.utils.rst +++ /dev/null @@ -1,117 +0,0 @@ -libs.utils package -================== - -Subpackages ------------ - -.. toctree:: - - libs.utils.android - -Submodules ----------- - -libs.utils.analysis_module module ---------------------------------- - -.. automodule:: libs.utils.analysis_module - :members: - :undoc-members: - :show-inheritance: - -libs.utils.analysis_register module ------------------------------------ - -.. automodule:: libs.utils.analysis_register - :members: - :undoc-members: - :show-inheritance: - -libs.utils.colors module ------------------------- - -.. automodule:: libs.utils.colors - :members: - :undoc-members: - :show-inheritance: - -libs.utils.conf module ----------------------- - -.. automodule:: libs.utils.conf - :members: - :undoc-members: - :show-inheritance: - -libs.utils.energy module ------------------------- - -.. automodule:: libs.utils.energy - :members: - :undoc-members: - :show-inheritance: - -libs.utils.env module ---------------------- - -.. automodule:: libs.utils.env - :members: - :undoc-members: - :show-inheritance: - -libs.utils.executor module --------------------------- - -.. automodule:: libs.utils.executor - :members: - :undoc-members: - :show-inheritance: - -libs.utils.perf_analysis module -------------------------------- - -.. automodule:: libs.utils.perf_analysis - :members: - :undoc-members: - :show-inheritance: - -libs.utils.report module ------------------------- - -.. automodule:: libs.utils.report - :members: - :undoc-members: - :show-inheritance: - -libs.utils.results module -------------------------- - -.. automodule:: libs.utils.results - :members: - :undoc-members: - :show-inheritance: - -libs.utils.test module ----------------------- - -.. automodule:: libs.utils.test - :members: - :undoc-members: - :show-inheritance: - -libs.utils.trace module ------------------------ - -.. automodule:: libs.utils.trace - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: libs.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/modules.rst b/doc/modules.rst deleted file mode 100644 index 6ebf22d803c8451670cb81feb50220c3f775832d..0000000000000000000000000000000000000000 --- a/doc/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -libs -==== - -.. toctree:: - :maxdepth: 4 - - libs diff --git a/doc/tests.rst b/doc/tests.rst new file mode 100644 index 0000000000000000000000000000000000000000..8b150db04404e297b4a2d103b647cb6b0064ed4e --- /dev/null +++ b/doc/tests.rst @@ -0,0 +1,70 @@ +********** +LISA tests +********** + +Introduction +============ + +These tests were developped to verify patches supporting Arm's big.LITTLE +in the Linux scheduler. You can see these test results being published +`here `_. + +Tests do not **have** to target Arm platforms nor the scheduler. The only real +requirement is to have a :class:`libs.devlib` target handle, and from there you +are free to implement tests as you see fit. + +They are commonly split into two steps: + 1) Collect some data by doing some work on the target + 2) Post-process the collected data + +In our case, the data usually consists of +`Ftrace `_ traces +that we then postprocess using :mod:`libs.trappy`. + +Writing tests +============= + +Writing scheduler tests can be difficult, especially when you're +trying to make them work without relying on custom tracepoints (which is +what you should aim for). Sometimes, a good chunk of the test code will be +about trying to get the target in an expected initial state, or preventing some +undesired mechanic from barging in. That's why we rely on the freezer cgroup to +reduce the amount of noise introduced by the userspace, but it's not solving all +of the issues. As such, your tests should be designed to: + +a. minimize the amount of non test-related noise (freezer, disable some module...) +b. withstand events we can't control (use error margins, averages...) + +Having tunable margins (such as for +:meth:`libs.utils.generic.GenericTestBundle.test_slack`) is also desirable, as +it allows things like parameter sweep in CI. + +API +=== + +Base classes +------------ + +.. automodule:: libs.utils.test_workload + :members: + +Generic tests +------------- + +.. autoclass:: libs.utils.generic.GenericTestBundle + :members: + +Hotplug tests +------------- +.. automodule:: libs.utils.cpu_hotplug + :members: + +Implemented tests +================= + +Generics +-------- + +.. automodule:: libs.utils.generic + :exclude-members: GenericTestBundle + :members: diff --git a/ipynb/examples/New test API example.ipynb b/ipynb/examples/New test API example.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f932c82610814aa0b64fc8f043cbe0628cb3e91c --- /dev/null +++ b/ipynb/examples/New test API example.ipynb @@ -0,0 +1,242 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "from conf import LisaLogging\n", + "LisaLogging.setup()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pylab inline\n", + "\n", + "import json\n", + "import os\n", + "\n", + "# Support to access the remote target\n", + "import devlib\n", + "from env import TestEnv\n", + "\n", + "# Import support for Android devices\n", + "from android import Screen, Workload\n", + "\n", + "# Support for trace events analysis\n", + "from trace import Trace\n", + "\n", + "# Suport for FTrace events parsing and visualization\n", + "import trappy\n", + "import pandas as pd\n", + "\n", + "from wlgen import RTA, Periodic, Ramp\n", + "from time import sleep\n", + "\n", + "from IPython.display import display" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "linux_hikey = {\n", + " \"platform\" : \"linux\",\n", + " \"board\" : \"hikey960\",\n", + " \"host\": \"192.168.0.1\",\n", + " \"username\" : \"root\",\n", + " \"password\" : \"root\",\n", + " \"modules\" : [\"sched\", \"cgroups\", \"hotplug\"],\n", + " \"tools\" : [\"taskset\", \"rt-app\"],\n", + " \"rtapp-calib\" : {\n", + " \"0\": 302, \"1\": 302, \"2\": 302, \"3\": 302, \"4\": 136, \"5\": 136, \"6\": 136, \"7\": 136\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "linux_juno = {\n", + " \"platform\" : \"linux\",\n", + " \"board\" : \"juno\",\n", + " \"host\": \"192.168.0.1\",\n", + " \"username\" : \"root\",\n", + " \"password\" : \"root\",\n", + " \"modules\" : [\"sched\", \"cgroups\", \"hotplug\"],\n", + " \"tools\" : [\"taskset\", \"rt-app\"]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "android_hikey = {\n", + " \"platform\" : \"android\",\n", + " \"board\" : \"hikey960\",\n", + " \"modules\" : [\"sched\", \"hotplug\"],\n", + " \"tools\" : [\"taskset\", \"rt-app\"]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "te = TestEnv(linux_hikey)\n", + "target = te.target" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from generic import OneSmallTask, ThreeSmallTasks, TwoBigTasks, TwoBigThreeSmall, RampUp, RampDown, EnergyModelWakeMigration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tests = [OneSmallTask, ThreeSmallTasks, TwoBigTasks, TwoBigThreeSmall, RampUp, RampDown, EnergyModelWakeMigration]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = {}\n", + "\n", + "for test in tests:\n", + " bundle = test.from_target(te)\n", + " res[test.__name__] = bundle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for test_name, bundle in res.iteritems():\n", + " print test_name\n", + " print \"---\"\n", + " print \"test_slack: {}\".format(\"PASSED\" if bundle.test_slack() else \"FAILED\")\n", + " print \"test_task_placement: {}\".format(\"PASSED\" if bundle.test_task_placement() else \"FAILED\")\n", + " print" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hotplug" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from cpu_hotplug import HotplugTestBundle" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bundle = HotplugTestBundle.from_target(te)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bundle.test_cpus_alive().passed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bundle.test_cpus_alive().metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bundle.test_target_alive().passed" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/libs/devlib b/libs/devlib index 8aa9d672a1ec15e3f1a516a8dda489f42badba0e..8cd1470bb835f4d264fb7bcf4ce0fdfbfa4d245b 160000 --- a/libs/devlib +++ b/libs/devlib @@ -1 +1 @@ -Subproject commit 8aa9d672a1ec15e3f1a516a8dda489f42badba0e +Subproject commit 8cd1470bb835f4d264fb7bcf4ce0fdfbfa4d245b diff --git a/libs/utils/__init__.py b/libs/utils/__init__.py index d84475dc9ae5879bd8204876b426ec319946a60e..c1ff22793f8f1525ab00ecd29d815fbc440fe168 100644 --- a/libs/utils/__init__.py +++ b/libs/utils/__init__.py @@ -22,7 +22,6 @@ """ from env import TestEnv -from executor import Executor from energy import EnergyMeter from conf import LisaLogging, JsonConf diff --git a/libs/utils/cpu_hotplug.py b/libs/utils/cpu_hotplug.py new file mode 100644 index 0000000000000000000000000000000000000000..7b74aa3e69ec8eac1206e1827dd6f533e45333cc --- /dev/null +++ b/libs/utils/cpu_hotplug.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import random + +from devlib.module.hotplug import HotplugModule +from devlib.exception import TimeoutError + +from test_workload import Metric, ResultBundle, TestBundle +from target_script import TargetScript + +class HotplugTestBundle(TestBundle): + + def __init__(self, target_alive, hotpluggable_cpus, live_cpus): + self.target_alive = target_alive + self.hotpluggable_cpus = hotpluggable_cpus + self.live_cpus = live_cpus + + @classmethod + def _random_cpuhp_seq(cls, seed, nr_operations, + hotpluggable_cpus, max_cpus_off): + """ + Yield a consistent random sequence of CPU hotplug operations + + :param seed: Seed of the RNG + :param nr_operations: Number of operations in the sequence + <= 0 will encode 'no sleep' + :param max_cpus_off: Max number of CPUs plugged-off + + "Consistent" means that a CPU will be plugged-in only if it was + plugged-off before (and vice versa). Moreover the state of the CPUs + once the sequence has completed should the same as it was before. + + The actual length of the sequence might differ from the requested one + by 1 because it's easier to implement and it shouldn't be an issue for + most test cases. + """ + cur_on_cpus = hotpluggable_cpus[:] + cur_off_cpus = [] + i = 0 + while i < nr_operations - len(cur_off_cpus): + if len(cur_on_cpus)<=1 or len(cur_off_cpus)>=max_cpus_off: + # Force plug IN when only 1 CPU is on or too many are off + plug_way = 1 + elif not cur_off_cpus: + # Force plug OFF if all CPUs are on + plug_way = 0 # Plug OFF + else: + plug_way = random.randint(0,1) + + src = cur_off_cpus if plug_way else cur_on_cpus + dst = cur_on_cpus if plug_way else cur_off_cpus + cpu = random.choice(src) + src.remove(cpu) + dst.append(cpu) + i += 1 + yield cpu, plug_way + + # Re-plug offline cpus to come back to original state + for cpu in cur_off_cpus: + yield cpu, 1 + + @classmethod + def _random_cpuhp_script(cls, te, res_dir, sequence, sleep_min_ms, + sleep_max_ms, timeout_s): + shift = ' ' + script = TargetScript(te, 'random_cpuhp.sh', res_dir) + + # Record configuration + # script.append('# File generated automatically') + # script.append('# Configuration:') + # script.append('# {}'.format(cls.hp_stress)) + # script.append('# Hotpluggable CPUs:') + # script.append('# {}'.format(cls.hotpluggable_cpus)) + + script.append('while true') + script.append('do') + for cpu, plug_way in sequence: + # Write in sysfs entry + cmd = 'echo {} > {}'.format(plug_way, HotplugModule._cpu_path(te.target, cpu)) + script.append(shift + cmd) + # Sleep if necessary + if sleep_max_ms > 0: + sleep_dur_sec = random.randint(sleep_min_ms, sleep_max_ms)/1000.0 + script.append(shift + 'sleep {}'.format(sleep_dur_sec)) + script.append('done &') + + # Make sure to stop the hotplug stress after timeout_s seconds + script.append('LOOP_PID=$!') + script.append('sleep {}'.format(timeout_s)) + script.append('[ $(ps -q $LOOP_PID | wc -l) -gt 1 ] && kill -9 $LOOP_PID') + + return script + + @classmethod + def _from_target(cls, te, res_dir=None, seed=None, nr_operations=100, + sleep_min_ms=10, sleep_max_ms=100, duration_s=10, + max_cpus_off=sys.maxint): + + if not seed: + random.seed() + seed = random.randint(0, sys.maxint) + + te.target.hotplug.online_all() + hotpluggable_cpus = te.target.hotplug.list_hotpluggable_cpus() + + sequence = cls._random_cpuhp_seq( + seed, nr_operations, hotpluggable_cpus, max_cpus_off + ) + + script = cls._random_cpuhp_script( + te, res_dir, sequence, sleep_min_ms, sleep_max_ms, duration_s + ) + + script.push() + + target_alive = True + timeout = duration_s + 60 + + try: + script.run(as_root=True, timeout=timeout) + te.target.hotplug.online_all() + except TimeoutError: + #msg = 'Target not responding after {} seconds ...' + #cls._log.info(msg.format(timeout)) + target_alive = False + + live_cpus = te.target.list_online_cpus() if target_alive else [] + + return cls(target_alive, hotpluggable_cpus, live_cpus) + + def test_target_alive(self): + """ + Test that the hotplugs didn't leave the target in an unusable state + """ + return ResultBundle(self.target_alive) + + def test_cpus_alive(self): + """ + Test that all CPUs came back online after the hotplug operations + """ + res = ResultBundle(self.hotpluggable_cpus == self.live_cpus) + res.add_metric(Metric("hotpluggable CPUs", self.hotpluggable_cpus)) + res.add_metric(Metric("Online CPUs", self.live_cpus)) + return res diff --git a/libs/utils/energy_model.py b/libs/utils/energy_model.py index c27a0e18ebd92a8d473adce621d64be21dd58bcd..cd5b90eff13e3612d170ca68af4b93a86ac2b150 100644 --- a/libs/utils/energy_model.py +++ b/libs/utils/energy_model.py @@ -20,6 +20,9 @@ from itertools import product import logging import operator import re +from ruamel.yaml import YAML, yaml_object + +yaml = YAML() import pandas as pd import numpy as np @@ -67,6 +70,7 @@ class EnergyModelCapacityError(Exception): """Used by :meth:`EnergyModel.get_optimal_placements`""" pass +@yaml_object(yaml) class ActiveState(namedtuple('ActiveState', ['capacity', 'power'])): """Represents power and compute capacity at a given frequency @@ -76,6 +80,19 @@ class ActiveState(namedtuple('ActiveState', ['capacity', 'power'])): def __new__(cls, capacity=None, power=None): return super(ActiveState, cls).__new__(cls, capacity, power) + # helpers for yaml serialization + yaml_tag = u'em_active_state:capacity,power' + + @classmethod + def to_yaml(cls, representer, node): + return representer.represent_scalar(cls.yaml_tag, + u'{.capacity},{.power}'.format(node, node)) + + @classmethod + def from_yaml(cls, constructor, node): + cap, power = (int(x) for x in node.value.split(',')) + return cls(capacity=cap, power=power) + class _CpuTree(object): """Internal class. Abstract representation of a CPU topology. @@ -128,6 +145,7 @@ class _CpuTree(object): """Iterate over leaves""" return self._iter(False) +@yaml_object(yaml) class EnergyModelNode(_CpuTree): """Describes topology and energy data for an EnergyModel. @@ -157,7 +175,7 @@ class EnergyModelNode(_CpuTree): cpu=None, children=None, name=None): super(EnergyModelNode, self).__init__(cpu, children) - self._log = logging.getLogger('EnergyModel') + _log = logging.getLogger('EnergyModel') def is_monotonic(l, decreasing=False): op = operator.ge if decreasing else operator.le @@ -167,14 +185,14 @@ class EnergyModelNode(_CpuTree): # Sanity check for active_states's frequencies freqs = active_states.keys() if not is_monotonic(freqs): - self._log.warning( + _log.warning( 'Active states frequencies are expected to be ' 'monotonically increasing. Freqs: {}'.format(freqs)) # Sanity check for active_states's powers power_vals = [s.power for s in active_states.values()] if not is_monotonic(power_vals): - self._log.warning( + _log.warning( 'Active states powers are expected to be ' 'monotonically increasing. Values: {}'.format(power_vals)) @@ -187,7 +205,7 @@ class EnergyModelNode(_CpuTree): # Sanity check for idle_states powers power_vals = idle_states.values() if not is_monotonic(power_vals, decreasing=True): - self._log.warning( + _log.warning( 'Idle states powers are expected to be ' 'monotonically decreasing. Values: {}'.format(power_vals)) @@ -212,6 +230,7 @@ class EnergyModelNode(_CpuTree): raise KeyError('No idle state with index {}'.format(idx)) +@yaml_object(yaml) class EnergyModelRoot(EnergyModelNode): """ Convenience class for root of an EnergyModelNode tree. @@ -224,6 +243,7 @@ class EnergyModelRoot(EnergyModelNode): return super(EnergyModelRoot, self).__init__( active_states, idle_states, cpu, children, name) +@yaml_object(yaml) class PowerDomain(_CpuTree): """Describes the power domain hierarchy for an EnergyModel. @@ -259,6 +279,7 @@ class PowerDomain(_CpuTree): super(PowerDomain, self).__init__(cpu, children) self.idle_states = idle_states +@yaml_object(yaml) class EnergyModel(object): """Represents hierarchical CPU topology with power and capacity data @@ -365,14 +386,15 @@ class EnergyModel(object): self.root = root_node self.cpu_nodes = sorted_leaves(root_node) + self.pd = root_power_domain self.cpu_pds = sorted_leaves(root_power_domain) assert len(self.cpu_pds) == len(self.cpu_nodes) - self._log = logging.getLogger('EnergyModel') + _log = logging.getLogger('EnergyModel') max_cap = max(n.max_capacity for n in self.cpu_nodes) if max_cap != self.capacity_scale: - self._log.debug( + _log.debug( 'Unusual max capacity (%s), overriding capacity_scale', max_cap) self.capacity_scale = max_cap @@ -679,7 +701,9 @@ class EnergyModel(object): tasks = capacities.keys() num_candidates = len(self.cpus) ** len(tasks) - self._log.debug( + + _log = logging.getLogger('EnergyModel') + _log.debug( '%14s - Searching %d configurations for optimal task placement...', 'EnergyModel', num_candidates) @@ -717,7 +741,7 @@ class EnergyModel(object): min_power = min(p for p in candidates.itervalues()) ret = [u for u, p in candidates.iteritems() if p == min_power] - self._log.debug('%14s - Done', 'EnergyModel') + _log.debug('%14s - Done', 'EnergyModel') return ret @classmethod @@ -1001,6 +1025,15 @@ class EnergyModel(object): root_power_domain=root_pd, freq_domains=freq_domains) + def to_disk(self, file_path): + with open(file_path, "w") as f: + yaml.dump(self, f) + + @classmethod + def from_disk(cls, file_path): + with open(file_path, "r") as f: + return yaml.load(f) + @classmethod def from_target(cls, target): """ diff --git a/libs/utils/env.py b/libs/utils/env.py index d0d60b392326cb947874f90b763063dde092494b..a013eef1cf9cac13fb5ae1f32222f4544565146c 100644 --- a/libs/utils/env.py +++ b/libs/utils/env.py @@ -49,26 +49,18 @@ LATEST_LINK = 'results_latest' basepath = os.path.dirname(os.path.realpath(__file__)) basepath = basepath.replace('/libs/utils', '') -class ShareState(object): - __shared_state = {} - - def __init__(self): - self.__dict__ = self.__shared_state +IFCFG_BCAST_RE = re.compile( + r'Bcast:(.*) ' +) -class TestEnv(ShareState): +class TestEnv(object): """ Represents the environment configuring LISA, the target, and the test setup The test environment is defined by: - - a target configuration (target_conf) defining which HW platform we + a target configuration (target_conf) defining which HW platform we want to use to run the experiments - - a test configuration (test_conf) defining which SW setups we need on - that HW target - - a folder to collect the experiments results, which can be specified using - the target_conf::results_dir option, or using LISA_RESULTS_DIR environment - variable and is by default wiped from all the previous contents - (if wipe=True) :param target_conf: Configuration defining the target to run experiments on. May be @@ -86,7 +78,7 @@ class TestEnv(ShareState): the relevant features aren't needed. Has the following keys: **host** - Target IP or MAC address for SSH access + Target IP or hostname for SSH access **username** For SSH access **keyfile** @@ -105,52 +97,10 @@ class TestEnv(ShareState): calibrate RT-App on the target. A message will be logged with a value that can be copied here to avoid having to re-run calibration on subsequent tests. - **tftp** - Directory path containing kernels and DTB images for the - target. LISA does *not* manage this TFTP server, it must be - provided externally. Optional. - **results_dir** - location of results of the experiments. **ftrace** - Ftrace configuration merged with test-specific configuration. + Ftrace configuration. Currently, only additional events through "events" key is supported. - :param test_conf: Configuration of software for target experiments. Takes - the same form as target_conf. Fields are: - - **modules** - Devlib modules to be enabled. Default is [] - **exclude_modules** - Devlib modules to be disabled. Default is []. - **tools** - List of tools (available under ./tools/$ARCH/) to install on - the target. Names, not paths (e.g. ['ftrace']). Default is []. - **ping_time**, **reboot_time** - Override parameters to :meth:`reboot` method - **__features__** - List of test environment features to enable. Options are: - - "no-kernel" - do not deploy kernel/dtb images - "no-reboot" - do not force reboot the target at each configuration change - "debug" - enable debugging messages - - **ftrace** - Configuration for ftrace. Dictionary with keys: - - events - events to enable. - functions - functions to enable in the function tracer. Optional. - buffsize - Size of buffer. Default is 10240. - - :param wipe: set true to cleanup all previous content from the output - folder - :type wipe: bool - :param force_new: Create a new TestEnv object even if there is one available for this session. By default, TestEnv only creates one object per session, use this to override this behaviour. @@ -186,15 +136,169 @@ class TestEnv(ShareState): _initialized = False - def __init__(self, target_conf=None, test_conf=None, wipe=True, - force_new=False): + def __init__(self, target_conf=None, force_new=False): super(TestEnv, self).__init__() if self._initialized and not force_new: return + # Setup logging + self._log = logging.getLogger('TestEnv') + + self._pre_target_init(target_conf) + self._init_target() + self._post_target_init() + + self._initialized = True + + def _load_em(self, board): + em_path = os.path.join(basepath, + 'libs/utils/platforms', board.lower() + '.json') + self._log.debug('Trying to load default EM from %s', em_path) + if not os.path.exists(em_path): + return None + self._log.info('Loading default EM:') + self._log.info(' %s', em_path) + board = JsonConf(em_path) + board.load() + if 'nrg_model' not in board.json: + return None + return board.json['nrg_model'] + + def _load_board(self, board): + board_path = os.path.join(basepath, + 'libs/utils/platforms', board.lower() + '.json') + self._log.debug('Trying to load board descriptor from %s', board_path) + if not os.path.exists(board_path): + return None + self._log.info('Loading board:') + self._log.info(' %s', board_path) + board = JsonConf(board_path) + board.load() + if 'board' not in board.json: + return None + return board.json['board'] + + def _build_topology(self): + # Initialize target Topology for behavior analysis + CLUSTERS = [] + + # Build topology for a big.LITTLE systems + if self.target.big_core and \ + (self.target.abi == 'arm64' or self.target.abi == 'armeabi'): + # Populate cluster for a big.LITTLE platform + if self.target.big_core: + # Load cluster of LITTLE cores + CLUSTERS.append( + [i for i,t in enumerate(self.target.core_names) + if t == self.target.little_core]) + # Load cluster of big cores + CLUSTERS.append( + [i for i,t in enumerate(self.target.core_names) + if t == self.target.big_core]) + # Build topology for an SMP systems + elif not self.target.big_core or \ + self.target.abi == 'x86_64': + for c in set(self.target.core_clusters): + CLUSTERS.append( + [i for i,v in enumerate(self.target.core_clusters) + if v == c]) + self.topology = Topology(clusters=CLUSTERS) + self._log.info('Topology:') + self._log.info(' %s', CLUSTERS) + + def _init_platform_bl(self): + self.platform = { + 'clusters' : { + 'little' : self.target.bl.littles, + 'big' : self.target.bl.bigs + }, + 'freqs' : { + 'little' : self.target.bl.list_littles_frequencies(), + 'big' : self.target.bl.list_bigs_frequencies() + } + } + self.platform['cpus_count'] = \ + len(self.platform['clusters']['little']) + \ + len(self.platform['clusters']['big']) + + def _init_platform_smp(self): + self.platform = { + 'clusters' : {}, + 'freqs' : {} + } + for cpu_id,node_id in enumerate(self.target.core_clusters): + if node_id not in self.platform['clusters']: + self.platform['clusters'][node_id] = [] + self.platform['clusters'][node_id].append(cpu_id) + + if 'cpufreq' in self.target.modules: + # Try loading frequencies using the cpufreq module + for cluster_id in self.platform['clusters']: + core_id = self.platform['clusters'][cluster_id][0] + self.platform['freqs'][cluster_id] = \ + self.target.cpufreq.list_frequencies(core_id) + else: + self._log.warning('Unable to identify cluster frequencies') + + # TODO: get the performance boundaries in case of intel_pstate driver + + self.platform['cpus_count'] = len(self.target.core_clusters) + + def _get_clusters(self, core_names): + idx = 0 + clusters = [] + ids_map = { core_names[0] : 0 } + for name in core_names: + idx = ids_map.get(name, idx+1) + ids_map[name] = idx + clusters.append(idx) + return clusters + + def _init_platform(self): + if 'bl' in self.target.modules: + self._init_platform_bl() + else: + self._init_platform_smp() + + # Adding energy model information + if 'nrg_model' in self.conf: + self.platform['nrg_model'] = self.conf['nrg_model'] + # Try to load the default energy model (if available) + else: + nrg_model = self._load_em(self.conf['board']) + # We shouldn't have an 'nrg_model' key if there is no energy model data + if nrg_model: + self.platform['nrg_model'] = nrg_model + + # Adding topology information + self.platform['topology'] = self.topology.get_level("cluster") + + # Adding kernel build information + kver = self.target.kernel_version + self.platform['kernel'] = {t: getattr(kver, t, None) + for t in [ + 'release', 'version', + 'version_number', 'major', 'minor', + 'rc', 'sha1', 'parts' + ] + } + self.platform['abi'] = self.target.abi + self.platform['os'] = self.target.os + + self._log.debug('Platform descriptor initialized\n%s', self.platform) + # self.platform_dump('./') + + def _init_energy(self, force): + # Initialize energy probe to board default + self.emeter = EnergyMeter.getInstance(self.target, self.conf, force) + + def _pre_target_init(self, target_conf): + """ + Initialization code that doesn't need a :class:`devlib.Target` instance + """ + self.conf = {} - self.test_conf = {} self.target = None self.ftrace = None self.workdir = None @@ -203,13 +307,8 @@ class TestEnv(ShareState): self.__connection_settings = None self._calib = None - # Keep track of target IP and MAC address + # Keep track of target IP self.ip = None - self.mac = None - - # Keep track of last installed kernel - self.kernel = None - self.dtb = None # Energy meter configuration self.emeter = None @@ -223,175 +322,37 @@ class TestEnv(ShareState): self.CATAPULT_HOME = os.environ.get('CATAPULT_HOME', os.path.join(self.LISA_HOME, 'tools', 'catapult')) - # Setup logging - self._log = logging.getLogger('TestEnv') - - # Compute base installation path - self._log.info('Using base path: %s', basepath) - # Setup target configuration if isinstance(target_conf, dict): self._log.info('Loading custom (inline) target configuration') self.conf = target_conf elif isinstance(target_conf, str): self._log.info('Loading %s target configuration', target_conf) - self.conf = self.loadTargetConfig(target_conf) + self.conf = self.load_target_config(target_conf) else: target_conf = os.environ.get('LISA_TARGET_CONF', '') self._log.info('Loading [%s] target configuration', target_conf or 'default') - self.conf = self.loadTargetConfig(target_conf) + self.conf = self.load_target_config(target_conf) self._log.debug('Target configuration %s', self.conf) - # Setup test configuration - if test_conf: - if isinstance(test_conf, dict): - self._log.info('Loading custom (inline) test configuration') - self.test_conf = test_conf - elif isinstance(test_conf, str): - self._log.info('Loading custom (file) test configuration') - self.test_conf = self.loadTargetConfig(test_conf) - else: - raise ValueError('test_conf must be either a dictionary or a filepath') - self._log.debug('Test configuration %s', self.conf) - # Setup target working directory if 'workdir' in self.conf: self.workdir = self.conf['workdir'] # Initialize binary tools to deploy - test_conf_tools = self.test_conf.get('tools', []) - target_conf_tools = self.conf.get('tools', []) - self.__tools = list(set(test_conf_tools + target_conf_tools)) + self.__tools = list(set(self.conf.get('tools', []))) # Initialize ftrace events - # test configuration override target one - test_ftrace = self.test_conf.get('ftrace', {}) - target_ftrace = self.conf.get('ftrace', {}) - ftrace = test_ftrace or target_ftrace - # Merge the events from target config and test config - ftrace['events'] = sorted( - set(test_ftrace.get('events', [])) - | set(target_ftrace.get('events', [])) - ) - self.conf['ftrace'] = ftrace - if ftrace['events']: - self.__tools.append('trace-cmd') - - # Initialize features - if '__features__' not in self.conf: - self.conf['__features__'] = [] - - # Initialize local results folder. - # The test configuration overrides the target's one and the environment - # variable overrides everything else. - self.res_dir = ( - os.getenv('LISA_RESULTS_DIR') or - self.conf.get('results_dir') - ) - # Default result dir based on the current time - if not self.res_dir: - self.res_dir = datetime.now().strftime( - os.path.join(basepath, OUT_PREFIX, '%Y%m%d_%H%M%S') - ) - - # Relative paths are interpreted as relative to a fixed root. - if not os.path.isabs(self.res_dir): - self.res_dir = os.path.join(basepath, OUT_PREFIX, self.res_dir) - - if wipe and os.path.exists(self.res_dir): - self._log.warning('Wipe previous contents of the results folder:') - self._log.warning(' %s', self.res_dir) - shutil.rmtree(self.res_dir, ignore_errors=True) - if not os.path.exists(self.res_dir): - os.makedirs(self.res_dir) - - res_lnk = os.path.join(basepath, LATEST_LINK) - if os.path.islink(res_lnk): - os.remove(res_lnk) - os.symlink(self.res_dir, res_lnk) - - self._init() + ftrace_conf = self.conf.get('ftrace', {}) + ftrace_conf['events'] = sorted(set(ftrace_conf.get('events', []))) + self.conf['ftrace'] = ftrace_conf - # Initialize FTrace events collection - self._init_ftrace(True) - - # Initialize RT-App calibration values - self.calibration() - - # Initialize energy probe instrument - self._init_energy(True) - - self._log.info('Set results folder to:') - self._log.info(' %s', self.res_dir) - self._log.info('Experiment results available also in:') - self._log.info(' %s', res_lnk) - - self._initialized = True - - def loadTargetConfig(self, filepath=None): + def _init_target(self): """ - Load the target configuration from the specified file. - - :param filepath: Path of the target configuration file. Relative to the - root folder of the test suite. - :type filepath: str - + Create a :class:`devlib.Target` object """ - - # "" and None are replaced by the default 'target.config' value - filepath = filepath or 'target.config' - - # Loading default target configuration - conf_file = os.path.join(basepath, filepath) - - self._log.info('Loading target configuration [%s]...', conf_file) - conf = JsonConf(conf_file) - conf.load() - return conf.json - - def _init(self, force = False): - - # Initialize target - self._init_target(force) - - # Initialize target Topology for behavior analysis - CLUSTERS = [] - - # Build topology for a big.LITTLE systems - if self.target.big_core and \ - (self.target.abi == 'arm64' or self.target.abi == 'armeabi'): - # Populate cluster for a big.LITTLE platform - if self.target.big_core: - # Load cluster of LITTLE cores - CLUSTERS.append( - [i for i,t in enumerate(self.target.core_names) - if t == self.target.little_core]) - # Load cluster of big cores - CLUSTERS.append( - [i for i,t in enumerate(self.target.core_names) - if t == self.target.big_core]) - # Build topology for an SMP systems - elif not self.target.big_core or \ - self.target.abi == 'x86_64': - for c in set(self.target.core_clusters): - CLUSTERS.append( - [i for i,v in enumerate(self.target.core_clusters) - if v == c]) - self.topology = Topology(clusters=CLUSTERS) - self._log.info('Topology:') - self._log.info(' %s', CLUSTERS) - - # Initialize the platform descriptor - self._init_platform() - - - def _init_target(self, force = False): - - if not force and self.target is not None: - return self.target - self.__connection_settings = {} # Configure username @@ -412,16 +373,10 @@ class TestEnv(ShareState): if 'port' in self.conf: self.__connection_settings['port'] = self.conf['port'] - # Configure the host IP/MAC address + # Configure the host IP if 'host' in self.conf: - try: - if ':' in self.conf['host']: - (self.mac, self.ip) = self.resolv_host(self.conf['host']) - else: - self.ip = self.conf['host'] - self.__connection_settings['host'] = self.ip - except KeyError: - raise ValueError('Config error: missing [host] parameter') + self.ip = self.conf['host'] + self.__connection_settings['host'] = self.ip try: platform_type = self.conf['platform'] @@ -529,11 +484,8 @@ class TestEnv(ShareState): # Refine modules list based on target.conf modules.update(self.conf.get('modules', [])) - # Merge tests specific modules - modules.update(self.test_conf.get('modules', [])) - remove_modules = set(self.conf.get('exclude_modules', []) + - self.test_conf.get('exclude_modules', [])) + remove_modules = set(self.conf.get('exclude_modules', [])) modules.difference_update(remove_modules) self.__modules = list(modules) @@ -675,7 +627,7 @@ class TestEnv(ShareState): gem5_bin = simulator['bin'], gem5_args = args, gem5_virtio = virtio_args, - host_output_dir = self.res_dir, + host_output_dir = self.get_res_dir('gem5'), core_names = board['cores'] if board else None, core_clusters = self._get_clusters(board['cores']) if board else None, big_core = board.get('big_core', None) if board else None, @@ -683,6 +635,72 @@ class TestEnv(ShareState): return platform + def _post_target_init(self): + """ + Initialization code that needs a :class:`devlib.Target` instance + """ + self._build_topology() + + # Initialize the platform descriptor + self._init_platform() + + # Initialize energy probe instrument + self._init_energy(True) + + def load_target_config(self, filepath=None): + """ + Load the target configuration from the specified file. + + :param filepath: Path of the target configuration file. Relative to the + root folder of the test suite. + :type filepath: str + + """ + + # "" and None are replaced by the default 'target.config' value + filepath = filepath or 'target.config' + + # Loading default target configuration + conf_file = os.path.join(basepath, filepath) + + self._log.info('Loading target configuration [%s]...', conf_file) + conf = JsonConf(conf_file) + conf.load() + return conf.json + + def get_res_dir(self, name=None): + """ + Returns a directory managed by LISA to store results + """ + # Initialize local results folder. + # The test configuration overrides the target's one and the environment + # variable overrides everything else. + res_dir = ( + os.getenv('LISA_RESULTS_DIR') or + self.conf.get('results_dir') + ) + + # Default result dir based on the current time + if not res_dir: + if not name: + name = datetime.now().strftime('%Y%m%d_%H%M%S') + + res_dir = os.path.join(basepath, OUT_PREFIX, name) + + # Relative paths are interpreted as relative to a fixed root. + if not os.path.isabs(res_dir): + res_dir = os.path.join(basepath, OUT_PREFIX, res_dir) + + if not os.path.exists(res_dir): + os.makedirs(res_dir) + + res_lnk = os.path.join(basepath, LATEST_LINK) + if os.path.islink(res_lnk): + os.remove(res_lnk) + os.symlink(res_dir, res_lnk) + + return res_dir + def install_tools(self, tools): """ Install tools additional to those specified in the test config 'tools' @@ -693,11 +711,7 @@ class TestEnv(ShareState): """ tools = set(tools) - # Add tools dependencies - if 'rt-app' in tools: - tools.update(['taskset', 'trace-cmd', 'perf', 'cgroup_run_into.sh']) - - # Remove duplicates and already-instaled tools + # Remove duplicates and already-installed tools tools.difference_update(self.__installed_tools) tools_to_install = [] @@ -713,26 +727,49 @@ class TestEnv(ShareState): self.__installed_tools.update(tools) - def ftrace_conf(self, conf): - self._init_ftrace(True, conf) + def configure_ftrace(self, events=None, functions=None, + buffsize=FTRACE_BUFSIZE_DEFAULT): + """ + Setup the environment's :class:`devlib.trace.FtraceCollector` - def _init_ftrace(self, force=False, conf=None): + :param events: The events to trace + :type events: list(str) - if not force and self.ftrace is not None: - return + :param functions: the kernel functions to trace + :type functions: list(str) - ftrace = conf or self.conf.get('ftrace') - if ftrace is None: - return + :param buffsize: The size of the Ftrace buffer + :type buffsize: int + + :raises RuntimeError: If no event nor function is to be traced + """ + + # Merge with setup from target config + target_conf = self.conf.get('ftrace', {}) + + if events is None: + events = [] + if functions is None: + functions = [] + + def merge_conf(value, index, default): + return sorted(set(value) | set(target_conf.get(index, default))) - events = ftrace.get('events', FTRACE_EVENTS_DEFAULT) - functions = ftrace.get('functions', None) - buffsize = ftrace.get('buffsize', FTRACE_BUFSIZE_DEFAULT) + events = merge_conf(events, 'events', []) + functions = merge_conf(functions, 'functions', []) + buffsize = max(buffsize, target_conf.get('buffsize', 0)) # If no events or functions have been specified: # do not create the FtraceCollector if not (events or functions): - return + raise RuntimeError( + "Tried to configure Ftrace, but no events nor functions were" + "provided from neither method parameters nor target_config" + ) + + # Ensure we have trace-cmd on the target + if 'trace-cmd' not in self.__installed_tools: + self.install_tools(['trace-cmd']) self.ftrace = devlib.FtraceCollector( self.target, @@ -752,124 +789,6 @@ class TestEnv(ShareState): for function in functions: self._log.info(' %s', function) - return - - def _init_energy(self, force): - - # Initialize energy probe to board default - self.emeter = EnergyMeter.getInstance(self.target, self.conf, force, - self.res_dir) - - def _init_platform_bl(self): - self.platform = { - 'clusters' : { - 'little' : self.target.bl.littles, - 'big' : self.target.bl.bigs - }, - 'freqs' : { - 'little' : self.target.bl.list_littles_frequencies(), - 'big' : self.target.bl.list_bigs_frequencies() - } - } - self.platform['cpus_count'] = \ - len(self.platform['clusters']['little']) + \ - len(self.platform['clusters']['big']) - - def _init_platform_smp(self): - self.platform = { - 'clusters' : {}, - 'freqs' : {} - } - for cpu_id,node_id in enumerate(self.target.core_clusters): - if node_id not in self.platform['clusters']: - self.platform['clusters'][node_id] = [] - self.platform['clusters'][node_id].append(cpu_id) - - if 'cpufreq' in self.target.modules: - # Try loading frequencies using the cpufreq module - for cluster_id in self.platform['clusters']: - core_id = self.platform['clusters'][cluster_id][0] - self.platform['freqs'][cluster_id] = \ - self.target.cpufreq.list_frequencies(core_id) - else: - self._log.warning('Unable to identify cluster frequencies') - - # TODO: get the performance boundaries in case of intel_pstate driver - - self.platform['cpus_count'] = len(self.target.core_clusters) - - def _load_em(self, board): - em_path = os.path.join(basepath, - 'libs/utils/platforms', board.lower() + '.json') - self._log.debug('Trying to load default EM from %s', em_path) - if not os.path.exists(em_path): - return None - self._log.info('Loading default EM:') - self._log.info(' %s', em_path) - board = JsonConf(em_path) - board.load() - if 'nrg_model' not in board.json: - return None - return board.json['nrg_model'] - - def _load_board(self, board): - board_path = os.path.join(basepath, - 'libs/utils/platforms', board.lower() + '.json') - self._log.debug('Trying to load board descriptor from %s', board_path) - if not os.path.exists(board_path): - return None - self._log.info('Loading board:') - self._log.info(' %s', board_path) - board = JsonConf(board_path) - board.load() - if 'board' not in board.json: - return None - return board.json['board'] - - def _get_clusters(self, core_names): - idx = 0 - clusters = [] - ids_map = { core_names[0] : 0 } - for name in core_names: - idx = ids_map.get(name, idx+1) - ids_map[name] = idx - clusters.append(idx) - return clusters - - def _init_platform(self): - if 'bl' in self.target.modules: - self._init_platform_bl() - else: - self._init_platform_smp() - - # Adding energy model information - if 'nrg_model' in self.conf: - self.platform['nrg_model'] = self.conf['nrg_model'] - # Try to load the default energy model (if available) - else: - nrg_model = self._load_em(self.conf['board']) - # We shouldn't have an 'nrg_model' key if there is no energy model data - if nrg_model: - self.platform['nrg_model'] = nrg_model - - # Adding topology information - self.platform['topology'] = self.topology.get_level("cluster") - - # Adding kernel build information - kver = self.target.kernel_version - self.platform['kernel'] = {t: getattr(kver, t, None) - for t in [ - 'release', 'version', - 'version_number', 'major', 'minor', - 'rc', 'sha1', 'parts' - ] - } - self.platform['abi'] = self.target.abi - self.platform['os'] = self.target.os - - self._log.debug('Platform descriptor initialized\n%s', self.platform) - # self.platform_dump('./') - def platform_dump(self, dest_dir, dest_file='platform.json'): plt_file = os.path.join(dest_dir, dest_file) self._log.debug('Dump platform descriptor in [%s]', plt_file) @@ -891,11 +810,9 @@ class TestEnv(ShareState): if not force and self._calib: return self._calib - required = force or 'rt-app' in self.__installed_tools - - if not required: - self._log.debug('No RT-App workloads, skipping calibration') - return + required_tools = ['rt-app', 'taskset', 'trace-cmd', 'perf', 'cgroup_run_into.sh'] + if not all([tool in self.__installed_tools for tool in required_tools]): + self.install_tools(required_tools) if not force and 'rtapp-calib' in self.conf: self._log.info('Using configuration provided RTApp calibration') @@ -913,226 +830,6 @@ class TestEnv(ShareState): for key in sorted(self._calib)) + "}") return self._calib - def resolv_host(self, host=None): - """ - Resolve a host name or IP address to a MAC address - - .. TODO Is my networking terminology correct here? - - :param host: IP address or host name to resolve. If None, use 'host' - value from target_config. - :type host: str - """ - if host is None: - host = self.conf['host'] - - # Refresh ARP for local network IPs - self._log.debug('Collecting all Bcast address') - output = os.popen(r'ifconfig').read().split('\n') - for line in output: - match = IFCFG_BCAST_RE.search(line) - if not match: - continue - baddr = match.group(1) - try: - cmd = r'nmap -T4 -sP {}/24 &>/dev/null'.format(baddr.strip()) - self._log.debug(cmd) - os.popen(cmd) - except RuntimeError: - self._log.warning('Nmap not available, try IP lookup using broadcast ping') - cmd = r'ping -b -c1 {} &>/dev/null'.format(baddr) - self._log.debug(cmd) - os.popen(cmd) - - return self.parse_arp_cache(host) - - def parse_arp_cache(self, host): - output = os.popen(r'arp -n') - if ':' in host: - # Assuming this is a MAC address - # TODO add a suitable check on MAC address format - # Query ARP for the specified HW address - ARP_RE = re.compile( - r'([^ ]*).*({}|{})'.format(host.lower(), host.upper()) - ) - macaddr = host - ipaddr = None - for line in output: - match = ARP_RE.search(line) - if not match: - continue - ipaddr = match.group(1) - break - else: - # Assuming this is an IP address - # TODO add a suitable check on IP address format - # Query ARP for the specified IP address - ARP_RE = re.compile( - r'{}.*ether *([0-9a-fA-F:]*)'.format(host) - ) - macaddr = None - ipaddr = host - for line in output: - match = ARP_RE.search(line) - if not match: - continue - macaddr = match.group(1) - break - else: - # When target is accessed via WiFi, there is not MAC address - # reported by arp. In these cases we can know only the IP - # of the remote target. - macaddr = 'UNKNOWN' - - if not ipaddr or not macaddr: - raise ValueError('Unable to lookup for target IP/MAC address') - self._log.info('Target (%s) at IP address: %s', macaddr, ipaddr) - return (macaddr, ipaddr) - - def reboot(self, reboot_time=120, ping_time=15): - """ - Reboot target. - - :param boot_time: Time to wait for the target to become available after - reboot before declaring failure. - :param ping_time: Period between attempts to ping the target while - waiting for reboot. - """ - # Send remote target a reboot command - if self._feature('no-reboot'): - self._log.warning('Reboot disabled by conf features') - else: - if 'reboot_time' in self.conf: - reboot_time = int(self.conf['reboot_time']) - - if 'ping_time' in self.conf: - ping_time = int(self.conf['ping_time']) - - # Before rebooting make sure to have IP and MAC addresses - # of the target - (self.mac, self.ip) = self.parse_arp_cache(self.ip) - - self.target.execute('sleep 2 && reboot -f &', as_root=True) - - # Wait for the target to complete the reboot - self._log.info('Waiting up to %s[s] for target [%s] to reboot...', - reboot_time, self.ip) - - ping_cmd = "ping -c 1 {} >/dev/null".format(self.ip) - elapsed = 0 - start = time.time() - while elapsed <= reboot_time: - time.sleep(ping_time) - self._log.debug('Trying to connect to [%s] target...', self.ip) - if os.system(ping_cmd) == 0: - break - elapsed = time.time() - start - if elapsed > reboot_time: - if self.mac: - self._log.warning('target [%s] not responding to PINGs, ' - 'trying to resolve MAC address...', - self.ip) - (self.mac, self.ip) = self.resolv_host(self.mac) - else: - self._log.warning('target [%s] not responding to PINGs, ' - 'trying to continue...', - self.ip) - - # Force re-initialization of all the devlib modules - force = True - - # Reset the connection to the target - self._init(force) - - # Initialize FTrace events collection - self._init_ftrace(force) - - # Initialize energy probe instrument - self._init_energy(force) - - def install_kernel(self, tc, reboot=False): - """ - Deploy kernel and DTB via TFTP, optionally rebooting - - :param tc: Dicionary containing optional keys 'kernel' and 'dtb'. Values - are paths to the binaries to deploy. - :type tc: dict - - :param reboot: Reboot thet target after deployment - :type reboot: bool - """ - - # Default initialize the kernel/dtb settings - tc.setdefault('kernel', None) - tc.setdefault('dtb', None) - - if self.kernel == tc['kernel'] and self.dtb == tc['dtb']: - return - - self._log.info('Install kernel [%s] on target...', tc['kernel']) - - # Install kernel/dtb via FTFP - if self._feature('no-kernel'): - self._log.warning('Kernel deploy disabled by conf features') - - elif 'tftp' in self.conf: - self._log.info('Deploy kernel via TFTP...') - - # Deploy kernel in TFTP folder (mandatory) - if 'kernel' not in tc or not tc['kernel']: - raise ValueError('Missing "kernel" parameter in conf: %s', - 'KernelSetup', tc) - self.tftp_deploy(tc['kernel']) - - # Deploy DTB in TFTP folder (if provided) - if 'dtb' not in tc or not tc['dtb']: - self._log.debug('DTB not provided, using existing one') - self._log.debug('Current conf:\n%s', tc) - self._log.warning('Using pre-installed DTB') - else: - self.tftp_deploy(tc['dtb']) - - else: - raise ValueError('Kernel installation method not supported') - - # Keep track of last installed kernel - self.kernel = tc['kernel'] - if 'dtb' in tc: - self.dtb = tc['dtb'] - - if not reboot: - return - - # Reboot target - self._log.info('Rebooting taget...') - self.reboot() - - - def tftp_deploy(self, src): - """ - .. TODO - """ - - tftp = self.conf['tftp'] - - dst = tftp['folder'] - if 'kernel' in src: - dst = os.path.join(dst, tftp['kernel']) - elif 'dtb' in src: - dst = os.path.join(dst, tftp['dtb']) - else: - dst = os.path.join(dst, os.path.basename(src)) - - cmd = 'cp {} {} && sync'.format(src, dst) - self._log.info('Deploy %s into %s', src, dst) - result = os.system(cmd) - if result != 0: - self._log.error('Failed to deploy image: %s', src) - raise ValueError('copy error') - - def _feature(self, feature): - return feature in self.conf['__features__'] - @contextlib.contextmanager def freeze_userspace(self): if 'cgroups' not in self.target.modules: @@ -1158,8 +855,16 @@ class TestEnv(ShareState): self._log.info('Un-freezing userspace tasks') self.target.cgroups.freeze(thaw=True) -IFCFG_BCAST_RE = re.compile( - r'Bcast:(.*) ' -) + @contextlib.contextmanager + def record_ftrace(self, output_file=None): + if not output_file: + output_file = os.path.join(self.get_res_dir(), "trace.dat") + + self.ftrace.start() + + yield + + self.ftrace.stop() + self.ftrace.get_trace(output_file) # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/libs/utils/executor.py b/libs/utils/executor.py deleted file mode 100644 index 0bf2175807c96e4f17b72b0cb87745139b8738d7..0000000000000000000000000000000000000000 --- a/libs/utils/executor.py +++ /dev/null @@ -1,753 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from bart.common.Analyzer import Analyzer -import collections -from collections import namedtuple -import datetime -import gzip -import json -import os -import re -import time -import trappy -from devlib import TargetError - -# Configure logging -import logging - -# Add JSON parsing support -from conf import JsonConf - -import wlgen - -from devlib import TargetError - -Experiment = namedtuple('Experiment', ['wload_name', 'wload', - 'conf', 'iteration', 'out_dir']) - -class Executor(): - """ - Abstraction for running sets of experiments and gathering data from targets - - An executor can be configured to run a set of workloads (wloads) in each - different target configuration of a specified set (confs). These wloads and - confs can be specified by the "experiments_conf" input dictionary. Each - (workload, conf, iteration) tuple is called an "experiment". - - After the workloads have been run, the Executor object's `experiments` - attribute is a list of Experiment objects. The `out_dir` attribute of these - objects can be used to find the results of the experiment. This output - directory follows this format: - - results//::/ - - where: - - test_id - Is the "tid" defined by the experiments_conf, or a timestamp based - folder in case "tid" is not specified. - wltype - Is the class of workload executed, e.g. rtapp or sched_perf. - conf - Is the "tag" of one of the specified **confs**. - wload - Is the identifier of one of the specified **wloads**. - run_id - Is the progressive execution number from 1 up to the specified - **iterations**. - - :param experiments_conf: Dict with experiment configuration. Keys are: - - **confs** - Mandatory. Platform configurations to be tested. List of dicts, - each with keys: - - tag - String to identify this configuration. Required, may be empty. - flags - List of strings describing features required for this - conf. Available flags are: - - "ftrace" - Enable collecting ftrace during the experiment. - "freeze_userspace" - Use the cgroups freezer to freeze as many userspace tasks as - possible during the experiment execution, in order to reduce - system noise. Some tasks cannot be frozen, such as those - required to maintain a connection to LISA. - - sched_features - Optional list of features to be written to - /sys/kernel/debug/sched_features. Prepend "NO\_" to a feature to - actively disable it. Requires ``CONFIG_SCHED_DEBUG`` in target - kernel. - cpufreq - Parameters to configure cpufreq via Devlib's cpufreq - module. Dictionary with fields: - - .. TODO link to devlib cpufreq module docs (which don't exist) - - governor - cpufreq governor to set (for all CPUs) before execution. The - previous governor is not restored when execution is finished. - governor_tunables - Dictionary of governor-specific tunables, expanded and passed as - kwargs to the cpufreq module's ``set_governor_tunables`` method. - freq - Requires "governor" to be "userspace". Dictionary mapping CPU - numbers to frequencies. Exact frequencies should be available on - those CPUs. It is not necessary to provide a frequency for every - CPU - the frequency for unspecified CPUs is not affected. Note - that cpufreq will transparrently set the frequencies of any - other CPUs sharing a clock domain. - - cgroups - Optional cgroups configuration. To use this, ensure the 'cgroups' - devlib module is enabled in your test_conf Contains fields: - - .. TODO reference test_conf - .. TODO link to devlib cgroup module's docs (which don't exist) - - conf - Dict specifying the cgroup controllers, cgroups, and cgroup - parameters to setup. If a controller listed here is not - enabled in the target kernel, a message is logged and the - configuration is **ignored**. Of the form: - - :: - - "" : { - "" : { " } - "" : { " } - } - - These cgroups can then be used in the "cgroup" field of workload - specifications. - - default - The default cgroup to run workloads in, if no "cgroup" is - specified. - - For example, to create a cpuset cgroup named "/big" which - restricts constituent tasks to CPUs 1 and 2: - - :: - - "cgroups" : { - "conf" : { - "cpuset" : { - "/big" : {"cpus" : "1-2"}, - } - }, - "default" : "/", - } - - **wloads** - .. TODO document wloads field. - - Mandatory. Workloads to run on each platform configuration - - **iterations** - Number of iterations for each workload/conf combination. Default - is 1. - :type experiments_conf: dict - - :ivar experiments: After calling :func:`run`, the list of - :class:`Experiment` s that were run - - :ivar iterations: The number of iterations run for each wload/conf pair - (i.e. ``experiments_conf['iterations']``. - - """ - - def __init__(self, test_env, experiments_conf): - # Initialize globals - self._default_cgroup = None - self._cgroup = None - self._old_selinux_mode = None - - # Setup logging - self._log = logging.getLogger('Executor') - - # Setup test configuration - if isinstance(experiments_conf, dict): - self._log.info('Loading custom (inline) test configuration') - self._experiments_conf = experiments_conf - elif isinstance(experiments_conf, str): - self._log.info('Loading custom (file) test configuration') - json_conf = JsonConf(experiments_conf) - self._experiments_conf = json_conf.load() - else: - raise ValueError( - 'experiments_conf must be either a dictionary or a filepath') - - # Check for mandatory configurations - if not self._experiments_conf.get('confs', None): - raise ValueError('Configuration error: ' - 'missing "conf" definitions') - if not self._experiments_conf.get('wloads', None): - raise ValueError('Configuration error: ' - 'missing "wloads" definitions') - - self.te = test_env - self.target = self.te.target - - self.iterations = self._experiments_conf.get('iterations', 1) - # Compute total number of experiments - self._exp_count = self.iterations \ - * len(self._experiments_conf['wloads']) \ - * len(self._experiments_conf['confs']) - - self._print_section('Experiments configuration') - - self._log.info('Configured to run:') - - self._log.info(' %3d target configurations:', - len(self._experiments_conf['confs'])) - target_confs = [conf['tag'] for conf in self._experiments_conf['confs']] - target_confs = ', '.join(target_confs) - self._log.info(' %s', target_confs) - - self._log.info(' %3d workloads (%d iterations each)', - len(self._experiments_conf['wloads']), - self.iterations) - wload_confs = ', '.join(self._experiments_conf['wloads']) - self._log.info(' %s', wload_confs) - - self._log.info('Total: %d experiments', self._exp_count) - - self._log.info('Results will be collected under:') - self._log.info(' %s', self.te.res_dir) - - if any(wl['type'] == 'rt-app' - for wl in self._experiments_conf['wloads'].values()): - self._log.info('rt-app workloads found, installing tool on target') - self.te.install_tools(['rt-app']) - - def run(self): - self._print_section('Experiments execution') - - self.experiments = [] - - # Run all the configured experiments - exp_idx = 0 - for tc in self._experiments_conf['confs']: - # TARGET: configuration - if not self._target_configure(tc): - continue - for wl_idx in self._experiments_conf['wloads']: - # TEST: configuration - wload, test_dir = self._wload_init(tc, wl_idx) - for itr_idx in range(1, self.iterations + 1): - exp = Experiment( - wload_name=wl_idx, - wload=wload, - conf=tc, - iteration=itr_idx, - out_dir=os.path.join(test_dir, str(itr_idx))) - self.experiments.append(exp) - - # WORKLOAD: execution - if self._target_conf_flag(tc, 'freeze_userspace'): - with self.te.freeze_userspace(): - self._wload_run(exp_idx, exp) - else: - self._wload_run(exp_idx, exp) - exp_idx += 1 - self._target_cleanup(tc) - - self._print_section('Experiments execution completed') - self._log.info('Results available in:') - self._log.info(' %s', self.te.res_dir) - - -################################################################################ -# Target Configuration -################################################################################ - - def _cgroups_init(self, tc): - self._default_cgroup = None - if 'cgroups' not in tc: - return True - if 'cgroups' not in self.target.modules: - raise RuntimeError('CGroups module not available. Please ensure ' - '"cgroups" is listed in your target/test modules') - self._log.info('Initialize CGroups support...') - errors = False - for kind in tc['cgroups']['conf']: - self._log.info('Setup [%s] CGroup controller...', kind) - controller = self.target.cgroups.controller(kind) - if not controller: - self._log.warning('CGroups controller [%s] NOT available', - kind) - errors = True - return not errors - - def _setup_kernel(self, tc): - # Deploy kernel on the device - self.te.install_kernel(tc, reboot=True) - # Setup the rootfs for the experiments - self._setup_rootfs(tc) - - def _setup_sched_features(self, tc): - if 'sched_features' not in tc: - self._log.debug('Scheduler features configuration not provided') - return - feats = tc['sched_features'].split(",") - for feat in feats: - self._log.info('Set scheduler feature: %s', feat) - self.target.execute('echo {} > /sys/kernel/debug/sched_features'.format(feat), - as_root=True) - - @staticmethod - def get_run_dir(target): - return os.path.join(target.working_directory, TGT_RUN_DIR) - - def _setup_rootfs(self, tc): - # Initialize CGroups if required - self._cgroups_init(tc) - # Setup target folder for experiments execution - self.te.run_dir = self.get_run_dir(self.target) - # Create run folder as tmpfs - self._log.debug('Setup RT-App run folder [%s]...', self.te.run_dir) - self.target.execute('[ -d {0} ] || mkdir {0}'\ - .format(self.te.run_dir)) - - if self.target.is_rooted: - self.target.execute( - 'grep schedtest /proc/mounts || '\ - ' mount -t tmpfs -o size=1024m {} {}'\ - .format('schedtest', self.te.run_dir), - as_root=True) - - # tmpfs mounts have an SELinux context with "tmpfs" as the type - # (while other files we create have "shell_data_file"). That - # prevents non-root users from creating files in tmpfs mounts. For - # now, just put SELinux in permissive mode to get around that. - try: - # First, save the old SELinux mode - self._old_selinux_mode = self.target.execute('getenforce') - except TargetError: - # Probably the target doesn't have SELinux. No problem. - pass - else: - - self._log.warning('Setting target SELinux in permissive mode') - self.target.execute('setenforce 0', as_root=True) - else: - self._log.warning('Not mounting tmpfs because no root') - - def _setup_cpufreq(self, tc): - if 'cpufreq' not in tc: - self._log.warning('cpufreq governor not specified, ' - 'using currently configured governor') - return - - cpufreq = tc['cpufreq'] - self._log.info('Configuring all CPUs to use [%s] cpufreq governor', - cpufreq['governor']) - - self.target.cpufreq.set_all_governors(cpufreq['governor']) - - if 'freqs' in cpufreq: - if cpufreq['governor'] != 'userspace': - raise ValueError('Must use userspace governor to set CPU freqs') - self._log.info(r'%14s - CPU frequencies: %s', - 'CPUFreq', str(cpufreq['freqs'])) - for cpu, freq in cpufreq['freqs'].iteritems(): - self.target.cpufreq.set_frequency(cpu, freq) - - if 'params' in cpufreq: - self._log.info('governor params: %s', str(cpufreq['params'])) - for cpu in self.target.list_online_cpus(): - self.target.cpufreq.set_governor_tunables( - cpu, - cpufreq['governor'], - **cpufreq['params']) - - def _setup_cgroups(self, tc): - if 'cgroups' not in tc: - return True - # Setup default CGroup to run tasks into - if 'default' in tc['cgroups']: - self._default_cgroup = tc['cgroups']['default'] - # Configure each required controller - if 'conf' not in tc['cgroups']: - return True - errors = False - for kind in tc['cgroups']['conf']: - controller = self.target.cgroups.controller(kind) - if not controller: - self._log.warning('Configuration error: ' - '[%s] contoller NOT supported', - kind) - errors = True - continue - self._setup_controller(tc, controller) - return not errors - - def _setup_controller(self, tc, controller): - kind = controller.kind - # Configure each required groups for that controller - errors = False - for name in tc['cgroups']['conf'][controller.kind]: - if name[0] != '/': - raise ValueError('Wrong CGroup name [{}]. ' - 'CGroups names must start by "/".' - .format(name)) - group = controller.cgroup(name) - if not group: - self._log.warning('Configuration error: ' - '[%s/%s] cgroup NOT available', - kind, name) - errors = True - continue - self._setup_group(tc, group) - return not errors - - def _setup_group(self, tc, group): - kind = group.controller.kind - name = group.name - # Configure each required attribute - group.set(**tc['cgroups']['conf'][kind][name]) - - def _setup_files(self, tc): - if 'files' not in tc: - self._log.debug('\'files\' Configuration block not provided') - return True - for name, value in tc['files'].iteritems(): - check = False - if name.startswith('!/'): - check = True - name = name[1:] - self._log.info('File Write(check=%s): \'%s\' -> \'%s\'', - check, value, name) - try: - self.target.write_value(name, value, True) - except TargetError: - self._log.info('File Write Failed: \'%s\' -> \'%s\'', - value, name) - if check: - raise - return False - - def _target_configure(self, tc): - self._print_header( - 'configuring target for [{}] experiments'\ - .format(tc['tag'])) - self._setup_kernel(tc) - self._setup_sched_features(tc) - self._setup_cpufreq(tc) - self._setup_files(tc) - return self._setup_cgroups(tc) - - def _target_conf_flag(self, tc, flag): - if 'flags' not in tc: - has_flag = False - else: - has_flag = flag in tc['flags'] - self._log.debug('Check if target configuration [%s] has flag [%s]: %s', - tc['tag'], flag, has_flag) - return has_flag - - def _target_cleanup(self, tc): - if self._old_selinux_mode is not None: - self._log.info('Restoring target SELinux mode: %s', - self._old_selinux_mode) - self.target.execute('setenforce ' + self._old_selinux_mode, - as_root=True) - -################################################################################ -# Workload Setup and Execution -################################################################################ - - def _wload_cpus(self, wl_idx, wlspec): - if not 'cpus' in wlspec['conf']: - return None - cpus = wlspec['conf']['cpus'] - - if type(cpus) == list: - return cpus - if type(cpus) == int: - return [cpus] - - # SMP target (or not bL module loaded) - if not hasattr(self.target, 'bl'): - if 'first' in cpus: - return [ self.target.list_online_cpus()[0] ] - if 'last' in cpus: - return [ self.target.list_online_cpus()[-1] ] - return self.target.list_online_cpus() - - # big.LITTLE target - if cpus.startswith('littles'): - if 'first' in cpus: - return [ self.target.bl.littles_online[0] ] - if 'last' in cpus: - return [ self.target.bl.littles_online[-1] ] - return self.target.bl.littles_online - if cpus.startswith('bigs'): - if 'first' in cpus: - return [ self.target.bl.bigs_online[0] ] - if 'last' in cpus: - return [ self.target.bl.bigs_online[-1] ] - return self.target.bl.bigs_online - raise ValueError('unsupported [{}] "cpus" value for [{}] ' - 'workload specification' - .format(cpus, wl_idx)) - - def _wload_task_idxs(self, wl_idx, tasks): - if type(tasks) == int: - return range(tasks) - if tasks == 'cpus': - return range(len(self.target.core_names)) - if tasks == 'little': - return range(len([t - for t in self.target.core_names - if t == self.target.little_core])) - if tasks == 'big': - return range(len([t - for t in self.target.core_names - if t == self.target.big_core])) - raise ValueError('unsupported "tasks" value for [{}] RT-App ' - 'workload specification' - .format(wl_idx)) - - def _wload_rtapp(self, wl_idx, wlspec, cpus): - conf = wlspec['conf'] - self._log.debug('Configuring [%s] rt-app...', conf['class']) - - # Setup a default "empty" task name prefix - if 'prefix' not in conf: - conf['prefix'] = 'task_' - - # Setup a default loadref CPU - loadref = None - if 'loadref' in wlspec: - loadref = wlspec['loadref'] - - if conf['class'] == 'profile': - params = {} - # Load each task specification - for task_name, task in conf['params'].items(): - if task['kind'] not in wlgen.__dict__: - self._log.error('RTA task of kind [%s] not supported', - task['kind']) - raise ValueError('unsupported "kind" value for task [{}] ' - 'in RT-App workload specification' - .format(task)) - task_ctor = getattr(wlgen, task['kind']) - num_tasks = task.get('tasks', 1) - task_idxs = self._wload_task_idxs(wl_idx, num_tasks) - for idx in task_idxs: - idx_name = "_{}".format(idx) if len(task_idxs) > 1 else "" - task_name_idx = conf['prefix'] + task_name + idx_name - params[task_name_idx] = task_ctor(**task['params']).get() - - rtapp = wlgen.RTA(self.target, - wl_idx, calibration = self.te.calibration()) - rtapp.conf(kind='profile', params=params, loadref=loadref, - cpus=cpus, run_dir=self.te.run_dir, - duration=conf.get('duration')) - return rtapp - - if conf['class'] == 'periodic': - task_idxs = self._wload_task_idxs(wl_idx, conf['tasks']) - params = {} - for idx in task_idxs: - task = conf['prefix'] + str(idx) - params[task] = wlgen.Periodic(**conf['params']).get() - rtapp = wlgen.RTA(self.target, - wl_idx, calibration = self.te.calibration()) - rtapp.conf(kind='profile', params=params, loadref=loadref, - cpus=cpus, run_dir=self.te.run_dir, - duration=conf.get('duration')) - return rtapp - - if conf['class'] == 'custom': - rtapp = wlgen.RTA(self.target, - wl_idx, calibration = self.te.calibration()) - rtapp.conf(kind='custom', - params=conf['json'], - duration=conf.get('duration'), - loadref=loadref, - cpus=cpus, run_dir=self.te.run_dir) - return rtapp - - raise ValueError('unsupported \'class\' value for [{}] ' - 'RT-App workload specification' - .format(wl_idx)) - - def _wload_perf_bench(self, wl_idx, wlspec, cpus): - conf = wlspec['conf'] - self._log.debug('Configuring perf_message...') - - if conf['class'] == 'messaging': - perf_bench = wlgen.PerfMessaging(self.target, wl_idx) - perf_bench.conf(**conf['params']) - return perf_bench - - if conf['class'] == 'pipe': - perf_bench = wlgen.PerfPipe(self.target, wl_idx) - perf_bench.conf(**conf['params']) - return perf_bench - - raise ValueError('unsupported "class" value for [{}] ' - 'perf bench workload specification' - .format(wl_idx)) - - def _wload_conf(self, wl_idx, wlspec): - - # CPUS: setup execution on CPUs if required by configuration - cpus = self._wload_cpus(wl_idx, wlspec) - - # CGroup: setup CGroups if requried by configuration - self._cgroup = self._default_cgroup - if 'cgroup' in wlspec: - if 'cgroups' not in self.target.modules: - raise RuntimeError('Target not supporting CGroups or CGroups ' - 'not configured for the current test configuration') - self._cgroup = wlspec['cgroup'] - - if wlspec['type'] == 'rt-app': - return self._wload_rtapp(wl_idx, wlspec, cpus) - if wlspec['type'] == 'perf_bench': - return self._wload_perf_bench(wl_idx, wlspec, cpus) - - - raise ValueError('unsupported "type" value for [{}] ' - 'workload specification' - .format(wl_idx)) - - def _wload_init(self, tc, wl_idx): - tc_idx = tc['tag'] - - # Configure the test workload - wlspec = self._experiments_conf['wloads'][wl_idx] - wload = self._wload_conf(wl_idx, wlspec) - - # Keep track of platform configuration - test_dir = '{}/{}:{}:{}'\ - .format(self.te.res_dir, wload.wtype, tc_idx, wl_idx) - os.makedirs(test_dir) - self.te.platform_dump(test_dir) - - # Keep track of kernel configuration and version - config = self.target.config - with gzip.open(os.path.join(test_dir, 'kernel.config'), 'wb') as fh: - fh.write(config.text) - output = self.target.execute('{} uname -a'\ - .format(self.target.busybox)) - with open(os.path.join(test_dir, 'kernel.version'), 'w') as fh: - fh.write(output) - - return wload, test_dir - - def _wload_run(self, exp_idx, experiment): - tc = experiment.conf - wload = experiment.wload - tc_idx = tc['tag'] - - self._print_title('Experiment {}/{}, [{}:{}] {}/{}'\ - .format(exp_idx, self._exp_count, - tc_idx, experiment.wload_name, - experiment.iteration, self.iterations)) - - # Setup local results folder - self._log.debug('out_dir set to [%s]', experiment.out_dir) - os.system('mkdir -p ' + experiment.out_dir) - - # FTRACE: start (if a configuration has been provided) - if self.te.ftrace and self._target_conf_flag(tc, 'ftrace'): - self._log.info('FTrace events collection enabled') - self.te.ftrace.start() - - # ENERGY: start sampling - if self.te.emeter: - self.te.emeter.reset() - - # WORKLOAD: Run the configured workload - wload.run(out_dir=experiment.out_dir, cgroup=self._cgroup) - - # ENERGY: collect measurements - if self.te.emeter: - self.te.emeter.report(experiment.out_dir) - - # FTRACE: stop and collect measurements - if self.te.ftrace and self._target_conf_flag(tc, 'ftrace'): - self.te.ftrace.stop() - - trace_file = experiment.out_dir + '/trace.dat' - self.te.ftrace.get_trace(trace_file) - self._log.info('Collected FTrace binary trace:') - self._log.info(' %s', - trace_file.replace(self.te.res_dir, '')) - - stats_file = experiment.out_dir + '/trace_stat.json' - self.te.ftrace.get_stats(stats_file) - self._log.info('Collected FTrace function profiling:') - self._log.info(' %s', - stats_file.replace(self.te.res_dir, '')) - - self._print_footer() - -################################################################################ -# Utility Functions -################################################################################ - - def _print_section(self, message): - self._log.info('') - self._log.info(FMT_SECTION) - self._log.info(message) - self._log.info(FMT_SECTION) - - def _print_header(self, message): - self._log.info('') - self._log.info(FMT_HEADER) - self._log.info(message) - - def _print_title(self, message): - self._log.info(FMT_TITLE) - self._log.info(message) - - def _print_footer(self, message=None): - if message: - self._log.info(message) - self._log.info(FMT_FOOTER) - - -################################################################################ -# Globals -################################################################################ - -# Regular expression for comments -JSON_COMMENTS_RE = re.compile( - '(^)?[^\S\n]*/(?:\*(.*?)\*/[^\S\n]*|/[^\n]*)($)?', - re.DOTALL | re.MULTILINE -) - -# Target specific paths -TGT_RUN_DIR = 'run_dir' - -# Logging formatters -FMT_SECTION = r'{:#<80}'.format('') -FMT_HEADER = r'{:=<80}'.format('') -FMT_TITLE = r'{:~<80}'.format('') -FMT_FOOTER = r'{:-<80}'.format('') - -# vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/libs/utils/generic.py b/libs/utils/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..7acfb009e5eb957cc2670ecf74c6420a9920a665 --- /dev/null +++ b/libs/utils/generic.py @@ -0,0 +1,560 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from math import isnan + +import pandas as pd +import matplotlib.pyplot as plt +import pylab as pl + +from bart.common.Utils import area_under_curve + +from wlgen.rta import RTA, Periodic, Ramp, Step +from trace import Trace +from test_workload import Metric, ResultBundle, TestBundle +from perf_analysis import PerfAnalysis + +class GenericTestBundle(TestBundle): + """ + "Abstract" class for generic synthetic tests. + + :param nrg_model: The energy model of the platform the synthetic workload + was run on + :type nrg_model: EnergyModel + + :param rtapp_params: The rtapp parameters used to create the synthetic + workload. That happens to be what is returned by :meth:`create_rtapp_params` + :type rtapp_params: dict + + This class provides :meth:`test_slack` and :meth:`test_task_placement` to + validate the basic behaviour of EAS. + """ + + ftrace_conf = { + "events" : ["sched_switch"], + } + """ + The FTrace configuration used to record a trace while the synthetic workload + is being run. + """ + + @property + def trace(self): + """ + + :returns: a Trace + + Having the trace as a property lets us defer the loading of the actual + trace to when it is first used. Also, this prevents it from being + serialized when calling :meth:`to_path` + """ + if not self._trace: + self._trace = Trace(self.res_dir, events=self.ftrace_conf["events"]) + + return self._trace + + def __init__(self, res_dir, nrg_model, rtapp_params): + super(GenericTestBundle, self).__init__(res_dir) + + # self.trace = Trace(res_dir, events=self.ftrace_conf["events"]) + #EnergyModel.from_path(os.path.join(res_dir, "nrg_model.yaml")) + self._trace = None + self.nrg_model = nrg_model + self.rtapp_params = rtapp_params + + @classmethod + def create_rtapp_params(cls, te): + """ + :returns: a :class:`dict` with task names as keys and :class:`RTATask` as values + + This is the method you want to override to specify what is + your synthetic workload. + """ + raise NotImplementedError() + + @classmethod + def _from_target(cls, te, res_dir): + rtapp_params = cls.create_rtapp_params(te) + + wload = RTA(te.target, "rta_{}".format(cls.__name__.lower()), te.calibration()) + wload.conf(kind='profile', params=rtapp_params, work_dir=res_dir) + + trace_path = os.path.join(res_dir, "trace.dat") + te.configure_ftrace(**cls.ftrace_conf) + + with te.record_ftrace(trace_path): + with te.freeze_userspace(): + wload.run(out_dir=res_dir) + + return cls(res_dir, te.nrg_model, rtapp_params) + + @classmethod + def min_cpu_capacity(cls, te): + """ + The smallest CPU capacity on the target + + :type te: TestEnv + + :returns: int + """ + return min(te.target.sched.get_capacities().values()) + + @classmethod + def max_cpu_capacity(cls, te): + """ + The highest CPU capacity on the target + + :type te: TestEnv + + :returns: int + """ + return max(te.target.sched.get_capacities().values()) + + def test_slack(self, negative_slack_allowed_pct=15): + """ + Assert that the RTApp workload was given enough performance + + :param out_dir: Output directory for test artefacts + :type out_dir: str + + :param negative_slack_allowed_pct: Allowed percentage of RT-app task + activations with negative slack. + :type negative_slack_allowed_pct: int + + Use :class:`PerfAnalysis` to find instances where the RT-App workload + wasn't able to complete its activations (i.e. its reported "slack" + was negative). Assert that this happened less than + :attr:`negative_slack_allowed_pct` percent of the time. + """ + pa = PerfAnalysis(self.res_dir) + + slacks = {} + passed = True + + # Data is only collected for rt-app tasks, so it's safe to iterate over + # all of them + for task in pa.tasks(): + slack = pa.df(task)["Slack"] + + bad_activations_pct = len(slack[slack < 0]) * 100. / len(slack) + if bad_activations_pct > negative_slack_allowed_pct: + passed = False + + slacks[task] = bad_activations_pct + + res = ResultBundle(passed) + for task, slack in slacks.iteritems(): + res.add_metric(Metric("slack_{}".format(task), slack, + units='%', lower_is_better=True)) + + return res + + def _get_start_time(self): + """ + Get the time where the first task spawned + """ + tasks = self.rtapp_params.keys() + sdf = self.trace.data_frame.trace_event('sched_switch') + start_time = self.trace.start_time + self.trace.time_range + + for task in tasks: + pid = self.trace.getTaskByName(task) + assert len(pid) == 1, "getTaskByName returned more than one PID" + pid = pid[0] + start_time = min(start_time, sdf[sdf.next_pid == pid].index[0]) + + return start_time + + def _get_expected_task_utils_df(self): + """ + Get a DataFrame with the *expected* utilization of each task over time + + :returns: A Pandas DataFrame with a column for each task, showing how + the utilization of that task varies over time + """ + util_scale = self.nrg_model.capacity_scale + + transitions = {} + def add_transition(time, task, util): + if time not in transitions: + transitions[time] = {task: util} + else: + transitions[time][task] = util + + # First we'll build a dict D {time: {task_name: util}} where D[t][n] is + # the expected utilization of task n from time t. + for task, params in self.rtapp_params.iteritems(): + # time = self.get_start_time(experiment) + params.get('delay', 0) + time = params.delay_s + add_transition(time, task, 0) + for _ in range(params.loops): + for phase in params.phases: + util = (phase.duty_cycle_pct * util_scale / 100.) + add_transition(time, task, util) + time += phase.duration_s + add_transition(time, task, 0) + + index = sorted(transitions.keys()) + df = pd.DataFrame([transitions[k] for k in index], index=index) + return df.fillna(method='ffill') + + def _get_task_cpu_df(self): + """ + Get a DataFrame mapping task names to the CPU they ran on + + Use the sched_switch trace event to find which CPU each task ran + on. Does not reflect idleness - tasks not running are shown as running + on the last CPU they woke on. + + :returns: A Pandas DataFrame with a column for each task, showing the + CPU that the task was "on" at each moment in time + """ + tasks = self.rtapp_params.keys() + + df = self.trace.ftrace.sched_switch.data_frame[['next_comm', '__cpu']] + df = df[df['next_comm'].isin(tasks)] + df = df.pivot(index=df.index, columns='next_comm').fillna(method='ffill') + cpu_df = df['__cpu'] + # Drop consecutive duplicates + cpu_df = cpu_df[(cpu_df.shift(+1) != cpu_df).any(axis=1)] + return cpu_df + + def _sort_power_df_columns(self, df): + """ + Helper method to re-order the columns of a power DataFrame + + This has no significance for code, but when examining DataFrames by hand + they are easier to understand if the columns are in a logical order. + """ + node_cpus = [node.cpus for node in self.nrg_model.root.iter_nodes()] + return pd.DataFrame(df, columns=[c for c in node_cpus if c in df]) + + def _plot_expected_util(self, util_df): + """ + Create a plot of the expected per-CPU utilization for the experiment + The plot is then outputted to the test results directory. + + :param experiment: The :class:Experiment to examine + :param util_df: A Pandas Dataframe with a column per CPU giving their + (expected) utilization at each timestamp. + """ + + fig, ax = plt.subplots( + len(self.nrg_model.cpus), 1, figsize=(16, 1.8 * len(self.nrg_model.cpus)) + ) + fig.suptitle('Per-CPU expected utilization') + + for cpu in self.nrg_model.cpus: + tdf = util_df[cpu] + + ax[cpu].set_ylim((0, 1024)) + tdf.plot(ax=ax[cpu], drawstyle='steps-post', title="CPU{}".format(cpu), color='red') + ax[cpu].set_ylabel('Utilization') + + # Grey-out areas where utilization == 0 + ffill = False + prev = 0.0 + for time, util in tdf.iteritems(): + if ffill: + ax[cpu].axvspan(prev, time, facecolor='gray', alpha=0.1, linewidth=0.0) + ffill = False + if util == 0.0: + ffill = True + + prev = time + + figname = os.path.join(self.res_dir, 'expected_placement.png') + pl.savefig(figname, bbox_inches='tight') + plt.close() + + def _get_expected_power_df(self): + """ + Estimate *optimal* power usage over time + + Examine a trace and use :meth:get_optimal_placements and + :meth:EnergyModel.estimate_from_cpu_util to get a DataFrame showing the + estimated power usage over time under ideal EAS behaviour. + + :meth:get_optimal_placements returns several optimal placements. They + are usually equivalent, but can be drastically different in some cases. + Currently only one of those placements is used (the first in the list). + + :returns: A Pandas DataFrame with a column each node in the energy model + (keyed with a tuple of the CPUs contained by that node) and a + "power" column with the sum of other columns. Shows the + estimated *optimal* power over time. + """ + task_utils_df = self._get_expected_task_utils_df() + + data = [] + index = [] + + def exp_power(row): + task_utils = row.to_dict() + expected_utils = self.nrg_model.get_optimal_placements(task_utils)[0] + power = self.nrg_model.estimate_from_cpu_util(expected_utils) + columns = power.keys() + + # Assemble a dataframe to plot the expected utilization + data.append(expected_utils) + index.append(row.name) + + return pd.Series([power[c] for c in columns], index=columns) + + res_df = self._sort_power_df_columns( + task_utils_df.apply(exp_power, axis=1)) + + #self._plot_expected_util(pd.DataFrame(data, index=index)) + + return res_df + + def _get_estimated_power_df(self): + """ + Considering only the task placement, estimate power usage over time + + Examine a trace and use :meth:EnergyModel.estimate_from_cpu_util to get + a DataFrame showing the estimated power usage over time. This assumes + perfect cpuidle and cpufreq behaviour. + + :returns: A Pandas DataFrame with a column node in the energy model + (keyed with a tuple of the CPUs contained by that node) Shows + the estimated power over time. + """ + task_cpu_df = self._get_task_cpu_df() + task_utils_df = self._get_expected_task_utils_df() + task_utils_df.index = [time + self._get_start_time() for time in task_utils_df.index] + tasks = self.rtapp_params.keys() + + # Create a combined DataFrame with the utilization of a task and the CPU + # it was running on at each moment. Looks like: + # utils cpus + # task_wmig0 task_wmig1 task_wmig0 task_wmig1 + # 2.375056 102.4 102.4 NaN NaN + # 2.375105 102.4 102.4 2.0 NaN + + df = pd.concat([task_utils_df, task_cpu_df], + axis=1, keys=['utils', 'cpus']) + df = df.sort_index().fillna(method='ffill') + + # Now make a DataFrame with the estimated power at each moment. + def est_power(row): + cpu_utils = [0 for cpu in self.nrg_model.cpus] + for task in tasks: + cpu = row['cpus'][task] + util = row['utils'][task] + if not isnan(cpu): + cpu_utils[int(cpu)] += util + power = self.nrg_model.estimate_from_cpu_util(cpu_utils) + columns = power.keys() + return pd.Series([power[c] for c in columns], index=columns) + return self._sort_power_df_columns(df.apply(est_power, axis=1)) + + + def test_task_placement(self, energy_est_threshold_pct=5): + """ + Test that task placement was energy-efficient + + :param energy_est_threshold_pct: Allowed margin for estimated vs + optimal task placement energy cost + :type energy_est_threshold_pct: int + + Compute optimal energy consumption (energy-optimal task placement) + and compare to energy consumption estimated from the trace. + Check that the estimated energy does not exceed the optimal energy by + more than :attr:`energy_est_threshold_pct` percents. + """ + exp_power = self._get_expected_power_df() + est_power = self._get_estimated_power_df() + + exp_energy = area_under_curve(exp_power.sum(axis=1), method='rect') + est_energy = area_under_curve(est_power.sum(axis=1), method='rect') + + msg = 'Estimated {} bogo-Joules to run workload, expected {}'.format( + est_energy, exp_energy) + threshold = exp_energy * (1 + (energy_est_threshold_pct / 100.)) + + passed = est_energy < threshold + res = ResultBundle(passed) + res.add_metric(Metric("estimated_energy", est_energy, units='bogo-joules', + lower_is_better=True)) + res.add_metric(Metric("energy_threshold", threshold, units='bogo-joules', + lower_is_better=True)) + return res + + +# TODO: factorize this crap out of these classes +class OneSmallTask(GenericTestBundle): + """ + A single 'small' task + """ + + task_name = "small" + + @classmethod + def create_rtapp_params(cls, te): + # 50% of the smallest CPU's capacity + duty = int((cls.min_cpu_capacity(te) / 1024.) * 50) + + rtapp_params = {} + rtapp_params[cls.task_name] = Periodic( + duty_cycle_pct=duty, + duration_s=1, + period_ms=16 + ) + + return rtapp_params + +class ThreeSmallTasks(GenericTestBundle): + """ + Three 'small' tasks + """ + task_prefix = "small" + + @classmethod + def create_rtapp_params(cls, te): + # 50% of the smallest CPU's capacity + duty = int((cls.min_cpu_capacity(te) / 1024.) * 50) + + rtapp_params = {} + for i in range(3): + rtapp_params["{}_{}".format(cls.task_prefix, i)] = Periodic( + duty_cycle_pct=duty, + duration_s=1, + period_ms=16 + ) + + return rtapp_params + +class TwoBigTasks(GenericTestBundle): + """ + Two 'big' tasks + """ + + task_prefix = "big" + + @classmethod + def create_rtapp_params(cls, te): + # 80% of the biggest CPU's capacity + duty = int((cls.max_cpu_capacity(te) / 1024.) * 80) + + rtapp_params = {} + for i in range(2): + rtapp_params["{}_{}".format(cls.task_prefix, i)] = Periodic( + duty_cycle_pct=duty, + duration_s=1, + period_ms=16 + ) + + return rtapp_params + +class TwoBigThreeSmall(GenericTestBundle): + """ + A mix of 'big' and 'small' tasks + """ + + small_prefix = "small" + big_prefix = "big" + + @classmethod + def create_rtapp_params(cls, te): + # 50% of the smallest CPU's capacity + small_duty = int((cls.min_cpu_capacity(te) / 1024.) * 50) + # 80% of the biggest CPU's capacity + big_duty = int((cls.max_cpu_capacity(te) / 1024.) * 80) + + rtapp_params = {} + + for i in range(3): + rtapp_params["{}_{}".format(cls.small_prefix, i)] = Periodic( + duty_cycle_pct=small_duty, + duration_s=1, + period_ms=16 + ) + + for i in range(2): + rtapp_params["{}_{}".format(cls.big_prefix, i)] = Periodic( + duty_cycle_pct=big_duty, + duration_s=1, + period_ms=16 + ) + + return rtapp_params + +class RampUp(GenericTestBundle): + """ + A single task whose utilisation slowly ramps up + """ + task_name = "ramp_up" + + @classmethod + def create_rtapp_params(cls, te): + rtapp_params = {} + rtapp_params[cls.task_name] = Ramp( + start_pct=5, + end_pct=70, + delta_pct=5, + time_s=.5, + period_ms=16 + ) + + return rtapp_params + +class RampDown(GenericTestBundle): + """ + A single task whose utilisation slowly ramps down + """ + task_name = "ramp_down" + + @classmethod + def create_rtapp_params(cls, te): + rtapp_params = {} + rtapp_params[cls.task_name] = Ramp( + start_pct=70, + end_pct=5, + delta_pct=5, + time_s=.5, + period_ms=16 + ) + + return rtapp_params + +class EnergyModelWakeMigration(GenericTestBundle): + """ + One task per big CPU, alternating between two phases: + * Low utilization phase (should run on LITTLE CPUs) + * High utilization phase (should run on a big CPU) + """ + task_prefix = "emwm" + + @classmethod + def create_rtapp_params(cls, te): + rtapp_params = {} + capacities = te.target.sched.get_capacities() + max_capa = cls.max_cpu_capacity(te) + bigs = [cpu for cpu, capacity in capacities.items() if capacity == max_capa] + + for i in range(len(bigs)): + rtapp_params["{}_{}".format(cls.task_prefix, i)] = Step( + start_pct=10, + end_pct=70, + time_s=2, + loops=2, + period_ms=16 + ) + + return rtapp_params diff --git a/libs/utils/new_load_tracking.py b/libs/utils/new_load_tracking.py new file mode 100644 index 0000000000000000000000000000000000000000..4235aa87c34d865160ece1245f7ee8b8fb502734 --- /dev/null +++ b/libs/utils/new_load_tracking.py @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2016, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import pandas as pd +import logging +import re +import json +from collections import OrderedDict +from functools import reduce + +from bart.common.Utils import select_window, area_under_curve +from bart.sched import pelt +from bart.sched.SchedAssert import SchedAssert +from devlib.utils.misc import memoized +from env import TestEnv +from test_workload import Metric, ResultBundle, TestBundle +from time import sleep +from trace import Trace +from trappy.stats.grammar import Parser +from wlgen.rta import Periodic, RTA +from wlgen.utils import SchedEntity, Task, Taskgroup +from trappy.stats.Topology import Topology + +UTIL_SCALE = 1024 +# Time in seconds to allow for util_avg to converge (i.e. ignored time) +UTIL_AVG_CONVERGENCE_TIME = 0.3 +# Allowed margin between expected and observed util_avg value +ERROR_MARGIN_PCT = 15 +# PELT half-life value in ms +HALF_LIFE_MS = 32 + +class LoadTrackingTestBundle(TestBundle): + """Base class for shared functionality of load tracking tests""" + + ftrace_conf = { + "events" : [ + "sched_switch", + "sched_load_avg_task", + "sched_load_avg_cpu", + "sched_pelt_se", + "sched_load_se", + "sched_load_cfs_rq", + ], + } + """ + The FTrace configuration used to record a trace while the synthetic workload + is being run. + """ + + @property + def trace(self): + """ + + :returns: a Trace + + Having the trace as a property lets us defer the loading of the actual + trace to when it is first used. Also, this prevents it from being + serialized when calling :meth:`to_path` + """ + if not self._trace: + self._trace = Trace(self.res_dir, events=self.ftrace_conf["events"]) + + return self._trace + + def __init__(self, res_dir, nrg_model, rtapp_params, cpufreq_params, caps): + super(LoadTrackingTestBundle, self).__init__(res_dir) + + # self.trace = Trace(res_dir, events=self.ftrace_conf["events"]) + #EnergyModel.from_path(os.path.join(res_dir, "nrg_model.yaml")) + self._trace = None + self.nrg_model = nrg_model + self.rtapp_params = rtapp_params + self.cpufreq_params = cpufreq_params + self.caps = caps + + @classmethod + def create_rtapp_params(cls, te): + """ + :returns: a :class:`dict` with task names as keys and :class:`RTATask` as values + + This is the method you want to override to specify what is + your synthetic workload. + """ + raise NotImplementedError() + + @classmethod + def create_cpufreq_params(cls, te): + """ + :returns: a list of :class:`dict` each containing a unique tag, + the cpufreq governor and any additional cpufreq params + + This is the method you want to override to specify what is + your cpufreq configurations for the workload run. + """ + raise NotImplementedError() + + @classmethod + def _from_target(cls, te, res_dir): + rtapp_params = cls.create_rtapp_params(te) + cpufreq_params = cls.create_cpufreq_params(te) + for cpufreq in cpufreq_params: + iter_res_dir = os.path.join(res_dir, cpufreq['tag']) + os.makedirs(iter_res_dir) + + wload = RTA(te.target, "rta_{}".format(cls.__name__.lower()), te.calibration()) + wload.conf(kind='profile', params=rtapp_params, work_dir=iter_res_dir) + + trace_path = os.path.join(iter_res_dir, "trace.dat") + te.configure_ftrace(**cls.ftrace_conf) + + # te.target.cpufreq.use_governor(cpufreq['governor'], **cpufreq['params']) + te.target.cpufreq.set_all_governors(cpufreq['governor']) + + if 'freqs' in cpufreq: + if cpufreq['governor'] != 'userspace': + raise ValueError('Must use userspace governor to set CPU freqs') + for cpu, freq in cpufreq['freqs'].iteritems(): + te.target.cpufreq.set_frequency(cpu, freq) + + if 'params' in cpufreq: + for cpu in te.target.list_online_cpus(): + te.target.cpufreq.set_governor_tunables( + cpu, + cpufreq['governor'], + **cpufreq['params']) + + with te.record_ftrace(trace_path): + with te.freeze_userspace(): + wload.run(out_dir=iter_res_dir) + + caps = [cls._get_cpu_capacity(te, cpu) + for cpu in range(te.target.number_of_cpus)] + + return cls(res_dir, te.nrg_model, rtapp_params, cpufreq_params, caps) + + @memoized + @staticmethod + def _get_cpu_capacity(te, cpu): + return te.target.sched.get_capacity(cpu) + + def get_sched_assert(self, cpufreq, cpu): + task = self.task_name + d = os.path.join(self.res_dir, cpufreq['tag']) + iter_trace = Trace(d, events=self.ftrace_conf["events"]) + t = Topology() + t.add_to_level('cpu', [[cpu]]) + return SchedAssert(iter_trace.ftrace, t, execname=task) + + def get_window(self, cpufreq, cpu): + sched_assert = self.get_sched_assert(cpufreq, cpu) + start_time = sched_assert.getStartTime() + end_time = sched_assert.getEndTime() + return (start_time, end_time) + + def get_expected_util_avg(self, cpufreq, cpu, cap): + """ + Examine trace to figure out an expected mean for util_avg + + Assumes an RT-App workload with a single task with a single phase + per each CPU + """ + # Find duty cycle of the workload task + sched_assert = self.get_sched_assert(cpufreq, cpu) + window = self.get_window(cpufreq, cpu) + duty_cycle_pct = sched_assert.getDutyCycle(window) + + cpu_capacity = cap + + # Scale the capacity linearly according to the frequency the workload + # was run at + if cpufreq['governor'] == 'userspace': + freq = cpufreq['freqs'][cpu] + max_freq = max(self.all_freqs) + cpu_capacity *= float(freq) / max_freq + else: + assert cpufreq['governor'] == 'performance' + + # Scale the relative CPU/freq capacity into the range 0..1 + scale = max(self.caps) + scaling_factor = float(cpu_capacity) / scale + + return UTIL_SCALE * (duty_cycle_pct / 100.) * scaling_factor + + def get_sched_task_signals(self, cpufreq, cpu, signals): + """ + Get a pandas.DataFrame with the sched signals for the workload task + + This examines scheduler load tracking trace events, supporting either + sched_load_avg_task or sched_pelt_se. You will need a target kernel that + includes these events. + + :param cpufreq: Cpufreq conf for the run to get trace for + :param signals: List of load tracking signals to extract. Probably a + subset of ``['util_avg', 'load_avg']`` + :returns: :class:`pandas.DataFrame` with a column for each signal for + the workload task + """ + task = self.task_name + d = os.path.join(self.res_dir, cpufreq['tag']) + trace = Trace(d, events=self.ftrace_conf["events"]) + + # There are two different scheduler trace events that expose the load + # tracking signals. Neither of them is in mainline. Eventually they + # should be unified but for now we'll just check for both types of + # event. + # TODO: Add support for this parsing in Trappy and/or tasks_analysis + signal_fields = signals + if 'sched_load_avg_task' in trace.available_events: + event = 'sched_load_avg_task' + elif 'sched_load_se' in trace.available_events: + event = 'sched_load_se' + # sched_load_se uses 'util' and 'load' instead of 'util_avg' and + # 'load_avg' + signal_fields = [s.replace('_avg', '') for s in signals] + elif 'sched_pelt_se' in trace.available_events: + event = 'sched_pelt_se' + else: + raise ValueError('No sched_load_avg_task or sched_load_se or sched_pelt_se events. ' + 'Does the kernel support them?') + + df = getattr(trace.ftrace, event).data_frame + df = df[df['comm'] == task][signal_fields] + df = select_window(df, self.get_window(cpufreq, cpu)) + return df.rename(columns=dict(zip(signal_fields, signals))) + + def get_signal_mean(self, cpufreq, cpu, signal, + ignore_first_s=UTIL_AVG_CONVERGENCE_TIME): + """ + Get the mean of a scheduler signal for the experiment's task + + Ignore the first `ignore_first_s` seconds of the signal. + """ + (wload_start, wload_end) = self.get_window(cpufreq, cpu) + window = (wload_start + ignore_first_s, wload_end) + + signal = self.get_sched_task_signals(cpufreq, cpu, [signal])[signal] + signal = select_window(signal, window) + return area_under_curve(signal) / (window[1] - window[0]) + + def isAlmostEqual(self, target, value, delta): + return (target - delta < value) and (value < target + delta) + + def _test_signal(self, signal_name): + res = ResultBundle() + passed = True + for (cpu, cpu_cap) in zip(self.target_cpus, self.target_cpus_capacity): + for cpufreq in self.cpufreq_params: + exp_util = self.get_expected_util_avg(cpufreq, cpu, cpu_cap) + signal_mean = self.get_signal_mean(cpufreq, cpu, signal_name) + + error_margin = exp_util * (ERROR_MARGIN_PCT / 100.) + + passed = passed and \ + self.isAlmostEqual(exp_util, signal_mean, error_margin) + + res.add_metric( + Metric("expected_util_avg, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), exp_util) + ) + res.add_metric( + Metric("signal_mean, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), signal_mean) + ) + + res.passed = passed + return res + + def test_task_util_avg(self): + """ + Test that the mean of the util_avg signal matched the expected value + """ + return self._test_signal('util_avg') + + def test_task_load_avg(self): + """ + Test that the mean of the load_avg signal matched the expected value + + Assuming that the system was under little stress (so the task was + RUNNING whenever it was RUNNABLE) and that the task was run with a + 'nice' value of 0, the load_avg should be similar to the util_avg. So, + this test does the same as test_task_util_avg but for load_avg. + """ + return self._test_signal('load_avg') + +class FreqInvarianceTest(LoadTrackingTestBundle): + """ + **Goal** + Basic check for frequency invariant load tracking + + **Detailed Description** + This test runs the same workload on the most capable CPU on the system at a + cross section of available frequencies. The trace is then examined to find + the average activation length of the workload, which is combined with the + known period to estimate an expected mean value for util_avg for each + frequency. The util_avg value is extracted from scheduler trace events and + its mean is compared with the expected value (ignoring the first 300ms so + that the signal can stabilize). The test fails if the observed mean is + beyond a certain error margin from the expected one. load_avg is then + similarly compared with the expected util_avg mean, under the assumption + that load_avg should equal util_avg when system load is light. + + **Expected Behaviour** + Load tracking signals are scaled so that the workload results in roughly the + same util & load values regardless of frequency. + """ + task_name = 'fie_10pct' + + @classmethod + def create_rtapp_params(cls, te): + """ + Get a specification for a rt-app workload with the specificied duty + cycle, pinned to the given CPU. + """ + # Run on one of the CPUs with highest capacity + cpu = max(range(te.target.number_of_cpus), + key=lambda c: cls._get_cpu_capacity(te, c)) + cls.target_cpus = [cpu] + cls.target_cpus_capacity = [cls._get_cpu_capacity(te, cpu)] + + rtapp_params = {} + rtapp_params[cls.task_name] = Periodic( + duty_cycle_pct=10, + duration_s=2, + period_ms=16, + cpus=[cpu] + ) + + return rtapp_params + + @classmethod + def create_cpufreq_params(cls, te): + # Create a set of confs with different frequencies + # We'll run the workload under each conf (i.e. at each frequency) + confs = [] + cpu = max(range(te.target.number_of_cpus), + key=lambda c: cls._get_cpu_capacity(te, c)) + cls.all_freqs = te.target.cpufreq.list_frequencies(cpu) + # If we have loads of frequencies just test a cross-section so it + # doesn't take all day + cls.freqs = cls.all_freqs[::len(cls.all_freqs) / 8 + 1] + for freq in cls.freqs: + confs.append({ + 'tag' : 'freq_{}'.format(freq), + 'freqs' : {cpu: freq}, + 'governor' : 'userspace', + }) + + return confs + +class CpuInvarianceTest(LoadTrackingTestBundle): + """ + **Goal** + Basic check for CPU invariant load and utilization tracking + + **Detailed Description** + This test runs the same workload on one CPU of each type in the system. The + trace is then examined to estimate an expected mean value for util_avg for + each CPU's workload. The util_avg value is extracted from scheduler trace + events and its mean is compared with the expected value (ignoring the first + 300ms so that the signal can stabilize). The test fails if the observed mean + is beyond a certain error margin from the expected one. load_avg is then + similarly compared with the expected util_avg mean, under the assumption + that load_avg should equal util_avg when system load is light. + + **Expected Behaviour** + Load tracking signals are scaled so that the workload results in roughly the + same util & load values regardless of compute power of the CPU + used. Moreover, assuming that the extraneous system load is negligible, the + load signal is similar to the utilization signal. + """ + task_name = 'cie_10pct' + + @classmethod + def create_rtapp_params(cls, te): + """ + Get a specification for a rt-app workload with the specificied duty + cycle, pinned to the given CPU. + """ + # Run the 10% workload on one CPU in each capacity group + cls.target_cpus = [] + cls.target_cpus_capacity = [] + phases = [] + tested_caps = set() + for cpu in range(te.target.number_of_cpus): + cap = cls._get_cpu_capacity(te, cpu) + # No need to test on every CPU, just one for each capacity value + if cap not in tested_caps: + tested_caps.add(cap) + cls.target_cpus.append(cpu) + cls.target_cpus_capacity.append(cap) + phase = Periodic( + duty_cycle_pct=10, + duration_s=2, + period_ms=16, + cpus=[cpu] + ) + phases.append(phase) + + rtapp_params = {} + rtapp_params[cls.task_name] = reduce((lambda x, y: x + y), phases) + + return rtapp_params + + @classmethod + def create_cpufreq_params(cls, te): + return [{ + 'tag' : 'cie_conf', + 'governor' : 'performance', + }] + +class PELTTaskTest(LoadTrackingTestBundle): + """ + **Goal** + Basic checks for tasks related PELT signals behaviour. + + **Detailed Description** + This test runs a synthetic periodic task on a CPU in the system and + collects a trace from the target device. The util_avg values are extracted + from scheduler trace events and the behaviour of the signal is compared + against a simulated value of PELT. + This class runs the following tests: + + - test_util_avg_range: test that util_avg's stable range matches with the + stable range of the simulated signal. In particular, this test compares + min, max and mean values of the two signals. + + - test_util_avg_behaviour: check behaviour of util_avg against the simualted + PELT signal. This test assumes that PELT is configured with 32 ms half + life time and the samples are 1024 us. Also, it assumes that the first + trace event related to the task used for testing is generated 'after' + the task starts (hence, we compute the initial PELT value when the task + started). + + **Expected Behaviour** + Simulated PELT signal and the signal extracted from the trace should have + very similar min, max and mean values in the stable range and the behaviour of + the signal should be very similar to simulated one. + """ + task_name = 'pelt_behv' + + @classmethod + def create_rtapp_params(cls, te): + # Run the 50% workload on a CPU with highest capacity + cpu = max(range(te.target.number_of_cpus), + key=lambda c: cls._get_cpu_capacity(te, c)) + cls.target_cpus = [cpu] + cls.target_cpus_capacity = [cls._get_cpu_capacity(te, cpu)] + + rtapp_params = {} + rtapp_params[cls.task_name] = Periodic( + duty_cycle_pct=50, + duration_s=2, + period_ms=16, + cpus=[cpu] + ) + + return rtapp_params + + @classmethod + def create_cpufreq_params(cls, te): + return [{ + 'tag' : 'pelt_behv_conf', + 'governor' : 'performance', + }] + + def get_simulated_pelt(self, task, init_value): + """ + Get simulated PELT signal and the periodic task used to model it. + + :returns: tuple of + - :mod:`bart.sched.pelt.Simulator` the PELT simulator object + - :mod:`bart.sched.pelt.PeriodicTask` simulated periodic task + - :mod:`pandas.DataFrame` instance which reports the computed + PELT values at each PELT sample interval. + """ + phase = self.rtapp_params[self.task_name].phases[0] + pelt_task = pelt.PeriodicTask(period_samples=phase.period_ms, + duty_cycle_pct=phase.duty_cycle_pct) + peltsim = pelt.Simulator(init_value=init_value, + half_life_ms=HALF_LIFE_MS) + df = peltsim.getSignal(pelt_task, 0, phase.duration_s + 1) + return peltsim, pelt_task, df + + def _test_range(self, signal_name): + res = ResultBundle() + passed = True + task = self.task_name + for cpu in self.target_cpus: + for cpufreq in self.cpufreq_params: + signal_df = self.get_sched_task_signals(cpufreq, cpu, [signal_name]) + # Get stats and stable range of the simulated PELT signal + start_time = self.get_sched_assert(cpufreq, cpu).getStartTime() + init_pelt = pelt.Simulator.estimateInitialPeltValue( + signal_df[signal_name].iloc[0], signal_df.index[0], + start_time, HALF_LIFE_MS + ) + peltsim, pelt_task, sim_df = self.get_simulated_pelt(task, init_pelt) + sim_range = peltsim.stableRange(pelt_task) + stable_time = peltsim.stableTime(pelt_task) + window = (start_time + stable_time, + start_time + stable_time + 0.5) + # Get signal statistics in a period of time where the signal is + # supposed to be stable + signal_stats = signal_df[window[0]:window[1]][signal_name].describe() + + # Narrow down simulated PELT signal to stable period + sim_df = sim_df[window[0]:window[1]].pelt_value + + # Check min + error_margin = sim_range.min_value * (ERROR_MARGIN_PCT / 100.) + passed = passed and self.isAlmostEqual(sim_range.min_value, signal_stats['min'], error_margin) + res.add_metric( + Metric("min_signal_value, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), signal_stats['min']) + ) + res.add_metric( + Metric("expected_min_signal_value, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), sim_range.min_value) + ) + + # Check max + error_margin = sim_range.max_value * (ERROR_MARGIN_PCT / 100.) + passed = passed and self.isAlmostEqual(sim_range.max_value, signal_stats['max'], error_margin) + res.add_metric( + Metric("max_signal_value, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), signal_stats['max']) + ) + res.add_metric( + Metric("expected_max_signal_value, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), sim_range.max_value) + ) + + # Check mean + sim_mean = sim_df.mean() + error_margin = sim_mean * (ERROR_MARGIN_PCT / 100.) + passed = passed and self.isAlmostEqual(sim_mean, signal_stats['mean'], error_margin) + res.add_metric( + Metric("mean_signal_value, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), signal_stats['mean']) + ) + res.add_metric( + Metric("expected_mean_signal, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), sim_mean) + ) + + res.passed = passed + return res + + def _test_behaviour(self, signal_name): + res = ResultBundle() + passed = True + task = self.task_name + + for cpu in self.target_cpus: + for cpufreq in self.cpufreq_params: + signal_df = self.get_sched_task_signals(cpufreq, cpu, [signal_name]) + # Get instant of time when the task starts running + start_time = self.get_sched_assert(cpufreq, cpu).getStartTime() + + # Get information about the task + phase = self.rtapp_params[self.task_name].phases[0] + + # Create simulated PELT signal for a periodic task + init_pelt = pelt.Simulator.estimateInitialPeltValue( + signal_df[signal_name].iloc[0], signal_df.index[0], + start_time, HALF_LIFE_MS + ) + peltsim, pelt_task, sim_df = self.get_simulated_pelt(task, init_pelt) + + # Compare actual PELT signal with the simulated one + margin = 0.05 + period_s = phase.period_ms / 1e3 + sim_period_ms = phase.period_ms * (peltsim._sample_us / 1e6) + n_errors = 0 + for entry in signal_df.iterrows(): + trace_val = entry[1][signal_name] + timestamp = entry[0] - start_time + # Next two instructions map the trace timestamp to a simulated + # signal timestamp. This is due to the fact that the 1 ms is + # actually 1024 us in the simulated signal. + n_periods = timestamp / period_s + nearest_timestamp = n_periods * sim_period_ms + sim_val_loc = sim_df.index.get_loc(nearest_timestamp, + method='nearest') + sim_val = sim_df.pelt_value.iloc[sim_val_loc] + res.add_metric( + Metric("trace_val at {}".format(timestamp), trace_val) + ) + res.add_metric( + Metric("sim_val at {}".format(timestamp), sim_val) + ) + if trace_val > (sim_val * (1 + margin)) or \ + trace_val < (sim_val * (1 - margin)): + n_errors += 1 + + total_no_errors = n_errors / len(signal_df) + res.add_metric( + Metric("total_no_errors, cpu {}, cpufreq conf {}" + .format(cpu, cpufreq['tag']), total_no_errors) + ) + # Exclude possible outliers (these may be due to a kernel thread that + # for some reason gets coscheduled with our workload). + passed = passed and (total_no_errors < margin) + + res.passed = passed + return res + + def test_util_avg_range(self): + """ + Test util_avg stable range for a 50% periodic task + """ + return self._test_range('util_avg') + + def test_util_avg_behaviour(self): + """ + Test util_avg behaviour for a 50% periodic task + """ + return self._test_behaviour('util_avg') + + def test_load_avg_range(self): + """ + Test load_avg stable range for a 50% periodic task + """ + return self._test_range('load_avg') + + def test_load_avg_behaviour(self, ): + """ + Test load_avg behaviour for a 50% periodic task + """ + return self._test_behaviour('load_avg') diff --git a/libs/utils/serialize.py b/libs/utils/serialize.py new file mode 100644 index 0000000000000000000000000000000000000000..521121e549dcf89b6ed56821ba3be4d3a3052b82 --- /dev/null +++ b/libs/utils/serialize.py @@ -0,0 +1,22 @@ +from ruamel.yaml import YAML + +class YAMLSerializable(object): + + _yaml = YAML(typ='unsafe') + _yaml.allow_unicode = True + + def to_stream(self, stream): + self._yaml.dump(self, stream) + + @classmethod + def from_stream(cls, stream): + return cls._yaml.load(stream) + + def to_path(self, filepath): + with open(filepath, "w") as fh: + self.to_stream(fh) + + @classmethod + def from_path(cls, filepath): + with open(filepath, "r") as fh: + return cls.from_stream(fh) diff --git a/libs/utils/target_script.py b/libs/utils/target_script.py index 04905ae7e0bc194e8e89b964f7225cfe613fc449..7d370c7bbbec96e0c6ea5403a438f4da9ee2bc07 100644 --- a/libs/utils/target_script.py +++ b/libs/utils/target_script.py @@ -32,17 +32,20 @@ class TargetScript(object): that must really be executed instead of accumulated. :type env: TestEnv - :param script_name: Name of the script that will be pushed on the target, - defaults to "remote_script.sh" + :param script_name: Name of the script that will be pushed on the target :type script_name: str + + :param local_dir: Local directory to use to prepare the script + :type local_dir: str """ _target_attrs = ['screen_resolution', 'android_id', 'abi', 'os_version', 'model'] - def __init__(self, env, script_name=SCRIPT_NAME): + def __init__(self, env, script_name=SCRIPT_NAME, local_dir='./'): self._env = env self._target = env.target self._script_name = script_name + self.local_dir = local_dir self.commands = [] # This is made to look like the devlib Target execute() @@ -80,13 +83,8 @@ class TargetScript(object): """ Push a script to the target - The script is created and stored on the host, - and is then sent to the target. - - :param path: Path where the script will be locally created - :type path: str - :param actions: List of actions(commands) to run - :type actions: list(str) + The script is created and stored on the host, and is then sent + to the target. """ actions = ['set -e'] + self.commands + ['set +e'] @@ -94,7 +92,7 @@ class TargetScript(object): actions = str.join('\n', actions) # Create script locally - self._local_path = os.path.join(self._env.res_dir, self._script_name) + self._local_path = os.path.join(self.local_dir, self._script_name) with open(self._local_path, 'w') as script: script.write(actions) diff --git a/libs/utils/test.py b/libs/utils/test.py deleted file mode 100644 index a3a35fb596ab58dc40632ab02a82d40f955e6e99..0000000000000000000000000000000000000000 --- a/libs/utils/test.py +++ /dev/null @@ -1,283 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# Copyright (C) 2015, ARM Limited and contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest -import logging - -from bart.sched.SchedAssert import SchedAssert -from bart.sched.SchedMultiAssert import SchedMultiAssert -from devlib.utils.misc import memoized -import wrapt - -from env import TestEnv -from executor import Executor -from trace import Trace - - -class LisaTest(unittest.TestCase): - """ - A base class for LISA tests - - This class is intended to be subclassed in order to create automated tests - for LISA. It sets up the TestEnv and Executor and provides convenience - methods for making assertions on results. - - Subclasses should provide a test_conf to configure the TestEnv and an - experiments_conf to configure the executor. - - Tests whose behaviour is dependent on target parameters, for example - presence of cpufreq governors or number of CPUs, can override - _getExperimentsConf to generate target-dependent experiments. - - Example users of this class can be found under LISA's tests/ directory. - - :ivar experiments: List of :class:`Experiment` s executed for the test. Only - available after :meth:`init` has been called. - """ - - test_conf = None - """Override this with a dictionary or JSON path to configure the TestEnv""" - - experiments_conf = None - """Override this with a dictionary or JSON path to configure the Executor""" - - permitted_fail_pct = 0 - """The percentage of iterations of each test that may be permitted to fail""" - - @classmethod - def _getTestConf(cls): - if cls.test_conf is None: - raise NotImplementedError("Override `test_conf` attribute") - return cls.test_conf - - @classmethod - def _getExperimentsConf(cls, test_env): - """ - Get the experiments_conf used to configure the Executor - - This method receives the initialized TestEnv as a parameter, so - subclasses can override it to configure workloads or target confs in a - manner dependent on the target. If not overridden, just returns the - experiments_conf attribute. - """ - if cls.experiments_conf is None: - raise NotImplementedError("Override `experiments_conf` attribute") - return cls.experiments_conf - - @classmethod - def runExperiments(cls): - """ - Set up logging and trigger running experiments - """ - cls._log = logging.getLogger('LisaTest') - - cls._log.info('Setup tests execution engine...') - test_env = TestEnv(test_conf=cls._getTestConf()) - - experiments_conf = cls._getExperimentsConf(test_env) - - if ITERATIONS_FROM_CMDLINE: - if 'iterations' in experiments_conf: - cls.logger.warning( - "Command line overrides iteration count in " - "{}'s experiments_conf".format(cls.__name__)) - experiments_conf['iterations'] = ITERATIONS_FROM_CMDLINE - - cls.executor = Executor(test_env, experiments_conf) - - # Alias tests and workloads configurations - cls.wloads = cls.executor._experiments_conf["wloads"] - cls.confs = cls.executor._experiments_conf["confs"] - - # Alias executor objects to make less verbose tests code - cls.te = cls.executor.te - cls.target = cls.executor.target - - # Execute pre-experiments code defined by the test - cls._experimentsInit() - - cls._log.info('Experiments execution...') - cls.executor.run() - - cls.experiments = cls.executor.experiments - - # Execute post-experiments code defined by the test - cls._experimentsFinalize() - - @classmethod - def _experimentsInit(cls): - """ - Code executed before running the experiments - """ - - @classmethod - def _experimentsFinalize(cls): - """ - Code executed after running the experiments - """ - - @memoized - def get_sched_assert(self, experiment, task): - """ - Return a SchedAssert over the task provided - """ - return SchedAssert( - self.get_trace(experiment).ftrace, self.te.topology, execname=task) - - @memoized - def get_multi_assert(self, experiment, task_filter=""): - """ - Return a SchedMultiAssert over the tasks whose names contain task_filter - - By default, this includes _all_ the tasks that were executed for the - experiment. - """ - tasks = experiment.wload.tasks.keys() - return SchedMultiAssert(self.get_trace(experiment).ftrace, - self.te.topology, - [t for t in tasks if task_filter in t]) - - def get_trace(self, experiment): - if not hasattr(self, "__traces"): - self.__traces = {} - if experiment.out_dir in self.__traces: - return self.__traces[experiment.out_dir] - - if ('ftrace' not in experiment.conf['flags'] - or 'ftrace' not in self.test_conf): - raise ValueError( - 'Tracing not enabled. If this test needs a trace, add "ftrace" ' - 'to your test/experiment configuration flags') - - events = self.test_conf['ftrace']['events'] - trace = Trace(experiment.out_dir, events, self.te.platform) - - self.__traces[experiment.out_dir] = trace - return trace - - def get_start_time(self, experiment): - """ - Get the time at which the experiment workload began executing - """ - start_times_dict = self.get_multi_assert(experiment).getStartTime() - return min([t["starttime"] for t in start_times_dict.itervalues()]) - - def get_task_start_time(self, experiment, task): - """ - Get the time at which the input task of the experiment workload began - executing - """ - start_times_dict = self.get_multi_assert(experiment, task_filter=task).getStartTime() - return min([t["starttime"] for t in start_times_dict.itervalues()]) - - def get_end_time(self, experiment): - """ - Get the time at which the experiment workload finished executing - """ - end_times_dict = self.get_multi_assert(experiment).getEndTime() - return max([t["endtime"] for t in end_times_dict.itervalues()]) - - def get_window(self, experiment): - return (self.get_start_time(experiment), self.get_end_time(experiment)) - - def get_end_times(self, experiment): - """ - Get the time at which each task in the workload finished - - Returned as a dict; {"task_name": finish_time, ...} - """ - - end_times = {} - ftrace = self.get_trace(experiment).ftrace - for task in experiment.wload.tasks.keys(): - sched_assert = SchedAssert(ftrace, self.te.topology, execname=task) - end_times[task] = sched_assert.getEndTime() - - return end_times - - def _dummy_method(self): - pass - - # In the Python unittest framework you instantiate TestCase objects passing - # the name of a test method that is going to be run to make assertions. We - # run our tests using nosetests, which automatically discovers these - # methods. However we also want to be able to instantiate LisaTest objects - # in notebooks without the inconvenience of having to provide a methodName, - # since we won't need any assertions. So we'll override __init__ with a - # default dummy test method that does nothing. - def __init__(self, methodName='_dummy_method', *args, **kwargs): - super(LisaTest, self).__init__(methodName, *args, **kwargs) - -@wrapt.decorator -def experiment_test(wrapped_test, instance, args, kwargs): - """ - Convert a LisaTest test method to be automatically called for each experiment - - The method will be passed the experiment object and a list of the names of - tasks that were run as the experiment's workload. - """ - failures = {} - for experiment in instance.executor.experiments: - tasks = experiment.wload.tasks.keys() - try: - wrapped_test(experiment, tasks, *args, **kwargs) - except AssertionError as e: - trace_relpath = os.path.join(experiment.out_dir, "trace.dat") - add_msg = "Check trace file: " + os.path.abspath(trace_relpath) - msg = str(e) + "\n\t" + add_msg - - test_key = (experiment.wload_name, experiment.conf['tag']) - failures[test_key] = failures.get(test_key, []) + [msg] - - for fails in failures.itervalues(): - iterations = instance.executor.iterations - fail_pct = 100. * len(fails) / iterations - - msg = "{} failures from {} iteration(s):\n{}".format( - len(fails), iterations, '\n'.join(fails)) - if fail_pct > instance.permitted_fail_pct: - raise AssertionError(msg) - else: - instance._log.warning(msg) - instance._log.warning( - 'ALLOWING due to permitted_fail_pct={}'.format( - instance.permitted_fail_pct)) - - -# Prevent nosetests from running experiment_test directly as a test case -experiment_test.__test__ = False - -# Allow the user to override the iterations setting from the command -# line. Nosetests does not support this kind of thing, so we use an -# evil hack: the lisa-test shell function takes an --iterations -# argument and exports an environment variable. If the test itself -# specifies an iterations count, we'll later print a warning and -# override it. We do this here in the root scope, rather than in -# runExperiments, so that if the value is invalid we print the error -# immediately instead of going ahead with target setup etc. -try: - iterations = os.getenv('LISA_TEST_ITERATIONS') - # Empty string or 0 will be replaced by 0, otherwise converted to int - ITERATIONS_FROM_CMDLINE = int(iterations) if iterations else 0 - - if ITERATIONS_FROM_CMDLINE < 0: - raise ValueError('Cannot be negative') -except ValueError as e: - raise ValueError("Couldn't read iterations count: {}".format(e)) - -# vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/libs/utils/test_workload.py b/libs/utils/test_workload.py new file mode 100644 index 0000000000000000000000000000000000000000..f5a4b512fed5448e72030092798e74845fefdf15 --- /dev/null +++ b/libs/utils/test_workload.py @@ -0,0 +1,306 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2018, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from serialize import YAMLSerializable + +# from wa import Metric + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!! FIXME !!! this is from workload-automation you bloody thief! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +class Metric(object): + """ + This is a single metric collected from executing a workload. + + :param name: the name of the metric. Uniquely identifies the metric + within the results. + :param value: The numerical value of the metric for this execution of a + workload. This can be either an int or a float. + :param units: Units for the collected value. Can be None if the value + has no units (e.g. it's a count or a standardised score). + :param lower_is_better: Boolean flag indicating where lower values are + better than higher ones. Defaults to False. + :param classifiers: A set of key-value pairs to further classify this + metric beyond current iteration (e.g. this can be used + to identify sub-tests). + + """ + + __slots__ = ['name', 'value', 'units', 'lower_is_better', 'classifiers'] + + @staticmethod + def from_pod(pod): + return Metric(**pod) + + def __init__(self, name, value, units=None, lower_is_better=False, + classifiers=None): + self.name = name + self.value = value + self.units = units + self.lower_is_better = lower_is_better + self.classifiers = classifiers or {} + + def to_pod(self): + return dict( + name=self.name, + value=self.value, + units=self.units, + lower_is_better=self.lower_is_better, + classifiers=self.classifiers, + ) + + def __str__(self): + result = '{}: {}'.format(self.name, self.value) + if self.units: + result += ' ' + self.units + result += ' ({})'.format('-' if self.lower_is_better else '+') + return result + + def __repr__(self): + text = self.__str__() + if self.classifiers: + return '<{} {}>'.format(text, self.classifiers) + else: + return '<{}>'.format(text) +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!! FIXME !!! this is from workload-automation you bloody thief! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +class ResultBundle(object): + """ + Bundle for storing test results + + :param passed: Indicates whether the associated test passed. + It will also be used as the truth-value of a ResultBundle. + :type passed: boolean + """ + def __init__(self, passed=True): + self.passed = passed + self.metrics = [] + + def __nonzero__(self): + return self.passed + + def add_metric(self, metric): + """ + Lets you append several test :class:`Metric` to the bundle. + + :param metric: Metric to add to the bundle + :type metric: Metric + """ + self.metrics.append(metric) + +class TestBundle(YAMLSerializable): + """ + A LISA test bundle. + + :param res_dir: Directory from where the target execution artifacts reside. + This will also be used to dump any artifact generated in the test code. + :type res_dir: str + + **Design notes:** + + * :meth:`from_target` will collect whatever artifacts are required + from a given target, and will then return a :class:`TestBundle`. + * :meth:`from_path` will use whatever artifacts are available in a + given directory (which should have been created by an earlier call + to :meth:`from_target` and then :meth:`to_path`), and will then return + a :class:`TestBundle`. + * :attr:`verify_serialization` is there to ensure both above methods remain + operationnal at all times. + + The point of a TestBundle is to bundle in a single object all of the + required data to run some test assertion (hence the name). When inheriting + from this class, you can define test methods that use this data, and return + a :class:`ResultBundle`. + + **Implementation example**:: + + class DummyTestBundle(TestBundle): + + def __init__(self, res_dir, shell_output): + super(DummyTestBundle, self).__init__(res_dir) + + self.shell_output = shell_output + + @classmethod + def _from_target(cls, te, res_dir): + output = te.target.execute('echo $((21+21))').split() + return cls(res_dir, output) + + def test_output(self): + passed = False + for line in self.shell_output: + if '42' in line: + passed = True + break + + return ResultBundle(passed) + + **Usage example**:: + + # Creating a Bundle from a live target + bundle = TestBundle.from_target(test_env, "/my/res/dir") + # Running some test on the bundle + res_bundle = bundle.test_foo() + + # Saving the bundle on the disk + bundle.to_path(test_env, "/my/res/dir") + + # Reloading the bundle from the disk + bundle = TestBundle.from_path("/my/res/dir") + res_bundle = bundle.test_foo() + """ + + verify_serialization = True + """ + When True, this enforces a serialization/deserialization step in :meth:`from_target`. + Although it hinders performance (we end up creating two :class:`TestBundle` + instances), it's very valuable to ensure :meth:`from_path` does not get broken + for some particular class. + """ + + def __init__(self, res_dir): + self.res_dir = res_dir + + @classmethod + def _from_target(cls, te, res_dir): + """ + Internals of the target factory method. + """ + raise NotImplementedError() + + @classmethod + def from_target(cls, te, res_dir=None, **kwargs): + """ + Factory method to create a bundle using a live target + + This is mostly boiler-plate code around :meth:`_from_target`, + which lets us introduce common functionalities for daughter classes. + Unless you know what you are doing, you should not override this method, + but the internal :meth:`_from_target` instead. + """ + if not res_dir: + res_dir = te.get_res_dir() + + # Logger stuff? + + bundle = cls._from_target(te, res_dir, **kwargs) + + # We've created the bundle from the target, and have all of + # the information we need to execute the test code. However, + # we enforce the use of the offline reloading path to ensure + # it does not get broken. + if cls.verify_serialization: + bundle.to_path(res_dir) + bundle = cls.from_path(res_dir) + + return bundle + + @classmethod + def _filepath(cls, res_dir): + return os.path.join(res_dir, "{}.yaml".format(cls.__name__)) + + @classmethod + def from_path(cls, res_dir): + """ + See :meth:`YAMLSerializable.from_path` + """ + bundle = YAMLSerializable.from_path(cls._filepath(res_dir)) + # We need to update the res_dir to the one we were given + bundle.res_dir = res_dir + + return bundle + + def to_path(self, res_dir): + """ + See :meth:`YAMLSerializable.to_path` + """ + super(TestBundle, self).to_path(self._filepath(res_dir)) + +################################################################################ +################################################################################ + +# class AndroidWorkload(LisaWorkload): + +# def _setup_wload(self): +# self.target.set_auto_brightness(0) +# self.target.set_brightness(0) + +# self.target.ensure_screen_is_on() +# self.target.swipe_to_unlock() + +# self.target.set_auto_rotation(0) +# self.target.set_rotation(1) + +# def _run_wload(self): +# pass + +# def _teardown_wload(self): +# self.target.set_auto_rotation(1) +# self.target.set_auto_brightness(1) + +# def run(self, trace_tool): +# if trace_tool == "ftrace": +# pass +# elif trace_tool == "systrace": +# pass + +# self._setup_wload() + +# with self.te.record_ftrace(): +# self._run_wload() + +# self._teardown_wload() + +# from target_script import TargetScript +# from devlib.target import AndroidTarget + +# class GmapsWorkload(AndroidWorkload): + +# def _setup_wload(self): +# super(GmapsWorkload, self)._setup_wload() + +# self.script = TargetScript(self.te, "gmaps_swiper.sh") + +# for i in range(self.swipe_count): +# # Swipe right +# self.script.input_swipe_pct(40, 50, 60, 60) +# #AndroidTarget.input_swipe_pct(self.script, 40, 50, 60, 60) +# AndroidTarget.sleep(self.script, 1) +# # Swipe down +# AndroidTarget.input_swipe_pct(self.script, 50, 60, 50, 40) +# AndroidTarget.sleep(self.script, 1) +# # Swipe left +# AndroidTarget.input_swipe_pct(self.script, 60, 50, 40, 50) +# AndroidTarget.sleep(self.script, 1) +# # Swipe up +# AndroidTarget.input_swipe_pct(self.script, 50, 40, 50, 60) +# AndroidTarget.sleep(self.script, 1) + +# # Push script to the target +# self.script.push() + +# def _run_wload(self): +# self.script.run() + +# def run(self, swipe_count=10): +# self.swipe_count = swipe_count + +# super(GmapsWorkload, self).run("ftrace") diff --git a/libs/utils/wlgen2/__init__.py b/libs/utils/wlgen2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/libs/utils/wlgen2/rta.py b/libs/utils/wlgen2/rta.py new file mode 100644 index 0000000000000000000000000000000000000000..1d96c958156ce0b4a09f1a0e5169785c26b70df7 --- /dev/null +++ b/libs/utils/wlgen2/rta.py @@ -0,0 +1,581 @@ +import json +import logging +import os +import re +from collections import OrderedDict + +from devlib.utils.misc import ranges_to_list + +from workload import Workload + +class RTA(Workload): + """ + :param tasks: Description of the workload using :class:`RTATask`, described + as {task_name : :class:`RTATask`} + :type tasks: dict(RTATask) + """ + + def __init__(self, te, res_dir, name, tasks=None, conf=None, default_policy=None, + max_duration_s=None, calibration=None): + super(RTA, self).__init__(te, res_dir) + + self.name = name + + self.tasks = tasks + self.conf = conf + self.default_policy = default_policy + self.max_duration_s = max_duration_s + self.calibration = calibration + + self.pload = None + + json_name = '{}.json'.format(self.name) + self.local_json = os.path.join(self.res_dir, json_name) + self.remote_json = self.te.target.path.join(self.run_dir, json_name) + + if tasks: + self._init_tasks() + else: + self._init_conf() + + # Move configuration file to target + self.te.target.push(self.local_json, self.remote_json) + + rta_cmd = self.te.target.which('rt-app') + self.command = '{0:s} {1:s} 2>&1'.format(rta_cmd, self.remote_json) + + def _select_calibration(self): + # Select CPU or pload value for task calibration + if self.calibration is not None: + return min(self.calibration.values()) + else: + cpus = range(self.te.target.number_of_cpus) + target_cpu = cpus[-1] + if 'bl'in self.te.target.modules: + candidates = sorted(set(self.te.target.bl.bigs).intersection(cpus)) + if candidates: + target_cpu = candidates[0] + + return 'CPU{0:d}'.format(target_cpu) + + def _init_tasks(self): + # Sanity check for task names + for task in self.tasks.keys(): + if len(task) > 15: + # rt-app uses pthread_setname_np(3) which limits the task name + # to 16 characters including the terminal '\0'. + msg = ('Task name "{}" too long, please configure your tasks ' + 'with names shorter than 16 characters').format(task) + raise ValueError(msg) + + # Task configuration + rta_profile = { + 'tasks': {}, + 'global': {} + } + + # Initialize global configuration + global_conf = { + 'default_policy': 'SCHED_OTHER', + 'duration': -1 if not self.max_duration_s else self.max_duration_s, + 'calibration': self._select_calibration(), + 'logdir': self.run_dir, + } + + # self._log.warn('Limiting workload duration to %d [s]', + # global_conf['duration']) + + # Setup default scheduling class + if self.default_policy: + policy = self.default_policy + if policy not in ['OTHER', 'FIFO', 'RR', 'DEADLINE']: + raise ValueError('scheduling class {} not supported'\ + .format(policy)) + global_conf['default_policy'] = 'SCHED_{}'.format(policy) + + #self._log.info('Default policy: %s', global_conf['default_policy']) + + # Setup global configuration + rta_profile['global'] = global_conf + + # Setup tasks parameters + for tid in sorted(self.tasks.keys()): + task = self.tasks[tid] + + # Initialize task configuration + task_conf = {} + + if not task.sched_policy: + policy = 'DEFAULT' + else: + policy = task.sched_policy.upper() + + if policy == 'DEFAULT': + task_conf['policy'] = global_conf['default_policy'] + sched_descr = 'sched: using default policy' + elif policy not in ['OTHER', 'FIFO', 'RR', 'DEADLINE']: + raise ValueError('scheduling class {} not supported'\ + .format(task['sclass'])) + else: + task_conf.update(task.sched_policy) + task_conf['policy'] = 'SCHED_' + policy + sched_descr = 'sched: {0:s}'.format(task['sched']) + + # Initialize task phases + task_conf['phases'] = OrderedDict() + + # self._log.info('------------------------') + # self._log.info('task [%s], %s', tid, sched_descr) + + if task.delay_s: + task_conf['delay'] = int(task['delay'] * 1e6) + # self._log.info(' | start delay: %.6f [s]', + # task['delay']) + + task_conf['loop'] = task.loops + self._log.info(' | loops count: %d', task.loops) + + # Setup task configuration + rta_profile['tasks'][tid] = task_conf + + # Getting task phase descriptor + pid=1 + for phase in task.phases: + + # Convert time parameters to integer [us] units + duration = int(phase.duration_s * 1e6) + + task_phase = OrderedDict() + + # A duty-cycle of 0[%] translates to a 'sleep' phase + if phase.duty_cycle_pct == 0: + # self._log.info(' + phase_%06d: sleep %.6f [s]', + # pid, duration/1e6) + + task_phase['loop'] = 1 + task_phase['sleep'] = duration + + # A duty-cycle of 100[%] translates on a 'run-only' phase + elif phase.duty_cycle_pct == 100: + # self._log.info(' + phase_%06d: batch %.6f [s]', + # pid, duration/1e6) + task_phase['loop'] = 1 + task_phase['run'] = duration + if phase.barrier_after: + task_phase['barrier'] = phase.barrier_after + + # A certain number of loops is requires to generate the + # proper load + else: + period = int(phase.period_ms * 1e3) + + cloops = -1 + if duration >= 0: + cloops = int(duration / period) + + sleep_time = period * (100 - phase.duty_cycle_pct) / 100 + running_time = period - sleep_time + + # self._log.info('+ phase_%06d: duration %.6f [s] (%d loops)', + # pid, duration/1e6, cloops) + # self._log.info('| period %6d [us], duty_cycle %3d %%', + # period, phase.duty_cycle_pct) + # self._log.info('| run_time %6d [us], sleep_time %6d [us]', + # running_time, sleep_time) + + task_phase['loop'] = cloops + task_phase['run'] = running_time + task_phase['timer'] = {'ref': tid, 'period': period} + + rta_profile['tasks'][tid]['phases']['p'+str(pid).zfill(6)] = task_phase + + if phase.cpus is not None: + if isinstance(phase.cpus, str): + task_phase['cpus'] = ranges_to_list(phase.cpus) + elif isinstance(phase.cpus, list): + task_phase['cpus'] = phase.cpus + elif isinstance(phase.cpus, int): + task_phase['cpus'] = [phase.cpus] + else: + raise ValueError('phases cpus must be a list or string \ + or int') + # self._log.info('| CPUs affinity: {}'.format(phase.cpus)) + pid += 1 + + # Generate JSON configuration on local file + with open(self.local_json, 'w') as outfile: + json.dump(rta_profile, outfile, indent=4, separators=(',', ': ')) + + def _init_conf(self): + rtapp_conf = self.conf + + ofile = open(self.local_json, 'w') + + calibration = self.getCalibrationConf() + # Calibration can either be a string like "CPU1" or an integer, if the + # former we need to quote it. + if type(calibration) != int: + calibration = '"{}"'.format(calibration) + + replacements = { + '__DURATION__' : str(self.duration), + '__PVALUE__' : str(calibration), + '__LOGDIR__' : str(self.run_dir), + '__WORKDIR__' : '"'+self.te.target.working_directory+'"', + } + + # Check for inline config + if not isinstance(self.params['custom'], basestring): + if isinstance(self.params['custom'], dict): + # Inline config present, turn it into a file repr + tmp_json = json.dumps(rtapp_conf, + indent=4, separators=(',', ': '), sort_keys=True) + ifile = tmp_json.splitlines(True) + else: + raise ValueError("Value specified for 'params' can only be " + "a filename or an embedded dictionary") + else: + # We assume we are being passed a filename instead of a dict + self._log.info('Loading custom configuration:') + self._log.info(' %s', rtapp_conf) + ifile = open(rtapp_conf, 'r') + + for line in ifile: + if '__DURATION__' in line and self.duration is None: + raise ValueError('Workload duration not specified') + for src, target in replacements.iteritems(): + line = line.replace(src, target) + ofile.write(line) + + if isinstance(ifile, file): + ifile.close() + ofile.close() + + with open(self.local_json) as f: + conf = json.load(f) + for tid in conf['tasks']: + self.tasks[tid] = {'pid': -1} + + @classmethod + def _calibrate(target): + pload_regexp = re.compile(r'pLoad = ([0-9]+)ns') + pload = {} + + # Setup logging + # log = logging.getLogger('RTApp') + + # Create calibration task + max_rtprio = int(target.execute('ulimit -Hr').split('\r')[0]) + # log.debug('Max RT prio: %d', max_rtprio) + if max_rtprio > 10: + max_rtprio = 10 + + calib_task = Periodic(period_ms=100, + duty_cycle_pct=50, + duration_s=1, + sched_policy={ + 'policy': 'FIFO', + 'prio': max_rtprio + }) + #rta = RTA(target, 'rta_calib') + + for cpu in target.list_online_cpus(): + # log.info('CPU%d calibration...', cpu) + + #rta.conf(kind='profile', params={'task1': calib_task}, cpus=[cpu]) + rta = RTA(target, tasks={'task1': calib_task}) + rta.run(as_root=True) + + for line in rta.getOutput().split('\n'): + pload_match = re.search(pload_regexp, line) + if pload_match is None: + continue + pload[cpu] = int(pload_match.group(1)) + # log.debug('>>> cpu%d: %d', cpu, pload[cpu]) + + # log.info('Target RT-App calibration:') + # log.info("{" + ", ".join('"%r": %r' % (key, pload[key]) + # for key in pload) + "}") + + # Sanity check calibration values for big.LITTLE systems + # if 'bl' in target.modules: + # bcpu = target.bl.bigs_online[0] + # lcpu = target.bl.littles_online[0] + # if pload[bcpu] > pload[lcpu]: + # log.warning('Calibration values reports big cores less ' + # 'capable than LITTLE cores') + # raise RuntimeError('Calibration failed: try again or file a bug') + # bigs_speedup = ((float(pload[lcpu]) / pload[bcpu]) - 1) * 100 + # log.info('big cores are ~%.0f%% more capable than LITTLE cores', + # bigs_speedup) + + return pload + + @classmethod + def get_calibration(target): + """ + Calibrate RT-App on each CPU in the system + + :param target: Devlib target to run calibration on. + :returns: Dict mapping CPU numbers to RT-App calibration values. + """ + + if 'cpufreq' not in target.modules: + logging.getLogger(cls.__name__).warning( + 'cpufreq module not loaded, skipping setting frequency to max') + return cls._calibrate(target) + + with target.cpufreq.use_governor('performance'): + return cls._calibrate(target) + +class Phase(object): + """ + Descriptor for an RT-App load phase + + :param duration_s: the phase duration in [s]. + :type duration_s: int + + :param period_ms: the phase period in [ms]. + :type period_ms: int + + :param duty_cycle_pct: the generated load in [%]. + :type duty_cycle_pct: int + + :param cpus: the list of cpus on which task execution is restricted during + this phase. + :type cpus: [int] or int + + :param barrier_after: if provided, the name of the barrier to sync against + when reaching the end of this phase. Currently only + supported when duty_cycle_pct=100 + :type barrier_after: str + """ + def __init__(self, duration_s, period_ms, duty_cycle_pct, cpus=None, barrier_after=None): + if barrier_after and duty_cycle_pct != 100: + # This could be implemented but currently don't foresee any use. + raise ValueError('Barriers only supported when duty_cycle_pct=100') + + self.duration_s = duration_s + self.period_ms = period_ms + self.duty_cycle_pct = duty_cycle_pct + self.cpus = cpus + self.barrier_after = barrier_after + +class RTATask(object): + """ + Base class for conveniently constructing params to :meth:`RTA.conf` + + This class represents an RT-App task which may contain multiple phases. It + implements ``__add__`` so that using ``+`` on two tasks concatenates their + phases. For example ``Ramp() + Periodic()`` would yield an ``RTATask`` that + executes the default phases for ``Ramp`` followed by the default phases for + ``Periodic``. + """ + + def __init__(self, delay_s=0, loops=1, sched_policy=None): + self.sched_policy = sched_policy + self.delay_s = delay_s + self.loops = loops + self.phases = [] + + def __add__(self, next_phases): + if next_phases.delay_s: + # This won't work, because rt-app's "delay" field is per-task and + # not per-phase. We might be able to implement it by adding a + # "sleep" event here, but let's not bother unless such a need + # arises. + raise ValueError("Can't compose rt-app tasks " + "when the second has nonzero 'delay_s'") + + self.phases.extend(next_phases.phases) + return self + + +class Ramp(RTATask): + """ + Configure a ramp load. + + This class defines a task which load is a ramp with a configured number + of steps according to the input parameters. + + :param start_pct: the initial load percentage. + :param end_pct: the final load percentage. + :param delta_pct: the load increase/decrease at each step, in percentage + points. + :param time_s: the duration in seconds of each load step. + :param period_ms: the period used to define the load in [ms]. + :param delay_s: the delay in seconds before ramp start. + :param loops: number of time to repeat the ramp, with the specified delay in + between. + + :param sched_policy: the scheduler configuration for this task. + :type sched_policy: dict + + :param cpus: the list of CPUs on which task can run. + .. note:: if not specified, it can run on all CPUs + :type cpus: list(int) + """ + + def __init__(self, start_pct=0, end_pct=100, delta_pct=10, time_s=1, + period_ms=100, delay_s=0, loops=1, sched_policy=None, cpus=None): + super(Ramp, self).__init__(delay_s, loops, sched_policy) + + if start_pct not in range(0,101) or end_pct not in range(0,101): + raise ValueError('start_pct and end_pct must be in [0..100] range') + + if start_pct >= end_pct: + if delta_pct > 0: + delta_pct = -delta_pct + delta_adj = -1 + if start_pct <= end_pct: + if delta_pct < 0: + delta_pct = -delta_pct + delta_adj = +1 + + phases = [] + steps = range(start_pct, end_pct+delta_adj, delta_pct) + for load in steps: + if load == 0: + phase = Phase(time_s, 0, 0, cpus) + else: + phase = Phase(time_s, period_ms, load, cpus) + phases.append(phase) + + self.phases = phases + +class Step(Ramp): + """ + Configure a step load. + + This class defines a task which load is a step with a configured initial and + final load. Using the ``loops`` param, this can be used to create a workload + that alternates between two load values. + + :param start_pct: the initial load percentage. + :param end_pct: the final load percentage. + :param time_s: the duration in seconds of each load step. + :param period_ms: the period used to define the load in [ms]. + :param delay_s: the delay in seconds before ramp start. + :param loops: number of time to repeat the step, with the specified delay in + between. + + :param sched: the scheduler configuration for this task. + :type sched: dict + + :param cpus: the list of CPUs on which task can run. + .. note:: if not specified, it can run on all CPUs + :type cpus: list(int) + """ + + def __init__(self, start_pct=0, end_pct=100, time_s=1, period_ms=100, + delay_s=0, loops=1, sched_policy=None, cpus=None): + delta_pct = abs(end_pct - start_pct) + super(Step, self).__init__(start_pct, end_pct, delta_pct, time_s, + period_ms, delay_s, loops, sched_policy, cpus) + +class Pulse(RTATask): + """ + Configure a pulse load. + + This class defines a task which load is a pulse with a configured + initial and final load. + + The main difference with the 'step' class is that a pulse workload is + by definition a 'step down', i.e. the workload switch from an finial + load to a final one which is always lower than the initial one. + Moreover, a pulse load does not generate a sleep phase in case of 0[%] + load, i.e. the task ends as soon as the non null initial load has + completed. + + :param start_pct: the initial load percentage. + :param end_pct: the final load percentage. Must be lower than ``start_pct`` + value. If end_pct is 0, the task end after the ``start_pct`` + period has completed. + :param time_s: the duration in seconds of each load step. + :param period_ms: the period used to define the load in [ms]. + :param delay_s: the delay in seconds before ramp start. + :param loops: number of time to repeat the pulse, with the specified delay + in between. + + :param sched: the scheduler configuration for this task. + :type sched: dict + + :param cpus: the list of CPUs on which task can run + .. note:: if not specified, it can run on all CPUs + :type cpus: list(int) + """ + + def __init__(self, start_pct=100, end_pct=0, time_s=1, period_ms=100, + delay_s=0, loops=1, sched_policy=None, cpus=None): + super(Pulse, self).__init__(delay_s, loops, sched_policy) + + if end_pct >= start_pct: + raise ValueError('end_pct must be lower than start_pct') + + if end_pct not in range(0,101) or start_pct not in range(0,101): + raise ValueError('end_pct and start_pct must be in [0..100] range') + if end_pct >= start_pct: + raise ValueError('end_pct must be lower than start_pct') + + phases = [] + for load in [start_pct, end_pct]: + if load == 0: + continue + phase = Phase(time_s, period_ms, load, cpus) + phases.append(phase) + + self.phases = phases + +class Periodic(Pulse): + """ + Configure a periodic load. This is the simplest type of RTA task. + + This class defines a task which load is periodic with a configured + period and duty-cycle. + + :param duty_cycle_pct: the load percentage. + :param duration_s: the total duration in seconds of the task. + :param period_ms: the period used to define the load in milliseconds. + :param delay_s: the delay in seconds before starting the periodic phase. + + :param sched: the scheduler configuration for this task. + :type sched: dict + + :param cpus: the list of CPUs on which task can run. + .. note:: if not specified, it can run on all CPUs + :type cpus: list(int) + """ + + def __init__(self, duty_cycle_pct=50, duration_s=1, period_ms=100, + delay_s=0, sched_policy=None, cpus=None): + super(Periodic, self).__init__(duty_cycle_pct, 0, duration_s, + period_ms, delay_s, 1, sched_policy, cpus) + +class RunAndSync(RTATask): + """ + Configure a task that runs 100% then waits on a barrier + + :param barrier: name of barrier to wait for. Sleeps until any other tasks + that refer to this barrier have reached the barrier too. + :type barrier: str + + :param time_s: time to run for + + :param delay_s: the delay in seconds before starting. + + :param sched: the scheduler configuration for this task. + :type sched: dict + + :param cpus: the list of CPUs on which task can run. + .. note:: if not specified, it can run on all CPUs + :type cpus: list(int) + + """ + def __init__(self, barrier, time_s=1, + delay_s=0, loops=1, sched_policy=None, cpus=None): + super(RunAndSync, self).__init__(delay_s, loops, sched_policy) + + # This should translate into a phase containing a 'run' event and a + # 'barrier' event + self.phases = [Phase(time_s, None, 100, cpus, + barrier_after=barrier)] diff --git a/libs/utils/wlgen2/workload.py b/libs/utils/wlgen2/workload.py new file mode 100644 index 0000000000000000000000000000000000000000..6adcaf8bec9d20a223726802bcd3225bb87c4184 --- /dev/null +++ b/libs/utils/wlgen2/workload.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2015, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +from devlib.utils.misc import list_to_mask + +class Workload(object): + """ + """ + + def __init__(self, te, res_dir): + self.te = te + self.res_dir = res_dir + + self.command = None + self.run_dir = self.te.target.working_directory + + self._log = logging.getLogger('Workload') + # self._log.info('Setup new workload %s', self.name) + + def run(self, cpus=None, cgroup=None, background=False, as_root=False): + + if not self.command: + raise RuntimeError("Workload does not specify any command to execute") + + _command = self.command + + if cpus: + taskset_bin = self.te.target.which('taskset') + if not taskset_bin: + raise RuntimeError("Could not find 'taskset' executable on the target") + + cpumask = list_to_mask(cpus) + taskset_cmd = '{} 0x{}'.format(taskset_bin, cpumask) + _command = '{} {}'.format(taskset_cmd, _command) + + if cgroup: + _command = self.te.target.cgroups.run_into_cmd(cgroup, _command) + + if background: + self.te.target.background(_command, as_root=as_root) + else: + self.te.target.execute(_command, as_root=as_root) diff --git a/libs/wlgen/wlgen/rta.py b/libs/wlgen/wlgen/rta.py index b26328b3781dae2a76f0a9780e49f9d8d3a78ade..f3042aeed93b1769109a3291af62473a20acbfe1 100644 --- a/libs/wlgen/wlgen/rta.py +++ b/libs/wlgen/wlgen/rta.py @@ -91,12 +91,10 @@ class RTA(Workload): self.executor = 'rt-app' # Default initialization - self.json = None self.rta_profile = None self.loadref = None self.rta_cmd = None self.rta_conf = None - self.test_label = None # Setup RTA callbacks self.setCallback('postrun', self.__postrun) @@ -137,7 +135,7 @@ class RTA(Workload): sched={ 'policy': 'FIFO', 'prio': max_rtprio - }).get() + }) rta = RTA(target, 'rta_calib') for cpu in target.list_online_cpus(): @@ -189,9 +187,6 @@ class RTA(Workload): logfile = self.target.path.join(self.run_dir, '*{}*.log'.format(task)) self.target.pull(logfile, destdir) - self._log.debug('Pulling JSON to [%s]...', destdir) - self.target.pull(self.target.path.join(self.run_dir, self.json), - destdir) logfile = self.target.path.join(destdir, 'output.log') self._log.debug('Saving output on [%s]...', logfile) with open(logfile, 'w') as ofile: @@ -222,8 +217,7 @@ class RTA(Workload): rtapp_conf = self.params['custom'] - self.json = '{0:s}_{1:02d}.json'.format(self.name, self.exc_id) - ofile = open(self.json, 'w') + ofile = open(self.local_json, 'w') calibration = self.getCalibrationConf() # Calibration can either be a string like "CPU1" or an integer, if the @@ -265,13 +259,11 @@ class RTA(Workload): ifile.close() ofile.close() - with open(self.json) as f: + with open(self.local_json) as f: conf = json.load(f) for tid in conf['tasks']: self.tasks[tid] = {'pid': -1} - return self.json - def _confProfile(self): # Sanity check for task names @@ -324,10 +316,11 @@ class RTA(Workload): # Initialize task configuration task_conf = {} - if 'sched' not in task: + if not task.sched: policy = 'DEFAULT' else: - policy = task['sched']['policy'].upper() + policy = task.sched['policy'].upper() + if policy == 'DEFAULT': task_conf['policy'] = global_conf['default_policy'] sched_descr = 'sched: using default policy' @@ -335,7 +328,7 @@ class RTA(Workload): raise ValueError('scheduling class {} not supported'\ .format(task['sclass'])) else: - task_conf.update(task['sched']) + task_conf.update(task.sched) task_conf['policy'] = 'SCHED_' + policy sched_descr = 'sched: {0:s}'.format(task['sched']) @@ -345,23 +338,23 @@ class RTA(Workload): self._log.info('------------------------') self._log.info('task [%s], %s', tid, sched_descr) - if 'delay' in task.keys(): - if task['delay'] > 0: - task_conf['delay'] = int(task['delay'] * 1e6) - self._log.info(' | start delay: %.6f [s]', - task['delay']) + if task.delay_s: + task_conf['delay'] = int(task['delay'] * 1e6) + self._log.info(' | start delay: %.6f [s]', + task['delay']) - if 'loops' not in task.keys(): + if not task.loops: task['loops'] = 1 - task_conf['loop'] = task['loops'] - self._log.info(' | loops count: %d', task['loops']) + + task_conf['loop'] = task.loops + self._log.info(' | loops count: %d', task.loops) # Setup task configuration self.rta_profile['tasks'][tid] = task_conf # Getting task phase descriptor pid=1 - for phase in task['phases']: + for phase in task.phases: # Convert time parameters to integer [us] units duration = int(phase.duration_s * 1e6) @@ -432,21 +425,18 @@ class RTA(Workload): self.tasks[tid] = {'pid': -1} # Generate JSON configuration on local file - self.json = '{0:s}_{1:02d}.json'.format(self.name, self.exc_id) - with open(self.json, 'w') as outfile: + with open(self.local_json, 'w') as outfile: json.dump(self.rta_profile, outfile, indent=4, separators=(',', ': ')) - return self.json - def conf(self, kind, params, duration=None, cpus=None, sched=None, + work_dir='./', run_dir=None, - exc_id=0, loadref='big'): """ Configure a workload of a specified kind. @@ -466,8 +456,7 @@ class RTA(Workload): Profile based workloads When ``kind`` is "profile", ``params`` is a dictionary mapping task - names to task specifications. The easiest way to create these task - specifications using :meth:`RTATask.get`. + names to task specifications. For example, the following configures an RTA workload with a single task, named 't1', using the default parameters for a Periodic RTATask: @@ -475,7 +464,7 @@ class RTA(Workload): :: wl = RTA(...) - wl.conf(kind='profile', params={'t1': Periodic().get()}) + wl.conf(kind='profile', params={'t1': Periodic()}) :param kind: Either 'custom' or 'profile' - see above. :param params: RT-App parameters - see above. @@ -492,6 +481,10 @@ class RTA(Workload): The default scheduler policy. Choose from 'OTHER', 'FIFO', 'RR', and 'DEADLINE'. + :param work_dir: Local directory in which to store the resulting rt-app + configuration + :type work_dir: str + :param run_dir: Target dir to store output and config files in. .. TODO: document or remove loadref @@ -501,10 +494,15 @@ class RTA(Workload): sched = {'policy' : 'OTHER'} super(RTA, self).conf(kind, params, duration, - cpus, sched, run_dir, exc_id) + cpus, sched, run_dir) + self.work_dir = work_dir self.loadref = loadref + json_name = '{}.json'.format(self.name) + self.local_json = os.path.join(self.work_dir, json_name) + self.remote_json = self.target.path.join(self.run_dir, json_name) + # Setup class-specific configuration if kind == 'custom': self._confCustom() @@ -512,15 +510,10 @@ class RTA(Workload): self._confProfile() # Move configuration file to target - self.target.push(self.json, self.run_dir) + self.target.push(self.local_json, self.remote_json) - self.rta_cmd = self.target.executables_directory + '/rt-app' - self.rta_conf = self.run_dir + '/' + self.json - self.command = '{0:s} {1:s} 2>&1'.format(self.rta_cmd, self.rta_conf) - - # Set and return the test label - self.test_label = '{0:s}_{1:02d}'.format(self.name, self.exc_id) - return self.test_label + self.rta_cmd = self.target.which('rt-app') + self.command = '{0:s} {1:s} 2>&1'.format(self.rta_cmd, self.remote_json) class RTATask(object): """ @@ -536,19 +529,13 @@ class RTATask(object): def __init__(self, delay_s=0, loops=1, sched=None): self._task = {} - self._task['sched'] = sched or {'policy' : 'DEFAULT'} - self._task['delay'] = delay_s - self._task['loops'] = loops - - def get(self): - """ - Return a dict that can be passed as an element of the ``params`` field - to :meth:`RTA.conf`. - """ - return self._task + self.sched = sched or {'policy' : 'DEFAULT'} + self.delay_s = delay_s + self.loops = loops + self.phases = [] def __add__(self, next_phases): - if next_phases._task.get('delay', 0): + if next_phases.delay_s: # This won't work, because rt-app's "delay" field is per-task and # not per-phase. We might be able to implement it by adding a # "sleep" event here, but let's not bother unless such a need @@ -556,7 +543,7 @@ class RTATask(object): raise ValueError("Can't compose rt-app tasks " "when the second has nonzero 'delay_s'") - self._task['phases'].extend(next_phases._task['phases']) + self.phases.extend(next_phases.phases) return self @@ -610,7 +597,7 @@ class Ramp(RTATask): phase = Phase(time_s, period_ms, load, cpus) phases.append(phase) - self._task['phases'] = phases + self.phases = phases class Step(Ramp): """ @@ -693,8 +680,7 @@ class Pulse(RTATask): phase = Phase(time_s, period_ms, load, cpus) phases.append(phase) - self._task['phases'] = phases - + self.phases = phases class Periodic(Pulse): """ @@ -747,5 +733,5 @@ class RunAndSync(RTATask): # This should translate into a phase containing a 'run' event and a # 'barrier' event - self._task['phases'] = [Phase(time_s, None, 100, cpus, + self.phases = [Phase(time_s, None, 100, cpus, barrier_after=barrier)] diff --git a/libs/wlgen/wlgen/workload.py b/libs/wlgen/wlgen/workload.py index ffd2d93f6283f8aa243a97927d02634db40ec4a8..c17392492b3d3a57825f6530717c835f7a7b26e5 100644 --- a/libs/wlgen/wlgen/workload.py +++ b/libs/wlgen/wlgen/workload.py @@ -80,7 +80,6 @@ class Workload(object): # Task specific configuration parameters self.duration = None self.run_dir = None - self.exc_id = None # Setup kind specific parameters self.kind = None @@ -137,15 +136,13 @@ class Workload(object): duration, cpus=None, sched={'policy': 'OTHER'}, - run_dir=None, - exc_id=0): + run_dir=None): """Configure workload. See documentation for subclasses""" self.cpus = cpus self.sched = sched self.duration = duration self.run_dir = run_dir - self.exc_id = exc_id # Setup kind specific parameters self.kind = kind @@ -288,7 +285,7 @@ class Workload(object): ftrace_dat = None if ftrace: ftrace.stop() - ftrace_dat = out_dir + '/' + self.test_label + '.dat' + ftrace_dat = os.path.join(out_dir, "{}_trace.dat".format(self.name)) dirname = os.path.dirname(ftrace_dat) if not os.path.exists(dirname): self._log.debug('Create ftrace results folder [%s]', diff --git a/target.config b/target.config index 770b15e49c7908a188ce85ef4a9f895157ce8df0..3fe3776649d767cabe6af7f903a82d7152ffafd7 100644 --- a/target.config +++ b/target.config @@ -47,17 +47,6 @@ "events": [] }, - /* FTFP Image server */ - /* This is the folder from where the target gets kernel/DTB */ - /* images at each boot. */ - /* The path of images to be deployed are specified by the */ - /* experiments configuration (e.g. tests/eas/rfc_eas.json) */ - "tftp" : { - "folder" : "/var/lib/tftpboot", - "kernel" : "kern.bin", - "dtb" : "dtb.bin" - }, - /* Devlib modules to enable/disbale for all the experiments */ "modules" : [], "exclude_modules" : [], @@ -69,17 +58,4 @@ /* architectures */ /* - shell scripts under './tools/scripts/ */ "tools" : [], - - /* Wait time before trying to access the target after reboot */ - // "ping_time" : "15", - - /* Maximum time to wait after rebooting the target */ - // "reboot_time" : "120", - - /* List of test environment features to enable */ - /* no-kernel : do not deploy kernel/dtb images */ - /* no-reboot : do not force reboot the target at each */ - /* configuration change */ - /* debug : enable debugging messages */ - "__features__" : "no-kernel no-reboot" }