diff --git a/doc/kernel_tests.rst b/doc/kernel_tests.rst index c5f659ff32d2dc1cedd09a62c7448e27c5b76bfb..c79c7bc9b4346db39cc0c579f19ae3f9da6def03 100644 --- a/doc/kernel_tests.rst +++ b/doc/kernel_tests.rst @@ -265,3 +265,6 @@ Staged tests Those are tests that have been merged into LISA but whose behaviour are being actively evaluated. + +.. automodule:: lisa.tests.staging.sched_android + :members: diff --git a/lisa/tests/base.py b/lisa/tests/base.py index cfcdc412cdc9f8712656360cbd1eab46abc4787d..39ea5db9bfbf035f88c2d2c7430b7f880f5a47ba 100644 --- a/lisa/tests/base.py +++ b/lisa/tests/base.py @@ -505,6 +505,13 @@ class RTATestBundle(TestBundle, metaclass=RTATestBundleMeta): """ return self.get_rtapp_profile(self.plat_info) + @property + def cgroup_configuration(self): + """ + Compute the cgroup configuration based on ``plat_info`` + """ + return self.get_cgroup_configuration(self.plat_info) + @TasksAnalysis.df_tasks_runtime.used_events def test_noisy_tasks(self, noise_threshold_pct=None, noise_threshold_ms=None): """ @@ -678,7 +685,45 @@ class RTATestBundle(TestBundle, metaclass=RTATestBundleMeta): pass @classmethod - def _run_rtapp(cls, target, res_dir, profile, ftrace_coll=None): + def get_cgroup_configuration(cls, plat_info): + """ + :returns: a :class:`dict` representing the configuration of a + particular cgroup. + + This is a method you may optionally override to configure a cgroup for + the synthetic workload. + + Example of return value:: + + { + 'name': 'lisa_test', + 'controller': 'schedtune', + 'attributes' : { + 'prefer_idle' : 1, + 'boost': 50 + } + } + + """ + return None + + @classmethod + def _target_configure_cgroup(cls, target, cfg): + if not cfg: + return None + + kind = cfg['controller'] + if kind not in target.cgroups.controllers: + raise CannotCreateError('"{}" cgroup controller unavailable'.format(kind)) + ctrl = target.cgroups.controllers[kind] + + cg = ctrl.cgroup(cfg['name']) + cg.set(**cfg['attributes']) + + return '/' + cg.name + + @classmethod + def _run_rtapp(cls, target, res_dir, profile, ftrace_coll=None, cg_cfg=None): wload = RTA.by_profile(target, "rta_{}".format(cls.__name__.lower()), profile, res_dir=res_dir) @@ -687,8 +732,12 @@ class RTATestBundle(TestBundle, metaclass=RTATestBundleMeta): ftrace_coll = ftrace_coll or FtraceCollector.from_conf(target, cls.ftrace_conf) dmesg_coll = DmesgCollector(target) + cgroup = cls._target_configure_cgroup(target, cg_cfg) + as_root = cgroup is not None + with dmesg_coll, ftrace_coll, target.freeze_userspace(): - wload.run() + wload.run(cgroup=cgroup, as_root=as_root) + ftrace_coll.get_trace(trace_path) dmesg_coll.get_trace(dmesg_path) return trace_path @@ -697,7 +746,8 @@ class RTATestBundle(TestBundle, metaclass=RTATestBundleMeta): def _from_target(cls, target, res_dir, ftrace_coll=None): plat_info = target.plat_info rtapp_profile = cls.get_rtapp_profile(plat_info) - cls._run_rtapp(target, res_dir, rtapp_profile, ftrace_coll) + cgroup_config = cls.get_cgroup_configuration(plat_info) + cls._run_rtapp(target, res_dir, rtapp_profile, ftrace_coll, cgroup_config) return cls(res_dir, plat_info) diff --git a/lisa/tests/scheduler/load_tracking.py b/lisa/tests/scheduler/load_tracking.py index bd4d0134dfdc7dbc3760aa9c9b970b30a4055155..41ccce63123e4b817f354e4b8639abb34171f4cf 100644 --- a/lisa/tests/scheduler/load_tracking.py +++ b/lisa/tests/scheduler/load_tracking.py @@ -899,7 +899,7 @@ class CPUMigrationBase(LoadTrackingBase): """ @classmethod - def _run_rtapp(cls, target, res_dir, profile, ftrace_coll): + def _run_rtapp(cls, target, res_dir, profile, ftrace_coll, cgroup=None): # Just do some validation on the profile for name, task in profile.items(): for phase in task.phases: @@ -907,7 +907,7 @@ class CPUMigrationBase(LoadTrackingBase): raise RuntimeError("Each phase must be tied to a single CPU. " "Task \"{}\" violates this".format(name)) - super()._run_rtapp(target, res_dir, profile, ftrace_coll) + super()._run_rtapp(target, res_dir, profile, ftrace_coll, cgroup) def __init__(self, res_dir, plat_info): super().__init__(res_dir, plat_info) diff --git a/lisa/tests/staging/sched_android.py b/lisa/tests/staging/sched_android.py new file mode 100644 index 0000000000000000000000000000000000000000..585e04001c31190108b22bc1861c41edb5925d93 --- /dev/null +++ b/lisa/tests/staging/sched_android.py @@ -0,0 +1,324 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2019, Arm Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import os.path +import abc + +from lisa.wlgen.rta import RTA, Periodic +from lisa.tests.base import TestBundle, Result, ResultBundle, RTATestBundle +from lisa.trace import Trace, FtraceCollector, FtraceConf, requires_events +from lisa.target import Target +from lisa.utils import ArtifactPath +from lisa.analysis.frequency import FrequencyAnalysis +from lisa.analysis.tasks import TasksAnalysis + +class SchedTuneItemBase(RTATestBundle): + """ + Abstract class enabling rtapp execution in a schedtune group + + :param boost: The boost level to set for the cgroup + :type boost: int + + :param prefer_idle: The prefer_idle flag to set for the cgroup + :type prefer_idle: bool + """ + + def __init__(self, res_dir, plat_info, boost, prefer_idle): + super().__init__(res_dir, plat_info) + self.boost = boost + self.prefer_idle = prefer_idle + + @property + def cgroup_configuration(self): + return self.get_cgroup_configuration(self.plat_info, self.boost, self.prefer_idle) + + @classmethod + def get_cgroup_configuration(cls, plat_info, boost, prefer_idle): + attributes = { + 'boost': boost, + 'prefer_idle': int(prefer_idle) + } + return { 'name': 'lisa_test', + 'controller': 'schedtune', + 'attributes': attributes } + + @classmethod + # Not annotated, to prevent exekall from picking it up. See + # SchedTuneBase.from_target + def from_target(cls, target, boost, prefer_idle, res_dir=None, ftrace_coll=None): + """ + .. warning:: `res_dir` is at the end of the parameter list, unlike most + other `from_target` where it is the second one. + """ + return super().from_target(target, res_dir, boost=boost, + prefer_idle=prefer_idle, ftrace_coll=ftrace_coll) + + @classmethod + def _from_target(cls, target, res_dir, boost, prefer_idle, ftrace_coll=None): + plat_info = target.plat_info + rtapp_profile = cls.get_rtapp_profile(plat_info) + cgroup_config = cls.get_cgroup_configuration(plat_info, boost, prefer_idle) + cls._run_rtapp(target, res_dir, rtapp_profile, ftrace_coll, cgroup_config) + + return cls(res_dir, plat_info, boost, prefer_idle) + +class SchedTuneBase(TestBundle): + """ + Abstract class enabling the aggregation of ``SchedTuneItemBase`` + + :param test_bundles: a list of test bundles generated by + multiple ``SchedTuneItemBase`` instances + :type test_bundles: list + """ + def __init__(self, res_dir, plat_info, test_bundles): + super().__init__(res_dir, plat_info) + + self.test_bundles = test_bundles + + @classmethod + def from_target(cls, target:Target, res_dir:ArtifactPath=None, + ftrace_coll:FtraceCollector=None) -> 'SchedTuneBase': + """ + Creates a SchedTuneBase bundle from the target. + """ + return super().from_target(target, res_dir, ftrace_coll=ftrace_coll) + + @classmethod + def _from_target(cls, target, res_dir, ftrace_coll): + return cls(res_dir, target.plat_info, + list(cls._create_test_bundles(target, res_dir, ftrace_coll)) + ) + + @classmethod + @abc.abstractmethod + def _create_test_bundles(cls, target, res_dir, ftrace_coll): + """ + Collects and yields a ResultBundle per test item. + """ + pass + + @classmethod + def _create_test_bundle_item(cls, target, res_dir, ftrace_coll, item_cls, + boost, prefer_idle): + """ + Creates and returns a TestBundle for a given item class, and a given + schedtune configuration + """ + item_dir = ArtifactPath.join(res_dir, 'boost_{}_prefer_idle_{}'.format( + boost, int(prefer_idle))) + os.makedirs(item_dir) + + logger = cls.get_logger() + logger.info('Running {} with boost={}, prefer_idle={}'.format( + item_cls.__name__, boost, prefer_idle)) + return item_cls.from_target(target, boost, prefer_idle, res_dir=item_dir, ftrace_coll=ftrace_coll) + + def _merge_res_bundles(self, res_bundles): + """ + Merge a set of result bundles + """ + overall_bundle = ResultBundle.from_bool(all(res_bundles.values())) + for name, bundle in res_bundles.items(): + overall_bundle.add_metric(name, bundle.metrics) + + overall_bundle.add_metric('failed', [ + name for name, bundle in res_bundles.items() + if bundle.result is Result.FAILED + ]) + return overall_bundle + +class SchedTuneFreqItem(SchedTuneItemBase): + """ + Runs a tiny RT rtapp task pinned to a big CPU at a given boost level and + checks the frequency selection was performed accordingly. + """ + + @classmethod + def get_rtapp_profile(cls, plat_info): + cpu = plat_info['capacity-classes'][-1][0] + rtapp_profile = {} + rtapp_profile['rta_stune'] = Periodic( + duty_cycle_pct = 1, # very small task, no impact on freq w/o boost + duration_s = 10, + period_ms = 16, + cpus = [cpu], # pin to big CPU, to focus on frequency selection + sched_policy = 'FIFO' # RT tasks have the boost holding feature so + # the frequency should be more stable, and we + # shouldn't go to max freq in Android + ) + return rtapp_profile + + @requires_events(SchedTuneItemBase.trace_window.used_events, "cpu_frequency") + def trace_window(self, trace): + """ + Set the boundaries of the trace window to ``cpu_frequency`` events + before/after the task's start/end time + """ + rta_start, rta_stop = super().trace_window(trace) + + cpu = self.plat_info['capacity-classes'][-1][0] + freq_df = trace.df_events('cpu_frequency') + freq_df = freq_df[freq_df.cpu == cpu] + + # Find the frequency events before and after the task runs + freq_start = freq_df[freq_df.index < rta_start].index[-1] + freq_stop = freq_df[freq_df.index > rta_stop].index[0] + + return (freq_start, freq_stop) + + @FrequencyAnalysis.get_average_cpu_frequency.used_events + def test_stune_frequency(self, freq_margin_pct=10) -> ResultBundle: + """ + Test that frequency selection followed the boost + + :param: freq_margin_pct: Allowed margin between estimated and measured + average frequencies + :type freq_margin_pct: int + + Compute the expected frequency given the boost level and compare to the + real average frequency from the trace. + Check that the difference between expected and measured frequencies is + no larger than ``freq_margin_pct``. + """ + kernel_version = self.plat_info['kernel']['version'] + if kernel_version.parts[:2] < (4, 14): + self.get_logger().warning('This test requires the RT boost hold, but it may be disabled in {}'.format(kernel_version)) + + cpu = self.plat_info['capacity-classes'][-1][0] + freqs = self.plat_info['freqs'][cpu] + max_freq = max(freqs) + + # Estimate the target frequency, including sugov's margin, and round + # into a real OPP + boost = self.boost + target_freq = min(max_freq, max_freq * boost / 80) + target_freq = list(filter(lambda f: f >= target_freq, freqs))[0] + + # Get the real average frequency + avg_freq = self.trace.analysis.frequency.get_average_cpu_frequency(cpu) + + distance = abs(target_freq - avg_freq) * 100 / target_freq + res = ResultBundle.from_bool(distance < freq_margin_pct) + res.add_metric("target freq", target_freq, 'kHz') + res.add_metric("average freq", avg_freq, 'kHz') + + return res + +class SchedTuneFrequencyTest(SchedTuneBase): + """ + Runs multiple ``SchedTuneFreqItem`` tests at various boost levels ranging + from 20% to 100%, then checks all succedeed. + """ + + # Make sure exekall will always collect all events required by items + ftrace_conf = SchedTuneFreqItem.ftrace_conf + + @classmethod + def _create_test_bundles(cls, target, res_dir, ftrace_coll): + for boost in range(20, 101, 20): + yield cls._create_test_bundle_item(target, res_dir, ftrace_coll, + SchedTuneFreqItem, boost, False) + + def test_stune_frequency(self, freq_margin_pct=10) -> ResultBundle: + """ + .. seealso:: :meth:`SchedTuneFreqItem.test_stune_frequency` + """ + res_bundles = { + 'boost{}'.format(b.boost): b.test_stune_frequency(freq_margin_pct) + for b in self.test_bundles + } + return self._merge_res_bundles(res_bundles) + + +class SchedTunePlacementItem(SchedTuneItemBase): + """ + Runs a tiny RT-App task marked 'prefer_idle' at a given boost level and + tests if it was placed on big-enough CPUs. + """ + + @classmethod + def get_rtapp_profile(cls, plat_info): + rtapp_profile = {} + rtapp_profile['rta_stune'] = Periodic( + duty_cycle_pct = 1, + duration_s = 3, + period_ms = 16, + ) + + return rtapp_profile + + @TasksAnalysis.df_task_total_residency.used_events + def test_stune_task_placement(self, bad_cpu_margin_pct=10) -> ResultBundle: + """ + Test that the task placement satisfied the boost requirement + + Check that top-app tasks spend no more than ``bad_cpu_margin_pct`` of + their time on CPUs that don't have enough capacity to serve their + boost. + """ + assert len(self.rtapp_profile) == 1 + task = list(self.rtapp_profile.keys())[0] + df = self.trace.analysis.tasks.df_task_total_residency(task) + + # Find CPUs without enough capacity to meet the boost + boost = self.boost + cpu_caps = self.plat_info['cpu-capacities'] + ko_cpus = list(filter(lambda x: (cpu_caps[x] / 10.24) < boost, cpu_caps)) + + # Count how much time was spend on wrong CPUs + time_ko = 0 + total_time = 0 + for cpu in cpu_caps: + t = df['runtime'][cpu] + if cpu in ko_cpus: + time_ko += t + total_time += t + + pct_ko = time_ko * 100 / total_time + res = ResultBundle.from_bool(pct_ko < bad_cpu_margin_pct) + res.add_metric("time spent on inappropriate CPUs", pct_ko, '%') + + return res + +class SchedTunePlacementTest(SchedTuneBase): + """ + Runs multiple ``SchedTunePlacementItem`` tests with prefer_idle set and + typical top-app boost levels, then checks all succedeed. + """ + + # Make sure exekall will always collect all events required by items + ftrace_conf = SchedTunePlacementItem.ftrace_conf + + @classmethod + def _create_test_bundles(cls, target, res_dir, ftrace_coll): + # Typically top-app tasks are boosted by 10%, or 50% during touchboost + for boost in [10, 50]: + yield cls._create_test_bundle_item(target, res_dir, ftrace_coll, + SchedTunePlacementItem, boost, True) + + def test_stune_task_placement(self, margin_pct=10) -> ResultBundle: + """ + .. seealso:: :meth:`SchedTunePlacementItem.test_stune_task_placement` + """ + res_bundles = { + 'boost{}'.format(b.boost): b.test_stune_task_placement(margin_pct) + for b in self.test_bundles + } + return self._merge_res_bundles(res_bundles) + +# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab