diff --git a/libs/utils/analysis/tasks_analysis.py b/libs/utils/analysis/tasks_analysis.py index 8faf169a346dcdec75ef0b1745b0cdc3e2fbbccd..137ff53f2a9b949b098015f7073ffdfb83836b15 100644 --- a/libs/utils/analysis/tasks_analysis.py +++ b/libs/utils/analysis/tasks_analysis.py @@ -57,15 +57,15 @@ class TasksAnalysis(AnalysisModule): default: capacity of a little cluster :type min_utilization: int """ - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found') + if self._dfg_task_load_events() is None: + self._log.warning('No trace events for task signals, plot DISABLED') return None if min_utilization is None: min_utilization = self._little_cap # Get utilization samples >= min_utilization - df = self._dfg_trace_event('sched_load_avg_task') + df = self._dfg_task_load_events() big_tasks_events = df[df.util_avg > min_utilization] if not len(big_tasks_events): self._log.warning('No tasks with with utilization samples > %d', @@ -186,6 +186,44 @@ class TasksAnalysis(AnalysisModule): return rt_tasks + def _dfg_task_load_events(self): + """ + Get a DataFrame with the scheduler's per-task load-tracking signals + + Parse the relevant trace event and return a DataFrame with the + scheduler's load tracking update events for each task. + + :returns: DataFrame with at least the following columns: + 'comm', 'pid', 'load_avg', 'util_avg'. + """ + df = None + + if 'sched_load_avg_task' in self._trace.available_events: + df = self._dfg_trace_event('sched_load_avg_task') + + elif 'sched_load_se' in self._trace.available_events: + df = self._trace._dfg_trace_event('sched_load_se') + df = df.rename(columns={'util': 'util_avg', 'load': 'load_avg'}) + # In sched_load_se, PID shows -1 for task groups. + df = df[df.pid != -1] + + if not self._trace.has_big_little: + return df + + df['cluster'] = np.select( + [df.cpu.isin(self._trace.platform['clusters']['little'])], + ['LITTLE'], 'big') + + if 'nrg_model' in self._trace.platform: + # Add a column which represents the max capacity of the smallest + # clustre which can accomodate the task utilization + little_cap = self._trace.platform['nrg_model']['little']['cpu']['cap_max'] + big_cap = self._trace.platform['nrg_model']['big']['cpu']['cap_max'] + df['min_cluster_cap'] = df.util_avg.map( + lambda util_avg: big_cap if util_avg > little_cap else little_cap + ) + + return df ############################################################################### # Plotting Methods @@ -233,9 +271,8 @@ class TasksAnalysis(AnalysisModule): 'residencies'] # Check for the minimum required signals to be available - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found, ' - 'plot DISABLED!') + if self._dfg_task_load_events() is None: + self._log.warning('No trace events for task signals, plot DISABLED') return # Defined list of tasks to plot @@ -374,6 +411,10 @@ class TasksAnalysis(AnalysisModule): # Get PID of big tasks big_frequent_task_df = self._dfg_top_big_tasks( min_samples, min_utilization) + if big_frequent_task_df is None: + # (Logged already) + return + if max_tasks > 0: big_frequent_task_df = big_frequent_task_df.head(max_tasks) big_frequent_task_pids = big_frequent_task_df.index.values @@ -384,7 +425,7 @@ class TasksAnalysis(AnalysisModule): return # Get the list of events for all big frequent tasks - df = self._dfg_trace_event('sched_load_avg_task') + df = self._dfg_task_load_events() big_frequent_tasks_events = df[df.pid.isin(big_frequent_task_pids)] # Define axes for side-by-side plottings @@ -537,13 +578,16 @@ class TasksAnalysis(AnalysisModule): :type big_cluster: bool """ - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found') - return if not self._trace.hasEvents('cpu_frequency'): self._log.warning('Events [cpu_frequency] not found') return + # Get all utilization update events + df = self._dfg_task_load_events() + if df is None: + self._log.warning('No trace events for task signals, plot DISABLED') + return + if big_cluster: cluster_correct = 'big' cpus = self._big_cpus @@ -551,9 +595,6 @@ class TasksAnalysis(AnalysisModule): cluster_correct = 'LITTLE' cpus = self._little_cpus - # Get all utilization update events - df = self._dfg_trace_event('sched_load_avg_task') - # Keep events of defined big tasks big_task_pids = self._dfg_top_big_tasks( min_samples, min_utilization) @@ -636,7 +677,10 @@ class TasksAnalysis(AnalysisModule): :type is_last: bool """ # Get dataframe for the required task - util_df = self._dfg_trace_event('sched_load_avg_task') + util_df = self._dfg_task_load_events() + if util_df is None: + self._log.warning('No trace events for task signals, plot DISABLED') + return # Plot load and util signals_to_plot = set(signals).difference({'boosted_util'}) @@ -699,15 +743,15 @@ class TasksAnalysis(AnalysisModule): :param is_last: if True this is the last plot :type is_last: bool """ - util_df = self._dfg_trace_event('sched_load_avg_task') - - if 'cluster' in util_df: - data = util_df[util_df.pid == tid][['cluster', 'cpu']] - for ccolor, clabel in zip('gr', ['LITTLE', 'big']): - cdata = data[data.cluster == clabel] - if len(cdata) > 0: - cdata.plot(ax=axes, style=[ccolor+'+'], legend=False) - + util_df = self._dfg_task_load_events() + if util_df is None: + self._log.warning('No trace events for task signals, plot DISABLED') + return + data = util_df[util_df.pid == tid][['cluster', 'cpu']] + for ccolor, clabel in zip('gr', ['LITTLE', 'big']): + cdata = data[data.cluster == clabel] + if len(cdata) > 0: + cdata.plot(ax=axes, style=[ccolor+'+'], legend=False) # Y Axis - placeholders for legend, acutal CPUs. topmost empty lane cpus = [str(n) for n in range(self._platform['cpus_count'])] ylabels = [''] + cpus @@ -737,6 +781,11 @@ class TasksAnalysis(AnalysisModule): :param signals: signals to be plot :param signals: list(str) """ + if not self._trace.hasEvents('sched_load_avg_task'): + self._log.warning( + 'No sched_load_avg_task events, skipping PELT plot') + return + util_df = self._dfg_trace_event('sched_load_avg_task') data = util_df[util_df.pid == tid][['load_sum', 'util_sum', diff --git a/libs/utils/trace.py b/libs/utils/trace.py index 9daba03a228951307b84eb4733ed6a7502a8c12a..57a6957692ca3c155c8a75d939c2bd0c29259fe3 100644 --- a/libs/utils/trace.py +++ b/libs/utils/trace.py @@ -514,24 +514,6 @@ class Trace(object): df.rename(columns={'runnable_avg_sum': 'load_sum'}, inplace=True) df.rename(columns={'running_avg_sum': 'util_sum'}, inplace=True) - if not self.has_big_little: - return - - df['cluster'] = np.select( - [df.cpu.isin(self.platform['clusters']['little'])], - ['LITTLE'], 'big') - - if 'nrg_model' not in self.platform: - return - - # Add a column which represents the max capacity of the smallest - # clustre which can accomodate the task utilization - little_cap = self.platform['nrg_model']['little']['cpu']['cap_max'] - big_cap = self.platform['nrg_model']['big']['cpu']['cap_max'] - df['min_cluster_cap'] = df.util_avg.map( - lambda util_avg: big_cap if util_avg > little_cap else little_cap - ) - def _sanitize_SchedBoostCpu(self): """ Add a boosted utilization signal as the sum of utilization and margin. diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..19e14f33a4ea8d6d51e8f5960df470ff1898d069 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2017, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import matplotlib +# Prevent matplotlib from trying to connect to X11 server, for headless testing. +# Must be done before importing matplotlib.pyplot or pylab +matplotlib.use('Agg') diff --git a/tests/lisa/__init__.py b/tests/lisa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/tests/lisa/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/lisa/test_trace.py b/tests/lisa/test_trace.py index 400f0f24330712b39aad19a207399d90fa113fb4..267268978978812a3e7868862ec89744d849ef6c 100644 --- a/tests/lisa/test_trace.py +++ b/tests/lisa/test_trace.py @@ -29,6 +29,8 @@ class TestTrace(TestCase): 'sched_switch', 'sched_overutilized', 'cpu_idle', + 'sched_load_avg_task', + 'sched_load_se' ] def __init__(self, *args, **kwargs): @@ -42,14 +44,30 @@ class TestTrace(TestCase): self.trace = Trace(self.platform, self.trace_path, self.events) def make_trace(self, in_data): + """ + Get a trace from an embedded string of textual trace data + """ with open(self.test_trace, "w") as fout: fout.write(in_data) return Trace(self.platform, self.test_trace, self.events, normalize_time=False) - def _get_platform(self): - with open(os.path.join(self.traces_dir, 'platform.json')) as f: + def get_trace(self, trace_name): + """ + Get a trace from a separate provided trace file + """ + dir = os.path.join(self.traces_dir, trace_name) + + trace_path = os.path.join(dir, 'trace.dat') + return Trace(self._get_platform(trace_name), trace_path, self.events) + + def _get_platform(self, trace_name=None): + trace_dir = self.traces_dir + if trace_name: + trace_dir = os.path.join(trace_dir, trace_name) + + with open(os.path.join(trace_dir, 'platform.json')) as f: return json.load(f) def test_getTaskByName(self): @@ -191,6 +209,42 @@ class TestTrace(TestCase): self.assertListEqual(df.index.tolist(), [519.022643]) self.assertListEqual(df.cpu.tolist(), [2]) + def _test_tasks_dfs(self, trace_name): + """Helper for smoke testing _dfg methods in tasks_analysis""" + trace = self.get_trace(trace_name) + + lt_df = trace.data_frame.task_load_events() + columns = ['comm', 'pid', 'load_avg', 'util_avg', 'cpu'] + if trace.has_big_little: + columns += ['cluster'] + if 'nrg_model' in trace.platform: + columns += ['min_cluster_cap'] + for column in columns: + msg = 'Task signals parsed from {} missing {} column'.format( + trace.data_dir, column) + self.assertIn(column, lt_df, msg=msg) + + if trace.has_big_little: + df = trace.data_frame.top_big_tasks(min_samples=1) + for column in ['samples', 'comm']: + msg = 'Big tasks parsed from {} missing {} column'.format( + trace.data_dir, column) + self.assertIn(column, df, msg=msg) + + # Pick an arbitrary PID to try plotting signals for. + pid = lt_df['pid'].unique()[0] + # Call plotTasks - although we won't check the results we can just check + # that things aren't totally borken. + trace.analysis.tasks.plotTasks(tasks=[pid]) + + def test_sched_load_signals(self): + """Test parsing sched_load_se events from EAS upstream integration""" + self._test_tasks_dfs('sched_load') + + def test_sched_load_avg_signals(self): + """Test parsing sched_load_avg_task events from EAS1.2""" + self._test_tasks_dfs('sched_load_avg') + class TestTraceNoClusterData(TestTrace): """ Test Trace without cluster data @@ -198,8 +252,8 @@ class TestTraceNoClusterData(TestTrace): Inherits from TestTrace, so all the tests are run again but with no cluster info the platform dict. """ - def _get_platform(self): - platform = super(TestTraceNoClusterData, self)._get_platform() + def _get_platform(self, trace_name=None): + platform = super(TestTraceNoClusterData, self)._get_platform(trace_name) del platform['clusters'] return platform @@ -210,5 +264,5 @@ class TestTraceNoPlatform(TestTrace): Inherits from TestTrace, so all the tests are run again but with platform=None """ - def _get_platform(self): + def _get_platform(self, trace_name=None): return None diff --git a/tests/lisa/traces/sched_load/platform.json b/tests/lisa/traces/sched_load/platform.json new file mode 100644 index 0000000000000000000000000000000000000000..d2c02c7a20a4b9a126d20fbd9c6a1d1ff334121b --- /dev/null +++ b/tests/lisa/traces/sched_load/platform.json @@ -0,0 +1,79 @@ +{ + "abi": "arm64", + "clusters": { + "big": [ + 1, + 2 + ], + "little": [ + 0, + 3, + 4, + 5 + ] + }, + "cpus_count": 6, + "freqs": { + "big": [ + 450000, + 625000, + 800000, + 950000, + 1100000 + ], + "little": [ + 450000, + 575000, + 700000, + 775000, + 850000 + ] + }, + "kernel": { + "major": 11, + "minor": 0, + "parts": [ + 4, + 11, + 0 + ], + "rc": 6, + "release": "4.11.0-rc6-00092-g9cc3141d9e4f-dirty", + "sha1": "9cc3141d9e4f", + "version": "58 SMP PREEMPT Wed May 24 18:37:50 BST 2017", + "version_number": 4 + }, + "nrg_model": { + "big": { + "cluster": { + "nrg_max": 64 + }, + "cpu": { + "cap_max": 1024, + "nrg_max": 616 + } + }, + "little": { + "cluster": { + "nrg_max": 57 + }, + "cpu": { + "cap_max": 447, + "nrg_max": 93 + } + } + }, + "os": "linux", + "topology": [ + [ + 0, + 3, + 4, + 5 + ], + [ + 1, + 2 + ] + ] +} \ No newline at end of file diff --git a/tests/lisa/traces/sched_load/trace.dat b/tests/lisa/traces/sched_load/trace.dat new file mode 100644 index 0000000000000000000000000000000000000000..0456cd04187bc53f7cc57ca99716b0234478c139 Binary files /dev/null and b/tests/lisa/traces/sched_load/trace.dat differ diff --git a/tests/lisa/traces/sched_load_avg/platform.json b/tests/lisa/traces/sched_load_avg/platform.json new file mode 100644 index 0000000000000000000000000000000000000000..80e06149bb0f004c999a48dd1c0cf7f3c1b665b2 --- /dev/null +++ b/tests/lisa/traces/sched_load_avg/platform.json @@ -0,0 +1,108 @@ +{ + "abi": "arm64", + "clusters": { + "big": [ + 2, + 3 + ], + "little": [ + 0, + 1 + ] + }, + "cpus_count": 4, + "freqs": { + "big": [ + 307200, + 384000, + 460800, + 537600, + 614400, + 691200, + 748800, + 825600, + 902400, + 979200, + 1056000, + 1132800, + 1209600, + 1286400, + 1363200, + 1440000, + 1516800, + 1593600, + 1670400, + 1747200, + 1824000, + 1900800, + 1977600, + 2054400, + 2150400 + ], + "little": [ + 307200, + 384000, + 460800, + 537600, + 614400, + 691200, + 768000, + 844800, + 902400, + 979200, + 1056000, + 1132800, + 1209600, + 1286400, + 1363200, + 1440000, + 1516800, + 1593600 + ] + }, + "kernel": { + "major": 18, + "minor": 31, + "parts": [ + 3, + 18, + 31 + ], + "rc": null, + "release": "3.18.31-gbd96fbf", + "sha1": "bd96fbf", + "version": "1 SMP PREEMPT Mon Nov 7 20:29:14 UTC 2016", + "version_number": 3 + }, + "nrg_model": { + "big": { + "cluster": { + "nrg_max": 96 + }, + "cpu": { + "cap_max": 1024, + "nrg_max": 1715 + } + }, + "little": { + "cluster": { + "nrg_max": 52 + }, + "cpu": { + "cap_max": 763, + "nrg_max": 925 + } + } + }, + "os": "android", + "topology": [ + [ + 0, + 1 + ], + [ + 2, + 3 + ] + ] +} \ No newline at end of file diff --git a/tests/lisa/traces/sched_load_avg/trace.dat b/tests/lisa/traces/sched_load_avg/trace.dat new file mode 100644 index 0000000000000000000000000000000000000000..9583ea8ad8ef4ef3cfe9f324dea6d17ac1e776d2 Binary files /dev/null and b/tests/lisa/traces/sched_load_avg/trace.dat differ