From 52ed8940a5cfeadc5234fa0f097df7d0891773b6 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 5 Dec 2017 17:50:39 +0000 Subject: [PATCH 1/3] tasks_analysis: factor out tasks signal tracepoint Right now we support reporting and plotting of tasks utilization and load using a single trace event: sched_load_avg_task. However, more recently we introduced a different set of more generic tracepoints for these signals. In preparation to support different tracepoints format to report task's utilziation and load metrics, this patch factors out all the hardheaded references to "sched_load_avg_task" and its fields into a more customizable setup defined at class build time. Signed-off-by: Patrick Bellasi --- libs/utils/analysis/tasks_analysis.py | 86 ++++++++++++++++++--------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/libs/utils/analysis/tasks_analysis.py b/libs/utils/analysis/tasks_analysis.py index 42beae6a2..8647e673c 100644 --- a/libs/utils/analysis/tasks_analysis.py +++ b/libs/utils/analysis/tasks_analysis.py @@ -39,6 +39,21 @@ class TasksAnalysis(AnalysisModule): def __init__(self, trace): super(TasksAnalysis, self).__init__(trace) + self.supported_events = [ + 'sched_load_avg_task', + ] + self._task_event = None + + # Check for the minimum required signals to be available + for event in self.supported_events: + if self._trace.hasEvents(event): + self._log.info("Using task signals from [%s] trace events", event) + self._task_event = event + break + else: + self._log.warning('Required events [%s] not found, ' + 'tasks signals not available', + self.supported_events) ############################################################################### # DataFrame Getter Methods @@ -56,15 +71,16 @@ class TasksAnalysis(AnalysisModule): default: capacity of a little cluster :type min_utilization: int """ - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found') + if self._task_event is None: + self._log.warning('Required events [%s] not available', + self.supported_events) return None if min_utilization is None: min_utilization = self._little_cap # Get utilization samples >= min_utilization - df = self._dfg_trace_event('sched_load_avg_task') + df = self._dfg_trace_event(self._task_event) big_tasks_events = df[df.util_avg > min_utilization] if not len(big_tasks_events): self._log.warning('No tasks with with utilization samples > %d', @@ -228,10 +244,9 @@ class TasksAnalysis(AnalysisModule): 'load_sum', 'util_sum', 'period_contrib', 'residencies'] - # Check for the minimum required signals to be available - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found, ' - 'plot DISABLED!') + if self._task_event is None: + self._log.warning('Required events [%s] not found, ' + 'plot DISABLED!', self.supported_events) return # Defined list of tasks to plot @@ -245,19 +260,31 @@ class TasksAnalysis(AnalysisModule): else: raise ValueError('No tasks to plot specified') + # Signals to use for each plot, depending on available events + if self._task_event == 'sched_load_avg_task': + utilization_signals = { + 'load_avg', 'util_avg', 'boosted_util', + } + residency_signals = {'residencies'} + load_signals = { + 'load_sum', 'util_sum', 'period_contrib', + } + # Compute number of plots to produce plots_count = 0 plots_signals = [ # Fist plot: task's utilization - {'load_avg', 'util_avg', 'boosted_util'}, + utilization_signals, # Second plot: task residency - {'residencies'}, + residency_signals, # Third plot: tasks's load - {'load_sum', 'util_sum', 'period_contrib'} + load_signals, ] hr = [] ysize = 0 for plot_id, signals_to_plot in enumerate(plots_signals): + if signals_to_plot is None: + continue signals_to_plot = signals_to_plot.intersection(signals) if len(signals_to_plot): plots_count = plots_count + 1 @@ -292,8 +319,9 @@ class TasksAnalysis(AnalysisModule): y=.94, fontsize=16, horizontalalignment='center') # Plot load and utilization - signals_to_plot = {'load_avg', 'util_avg', 'boosted_util'} - signals_to_plot = list(signals_to_plot.intersection(signals)) + signals_to_plot = utilization_signals + signals_to_plot = list(signals_to_plot.intersection(signals)) \ + if utilization_signals else [] if len(signals_to_plot) > 0: axes = plt.subplot(gs[plot_id, 0]) axes.set_title('Task [{0:d}:{1:s}] Signals' @@ -304,8 +332,9 @@ class TasksAnalysis(AnalysisModule): savefig = True # Plot CPUs residency - signals_to_plot = {'residencies'} - signals_to_plot = list(signals_to_plot.intersection(signals)) + signals_to_plot = residency_signals + signals_to_plot = list(signals_to_plot.intersection(signals)) \ + if residency_signals else [] if len(signals_to_plot) > 0: axes = plt.subplot(gs[plot_id, 0]) axes.set_title( @@ -320,8 +349,9 @@ class TasksAnalysis(AnalysisModule): savefig = True # Plot PELT signals - signals_to_plot = {'load_sum', 'util_sum', 'period_contrib'} - signals_to_plot = list(signals_to_plot.intersection(signals)) + signals_to_plot = load_signals + signals_to_plot = list(signals_to_plot.intersection(signals)) \ + if load_signals else [] if len(signals_to_plot) > 0: axes = plt.subplot(gs[plot_id, 0]) axes.set_title('Task [{0:d}:{1:s}] PELT Signals' @@ -362,6 +392,10 @@ class TasksAnalysis(AnalysisModule): default: capacity of a little cluster :type min_utilization: int """ + if self._task_event is None: + self._log.warning('Required events [%s] not available', + self.supported_events) + return None # Get PID of big tasks big_frequent_task_df = self._dfg_top_big_tasks( @@ -376,7 +410,7 @@ class TasksAnalysis(AnalysisModule): return # Get the list of events for all big frequent tasks - df = self._dfg_trace_event('sched_load_avg_task') + df = self._dfg_trace_event(self._task_event) big_frequent_tasks_events = df[df.pid.isin(big_frequent_task_pids)] # Define axes for side-by-side plottings @@ -528,9 +562,9 @@ class TasksAnalysis(AnalysisModule): :param big_cluster: :type big_cluster: bool """ - - if not self._trace.hasEvents('sched_load_avg_task'): - self._log.warning('Events [sched_load_avg_task] not found') + if self._task_event is None: + self._log.warning('Required events [%s] not found, ' + 'plot DISABLED!', self.supported_events) return if not self._trace.hasEvents('cpu_frequency'): self._log.warning('Events [cpu_frequency] not found') @@ -544,7 +578,7 @@ class TasksAnalysis(AnalysisModule): cpus = self._little_cpus # Get all utilization update events - df = self._dfg_trace_event('sched_load_avg_task') + df = self._dfg_trace_event(self._task_event) # Keep events of defined big tasks big_task_pids = self._dfg_top_big_tasks( @@ -628,7 +662,7 @@ class TasksAnalysis(AnalysisModule): :type is_last: bool """ # Get dataframe for the required task - util_df = self._dfg_trace_event('sched_load_avg_task') + util_df = self._dfg_trace_event(self._task_event) # Plot load and util signals_to_plot = set(signals).difference({'boosted_util'}) @@ -691,7 +725,7 @@ class TasksAnalysis(AnalysisModule): :param is_last: if True this is the last plot :type is_last: bool """ - util_df = self._dfg_trace_event('sched_load_avg_task') + util_df = self._dfg_trace_event(self._task_event) if 'cluster' in util_df: data = util_df[util_df.pid == tid][['cluster', 'cpu']] @@ -729,10 +763,8 @@ class TasksAnalysis(AnalysisModule): :param signals: signals to be plot :param signals: list(str) """ - util_df = self._dfg_trace_event('sched_load_avg_task') - data = util_df[util_df.pid == tid][['load_sum', - 'util_sum', - 'period_contrib']] + util_df = self._dfg_trace_event(self._task_event) + data = util_df[util_df.pid == tid][signals_to_plot] data.plot(ax=axes, drawstyle='steps-post') axes.set_xlim(self._trace.x_min, self._trace.x_max) axes.ticklabel_format(style='scientific', scilimits=(0, 0), -- GitLab From eb4e063f480e18f242ce5a400167f67dc2e38296 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 5 Dec 2017 17:53:53 +0000 Subject: [PATCH 2/3] tasks_analysis: add support for sched_load_se tracepoints This allows to use the Task's Analysis API on more recent kernels where a sched_load_se is available instead of sched_load_avg_task. Signed-off-by: Patrick Bellasi --- libs/utils/analysis/tasks_analysis.py | 10 +++++++++- libs/utils/trace.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/libs/utils/analysis/tasks_analysis.py b/libs/utils/analysis/tasks_analysis.py index 8647e673c..101227099 100644 --- a/libs/utils/analysis/tasks_analysis.py +++ b/libs/utils/analysis/tasks_analysis.py @@ -40,7 +40,7 @@ class TasksAnalysis(AnalysisModule): super(TasksAnalysis, self).__init__(trace) self.supported_events = [ - 'sched_load_avg_task', + 'sched_load_avg_task', 'sched_load_se', ] self._task_event = None @@ -269,6 +269,14 @@ class TasksAnalysis(AnalysisModule): load_signals = { 'load_sum', 'util_sum', 'period_contrib', } + elif self._task_event == 'sched_load_se': + utilization_signals = { + 'util_avg', 'boosted_util', + } + residency_signals = {'residencies'} + load_signals = { + 'load', 'runnable_load_avg', + } # Compute number of plots to produce plots_count = 0 diff --git a/libs/utils/trace.py b/libs/utils/trace.py index a78916754..777c0e8ab 100644 --- a/libs/utils/trace.py +++ b/libs/utils/trace.py @@ -253,6 +253,7 @@ class Trace(object): # Setup internal data reference to interesting events/dataframes self._sanitize_SchedLoadAvgCpu() self._sanitize_SchedLoadAvgTask() + self._sanitize_SchedLoadSe() self._sanitize_SchedCpuCapacity() self._sanitize_SchedBoostCpu() self._sanitize_SchedBoostTask() @@ -530,6 +531,29 @@ class Trace(object): lambda util_avg: big_cap if util_avg > little_cap else little_cap ) + def _sanitize_SchedLoadSe(self): + if not self.hasEvents('sched_load_se'): + return + + if not self.has_big_little: + return + + df = self._dfg_trace_event('sched_load_se') + df['cluster'] = np.select( + [df.cpu.isin(self.platform['clusters']['little'])], + ['LITTLE'], 'big') + + if 'nrg_model' not in self.platform: + return + + # Add a column which represents the max capacity of the smallest + # clustre which can accomodate the task utilization + little_cap = self.platform['nrg_model']['little']['cpu']['cap_max'] + big_cap = self.platform['nrg_model']['big']['cpu']['cap_max'] + df['min_cluster_cap'] = df.util_avg.map( + lambda util_avg: big_cap if util_avg > little_cap else little_cap + ) + def _sanitize_SchedBoostCpu(self): """ Add a boosted utilization signal as the sum of utilization and margin. -- GitLab From 79657cc7e0c3b6c5b39aeebc1b82d8ddd435e570 Mon Sep 17 00:00:00 2001 From: Patrick Bellasi Date: Tue, 5 Dec 2017 17:56:19 +0000 Subject: [PATCH 3/3] tasks_analysis: add support for sched_util_est_task tracepoints This allows to use the Task's Analysis API on kernels with UtilEst supprot where a sched_util_est_task tracepoint can be used to report both task PELT's utilization and the corresponding estimated utilization. Signed-off-by: Patrick Bellasi --- libs/utils/analysis/tasks_analysis.py | 18 +++++++++++++---- libs/utils/trace.py | 28 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/libs/utils/analysis/tasks_analysis.py b/libs/utils/analysis/tasks_analysis.py index 101227099..3cd156e98 100644 --- a/libs/utils/analysis/tasks_analysis.py +++ b/libs/utils/analysis/tasks_analysis.py @@ -40,7 +40,7 @@ class TasksAnalysis(AnalysisModule): super(TasksAnalysis, self).__init__(trace) self.supported_events = [ - 'sched_load_avg_task', 'sched_load_se', + 'sched_load_avg_task', 'sched_load_se', 'sched_util_est_task', ] self._task_event = None @@ -81,7 +81,10 @@ class TasksAnalysis(AnalysisModule): # Get utilization samples >= min_utilization df = self._dfg_trace_event(self._task_event) - big_tasks_events = df[df.util_avg > min_utilization] + if 'util_est' in df.columns: + big_tasks_events = df[df.util_est > min_utilization] + else: + big_tasks_events = df[df.util_avg > min_utilization] if not len(big_tasks_events): self._log.warning('No tasks with with utilization samples > %d', min_utilization) @@ -239,10 +242,11 @@ class TasksAnalysis(AnalysisModule): :type signals: list(str) """ if not signals: - signals = ['load_avg', 'util_avg', 'boosted_util', + signals = ['load_avg', 'util_avg', 'boosted_util', 'util_est', 'sched_overutilized', 'load_sum', 'util_sum', 'period_contrib', - 'residencies'] + 'residencies' + ] if self._task_event is None: self._log.warning('Required events [%s] not found, ' @@ -277,6 +281,12 @@ class TasksAnalysis(AnalysisModule): load_signals = { 'load', 'runnable_load_avg', } + elif self._task_event == 'sched_util_est_task': + utilization_signals = { + 'util_avg', 'util_runnable', + } + residency_signals = {'residencies'} + load_signals = None # Compute number of plots to produce plots_count = 0 diff --git a/libs/utils/trace.py b/libs/utils/trace.py index 777c0e8ab..4057ac87d 100644 --- a/libs/utils/trace.py +++ b/libs/utils/trace.py @@ -254,6 +254,7 @@ class Trace(object): self._sanitize_SchedLoadAvgCpu() self._sanitize_SchedLoadAvgTask() self._sanitize_SchedLoadSe() + self._sanitize_SchedUtilEstTask() self._sanitize_SchedCpuCapacity() self._sanitize_SchedBoostCpu() self._sanitize_SchedBoostTask() @@ -554,6 +555,33 @@ class Trace(object): lambda util_avg: big_cap if util_avg > little_cap else little_cap ) + def _sanitize_SchedUtilEstTask(self): + if not self.hasEvents('sched_util_est_task'): + return + + # Add column with util_max = max(util_avg, util_est.last) + df = self._dfg_trace_event('sched_util_est_task') + df['util_est'] = df[['util_avg', 'est_ewma', 'est_last']].max(axis=1) + + if not self.has_big_little: + return + + df = self._dfg_trace_event('sched_util_est_task') + df['cluster'] = np.select( + [df.cpu.isin(self.platform['clusters']['little'])], + ['LITTLE'], 'big') + + if 'nrg_model' not in self.platform: + return + + # Add a column which represents the max capacity of the smallest + # clustre which can accomodate the task utilization + little_cap = self.platform['nrg_model']['little']['cpu']['cap_max'] + big_cap = self.platform['nrg_model']['big']['cpu']['cap_max'] + df['min_cluster_cap'] = df.util_avg.map( + lambda util_avg: big_cap if util_avg > little_cap else little_cap + ) + def _sanitize_SchedBoostCpu(self): """ Add a boosted utilization signal as the sum of utilization and margin. -- GitLab