diff --git a/libs/utils/filters.py b/libs/utils/filters.py index 1a614d8e67615d2262ab812884b4de2b290365bf..45ccfbe5e878e72119b1fe6201f974880036810f 100644 --- a/libs/utils/filters.py +++ b/libs/utils/filters.py @@ -73,7 +73,7 @@ class Filters(object): if min_utilization is None: min_utilization = self.little_cap - df = self.trace.df('tload') + df = self.trace.df('sched_load_avg_task') big_tasks_events = df[df.util_avg > min_utilization] big_tasks = big_tasks_events.pid.unique() @@ -94,7 +94,7 @@ class Filters(object): self.big_frequent_tasks_pids = list(big_topmost.index) # Keep track of big tasks tload events - self.big_tasks['tload'] = big_tasks_events + self.big_tasks['sched_load_avg_task'] = big_tasks_events return self.big_frequent_tasks_pids def _taskIsBig(self, utilization): @@ -114,7 +114,7 @@ class Filters(object): return # Get the list of events for all big frequent tasks - df = self.trace.df('tload') + df = self.trace.df('sched_load_avg_task') big_frequent_tasks_events = df[df.pid.isin(self.big_frequent_tasks_pids)] # Add a column to represent big status @@ -164,7 +164,7 @@ class Filters(object): Tasks which wakeups more frequent than a specified threshold """ - df = self.trace.df('swkp') + df = self.trace.df('sched_wakeup') wkp_tasks_stats = df.groupby('pid').describe(include=['object']) wkp_tasks_pids = wkp_tasks_stats.unstack()['comm']\ @@ -196,7 +196,7 @@ class Filters(object): if per_cluster: # Get per cluster wakeup events - df = self.trace.df('swkpn') + df = self.trace.df('sched_wakeup_new') big_frequent = ( (df.target_cpu.isin(self.big_cpus)) ) @@ -220,14 +220,14 @@ class Filters(object): ax = axes[0] ax.set_title('Tasks WakeUps Events'); - df = self.trace.df('swkp') + df = self.trace.df('sched_wakeup') df.pid.astype(int).plot(style=['b.'], ax=ax); ax.set_xlim(self.x_min, self.x_max); ax.xaxis.set_visible(False); ax = axes[1] ax.set_title('Tasks Forks Events'); - df = self.trace.df('swkpn') + df = self.trace.df('sched_wakeup_new') df.pid.astype(int).plot(style=['r.'], ax=ax); ax.set_xlim(self.x_min, self.x_max); ax.xaxis.set_visible(False); @@ -246,7 +246,7 @@ class Filters(object): fig, axes = plt.subplots(2, 1, figsize=(14, 5)); plt.subplots_adjust(wspace=0.2, hspace=0.3); - df_wkp = self.big_tasks['tload'] + df_wkp = self.big_tasks['sched_load_avg_task'] # Add column of expected cluster, depending on utilization value and # capacity of the selected cluster bu_bc = ( \ @@ -260,7 +260,7 @@ class Filters(object): df_wkp.loc[:,'ccap_mutil'] = np.select( [(bu_bc | su_lc)], [True], False) - df_freq = self.trace.df('pfreq') + df_freq = self.trace.df('cpu_frequency') rd_freq = df_freq[df_freq['cpu'].isin(cpus)] ax = axes[0] @@ -279,7 +279,7 @@ class Filters(object): ax.set_xlim(self.x_min, self.x_max); def rtTasks(self, max_prio = 100): - df = self.trace.df('sswitch') + df = self.trace.df('sched_switch') df = df[df.next_prio <= max_prio] df = df[['next_pid', 'next_comm']] df = df.drop_duplicates() diff --git a/libs/utils/trace.py b/libs/utils/trace.py index 1df6cc98a8e1c3d5a954b8e1d7903b715901b228..f1cff7e3d07faf75f9b9ab26298fc7469dd1f135 100644 --- a/libs/utils/trace.py +++ b/libs/utils/trace.py @@ -36,14 +36,11 @@ class Trace(object): # The platform used to run the experiments self.platform = None - # Dataframe of all events data - self.trace_data = {} - # Folder containing all perf data self.datadir = None - # TRAPpy run object - self.run = None + # TRAPpy FTrace object + self.ftrace = None # The time window used to limit trace parsing to self.window = window @@ -86,7 +83,7 @@ class Trace(object): def __parseTrace(self, datadir, tasks, window): logging.debug('Loading [sched] events from trace in [%s]...', datadir) logging.debug("Parsing events: %s", self.events) - self.run = trappy.Run(datadir, scope="custom", events=self.events, window=window) + self.ftrace = trappy.FTrace(datadir, scope="custom", events=self.events, window=window) # Check for events available on the parsed trace self.__checkAvailableEvents() @@ -95,70 +92,20 @@ class Trace(object): # Setup internal data reference to interesting events/dataframes - if self.hasEvents('sched_switch'): - self.trace_data['sswitch'] = \ - self.run.sched_switch.data_frame - - if self.hasEvents('sched_wakeup'): - self.trace_data['swkp'] = \ - self.run.sched_wakeup.data_frame - - if self.hasEvents('sched_wakeup_new'): - self.trace_data['swkpn'] = \ - self.run.sched_wakeup_new.data_frame - - if self.hasEvents('cpu_frequency'): - self.trace_data['pfreq'] = \ - self.run.cpu_frequency.data_frame - - if self.hasEvents('sched_load_avg_cpu'): - self.trace_data['cload'] = \ - self.run.sched_load_avg_cpu.data_frame - self._sanitize_SchedLoadAvgCpu() - - if self.hasEvents('sched_load_avg_task'): - self.trace_data['tload'] = \ - self.run.sched_load_avg_task.data_frame - self._sanitize_SchedLoadAvgTask() - - if self.hasEvents('cpu_capacity'): - self.trace_data['ccap'] = \ - self.run.cpu_capacity.data_frame - self._sanitize_SchedCpuCapacity() - - if self.hasEvents('sched_boost_cpu'): - self.trace_data['cboost'] = \ - self.run.sched_boost_cpu.data_frame - self._sanitize_SchedBoostCpu() - - if self.hasEvents('sched_boost_task'): - self.trace_data['tboost'] = \ - self.run.sched_boost_task.data_frame - self._sanitize_SchedBoostTask() - - if self.hasEvents('sched_contrib_scale_f'): - self.trace_data['scalef'] = \ - self.run.sched_contrib_scale_f.data_frame - - if self.hasEvents('sched_energy_diff'): - self.trace_data['ediff'] = \ - self.run.sched_energy_diff.data_frame - self._sanitize_SchedEnergyDiff() - - if self.hasEvents('sched_tune_config'): - self.trace_data['stune'] = \ - self.run.sched_tune_config.data_frame - - if self.hasEvents('sched_tune_tasks_update'): - self.trace_data['utask'] = \ - self.run.sched_tune_tasks_update.data_frame + self._sanitize_SchedLoadAvgCpu() + self._sanitize_SchedLoadAvgTask() + self._sanitize_SchedCpuCapacity() + self._sanitize_SchedBoostCpu() + self._sanitize_SchedBoostTask() + self._sanitize_SchedEnergyDiff() + self._sanitize_SchedOverutilized() self.__loadTasksNames(tasks) def __checkAvailableEvents(self): - for val in trappy.Run.get_filters(self.run): - obj = getattr(self.run, val) + for val in trappy.FTrace.get_filters(self.ftrace): + obj = getattr(self.ftrace, val) if len(obj.data_frame): self.available_events.append(val) logging.debug('Events found on trace:') @@ -169,11 +116,11 @@ class Trace(object): def __loadTasksNames(self, tasks): # Try to load tasks names using one of the supported events if 'sched_switch' in self.available_events: - self.getTasks(self.trace_data['sswitch'], tasks, + self.getTasks(self.df('sched_switch'), tasks, name_key='next_comm', pid_key='next_pid') return if 'sched_load_avg_task' in self.available_events: - self.getTasks(self.trace_data['tload'], tasks) + self.getTasks(self.df('sched_load_avg_task'), tasks) return logging.warning('Failed to load tasks names from trace events') @@ -187,8 +134,8 @@ class Trace(object): ts = sys.maxint te = 0 - for key in self.trace_data.keys(): - df = self.trace_data[key] + for events in self.available_events: + df = self.df(events) if len(df) == 0: continue if (df.index[0]) < ts: @@ -255,19 +202,20 @@ class Trace(object): """ if self.datadir is None: raise ValueError("trace data not (yet) loaded") - if event not in self.trace_data: - raise ValueError('Event [{}] not supported. '\ - 'Supported events are: {}'\ - .format(event, self.trace_data.keys())) - return self.trace_data[event] + if self.ftrace and hasattr(self.ftrace, event): + return getattr(self.ftrace, event).data_frame + raise ValueError('Event [{}] not supported. '\ + 'Supported events are: {}'\ + .format(event, self.available_events)) def _sanitize_SchedCpuCapacity(self): - df = self.df('ccap') - # Add more columns if the energy model is available - if 'nrg_model' not in self.platform: + if not self.hasEvents('cpu_capacity') \ + or 'nrg_model' not in self.platform: return + df = self.df('cpu_capacity') + # Add column with LITTLE and big CPUs max capacities nrg_model = self.platform['nrg_model'] max_lcap = nrg_model['little']['cpu']['cap_max'] @@ -283,14 +231,18 @@ class Trace(object): [tip_lcap], tip_bcap) def _sanitize_SchedLoadAvgCpu(self): - df = self.df('cload') + if not self.hasEvents('sched_load_avg_cpu'): + return + df = self.df('sched_load_avg_cpu') if 'utilization' in df: # Convert signals name from v5.0 to v5.1 format df.rename(columns={'utilization':'util_avg'}, inplace=True) df.rename(columns={'load':'load_avg'}, inplace=True) def _sanitize_SchedLoadAvgTask(self): - df = self.df('tload') + if not self.hasEvents('sched_load_avg_task'): + return + df = self.df('sched_load_avg_task') if 'utilization' in df: # Convert signals name from v5.0 to v5.1 format df.rename(columns={'utilization':'util_avg'}, inplace=True) @@ -303,7 +255,9 @@ class Trace(object): ['LITTLE'], 'big') def _sanitize_SchedBoostCpu(self): - df = self.df('cboost') + if not self.hasEvents('sched_boost_cpu'): + return + df = self.df('sched_boost_cpu') if 'usage' in df: # Convert signals name from to v5.1 format df.rename(columns={'usage':'util'}, inplace=True) @@ -311,14 +265,17 @@ class Trace(object): def _sanitize_SchedBoostTask(self): - df = self.df('tboost') + if not self.hasEvents('sched_boost_task'): + return + df = self.df('sched_boost_task') if 'utilization' in df: # Convert signals name from to v5.1 format df.rename(columns={'utilization':'util'}, inplace=True) df['boosted_util'] = df['util'] + df['margin'] def _sanitize_SchedEnergyDiff(self): - if 'nrg_model' not in self.platform: + if not self.hasEvents('sched_energy_diff') \ + or 'nrg_model' not in self.platform: return nrg_model = self.platform['nrg_model'] em_lcluster = nrg_model['little']['cluster'] @@ -333,7 +290,7 @@ class Trace(object): em_lcluster['nrg_max'] + em_bcluster['nrg_max'] print "Maximum estimated system energy: {0:d}".format(power_max) - df = self.df('ediff') + df = self.df('sched_energy_diff') df['nrg_diff_pct'] = SCHED_LOAD_SCALE * df.nrg_diff / power_max # Tag columns by usage_delta diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index 07adadd8a47a9fc26017707959fd4f477d0f6339..35dfa4a4cca0840c1662b1883f3ac4d171668fd7 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -69,7 +69,7 @@ class TraceAnalysis(object): logging.warn('Events [cpu_frequency] not found, '\ 'plot DISABLED!') return - df = self.trace.df('pfreq') + df = self.trace.df('cpu_frequency') pd.options.mode.chained_assignment = None @@ -156,7 +156,7 @@ class TraceAnalysis(object): return (avg_lfreq/1e3, avg_bfreq/1e3) def __addCapacityColum(self): - df = self.trace.df('ccap') + df = self.trace.df('cpu_capacity') # Rename CPU and Capacity columns df.rename(columns={'cpu_id':'cpu'}, inplace=True) # Add column with LITTLE and big CPUs max capacities @@ -179,7 +179,6 @@ class TraceAnalysis(object): if label != '': label1 = '{} '.format(label) label2 = '_{}s'.format(label.lower()) - df = self.trace.trace_data # Plot required CPUs fig, pltaxes = plt.subplots(len(cpus), 1, figsize=(16, 3*(len(cpus)))); @@ -196,27 +195,30 @@ class TraceAnalysis(object): # Add CPU utilization axes.set_title('{0:s}CPU [{1:d}]'.format(label1, cpu)); - df1 = df['cload'][df['cload'].cpu == cpu] - if (len(df1)): - df1[['util_avg']].plot(ax=axes, drawstyle='steps-post', alpha=0.4); + df = self.trace.df('sched_load_avg_cpu') + df = df[df.cpu == cpu] + if len(df): + df[['util_avg']].plot(ax=axes, drawstyle='steps-post', alpha=0.4); # if self.trace.hasEvents('sched_boost_cpu'): - # df2 = df['cboost'][df['cboost'].cpu == cpu] - # if (len(df2)): - # df2[['usage', 'boosted_usage']].plot( - # ax=axes, - # style=['m-', 'r-'], - # drawstyle='steps-post'); + # df = self.trace.df('sched_boost_cpu') + # df = df[df.cpu == cpu] + # if len(df): + # df[['usage', 'boosted_usage']].plot( + # ax=axes, + # style=['m-', 'r-'], + # drawstyle='steps-post'); # Add Capacities data if avilable if self.trace.hasEvents('cpu_capacity'): - df2 = df['ccap'][df['ccap'].cpu == cpu] - if (len(df2)): - # data = df2[['capacity', 'tip_capacity', 'max_capacity']] + df = self.trace.df('cpu_capacity') + df = df[df.cpu == cpu] + if len(df): + # data = df[['capacity', 'tip_capacity', 'max_capacity']] # data.plot(ax=axes, style=['m', 'y', 'r'], - data = df2[['capacity', 'tip_capacity' ]] + data = df[['capacity', 'tip_capacity' ]] data.plot(ax=axes, style=['m', '--y' ], - drawstyle='steps-post') + drawstyle='steps-post') axes.set_ylim(0, 1100); axes.set_xlim(self.x_min, self.x_max); @@ -256,7 +258,7 @@ class TraceAnalysis(object): logging.warn('Events [sched_load_avg_task] not found, '\ 'plot DISABLED!') return - df = self.trace.trace_data['tload'] + df = self.trace.df('sched_load_avg_task') self.trace.getTasks(df, tasks) tasks_to_plot = sorted(self.tasks) if tasks: @@ -282,7 +284,7 @@ class TraceAnalysis(object): data.plot(ax=axes, drawstyle='steps-post'); # Plot boost utilization if available if self.trace.hasEvents('sched_boost_task'): - df2 = self.trace.trace_data['tboost'] + df2 = self.trace.df('sched_boost_task') data = df2[df2.comm == task_name][['boosted_util']] if len(data): data.plot(ax=axes, style=['y-'], drawstyle='steps-post'); @@ -342,7 +344,7 @@ class TraceAnalysis(object): if not self.trace.hasEvents('sched_energy_diff'): logging.warn('Events [sched_energy_diff] not found, plot DISABLED!') return - df = self.trace.df('ediff') + df = self.trace.df('sched_energy_diff') # Filter on 'tasks' if tasks is not None: @@ -467,7 +469,7 @@ class TraceAnalysis(object): if not self.trace.hasEvents('sched_energy_diff'): logging.warn('Events [sched_energy_diff] not found, plot DISABLED!') return - df = self.trace.df('ediff') + df = self.trace.df('sched_energy_diff') # Filter on 'tasks' if tasks is not None: @@ -633,7 +635,7 @@ class TraceAnalysis(object): # Plot: Margin axes = plt.subplot(gs[0,0]); axes.set_title('Margin'); - data = self.trace.df('stune')[['margin']] + data = self.trace.df('sched_tune_config')[['margin']] data.plot(ax=axes, drawstyle='steps-post', style=['b']); axes.set_ylim(0, 110); axes.set_xlim(self.x_min, self.x_max); @@ -642,7 +644,7 @@ class TraceAnalysis(object): # Plot: Boost mode axes = plt.subplot(gs[1,0]); axes.set_title('Boost mode'); - data = self.trace.df('stune')[['boostmode']] + data = self.trace.df('sched_tune_config')[['boostmode']] data.plot(ax=axes, drawstyle='steps-post'); axes.set_ylim(0, 4); axes.set_xlim(self.x_min, self.x_max);