From 18d3aaa46c1c82c45c35d0cb4f4e152a782ec4a8 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Mon, 6 Jun 2016 16:07:22 +0100 Subject: [PATCH 1/4] libs/utils/trace_analysis: add per-CPU frequency residency plots Generate horizontal bar plots for per-CPU frequency residency. Frequency residency is the amount of time spent at a each frequency by a given CPU. By default all CPUs are taken into account. For each CPU both total and active (i.e. non-idle) times are plot on the same graph, such that the user has a clear view of the total time spent at a given frequency and the amount of time in which the CPU was executing instructions (not idle). Signed-off-by: Michele Di Giorgio --- libs/utils/trace_analysis.py | 165 +++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index de0b39575..ad803a191 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -29,6 +29,8 @@ import trappy # Configure logging import logging +NON_IDLE_STATE = 4294967295 + class TraceAnalysis(object): def __init__(self, trace, tasks=None, plotsdir=None, prefix=''): @@ -785,3 +787,166 @@ class TraceAnalysis(object): # Save generated plots into datadir figname = '{}/{}schedtune_conf.png'.format(self.plotsdir, self.prefix) pl.savefig(figname, bbox_inches='tight') + + def _computeNonIdleTime(self, t_prev, t, tasks_time_intervals): + """ + Compute the non-idle time spent by tasks within the time interval + [t_prev. t]. + + :param t_prev: Time interval init boundary + :type t_prev: double + + :param t: Time interval end boundary + :type t: double + + :param tasks_time_intervals: DataFrame containing start and end times + of tasks + :type tasks_time_intervals: :mod:`pandas.DataFrame` + """ + nonidle_time = 0.0 + for _, row in tasks_time_intervals.iterrows(): + start = row['start'] + end = row['end'] + if start > t: + # Task not running at f_prev + continue + elif start >= t_prev and start <= t: + # Task starting at instant of time where frequency is f_prev + if end <= t: + # Task runs at f_prev for the whole interval + nonidle_time += end - start + else: + # Task continues running at different frequency + nonidle_time += t - start + else: + # Task started with previous frequency + if end > t_prev: + # Task is still running now, at f_prev + if end <= t: + nonidle_time += end - t_prev + else: + # Task will continue running at different frequency + nonidle_time += t - t_prev + + return nonidle_time + + def getCPUFrequencyResidency(self, cpu): + """ + Get a DataFrame with per-CPU frequency residency, i.e. amount of + time CPU `cpu` spent at each frequency. Both total and active times + will be computed. + + :param cpu: CPU ID + :type cpu: int + """ + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + if not self.trace.hasEvents('cpu_idle'): + logging.warn('Events [cpu_idle] not found, '\ + 'plot DISABLED!') + return None + + freq_dfr = self.trace.df('cpu_frequency') + cpu_freqs = freq_dfr[freq_dfr.cpu == cpu] + idle_dfr = self.trace.df('cpu_idle') + cpu_idle = idle_dfr[idle_dfr.cpu_id == cpu] + + ### Compute TOTAL time spent at each frequency ### + time_intervals = cpu_freqs.index[1:] - cpu_freqs.index[:-1] + total_time = pd.DataFrame({ + 'TOTAL Time' : time_intervals, + 'Frequency [MHz]' : [f/1000 for f in cpu_freqs.iloc[:-1].frequency] + }) + total_time = total_time.groupby(['Frequency [MHz]']).sum() + + ### Compute ACTIVE time spent at each frequency ### + # time instants where CPU is idle + idle_times = cpu_idle[cpu_idle['state'] != NON_IDLE_STATE] + # time instants where CPU is not idle + non_idle_times = cpu_idle[cpu_idle['state'] == NON_IDLE_STATE] + + if idle_times.index[0] < non_idle_times.index[0]: + idle_times = idle_times.iloc[1:] + + time_intervals = [t2 - t1 for t1, t2 in zip(non_idle_times.index, + idle_times.index)] + + times_len = min(len(idle_times), len(non_idle_times)) + tasks_time_intervals = pd.DataFrame({ + "start": non_idle_times.index[:times_len], + "end": idle_times.index[:times_len] + }) + + # Compute the non-IDLE time spent at a certain frequency + time_intervals = [] + frequencies = [] + row_iterator = cpu_freqs.iterrows() + # Take first item from row_iterator + t_prev, f_prev = row_iterator.next() + for t, f in row_iterator: + nonidle_time = self._computeNonIdleTime(t_prev, + t, + tasks_time_intervals) + freq = f_prev['frequency'] + time_intervals.append(nonidle_time) + frequencies.append(freq) + t_prev = t + f_prev = f + + active_time = pd.DataFrame({ + 'ACTIVE Time' : time_intervals, + 'Frequency [MHz]' : [f/1000 for f in frequencies] + }) + # Sum the time intervals with same frequncy + active_time = active_time.groupby(['Frequency [MHz]']).sum() + return total_time, active_time + + def plotCPUFrequencyResidency(self, cpus=None): + """ + Plot per-CPU frequency residency. + + :param cpus: List of cpus. By default plot all CPUs + :type cpus: list(str) + """ + + if not cpus: + cpus = sorted(self.platform['clusters']['little'] + self.platform['clusters']['big']) + else: + cpus.sort() + + n_cpus = len(cpus) + if n_cpus == 1: + gs = gridspec.GridSpec(1, 1) + else: + gs = gridspec.GridSpec(n_cpus/2 + n_cpus%2, 2) + + fig = plt.figure() + + axis = None + for idx, cpu in enumerate(cpus): + total_time, active_time = self.getCPUFrequencyResidency(cpu) + + axis = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax=axis, + color='g', + legend=False, + figsize=(16,8), + fontsize=16); + active_time.plot.barh(ax=axis, + color='r', + legend=False, + figsize=(16,8), + fontsize=16); + axis.set_title('CPU{}'.format(cpu)) + axis.set_xlabel('Time [s]') + axis.set_xlim(0, self.x_max) + + if axis: + handles, labels = axis.get_legend_handles_labels() + plt.figlegend(handles, + labels, + loc='lower right') + plt.subplots_adjust(hspace=0.5) -- GitLab From 2a1e8c2ac2e943c3f83befc2d260ef3546b5c4f6 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Mon, 6 Jun 2016 16:08:40 +0100 Subject: [PATCH 2/4] libs/utils/trace_analysis: add cluster frequency residency plots Add a plot method to show the frequency residency in each cluster. Same information as for per-CPU plots are shown (total and active time). Signed-off-by: Michele Di Giorgio --- libs/utils/trace_analysis.py | 155 +++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index ad803a191..a0f8c6b7b 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -950,3 +950,158 @@ class TraceAnalysis(object): labels, loc='lower right') plt.subplots_adjust(hspace=0.5) + + def getClusterFrequencyResidency(self, cluster): + """ + Get a DataFrame with per cluster frequency residency, i.e. amount of + time spent at a given frequency in each cluster. + + :param cluster: list of CPU IDs belonging to the cluster + :type cluster: list(int) + """ + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + if not self.trace.hasEvents('cpu_idle'): + logging.warn('Events [cpu_idle] not found, '\ + 'plot DISABLED!') + return None + + freq_dfr = self.trace.df('cpu_frequency') + cluster_freqs = freq_dfr[freq_dfr['cpu'].isin(cluster)] + idle_dfr = self.trace.df('cpu_idle') + + ### Computer TOTAL Time ### + time_intervals = cluster_freqs.index[1:] - cluster_freqs.index[:-1] + total_time = pd.DataFrame({ + 'TOTAL Time' : time_intervals, + 'Frequency [MHz]' : [f/1000 for f in cluster_freqs.iloc[:-1].frequency] + }) + total_time = total_time.groupby(['Frequency [MHz]']).sum() + + ### Compute ACTIVE Time ### + cpu_activation = {} + for cpu in cluster: + cpu_freqs = freq_dfr[freq_dfr['cpu'] == cpu] + cpu_idle = idle_dfr[idle_dfr['cpu_id'] == cpu] + + # time instants where CPU goes to an idle state + idle_times = cpu_idle[cpu_idle['state'] != IDLE_STATE] + # time instants where CPU leaves an idle state + non_idle_times = cpu_idle[cpu_idle['state'] == IDLE_STATE] + + if idle_times.index[0] < non_idle_times.index[0]: + idle_times = idle_times.iloc[1:] + + activation_indexes = sorted( + non_idle_times.index.values.tolist() + \ + idle_times.index.values.tolist() + ) + if activation_indexes[0] != 0.0: + activation_indexes.insert(0, 0.0) + activations = [i%2 for i in range(len(activation_indexes))] + else: + activations = [(i+1)%2 for i in range(len(activation_indexes))] + cpu_activation[cpu] = pd.Series(activations, + index=activation_indexes) + + activations = pd.DataFrame(cpu_activation) + activations.loc[0.0].fillna(0, inplace=True) + activations.fillna(method="ffill", inplace=True) + + cluster_active = activations[activations.columns[0]].astype(int) + for c in range(1, len(activations.columns)): + cluster_active |= activations[activations.columns[c]].astype(int) + + start = [] + end = [] + active = False + for idx, val in cluster_active.iteritems(): + if val == 1 and not active: + start.append(idx) + active = True + elif val == 0 and active: + end.append(idx) + active = False + if len(start) > len(end): + end.append(self.x_max) + + tasks_time_intervals = pd.DataFrame({ + "start" : start, + "end" : end + }) + + # Compute the non-IDLE time spent at a certain frequency + time_intervals = [] + frequencies = [] + row_iterator = cluster_freqs.iterrows() + # Take first item from row_iterator + t_prev, f_prev = row_iterator.next() + for t, f in row_iterator: + nonidle_time = self._computeNonIdleTime(t_prev, + t, + tasks_time_intervals) + freq = f_prev['frequency'] + time_intervals.append(nonidle_time) + frequencies.append(freq) + t_prev = t + f_prev = f + + active_time = pd.DataFrame({ + 'ACTIVE Time' : time_intervals, + 'Frequency [MHz]' : [f/1000 for f in frequencies] + }) + # Sum the time intervals with same frequncy + active_time = active_time.groupby(['Frequency [MHz]']).sum() + return total_time, active_time + + def plotClusterFrequencyResidency(self, clusters=['big', 'little']): + """ + Plot the frequency residency in a given cluster, i.e. the amount of + time cluster `cluster` spent at frequency `f_i`. By default, both 'big' + and 'LITTLE' clusters data are plotted + + :param clusters: force to only plot one cluster + :type clusters: list(str) + """ + + if len(clusters) == 1: + gs = gridspec.GridSpec(1, 1) + else: + gs = gridspec.GridSpec(1, 2) + fig = plt.figure() + fig.subplots_adjust(hspace=.5); + + axis = None + for idx, c in enumerate(clusters): + total_time, active_time = self.getClusterFrequencyResidency( + self.platform['clusters'][c.lower()]) + + if total_time is None or active_time is None: + plt.close(fig) + return + + axis = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax = axis, + color='g', + legend=False, + figsize=(16,8), + fontsize=16); + + active_time.plot.barh(ax = axis, + color='r', + legend=False, + figsize=(16,8), + fontsize=16); + + axis.set_title('{} Cluster'.format(c)) + axis.set_xlabel('Time [s]') + axis.set_xlim(0, self.x_max) + + if axis: + handles, labels = axis.get_legend_handles_labels() + plt.figlegend(handles, + labels, + loc='lower right') -- GitLab From 2cdddecc8386e9b4acb081fc957dad8ddf78b715 Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Wed, 8 Jun 2016 17:33:54 +0100 Subject: [PATCH 3/4] libs/utils/trace_analysis: optimise frequency residency computation This optimises the frequency residency computations in both cluster and per-CPU cases. Using the %timeit magic on IPython Notebook we have: - all clusters plot: 40 times faster - all CPU plot: 7 times faster Most of the operations are now done in linear time, that is why the big improvement. Signed-off-by: Michele Di Giorgio --- libs/utils/trace_analysis.py | 536 +++++++++++++++++++---------------- 1 file changed, 296 insertions(+), 240 deletions(-) diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index a0f8c6b7b..10ecdd6e2 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -25,6 +25,9 @@ import pylab as pl import re import sys import trappy +import operator +from trappy.utils import listify +import copy # Configure logging import logging @@ -788,48 +791,6 @@ class TraceAnalysis(object): figname = '{}/{}schedtune_conf.png'.format(self.plotsdir, self.prefix) pl.savefig(figname, bbox_inches='tight') - def _computeNonIdleTime(self, t_prev, t, tasks_time_intervals): - """ - Compute the non-idle time spent by tasks within the time interval - [t_prev. t]. - - :param t_prev: Time interval init boundary - :type t_prev: double - - :param t: Time interval end boundary - :type t: double - - :param tasks_time_intervals: DataFrame containing start and end times - of tasks - :type tasks_time_intervals: :mod:`pandas.DataFrame` - """ - nonidle_time = 0.0 - for _, row in tasks_time_intervals.iterrows(): - start = row['start'] - end = row['end'] - if start > t: - # Task not running at f_prev - continue - elif start >= t_prev and start <= t: - # Task starting at instant of time where frequency is f_prev - if end <= t: - # Task runs at f_prev for the whole interval - nonidle_time += end - start - else: - # Task continues running at different frequency - nonidle_time += t - start - else: - # Task started with previous frequency - if end > t_prev: - # Task is still running now, at f_prev - if end <= t: - nonidle_time += end - t_prev - else: - # Task will continue running at different frequency - nonidle_time += t - t_prev - - return nonidle_time - def getCPUFrequencyResidency(self, cpu): """ Get a DataFrame with per-CPU frequency residency, i.e. amount of @@ -839,70 +800,7 @@ class TraceAnalysis(object): :param cpu: CPU ID :type cpu: int """ - - if not self.trace.hasEvents('cpu_frequency'): - logging.warn('Events [cpu_frequency] not found, '\ - 'plot DISABLED!') - return None - if not self.trace.hasEvents('cpu_idle'): - logging.warn('Events [cpu_idle] not found, '\ - 'plot DISABLED!') - return None - - freq_dfr = self.trace.df('cpu_frequency') - cpu_freqs = freq_dfr[freq_dfr.cpu == cpu] - idle_dfr = self.trace.df('cpu_idle') - cpu_idle = idle_dfr[idle_dfr.cpu_id == cpu] - - ### Compute TOTAL time spent at each frequency ### - time_intervals = cpu_freqs.index[1:] - cpu_freqs.index[:-1] - total_time = pd.DataFrame({ - 'TOTAL Time' : time_intervals, - 'Frequency [MHz]' : [f/1000 for f in cpu_freqs.iloc[:-1].frequency] - }) - total_time = total_time.groupby(['Frequency [MHz]']).sum() - - ### Compute ACTIVE time spent at each frequency ### - # time instants where CPU is idle - idle_times = cpu_idle[cpu_idle['state'] != NON_IDLE_STATE] - # time instants where CPU is not idle - non_idle_times = cpu_idle[cpu_idle['state'] == NON_IDLE_STATE] - - if idle_times.index[0] < non_idle_times.index[0]: - idle_times = idle_times.iloc[1:] - - time_intervals = [t2 - t1 for t1, t2 in zip(non_idle_times.index, - idle_times.index)] - - times_len = min(len(idle_times), len(non_idle_times)) - tasks_time_intervals = pd.DataFrame({ - "start": non_idle_times.index[:times_len], - "end": idle_times.index[:times_len] - }) - - # Compute the non-IDLE time spent at a certain frequency - time_intervals = [] - frequencies = [] - row_iterator = cpu_freqs.iterrows() - # Take first item from row_iterator - t_prev, f_prev = row_iterator.next() - for t, f in row_iterator: - nonidle_time = self._computeNonIdleTime(t_prev, - t, - tasks_time_intervals) - freq = f_prev['frequency'] - time_intervals.append(nonidle_time) - frequencies.append(freq) - t_prev = t - f_prev = f - - active_time = pd.DataFrame({ - 'ACTIVE Time' : time_intervals, - 'Frequency [MHz]' : [f/1000 for f in frequencies] - }) - # Sum the time intervals with same frequncy - active_time = active_time.groupby(['Frequency [MHz]']).sum() - return total_time, active_time + return self.getClusterFrequencyResidency(cpu) def plotCPUFrequencyResidency(self, cpus=None): """ @@ -912,170 +810,301 @@ class TraceAnalysis(object): :type cpus: list(str) """ + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + + _cpus = copy.deepcopy(cpus) + if not cpus: - cpus = sorted(self.platform['clusters']['little'] + self.platform['clusters']['big']) + # Generate plots only for available CPUs + cpufreq_data = self.trace.df('cpu_frequency') + _cpus = range(cpufreq_data.cpu.max()+1) else: - cpus.sort() + _cpus.sort() - n_cpus = len(cpus) - if n_cpus == 1: - gs = gridspec.GridSpec(1, 1) - else: - gs = gridspec.GridSpec(n_cpus/2 + n_cpus%2, 2) + # Split between big and LITTLE CPUs ordered from higher to lower ID + _cpus.reverse() + big_cpus = [c for c in _cpus if c in self.platform['clusters']['big']] + little_cpus = [c for c in _cpus if c in + self.platform['clusters']['little']] + _cpus = big_cpus + little_cpus + n_cpus = len(_cpus) + gs = gridspec.GridSpec(n_cpus, 1) fig = plt.figure() - axis = None - for idx, cpu in enumerate(cpus): + # Plot CPUs residency, big CPUs first and then LITTLEs + axis = [] + xmax = 0.0 + for idx, cpu in enumerate(_cpus): total_time, active_time = self.getCPUFrequencyResidency(cpu) - axis = fig.add_subplot(gs[idx]) - total_time.plot.barh(ax=axis, + # Compute maximum x-axes limit + max_time = total_time.max().values[0] + if xmax < max_time: + xmax = max_time + yrange = 0.3 * max(6, len(total_time)) * n_cpus + + axes = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax=axes, color='g', legend=False, - figsize=(16,8), - fontsize=16); - active_time.plot.barh(ax=axis, + figsize=(16,yrange)); + active_time.plot.barh(ax=axes, color='r', legend=False, - figsize=(16,8), - fontsize=16); - axis.set_title('CPU{}'.format(cpu)) - axis.set_xlabel('Time [s]') - axis.set_xlim(0, self.x_max) - - if axis: - handles, labels = axis.get_legend_handles_labels() - plt.figlegend(handles, - labels, - loc='lower right') - plt.subplots_adjust(hspace=0.5) + figsize=(16,yrange)); + + axes.set_title('CPU{}'.format(cpu)) + + axes.grid(True); + if (idx+1 < n_cpus): + axes.set_xticklabels([]) + else: + axes.set_xlabel('Time [s]') + axis.append(axes) + + # Set same x-axes limit for all plots + for ax in axis: + ax.set_xlim(0, 1.05 * xmax) + + # Put title on top of the figure. As of now there is no clean way to + # make the title appear always in the same position in the figure + # because figure heights may vary between different platforms + # (different number of OPPs). Hence, we use annotation + legend_y = axis[0].get_ylim()[1] + axis[0].annotate('OPP Residency Time', + xy=(0, legend_y), + xytext=(-50, 45), + textcoords='offset points', + fontsize=18) + axis[0].annotate('GREEN: Total', + xy=(0, legend_y), + xytext=(-50, 25), + textcoords='offset points', + color='g', + fontsize=14) + axis[0].annotate('RED: Active', + xy=(0, legend_y), + xytext=(50, 25), + textcoords='offset points', + color='r', + fontsize=14) + figname = '{}/{}cpu_freq_residency.png'.format(self.plotsdir, + self.prefix) + pl.savefig(figname, bbox_inches='tight') + + def _getClusterActiveSignal(self, cluster): + """ + Build a square wave representing the active (i.e. non-idle) cluster + time, i.e.: + cluster_active[t] == 1 if at least one CPU is reported to be + non-idle by CPUFreq at time t + cluster_active[t] == 0 otherwise + + :param cluster: list of CPU IDs belonging to a cluster + :type cluster: list(int) + """ + idle_df = self.trace.df('cpu_idle') + + cpu_active = {} + for cpu in cluster: + cpu_idle = idle_df[idle_df.cpu_id == cpu] + + # time instants where CPU goes to an idle state + idle_times = cpu_idle[cpu_idle.state != NON_IDLE_STATE] + # time instants where CPU leaves an idle state + non_idle_times = cpu_idle[cpu_idle.state == NON_IDLE_STATE] + + active_indexes = cpu_idle.index.values.tolist() + + exist_zero = active_indexes[0] == 0.0 + if not exist_zero: + active_indexes.insert(0, 0.0) + + if idle_times.index[0] < non_idle_times.index[0]: + # First cpu_idle entry reports an enter in idle + if not exist_zero: + # If there wasn't a 0.0 entry in the cpu_idle trace it + # means that from time 0.0 until the first cpu_idle event + # there was a task running, hence first value of the square + # wave must be a 1 + active = [(i+1)%2 for i in range(len(active_indexes))] + else: + # If we are here first the first cpu_idle event reports an + # entry in idle at time 0.0, hence first value of the + # square wave must be a 0 + active = [i%2 for i in range(len(active_indexes))] + else: + # First cpu_idle entry reports an exit from idle + if not exist_zero: + # If there wasn't a 0.0 entry in the cpu_idle trace it + # means that from time 0.0 until the first cpu_idle event + # the CPU was idle, hence first value of the square wave + # must be a 0 + active = [i%2 for i in range(len(active_indexes))] + else: + # If we are here first the first cpu_idle event reports an + # exit from idle at time 0.0, hence first value of the + # square wave must be a 1 + active = [(i+1)%2 for i in range(len(active_indexes))] + + cpu_active[cpu] = pd.Series(active, index=active_indexes) + + active = pd.DataFrame(cpu_active) + active.fillna(method='ffill', inplace=True) + + # Cluster active is the OR between the actives on each CPU + # belonging to that specific cluster + cluster_active = reduce( + operator.or_, + [cpu_active.astype(int) for _, cpu_active in + active.iteritems()] + ) + + cluster_active = active[active.columns[0]].astype(int) + for c in active.columns[1:]: + cluster_active |= active[c].astype(int) + + return cluster_active + + def _getCPUActiveSignal(self, cpu): + """ + Build a square wave representing the active (i.e. non-idle) CPU time, + i.e.: + cpu_active[t] = 1 if there is a task running in the CPU at time t + cpu_active[t] = 0 otherwise + + :param cpu: CPU ID + :type cpu: int + """ + return self._getClusterActiveSignal([cpu]) + + def _integrate_square_wave(self, sq_wave): + """ + Compute the integral of a square wave time series. + + :param sq_wave: square wave assuming only 1.0 and 0.0 values + :type sq_wave: :mod:`pandas.Series` + """ + sq_wave.iloc[-1] = 0.0 + # Compact signal to obtain only 1-0-1-0 sequences + comp_sig = sq_wave.loc[sq_wave.shift() != sq_wave] + # First value for computing the difference must be a 1 + if comp_sig.iloc[0] == 0.0: + return sum(comp_sig.iloc[2::2].index - comp_sig.iloc[1:-1:2].index) + else: + return sum(comp_sig.iloc[1::2].index - comp_sig.iloc[:-1:2].index) def getClusterFrequencyResidency(self, cluster): """ Get a DataFrame with per cluster frequency residency, i.e. amount of time spent at a given frequency in each cluster. - :param cluster: list of CPU IDs belonging to the cluster - :type cluster: list(int) + :param cluster: this can be either a single CPU ID or a list of CPU IDs + belonging to a cluster or the cluster name as specified in the + platform description + :type cluster: str or int or list(int) """ + if isinstance(cluster, str): + try: + _cluster = self.platform['clusters'][cluster.lower()] + except KeyError: + logging.warn('%s cluster not found!', cluster) + return None + else: + _cluster = listify(cluster) + if not self.trace.hasEvents('cpu_frequency'): logging.warn('Events [cpu_frequency] not found, '\ - 'plot DISABLED!') + 'frequency residency computation not possible!') return None if not self.trace.hasEvents('cpu_idle'): logging.warn('Events [cpu_idle] not found, '\ - 'plot DISABLED!') + 'frequency residency computation not possible!') return None - freq_dfr = self.trace.df('cpu_frequency') - cluster_freqs = freq_dfr[freq_dfr['cpu'].isin(cluster)] - idle_dfr = self.trace.df('cpu_idle') + freq_df = self.trace.df('cpu_frequency') + # We assume that every CPU in a cluster always run at the same + # frequency, i.e. the frequency is scaled per-cluster not per-CPU. + # Hence, we can limit the cluster frequencies data to a single CPU + #TODO: we could check that this is valid in the trace and return a + # warning in case it is not + cluster_freqs = freq_df[freq_df.cpu == _cluster[0]] - ### Computer TOTAL Time ### + ### Compute TOTAL Time ### time_intervals = cluster_freqs.index[1:] - cluster_freqs.index[:-1] total_time = pd.DataFrame({ 'TOTAL Time' : time_intervals, - 'Frequency [MHz]' : [f/1000 for f in cluster_freqs.iloc[:-1].frequency] + 'Frequency [MHz]' : [ + f/1000 for f in cluster_freqs.iloc[:-1].frequency + ] }) total_time = total_time.groupby(['Frequency [MHz]']).sum() ### Compute ACTIVE Time ### - cpu_activation = {} - for cpu in cluster: - cpu_freqs = freq_dfr[freq_dfr['cpu'] == cpu] - cpu_idle = idle_dfr[idle_dfr['cpu_id'] == cpu] - - # time instants where CPU goes to an idle state - idle_times = cpu_idle[cpu_idle['state'] != IDLE_STATE] - # time instants where CPU leaves an idle state - non_idle_times = cpu_idle[cpu_idle['state'] == IDLE_STATE] - - if idle_times.index[0] < non_idle_times.index[0]: - idle_times = idle_times.iloc[1:] - - activation_indexes = sorted( - non_idle_times.index.values.tolist() + \ - idle_times.index.values.tolist() + cluster_active = self._getClusterActiveSignal(_cluster) + + # In order to compute the active time spent at each frequency we + # multiply 2 square waves: + # - cluster_active, a square of the form: + # cluster_active[t] == 1 if at least one CPU is reported to be + # non-idle by CPUFreq at time t + # cluster_active[t] == 0 otherwise + # - freq_active, square wave of the form: + # freq_active[t] == 1 if at time t the frequency is f + # freq_active[t] == 0 otherwise + available_freqs = sorted(cluster_freqs.frequency.unique()) + new_idx = sorted(cluster_freqs.index.tolist() + \ + cluster_active.index.tolist()) + cluster_freqs = cluster_freqs.reindex( + new_idx, method='ffill' + ).fillna(method='bfill') + cluster_active = cluster_active.reindex(new_idx, method='ffill') + nonidle_time = [] + for f in available_freqs: + # Build square wave for the current frequency such that: + freq_active = cluster_freqs.frequency.apply( + lambda x: 1 if x == f else 0 ) - if activation_indexes[0] != 0.0: - activation_indexes.insert(0, 0.0) - activations = [i%2 for i in range(len(activation_indexes))] - else: - activations = [(i+1)%2 for i in range(len(activation_indexes))] - cpu_activation[cpu] = pd.Series(activations, - index=activation_indexes) - - activations = pd.DataFrame(cpu_activation) - activations.loc[0.0].fillna(0, inplace=True) - activations.fillna(method="ffill", inplace=True) - - cluster_active = activations[activations.columns[0]].astype(int) - for c in range(1, len(activations.columns)): - cluster_active |= activations[activations.columns[c]].astype(int) - - start = [] - end = [] - active = False - for idx, val in cluster_active.iteritems(): - if val == 1 and not active: - start.append(idx) - active = True - elif val == 0 and active: - end.append(idx) - active = False - if len(start) > len(end): - end.append(self.x_max) - - tasks_time_intervals = pd.DataFrame({ - "start" : start, - "end" : end - }) - - # Compute the non-IDLE time spent at a certain frequency - time_intervals = [] - frequencies = [] - row_iterator = cluster_freqs.iterrows() - # Take first item from row_iterator - t_prev, f_prev = row_iterator.next() - for t, f in row_iterator: - nonidle_time = self._computeNonIdleTime(t_prev, - t, - tasks_time_intervals) - freq = f_prev['frequency'] - time_intervals.append(nonidle_time) - frequencies.append(freq) - t_prev = t - f_prev = f - - active_time = pd.DataFrame({ - 'ACTIVE Time' : time_intervals, - 'Frequency [MHz]' : [f/1000 for f in frequencies] - }) - # Sum the time intervals with same frequncy - active_time = active_time.groupby(['Frequency [MHz]']).sum() + active_t = cluster_active * freq_active + # Sum up the time intervals by computing the integral of the square + # wave + nonidle_time.append(self._integrate_square_wave(active_t)) + + active_time = pd.DataFrame({"ACTIVE Time" : nonidle_time}, + index=[f/1000 for f in available_freqs]) + active_time.index.name = 'Frequency [MHz]' return total_time, active_time - def plotClusterFrequencyResidency(self, clusters=['big', 'little']): + def plotClusterFrequencyResidency(self, clusters=['big', 'LITTLE']): """ Plot the frequency residency in a given cluster, i.e. the amount of time cluster `cluster` spent at frequency `f_i`. By default, both 'big' and 'LITTLE' clusters data are plotted - :param clusters: force to only plot one cluster - :type clusters: list(str) + :param clusters: name of the clusters to be plotted (all of them by + default) + :type clusters: str ot list(str) """ - if len(clusters) == 1: - gs = gridspec.GridSpec(1, 1) - else: - gs = gridspec.GridSpec(1, 2) + _clusters = listify(clusters) + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + + n_clusters = len(_clusters) + gs = gridspec.GridSpec(n_clusters, 1) fig = plt.figure() - fig.subplots_adjust(hspace=.5); - axis = None - for idx, c in enumerate(clusters): + axis = [] + xmax = 0.0 + for idx, c in enumerate(_clusters): total_time, active_time = self.getClusterFrequencyResidency( self.platform['clusters'][c.lower()]) @@ -1083,25 +1112,52 @@ class TraceAnalysis(object): plt.close(fig) return - axis = fig.add_subplot(gs[idx]) - total_time.plot.barh(ax = axis, + max_time = total_time.max().values[0] + if xmax < max_time: + xmax = max_time + yrange = 0.4 * max(6, len(total_time)) * n_clusters + + axes = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax = axes, color='g', legend=False, - figsize=(16,8), - fontsize=16); + figsize=(16,yrange)); - active_time.plot.barh(ax = axis, + active_time.plot.barh(ax = axes, color='r', legend=False, - figsize=(16,8), - fontsize=16); - - axis.set_title('{} Cluster'.format(c)) - axis.set_xlabel('Time [s]') - axis.set_xlim(0, self.x_max) - - if axis: - handles, labels = axis.get_legend_handles_labels() - plt.figlegend(handles, - labels, - loc='lower right') + figsize=(16,yrange)); + + axes.set_title('{} Cluster'.format(c)) + + axes.grid(True); + if (idx+1 < n_clusters): + axes.set_xticklabels([]) + else: + axes.set_xlabel('Time [s]') + axis.append(axes) + + for ax in axis: + ax.set_xlim(0, 1.05 * xmax) + + legend_y = axis[0].get_ylim()[1] + axis[0].annotate('OPP Residency Time', + xy=(0, legend_y), + xytext=(-50, 45), + textcoords='offset points', + fontsize=18) + axis[0].annotate('GREEN: Total', + xy=(0, legend_y), + xytext=(-50, 25), + textcoords='offset points', + color='g', + fontsize=14) + axis[0].annotate('RED: Active', + xy=(0, legend_y), + xytext=(50, 25), + textcoords='offset points', + color='r', + fontsize=14) + figname = '{}/{}cluster_freq_residency.png'.format(self.plotsdir, + self.prefix) + pl.savefig(figname, bbox_inches='tight') -- GitLab From 046f4f824f52cdb5374b50f69cd279d9b1cb427c Mon Sep 17 00:00:00 2001 From: Michele Di Giorgio Date: Thu, 16 Jun 2016 13:39:44 +0100 Subject: [PATCH 4/4] libs/utils/trace_analysis: add memoize decorator to getClusterFrequencyResidency Avoid redoing computations to obtain cluster frequency residency by using the Memoize decorator. Signed-off-by: Michele Di Giorgio --- libs/utils/trace_analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index 10ecdd6e2..c4f1db9e5 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -28,6 +28,7 @@ import trappy import operator from trappy.utils import listify import copy +from devlib.utils.misc import memoized # Configure logging import logging @@ -998,6 +999,7 @@ class TraceAnalysis(object): else: return sum(comp_sig.iloc[1::2].index - comp_sig.iloc[:-1:2].index) + @memoized def getClusterFrequencyResidency(self, cluster): """ Get a DataFrame with per cluster frequency residency, i.e. amount of -- GitLab