diff --git a/libs/utils/trace_analysis.py b/libs/utils/trace_analysis.py index de0b39575d430c7a6325e8121dc185589068ab90..c4f1db9e5179ff1eacf202eba03d260124637883 100644 --- a/libs/utils/trace_analysis.py +++ b/libs/utils/trace_analysis.py @@ -25,10 +25,16 @@ import pylab as pl import re import sys import trappy +import operator +from trappy.utils import listify +import copy +from devlib.utils.misc import memoized # Configure logging import logging +NON_IDLE_STATE = 4294967295 + class TraceAnalysis(object): def __init__(self, trace, tasks=None, plotsdir=None, prefix=''): @@ -785,3 +791,375 @@ class TraceAnalysis(object): # Save generated plots into datadir figname = '{}/{}schedtune_conf.png'.format(self.plotsdir, self.prefix) pl.savefig(figname, bbox_inches='tight') + + def getCPUFrequencyResidency(self, cpu): + """ + Get a DataFrame with per-CPU frequency residency, i.e. amount of + time CPU `cpu` spent at each frequency. Both total and active times + will be computed. + + :param cpu: CPU ID + :type cpu: int + """ + return self.getClusterFrequencyResidency(cpu) + + def plotCPUFrequencyResidency(self, cpus=None): + """ + Plot per-CPU frequency residency. + + :param cpus: List of cpus. By default plot all CPUs + :type cpus: list(str) + """ + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + + _cpus = copy.deepcopy(cpus) + + if not cpus: + # Generate plots only for available CPUs + cpufreq_data = self.trace.df('cpu_frequency') + _cpus = range(cpufreq_data.cpu.max()+1) + else: + _cpus.sort() + + # Split between big and LITTLE CPUs ordered from higher to lower ID + _cpus.reverse() + big_cpus = [c for c in _cpus if c in self.platform['clusters']['big']] + little_cpus = [c for c in _cpus if c in + self.platform['clusters']['little']] + _cpus = big_cpus + little_cpus + + n_cpus = len(_cpus) + gs = gridspec.GridSpec(n_cpus, 1) + fig = plt.figure() + + # Plot CPUs residency, big CPUs first and then LITTLEs + axis = [] + xmax = 0.0 + for idx, cpu in enumerate(_cpus): + total_time, active_time = self.getCPUFrequencyResidency(cpu) + + # Compute maximum x-axes limit + max_time = total_time.max().values[0] + if xmax < max_time: + xmax = max_time + yrange = 0.3 * max(6, len(total_time)) * n_cpus + + axes = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax=axes, + color='g', + legend=False, + figsize=(16,yrange)); + active_time.plot.barh(ax=axes, + color='r', + legend=False, + figsize=(16,yrange)); + + axes.set_title('CPU{}'.format(cpu)) + + axes.grid(True); + if (idx+1 < n_cpus): + axes.set_xticklabels([]) + else: + axes.set_xlabel('Time [s]') + axis.append(axes) + + # Set same x-axes limit for all plots + for ax in axis: + ax.set_xlim(0, 1.05 * xmax) + + # Put title on top of the figure. As of now there is no clean way to + # make the title appear always in the same position in the figure + # because figure heights may vary between different platforms + # (different number of OPPs). Hence, we use annotation + legend_y = axis[0].get_ylim()[1] + axis[0].annotate('OPP Residency Time', + xy=(0, legend_y), + xytext=(-50, 45), + textcoords='offset points', + fontsize=18) + axis[0].annotate('GREEN: Total', + xy=(0, legend_y), + xytext=(-50, 25), + textcoords='offset points', + color='g', + fontsize=14) + axis[0].annotate('RED: Active', + xy=(0, legend_y), + xytext=(50, 25), + textcoords='offset points', + color='r', + fontsize=14) + figname = '{}/{}cpu_freq_residency.png'.format(self.plotsdir, + self.prefix) + pl.savefig(figname, bbox_inches='tight') + + def _getClusterActiveSignal(self, cluster): + """ + Build a square wave representing the active (i.e. non-idle) cluster + time, i.e.: + cluster_active[t] == 1 if at least one CPU is reported to be + non-idle by CPUFreq at time t + cluster_active[t] == 0 otherwise + + :param cluster: list of CPU IDs belonging to a cluster + :type cluster: list(int) + """ + idle_df = self.trace.df('cpu_idle') + + cpu_active = {} + for cpu in cluster: + cpu_idle = idle_df[idle_df.cpu_id == cpu] + + # time instants where CPU goes to an idle state + idle_times = cpu_idle[cpu_idle.state != NON_IDLE_STATE] + # time instants where CPU leaves an idle state + non_idle_times = cpu_idle[cpu_idle.state == NON_IDLE_STATE] + + active_indexes = cpu_idle.index.values.tolist() + + exist_zero = active_indexes[0] == 0.0 + if not exist_zero: + active_indexes.insert(0, 0.0) + + if idle_times.index[0] < non_idle_times.index[0]: + # First cpu_idle entry reports an enter in idle + if not exist_zero: + # If there wasn't a 0.0 entry in the cpu_idle trace it + # means that from time 0.0 until the first cpu_idle event + # there was a task running, hence first value of the square + # wave must be a 1 + active = [(i+1)%2 for i in range(len(active_indexes))] + else: + # If we are here first the first cpu_idle event reports an + # entry in idle at time 0.0, hence first value of the + # square wave must be a 0 + active = [i%2 for i in range(len(active_indexes))] + else: + # First cpu_idle entry reports an exit from idle + if not exist_zero: + # If there wasn't a 0.0 entry in the cpu_idle trace it + # means that from time 0.0 until the first cpu_idle event + # the CPU was idle, hence first value of the square wave + # must be a 0 + active = [i%2 for i in range(len(active_indexes))] + else: + # If we are here first the first cpu_idle event reports an + # exit from idle at time 0.0, hence first value of the + # square wave must be a 1 + active = [(i+1)%2 for i in range(len(active_indexes))] + + cpu_active[cpu] = pd.Series(active, index=active_indexes) + + active = pd.DataFrame(cpu_active) + active.fillna(method='ffill', inplace=True) + + # Cluster active is the OR between the actives on each CPU + # belonging to that specific cluster + cluster_active = reduce( + operator.or_, + [cpu_active.astype(int) for _, cpu_active in + active.iteritems()] + ) + + cluster_active = active[active.columns[0]].astype(int) + for c in active.columns[1:]: + cluster_active |= active[c].astype(int) + + return cluster_active + + def _getCPUActiveSignal(self, cpu): + """ + Build a square wave representing the active (i.e. non-idle) CPU time, + i.e.: + cpu_active[t] = 1 if there is a task running in the CPU at time t + cpu_active[t] = 0 otherwise + + :param cpu: CPU ID + :type cpu: int + """ + return self._getClusterActiveSignal([cpu]) + + def _integrate_square_wave(self, sq_wave): + """ + Compute the integral of a square wave time series. + + :param sq_wave: square wave assuming only 1.0 and 0.0 values + :type sq_wave: :mod:`pandas.Series` + """ + sq_wave.iloc[-1] = 0.0 + # Compact signal to obtain only 1-0-1-0 sequences + comp_sig = sq_wave.loc[sq_wave.shift() != sq_wave] + # First value for computing the difference must be a 1 + if comp_sig.iloc[0] == 0.0: + return sum(comp_sig.iloc[2::2].index - comp_sig.iloc[1:-1:2].index) + else: + return sum(comp_sig.iloc[1::2].index - comp_sig.iloc[:-1:2].index) + + @memoized + def getClusterFrequencyResidency(self, cluster): + """ + Get a DataFrame with per cluster frequency residency, i.e. amount of + time spent at a given frequency in each cluster. + + :param cluster: this can be either a single CPU ID or a list of CPU IDs + belonging to a cluster or the cluster name as specified in the + platform description + :type cluster: str or int or list(int) + """ + + if isinstance(cluster, str): + try: + _cluster = self.platform['clusters'][cluster.lower()] + except KeyError: + logging.warn('%s cluster not found!', cluster) + return None + else: + _cluster = listify(cluster) + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'frequency residency computation not possible!') + return None + if not self.trace.hasEvents('cpu_idle'): + logging.warn('Events [cpu_idle] not found, '\ + 'frequency residency computation not possible!') + return None + + freq_df = self.trace.df('cpu_frequency') + # We assume that every CPU in a cluster always run at the same + # frequency, i.e. the frequency is scaled per-cluster not per-CPU. + # Hence, we can limit the cluster frequencies data to a single CPU + #TODO: we could check that this is valid in the trace and return a + # warning in case it is not + cluster_freqs = freq_df[freq_df.cpu == _cluster[0]] + + ### Compute TOTAL Time ### + time_intervals = cluster_freqs.index[1:] - cluster_freqs.index[:-1] + total_time = pd.DataFrame({ + 'TOTAL Time' : time_intervals, + 'Frequency [MHz]' : [ + f/1000 for f in cluster_freqs.iloc[:-1].frequency + ] + }) + total_time = total_time.groupby(['Frequency [MHz]']).sum() + + ### Compute ACTIVE Time ### + cluster_active = self._getClusterActiveSignal(_cluster) + + # In order to compute the active time spent at each frequency we + # multiply 2 square waves: + # - cluster_active, a square of the form: + # cluster_active[t] == 1 if at least one CPU is reported to be + # non-idle by CPUFreq at time t + # cluster_active[t] == 0 otherwise + # - freq_active, square wave of the form: + # freq_active[t] == 1 if at time t the frequency is f + # freq_active[t] == 0 otherwise + available_freqs = sorted(cluster_freqs.frequency.unique()) + new_idx = sorted(cluster_freqs.index.tolist() + \ + cluster_active.index.tolist()) + cluster_freqs = cluster_freqs.reindex( + new_idx, method='ffill' + ).fillna(method='bfill') + cluster_active = cluster_active.reindex(new_idx, method='ffill') + nonidle_time = [] + for f in available_freqs: + # Build square wave for the current frequency such that: + freq_active = cluster_freqs.frequency.apply( + lambda x: 1 if x == f else 0 + ) + active_t = cluster_active * freq_active + # Sum up the time intervals by computing the integral of the square + # wave + nonidle_time.append(self._integrate_square_wave(active_t)) + + active_time = pd.DataFrame({"ACTIVE Time" : nonidle_time}, + index=[f/1000 for f in available_freqs]) + active_time.index.name = 'Frequency [MHz]' + return total_time, active_time + + def plotClusterFrequencyResidency(self, clusters=['big', 'LITTLE']): + """ + Plot the frequency residency in a given cluster, i.e. the amount of + time cluster `cluster` spent at frequency `f_i`. By default, both 'big' + and 'LITTLE' clusters data are plotted + + :param clusters: name of the clusters to be plotted (all of them by + default) + :type clusters: str ot list(str) + """ + + _clusters = listify(clusters) + + if not self.trace.hasEvents('cpu_frequency'): + logging.warn('Events [cpu_frequency] not found, '\ + 'plot DISABLED!') + return None + + n_clusters = len(_clusters) + gs = gridspec.GridSpec(n_clusters, 1) + fig = plt.figure() + + axis = [] + xmax = 0.0 + for idx, c in enumerate(_clusters): + total_time, active_time = self.getClusterFrequencyResidency( + self.platform['clusters'][c.lower()]) + + if total_time is None or active_time is None: + plt.close(fig) + return + + max_time = total_time.max().values[0] + if xmax < max_time: + xmax = max_time + yrange = 0.4 * max(6, len(total_time)) * n_clusters + + axes = fig.add_subplot(gs[idx]) + total_time.plot.barh(ax = axes, + color='g', + legend=False, + figsize=(16,yrange)); + + active_time.plot.barh(ax = axes, + color='r', + legend=False, + figsize=(16,yrange)); + + axes.set_title('{} Cluster'.format(c)) + + axes.grid(True); + if (idx+1 < n_clusters): + axes.set_xticklabels([]) + else: + axes.set_xlabel('Time [s]') + axis.append(axes) + + for ax in axis: + ax.set_xlim(0, 1.05 * xmax) + + legend_y = axis[0].get_ylim()[1] + axis[0].annotate('OPP Residency Time', + xy=(0, legend_y), + xytext=(-50, 45), + textcoords='offset points', + fontsize=18) + axis[0].annotate('GREEN: Total', + xy=(0, legend_y), + xytext=(-50, 25), + textcoords='offset points', + color='g', + fontsize=14) + axis[0].annotate('RED: Active', + xy=(0, legend_y), + xytext=(50, 25), + textcoords='offset points', + color='r', + fontsize=14) + figname = '{}/{}cluster_freq_residency.png'.format(self.plotsdir, + self.prefix) + pl.savefig(figname, bbox_inches='tight')