diff --git a/lisa/_cli_tools/lisa_plot.py b/lisa/_cli_tools/lisa_plot.py index 90ad3b8b9de689e7ce60ca380e89f0377cdf83cf..132df2de16e8a7b058d1bcefdae5862313d44701 100755 --- a/lisa/_cli_tools/lisa_plot.py +++ b/lisa/_cli_tools/lisa_plot.py @@ -147,7 +147,11 @@ def handle_plot_excep(exit_on_error=True): else: excep_msg = 'Please specify --plat-info with the "{}" filled in'.format(e.args[1]) except Exception as e: - excep_msg = str(e) + msg = str(e) + if msg: + excep_msg = msg + else: + excep_msg = e.__class__.__qualname__ else: excep_msg = None diff --git a/lisa/analysis/frequency.py b/lisa/analysis/frequency.py index 8a056baf71a15911b4c388c8005b30fafb7675b6..6c293210a3765be945110ae595800444477a0790 100644 --- a/lisa/analysis/frequency.py +++ b/lisa/analysis/frequency.py @@ -391,7 +391,7 @@ class FrequencyAnalysis(TraceAnalysisBase): @TraceAnalysisBase.plot_method @df_cpu_frequency.used_events - def plot_cpu_frequencies(self, cpu: CPU, average: bool=True): + def plot_cpu_frequencies(self, cpu: CPU, average: bool=True, overutilized: bool=True): """ Plot frequency for the specified CPU @@ -402,17 +402,22 @@ class FrequencyAnalysis(TraceAnalysisBase): frequency average. :type average: bool + :param overutilized: If ``True``, add the overutilized state as an overlay. + :type overutilized: bool + If ``sched_overutilized`` events are available, the plots will also show the intervals of time where the system was overutilized. """ logger = self.logger - df = self.df_cpu_frequency(cpu) + df = self.df_cpu_frequency(cpu, df_fmt='polars-lazyframe') if "freqs" in self.trace.plat_info: frequencies = self.trace.plat_info['freqs'][cpu] else: logger.info(f"Estimating CPU{cpu} frequencies from trace") - frequencies = sorted(list(df.frequency.unique())) + frequencies = sorted( + df.select(pl.col('frequency').unique()).collect()['frequency'].to_list() + ) logger.debug(f"Estimated frequencies: {frequencies}") avg = self.get_average_cpu_frequency(cpu) @@ -420,14 +425,18 @@ class FrequencyAnalysis(TraceAnalysisBase): "Average frequency for CPU{} : {:.3f} GHz".format(cpu, avg / 1e6)) df = df_refit_index(df, window=self.trace.window) - fig = plot_signal(df['frequency'], name=f'Frequency of CPU{cpu} (Hz)') + fig = plot_signal( + df.select(('Time', 'frequency')), + name=f'Frequency of CPU{cpu} (Hz)', + ) if average and avg > 0: fig *= hv.HLine(avg, group='average').opts(color='red') - plot_overutilized = self.ana.status.plot_overutilized - if self.trace.has_events(plot_overutilized.used_events): - fig *= plot_overutilized() + if overutilized: + plot_overutilized = self.ana.status.plot_overutilized + if self.trace.has_events(plot_overutilized.used_events): + fig *= plot_overutilized() return fig diff --git a/lisa/analysis/latency.py b/lisa/analysis/latency.py index f4a96649e0f26763ef5add4cb7d9c0446a5c18f1..69a32f80066c08d9404c326ca5455a8688be38b1 100644 --- a/lisa/analysis/latency.py +++ b/lisa/analysis/latency.py @@ -406,20 +406,14 @@ class LatencyAnalysis(TraceAnalysisBase): df = df_refit_index(df, window=self.trace.window) if df.empty: return _hv_neutral() - - return hv.Overlay( - [ - hv.VSpan( - start, - start + duration, - label=label, - ).options( - alpha=0.5, - ) - for start, duration in df[[column]].itertuples() - ] - ) - + else: + df = df[[column]].reset_index() + return hv.VSpans( + (df['Time'], df['Time'] + df[column]), + label=label, + ).options( + alpha=0.5, + ) return ( plot_bands(wkl_df, "wakeup_latency", "Wakeup latencies") * plot_bands(prt_df, "preempt_latency", "Preemption latencies") diff --git a/lisa/analysis/status.py b/lisa/analysis/status.py index 5a01ea845d46c9882dce5b8f54d97c373239c948..d1679869d901ea7674b3c212f2800010f4d84de7 100644 --- a/lisa/analysis/status.py +++ b/lisa/analysis/status.py @@ -20,10 +20,11 @@ """ System Status Analaysis Module """ import holoviews as hv +import polars as pl from lisa.analysis.base import TraceAnalysisBase from lisa.trace import requires_events -from lisa.datautils import df_refit_index, df_add_delta, df_deduplicate +from lisa.datautils import df_refit_index, df_add_delta, df_deduplicate, _df_to from lisa.notebook import _hv_neutral @@ -52,29 +53,48 @@ class StatusAnalysis(TraceAnalysisBase): * A ``overutilized`` column (the overutilized status at a given time) * A ``len`` column (the time spent in that overutilized status) """ + trace = self.trace.get_view(df_fmt='polars-lazyframe') # Build sequence of overutilization "bands" - df = self.trace.df_event('sched_overutilized') + df = trace.df_event('sched_overutilized') + # Deduplicate before calling df_refit_index() since it will likely add + # a row with duplicated state to have the expected window end + # timestamp. + df = df.filter( + pl.col('overutilized') != + pl.col('overutilized').shift( + 1, + # We want to select the first row, so make sure the filter + # evaluates to true at that index. + fill_value=pl.col('overutilized').not_(), + ) + ) + df = df_refit_index(df, window=trace.window) + # There might be a race between multiple CPUs to emit the # sched_overutilized event, so get rid of duplicated events - df = df_deduplicate(df, cols=['overutilized'], keep='first', consecutives=True) - df = df_add_delta(df, col='len', window=self.trace.window) - # Ignore the last line added by df_refit_index() with a NaN len - df = df.iloc[:-1] - return df[['len', 'overutilized']] + df = df.with_columns( + overutilized=pl.col('overutilized').cast(pl.Boolean), + len=pl.col('Time').diff().shift(-1), + ) + return df.select(('Time', 'overutilized', 'len')) def get_overutilized_time(self): """ Return the time spent in overutilized state. """ - df = self.df_overutilized() - return df[df['overutilized'] == 1]['len'].sum() + df = self.df_overutilized(df_fmt='polars-lazyframe') + df = df.filter(pl.col('overutilized')) + duration = df.select( + pl.col('len').dt.total_nanoseconds().sum() / 1e9 + ).collect().item() + return float(duration) def get_overutilized_pct(self): """ The percentage of the time spent in overutilized state. """ ou_time = self.get_overutilized_time() - return 100 * ou_time / self.trace.time_range + return float(100 * ou_time / self.trace.time_range) ############################################################################### # Plotting Methods @@ -86,28 +106,24 @@ class StatusAnalysis(TraceAnalysisBase): """ Draw the system's overutilized status as colored bands """ - df = self.df_overutilized() - if not df.empty: - df = df_refit_index(df, window=self.trace.window) - - # Compute intervals in which the system is reported to be overutilized - return hv.Overlay( - [ - hv.VSpan( - start, - start + delta, - label='Overutilized' - ).options( - color='red', - alpha=0.05, - ) - for start, delta, overutilized in df[['len', 'overutilized']].itertuples() - if overutilized - ] - ).options( - title='System-wide overutilized status' - ) - else: - return _hv_neutral() + df = self.df_overutilized(df_fmt='polars-lazyframe') + + df = df.filter(pl.col('overutilized')) + df = df.select( + pl.col('Time'), + (pl.col('Time') + pl.col('len')).alias('width'), + ) + df = _df_to(df, fmt='pandas') + df.reset_index(inplace=True) + + # Compute intervals in which the system is reported to be overutilized + return hv.VSpans( + (df['Time'], df['width']), + label='Overutilized' + ).options( + color='red', + alpha=0.05, + title='System-wide overutilized status', + ) # vim :set tabstop=4 shiftwidth=4 expandtab textwidth=80 diff --git a/lisa/datautils.py b/lisa/datautils.py index ecf9f2f3ccd5830134966a13a8da74eaede95e4f..ee9180de9661a9a27e80ea13d2db465f8a472a6c 100644 --- a/lisa/datautils.py +++ b/lisa/datautils.py @@ -460,7 +460,7 @@ class SeriesAccessor(DataAccessor): @SeriesAccessor.register_accessor -def series_refit_index(series, start=None, end=None, window=None, method='inclusive', clip_window=True): +def series_refit_index(series, start=None, end=None, window=None, clip_window=True): """ Slice a series using :func:`series_window` and ensure we have a value at exactly the specified boundaries, unless the signal started after the @@ -480,13 +480,6 @@ def series_refit_index(series, start=None, end=None, window=None, method='inclus exclusive. :type window: tuple(float or None, float or None) or None - :param method: Windowing method used to select the first and last values of - the series using :func:`series_window`. Defaults to ``inclusive``, - which is suitable for signals where all the value changes have a - corresponding row without any fixed sample-rate constraints. If they - have been downsampled, ``nearest`` might be a better choice.). - :type method: str - .. note:: If ``end`` is past the end of the data, the last row will be duplicated so that we can have a start and end index at the right location, without moving the point at which the transition to the last @@ -496,11 +489,11 @@ def series_refit_index(series, start=None, end=None, window=None, method='inclus :param clip_window: Passed down to :func:`series_refit_index`. """ window = _make_window(start, end, window) - return _pandas_refit_index(series, window, method=method) + return _pandas_refit_index(series, window) @DataFrameAccessor.register_accessor -def df_refit_index(df, start=None, end=None, window=None, method='inclusive'): +def df_refit_index(df, start=None, end=None, window=None): """ Same as :func:`series_refit_index` but acting on :class:`pandas.DataFrame` """ @@ -509,7 +502,7 @@ def df_refit_index(df, start=None, end=None, window=None, method='inclusive'): return _dispatch( _polars_refit_index, _pandas_refit_index, - df, window, method + df, window ) @@ -580,17 +573,19 @@ def df_split_signals(df, signal_cols, align_start=False, window=None): cols_val = {signal_cols[0]: group} if window: - signal = df_refit_index(signal, window=window, method='inclusive') + signal = df_refit_index(signal, window=window) yield (cols_val, signal) -def _polars_refit_index(data, window, method): +def _polars_refit_index(data, window): # TODO: maybe expose that as a param index = _polars_index_col(data, index='Time') start, end = _polars_duration_window(window) - data = _polars_window(data, window, method=method, col=index) index_col = pl.col(index) + # Ensure the data is sorted, which should be free if they already are. + data = data.sort(index_col) + data = _polars_window(data, window, method='pre', col=index) if start is not None: data = data.with_columns( @@ -620,7 +615,7 @@ def _polars_refit_index(data, window, method): return data -def _pandas_refit_index(data, window, method): +def _pandas_refit_index(data, window): if data.empty: raise ValueError('Cannot refit the index of an empty dataframe or series') @@ -629,7 +624,7 @@ def _pandas_refit_index(data, window, method): duplicate_last = False else: duplicate_last = end > data.index[-1] - data = _pandas_window(data, window, method=method) + data = _pandas_window(data, window, method='pre') if data.empty: return data diff --git a/tests/test_trace.py b/tests/test_trace.py index e6ed30ee0a407185fee6bbc96c279c2e1c26cfe1..9b1c96aa48f8bb12f8911c88d1a23076d2fdde03 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -406,16 +406,18 @@ class TestTraceView(TraceTestCase): def test_lower_slice(self): view = self.trace[81:] - assert len(view.ana.status.df_overutilized()) == 2 + df = view.ana.status.df_overutilized() + assert len(df) == 3 def test_upper_slice(self): view = self.trace[:80.402065] df = view.ana.status.df_overutilized() - assert len(view.ana.status.df_overutilized()) == 1 + assert len(df) == 2 def test_full_slice(self): view = self.trace[80:81] - assert len(view.ana.status.df_overutilized()) == 2 + df = view.ana.status.df_overutilized() + assert len(df) == 3 def test_time_range(self): expected_duration = np.nextafter(4.0, math.inf)