From fdf6a3c7a4046a0f90584352d56bf59d2cfffda1 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 23 Oct 2019 11:26:52 +0100 Subject: [PATCH 1/4] lisa.wlgen.rta: Allow JSON files outside of the res_dir The way the RTA class is designed is that its base constructor is very simple - it only really requires a target and an rt-app JSON file. Alternative constructors then come in handy to create the JSON for us. I wanted to try out an rt-app JSON handed out to me by someone else, but wlgen complains when trying to join res_dir and the jsson_file. The docstring of the class does say it is a path to a file, not a name, so make it so. --- lisa/wlgen/rta.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lisa/wlgen/rta.py b/lisa/wlgen/rta.py index 158d47af6..2a6ca123e 100644 --- a/lisa/wlgen/rta.py +++ b/lisa/wlgen/rta.py @@ -75,10 +75,10 @@ class RTA(Workload): self.trace_events = trace_events or [] if not json_file: - json_file = '{}.json'.format(self.name) + json_file = ArtifactPath.join(self.res_dir, '{}.json'.format(self.name)) - self.local_json = ArtifactPath.join(self.res_dir, json_file) - self.remote_json = self.target.path.join(self.run_dir, json_file) + self.local_json = json_file + self.remote_json = self.target.path.join(self.run_dir, os.path.basename(json_file)) rta_cmd = self.target.which('rt-app') if not rta_cmd: -- GitLab From b72f3ffcc01016ee7af6c91bf16e6c136692c910 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 23 Oct 2019 11:31:58 +0100 Subject: [PATCH 2/4] lisa.analysis.idle: Make the CPU idle signal boolean It really just is a boolean signal (on or off), and let's just say not doing this is quite wasteful: ``` 143.850158 0 143.850233 1 143.850378 0 143.854415 1 143.854635 0 dtype: int64 ``` ``` 143.850158 True 143.850233 True 143.850378 False 143.854340 False 143.854415 True dtype: bool ``` --- lisa/analysis/idle.py | 14 +++++++------- lisa/analysis/tasks.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lisa/analysis/idle.py b/lisa/analysis/idle.py index ba3b0e37d..4876a866f 100644 --- a/lisa/analysis/idle.py +++ b/lisa/analysis/idle.py @@ -51,22 +51,22 @@ class IdleAnalysis(TraceAnalysisBase): :param cpu: CPU ID :type cpu: int - :returns: A :class:`pandas.Series` that equals 1 at timestamps where the - CPU is reported to be non-idle, 0 otherwise + :returns: A :class:`pandas.Series` that equals True at timestamps where the + CPU is reported to be non-idle, False otherwise """ idle_df = self.trace.df_events('cpu_idle') cpu_df = idle_df[idle_df.cpu_id == cpu] cpu_active = cpu_df.state.apply( - lambda s: 1 if s == -1 else 0 + lambda s: True if s == -1 else False ) start_time = self.trace.start if cpu_active.empty: - cpu_active = pd.Series([0], index=[start_time]) + cpu_active = pd.Series([False], index=[start_time]) elif cpu_active.index[0] != start_time: - entry_0 = pd.Series(cpu_active.iloc[0] ^ 1, index=[start_time]) + entry_0 = pd.Series(not cpu_active.iloc[0], index=[start_time]) cpu_active = pd.concat([entry_0, cpu_active]) # Fix sequences of wakeup/sleep events reported with the same index @@ -123,8 +123,8 @@ class IdleAnalysis(TraceAnalysisBase): sr = pd.Series() for cpu in cpus: cpu_sr = self.signal_cpu_active(cpu) - cpu_sr = cpu_sr[cpu_sr == 1] - cpu_sr = cpu_sr.replace(1, cpu) + cpu_sr = cpu_sr[cpu_sr] + cpu_sr = cpu_sr.replace(True, cpu) sr = sr.append(cpu_sr) return pd.DataFrame({'cpu': sr}).sort_index() diff --git a/lisa/analysis/tasks.py b/lisa/analysis/tasks.py index b5c05cf5f..ae2c8990b 100644 --- a/lisa/analysis/tasks.py +++ b/lisa/analysis/tasks.py @@ -488,7 +488,7 @@ class TasksAnalysis(TraceAnalysisBase): return res_df[:count] @df_task_states.used_events - def df_task_activation(self, task, cpu=None, active_value=1, sleep_value=0): + def df_task_activation(self, task, cpu=None, active_value=True, sleep_value=False): """ DataFrame of a task's active time on a given CPU -- GitLab From 8651681de329fc42aa07f1e3a2cfdfcf863830fd Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 10 Oct 2019 17:20:21 +0100 Subject: [PATCH 3/4] lisa.datautils: Add adjacent index helpers --- lisa/datautils.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lisa/datautils.py b/lisa/datautils.py index 9b7816973..29bd83704 100644 --- a/lisa/datautils.py +++ b/lisa/datautils.py @@ -504,6 +504,46 @@ def df_window(df, window, method='inclusive', clip_window=False): """ return _data_window(df, window, method, clip_window) +def _data_adjacent_index(data, ref, method): + """ + ``data`` can either be a :class:`pandas.DataFrame` or :class:`pandas.Series`. + """ + res = None + + if method not in ["prev", "next"]: + raise ValueError("Adjacent index search method not supported: {}".format(method)) + + if method == "prev" and ref > data.index[0]: + res = data.index.get_loc(ref, "ffill") + elif method == "next" and ref < data.index[-1]: + res = data.index.get_loc(ref, "bfill") + + return res + +def df_adjacent_index(df, ref, method): + """ + :param ref: The index value to look around + + :param method: Choose how adjacent index is selected: + * `prev`: find the index value directly preceding ``ref`` (will be + ``ref`` if it is a valid index value). + * `next`: find the index value directly following ``ref`` (will be + ``ref`` if it is a valid index value). + :type method: str + + :returns: An index value following the `method` choice. Will be `None` if + there is no valid index to return (e.g. `prev` was selected but there is + no index value preceding ``ref``). + + .. note:: ``ref`` doesn't have to be an existing index itself. + """ + return _data_adjacent_index(df, ref, method) + +def series_adjacent_index(series, ref, method): + """ + .. seealso:: :func:`df_adjacent_index` + """ + return _data_adjacent_index(series, ref, method) def series_align_signal(ref, to_align, max_shift=None): """ -- GitLab From 2bcf4ba33d5a87746a550ff743affe002dd62b14 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 10 Oct 2019 17:21:24 +0100 Subject: [PATCH 4/4] lisa.analysis.idle: Add helper to get idle CPUs at given time --- lisa/analysis/idle.py | 77 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/lisa/analysis/idle.py b/lisa/analysis/idle.py index 4876a866f..84bb76965 100644 --- a/lisa/analysis/idle.py +++ b/lisa/analysis/idle.py @@ -23,8 +23,9 @@ import pandas as pd from trappy.utils import handle_duplicate_index from lisa.utils import memoized -from lisa.datautils import series_integrate +from lisa.datautils import series_integrate, series_adjacent_index from lisa.analysis.base import TraceAnalysisBase +from lisa.analysis.tasks import TaskState, TasksAnalysis from lisa.trace import requires_events @@ -72,6 +73,50 @@ class IdleAnalysis(TraceAnalysisBase): # Fix sequences of wakeup/sleep events reported with the same index return handle_duplicate_index(cpu_active) + @memoized + @signal_cpu_active.used_events + @TasksAnalysis.df_tasks_states.used_events + def signal_cpu_sched_active(self, cpu): + """ + Build a square wave representing the active (i.e. non-idle) CPU time + from the scheduler's point of view. + + :param cpu: CPU ID + :type cpu: int + + :returns: A :class:`pandas.Series` that equals True at timestamps where the + CPU is reported to be non-idle, False otherwise + """ + # Find the earliest time at which the scheduler placed a task on + # this CPU's rq. The scheduler won't consider this CPU idle at this + # time. A proper rq depth signal would make this more accurate. + signal_active = self.signal_cpu_active(cpu).copy() + df_states = self.trace.analysis.tasks.df_tasks_states() + + df_states = df_states[df_states.target_cpu == cpu] + sched_active = pd.Series(index=df_states.index, data=1) + + # If there's an enqueue when the CPU is deemed inactive, we want to + # "move" the active signal earlier in the df. + # To do that, first set the signal to active whenever there's such a + # wakeup, then clear up redundant values. + # + # XXX: bitwise 'or' has surprising results: + # s1 = pd.Series(index=[1.25], data=[True]) + # s2 = pd.Series(index=[1.05], data=[True]) + # print(s1 | s2) + # 1.05 False + # 1.25 True + def orify(a, b): + return a or b + + signal_active = signal_active.combine(sched_active, orify, fill_value=False) + + # Delete successive duplicate values + signal_active = signal_active[signal_active.shift() != signal_active].dropna() + + return signal_active + @signal_cpu_active.used_events def signal_cluster_active(self, cluster): """ @@ -236,6 +281,36 @@ class IdleAnalysis(TraceAnalysisBase): idle_time_df.index.name = 'idle_state' return idle_time_df + def _idle_cpus_at(self, timestamp, active_signal_fn): + idles = [] + for cpu in range(self.trace.cpus_count): + series_active = active_signal_fn(cpu) + + # Get very first event before 'timestamp' + # + # XXX: NaN if 'timestamp' is before the first event in the series. + # This could be fixed by checking the first event after 'timestamp': + # if the CPU goes out of idle later, it was idle during 'timestamp' + # and vice-versa. + prev_idx = series_adjacent_index(series_active, timestamp, method="prev") + + if prev_idx is None: + continue + + if not series_active.iloc[prev_idx]: + idles.append(cpu) + + return sorted(idles) + + @signal_cpu_active.used_events + def idle_cpus_at(self, timestamp, as_sched=False): + """ + :returns: list of CPUs that were idle at the given ``timestamp`` + """ + return self._idle_cpus_at( + timestamp, + self.signal_cpu_sched_active if as_sched else self.signal_cpu_active) + ############################################################################### # Plotting Methods -- GitLab