From c5a7084ad52647cd50494db780491e9b59521a95 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Fri, 6 Apr 2018 11:49:47 +0200 Subject: [PATCH 1/3] WaResultsCollector: handle failed iterations Handle correctly when an agenda hasn't been fully executed or when some iterations have empty file because something wrong happened during the test. Instead of stopping and returning error, WaResultsCollector continues and post processes available files and results Signed-off-by: Vincent Guittot --- libs/utils/wa_results_collector.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/utils/wa_results_collector.py b/libs/utils/wa_results_collector.py index fbb035ca2..1a2389ee4 100644 --- a/libs/utils/wa_results_collector.py +++ b/libs/utils/wa_results_collector.py @@ -295,7 +295,12 @@ class WaResultsCollector(object): # Jobs can fail due to target misconfiguration or other problems, # without preventing us from collecting the results for the jobs # that ran OK. - with open(os.path.join(job_dir, 'result.json')) as f: + my_file = os.path.join(job_dir, 'result.json') + if not os.path.isfile(my_file): + skipped_jobs[iteration].append(job_id) + continue + + with open(my_file) as f: job_result = json.load(f) if job_result['status'] == 'FAILED': skipped_jobs[iteration].append(job_id) @@ -477,7 +482,12 @@ class WaResultsCollector(object): continue if artifact_name.startswith('energy_instrument_output'): - df = pd.read_csv(path) + + try: + df = pd.read_csv(path) + except pandas.errors.ParserError as e: + self._log.info(" no data for %s", path) + continue if 'device_power' in df.columns: # Looks like this is from an ACME -- GitLab From 3413bde682b61fce0c03f27d104d811356c8c0f5 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Fri, 6 Apr 2018 11:53:20 +0200 Subject: [PATCH 2/3] WaResultsCollector: add parameter to skip chart rendering Add a new parameter to WaResultsCollector to skip all kind of charts rendering. This helps to easily disable any kind of graphic rendering when the class is used to only get raw data results in a csv file as an example and doesn't want or support graphic rendering Signed-off-by: Vincent Guittot --- libs/utils/wa_results_collector.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/libs/utils/wa_results_collector.py b/libs/utils/wa_results_collector.py index 1a2389ee4..4dcf1d923 100644 --- a/libs/utils/wa_results_collector.py +++ b/libs/utils/wa_results_collector.py @@ -111,12 +111,19 @@ class WaResultsCollector(object): this can take some time, so the extracted metrics are cached in the provided output directories. Set this param to False to disable this caching. + + :param display_charts: This class uses IPython.display module to render some + charts of workloads' results. But we also want to use + this class without rendering any charts when we are + only interested in table of figures. Set this param + to False if you only want table of results but not + display them. """ RE_WLTEST_DIR = re.compile(r"wa\.(?P\w+)_(?P.+)") def __init__(self, base_dir=None, wa_dirs=".*", platform=None, kernel_repo_path=None, parse_traces=True, - use_cached_trace_metrics=True): + use_cached_trace_metrics=True, display_charts=True): self._log = logging.getLogger('WaResultsCollector') @@ -143,6 +150,7 @@ class WaResultsCollector(object): if not self.parse_traces: self._log.warning("Trace parsing disabled") self.use_cached_trace_metrics = use_cached_trace_metrics + self.display_charts = display_charts df = pd.DataFrame() df_list = [] @@ -674,6 +682,9 @@ class WaResultsCollector(object): (lowest-valued boxplot at the top of the graph) of the specified `sort_on` statistic. """ + if not self.display_charts: + return + sp = self._get_sort_params(sort_on) df = self._get_metric_df(workload, metric, tag, kernel, test) if df is None: @@ -814,7 +825,8 @@ class WaResultsCollector(object): by, sort_on, ascending, xlim) stats_df = self.describe(workload, metric, tag, kernel, test, by, sort_on, ascending) - display(stats_df) + if self.display_charts: + display(stats_df) return (axes, stats_df) @@ -866,6 +878,10 @@ class WaResultsCollector(object): :param by: List of identifiers to group output as in DataFrame.groupby. """ + + if not self.display_charts: + return + df = self._get_metric_df(workload, metric, tag, kernel, test) if df is None: return @@ -1009,6 +1025,9 @@ class WaResultsCollector(object): 'kernel' column in the results_df uses). If by='tag' then `base_id` should be a WA 'tag id' (as named in the WA agenda). """ + if not self.display_charts: + return + df = self.find_comparisons(base_id=base_id, by=by) if df.empty: -- GitLab From 3e1b36b9df3c8ca2ecf447577dcbaf28aa67ef5c Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Fri, 6 Apr 2018 15:40:40 +0200 Subject: [PATCH 3/3] WaResultsCollector: some error with sched-full-evaluation.ipnb, plot_comparisons returns an error in my setup (vagrant vm image provided by the tools) Signed-off-by: Vincent Guittot --- libs/utils/wa_results_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/utils/wa_results_collector.py b/libs/utils/wa_results_collector.py index 4dcf1d923..4a280b02d 100644 --- a/libs/utils/wa_results_collector.py +++ b/libs/utils/wa_results_collector.py @@ -1006,7 +1006,7 @@ class WaResultsCollector(object): # Find a p-value which hopefully represents the # (complement of the) certainty that any difference in # the mean represents something real. - pvalue = ttest_ind(group_results, base_results, equal_var=False).pvalue + _, pvalue = ttest_ind(group_results, base_results, equal_var=False) comparisons.append(Comparison( metric, test, inv_id, -- GitLab