From f9815657a2682e50f954cef57d5ad754dc518c83 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 20 Dec 2018 17:56:23 +0000 Subject: [PATCH 1/8] wa_results_collector: Fix FutureWarning The warning in question: """ data/work/lisa/lisa/wa_results_collector.py:1086: FutureWarning: Interpreting tuple 'by' as a list of keys, rather than a single key. Use 'by=[...]' instead of 'by=(...)'. In the future, a tuple will always mean a single key. """ Do what it says. --- lisa/wa_results_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/wa_results_collector.py b/lisa/wa_results_collector.py index e98150955..1ce2eb7e4 100644 --- a/lisa/wa_results_collector.py +++ b/lisa/wa_results_collector.py @@ -1083,7 +1083,7 @@ class WaResultsCollector(Loggable): return # Separate plot for each test (e.g. one plot for Jankbench list_view) - for (test, inv_id), test_comparisons in df.groupby(('test', 'inv_id')): + for (test, inv_id), test_comparisons in df.groupby(['test', 'inv_id']): # Vertical size of plot depends on how many metrics we're comparing # and how many things (kernels/tags) we're comparing metrics for. # a.k.a the total length of the comparisons df. -- GitLab From 0dd712f4698ca608c69d5d9a4b39c86a9a4a2c02 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 21 Dec 2018 12:19:51 +0000 Subject: [PATCH 2/8] wa_results_collector: Fix comparison plotting --- lisa/wa_results_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lisa/wa_results_collector.py b/lisa/wa_results_collector.py index 1ce2eb7e4..b67c9e910 100644 --- a/lisa/wa_results_collector.py +++ b/lisa/wa_results_collector.py @@ -1134,7 +1134,7 @@ class WaResultsCollector(Loggable): # For each of the things we're comparing we'll plot a bar chart # but slightly shifted. That's how we get multiple bars on each # y-axis point. - bars = ax.barh(bottom=pos + (i * thickness), + bars = ax.barh(pos + (i * thickness), width=gdf['diff_pct'], height=thickness, label=group, color=colors[i % len(colors)], align='center') -- GitLab From fbc7d41d07035026d4e5bf6249eeab90eeb0244c Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 21 Dec 2018 12:28:30 +0000 Subject: [PATCH 3/8] ipynb: Update wltests notebook --- ipynb/wltests/sched-evaluation-full.ipynb | 64 +++++++++++------------ 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/ipynb/wltests/sched-evaluation-full.ipynb b/ipynb/wltests/sched-evaluation-full.ipynb index ec8d5e76b..59487566b 100644 --- a/ipynb/wltests/sched-evaluation-full.ipynb +++ b/ipynb/wltests/sched-evaluation-full.ipynb @@ -14,23 +14,22 @@ "metadata": {}, "outputs": [], "source": [ - "from conf import LisaLogging\n", - "LisaLogging.setup()" + "from lisa.utils import setup_logging\n", + "setup_logging()" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import logging\n", + "import pandas as pd\n", + "\n", "from IPython.display import display\n", "\n", - "from wa_results_collector import WaResultsCollector\n", - "import pandas as pd\n", + "from lisa.wa_results_collector import WaResultsCollector\n", "\n", "%pylab inline" ] @@ -253,9 +252,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "# Get Geekbench scores\n", @@ -263,11 +260,11 @@ "gb_scores_db = df[df.test == 'geekbench']\n", "\n", "# Group scores\n", - "grouped_df = gb_scores_db.groupby(['test', 'tag', 'kernel', 'metric'])\n", + "grouped_df = gb_scores_db.groupby(['test', 'kernel', 'metric'])\n", "\n", "# Get stats for grouped scores\n", - "stats_df = pd.DataFrame(grouped_df.describe(percentiles=[.95, .99]))\n", - "stats_df = stats_df.reset_index().rename(columns={'level_4': 'stats'})" + "stats_df = pd.DataFrame(grouped_df.value.describe(percentiles=[.95, .99])).reset_index()\n", + "#stats_df = stats_df.reset_index().rename(columns={'level_4': 'stats'})" ] }, { @@ -280,12 +277,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ - "single_score_df = stats_df[stats_df.metric.str.match('Single.*')][['metric', 'kernel', 'stats', 'value']]\n", - "single_score_df['metric'] = single_score_df.metric.apply(lambda s : s.replace('Single-Core_', '').replace('_score', ''))\n", - "single_score_df = single_score_df.set_index(['metric', 'kernel', 'stats']).unstack()\n", + "single_score_df = stats_df[stats_df.metric.str.match(\"Single.*\")]\n", + "single_score_df.loc[:, \"metric\"] = single_score_df.metric.apply(lambda s : s.replace('Single-Core_', '').replace('_score', ''))\n", + "single_score_df = single_score_df.set_index(['kernel', 'test', 'metric'])\n", "logging.info(\"Detailed SINGLE core scores:\")\n", "single_score_df" ] @@ -303,10 +302,10 @@ "metadata": {}, "outputs": [], "source": [ - "multi_score_df = stats_df[stats_df.metric.str.match('Multi.*')][['metric', 'kernel', 'stats', 'value']]\n", - "multi_score_df['metric'] = multi_score_df.metric.apply(lambda s : s.replace('Multi-Core_', '').replace('_score', ''))\n", - "multi_score_df = multi_score_df.set_index(['metric', 'kernel', 'stats']).unstack()\n", - "logging.info(\"Detailed SINGLE core scores:\")\n", + "multi_score_df = stats_df[stats_df.metric.str.match(\"Multi.*\")]\n", + "multi_score_df.loc[:, \"metric\"] = multi_score_df.metric.apply(lambda s : s.replace('Multi-Core_', '').replace('_score', ''))\n", + "multi_score_df = multi_score_df.set_index(['kernel', 'test', 'metric'])\n", + "logging.info(\"Detailed MULTI core scores:\")\n", "multi_score_df" ] }, @@ -350,11 +349,11 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "scrolled": true }, "outputs": [], "source": [ - "# Get Geekbench scores\n", + "# Get PCMark scores\n", "df = collector.results_df\n", "pm_scores_db = df[df.workload == 'pcmark']\n", "\n", @@ -362,8 +361,7 @@ "grouped_df = pm_scores_db.groupby(['test', 'tag', 'kernel', 'metric'])\n", "\n", "# Get stats for grouped scores\n", - "stats_df = pd.DataFrame(grouped_df.describe(percentiles=[.95, .99]))\n", - "stats_df = stats_df.reset_index().rename(columns={'level_4': 'stats'})" + "stats_df = pd.DataFrame(grouped_df.value.describe(percentiles=[.95, .99])).reset_index()" ] }, { @@ -372,9 +370,9 @@ "metadata": {}, "outputs": [], "source": [ - "pm_score_df = stats_df[stats_df.metric.str.match('pcmark_.*')][['metric', 'kernel', 'stats', 'value']]\n", - "pm_score_df['metric'] = pm_score_df.metric.apply(lambda s : s.replace('pcmark_', ''))\n", - "pm_score_df = pm_score_df.set_index(['metric', 'kernel', 'stats']).unstack()\n", + "pm_score_df = stats_df[stats_df.metric.str.match('pcmark_.*')]\n", + "pm_score_df.loc[:, 'metric'] = pm_score_df.metric.apply(lambda s : s.replace('pcmark_', ''))\n", + "pm_score_df = pm_score_df.set_index(['kernel', 'test'])\n", "logging.info(\"Detailed scores:\")\n", "pm_score_df" ] @@ -418,21 +416,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" }, "toc": { "colors": { -- GitLab From e7cba259d3a4d3280c634fbfc8fa475d7a605a19 Mon Sep 17 00:00:00 2001 From: Qais Yousef Date: Fri, 28 Sep 2018 13:53:22 +0100 Subject: [PATCH 4/8] Update wa agendas to fix several breakges 1. pcmark needs internet access, so disable airplane mode 2. use v2.4 of exoplayer inline with sched-evaluation-full.yaml 3. s/test/test_ids/ in jankbench attributes Signed-off-by: Qais Yousef (cherry picked from commit c1a4bf35e8b0ea570cb32f1ba082444ad434c6bb) --- .../agendas/example-exoplayer-simple.yaml | 2 ++ tools/wltests/agendas/example-rich.yaml | 23 +++++++++++-------- .../agendas/sched-evaluation-full-traced.yaml | 14 +++++++---- .../agendas/sched-evaluation-full.yaml | 2 ++ .../agendas/sched-evaluation-small.yaml | 14 +++++++---- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/tools/wltests/agendas/example-exoplayer-simple.yaml b/tools/wltests/agendas/example-exoplayer-simple.yaml index d2e73474e..da8103c05 100644 --- a/tools/wltests/agendas/example-exoplayer-simple.yaml +++ b/tools/wltests/agendas/example-exoplayer-simple.yaml @@ -7,3 +7,5 @@ workloads: - name: exoplayer iterations: 3 + workload_parameters: + version: 2.4 diff --git a/tools/wltests/agendas/example-rich.yaml b/tools/wltests/agendas/example-rich.yaml index 21fcb8f2a..3e219d935 100644 --- a/tools/wltests/agendas/example-rich.yaml +++ b/tools/wltests/agendas/example-rich.yaml @@ -53,10 +53,13 @@ workloads: - name: exoplayer id: exoplayer_30s workload_parameters: + version: 2.4 duration: 30 - name: pcmark id: pcmark + runtime_parameters: + airplane_mode: false - name: geekbench id: geekbench @@ -69,31 +72,31 @@ workloads: # easier to read/parse id: jb_list_view classifiers: - test: jb_list_view + test_ids: jb_list_view # workload_parameters are the real parameters that influence what gets run workload_parameters: - test: list_view + test_ids: list_view - name: jankbench id: jb_image_list_view classifiers: - test: jb_image_list_view + test_ids: jb_image_list_view workload_parameters: - test: image_list_view + test_ids: image_list_view - name: jankbench id: jb_shadow_grid classifiers: - test: jb_shadow_grid + test_ids: jb_shadow_grid workload_parameters: - test: shadow_grid + test_ids: shadow_grid - name: jankbench id: jb_low_hitrate_text classifiers: - test: jb_low_hitrate_text + test_ids: jb_low_hitrate_text workload_parameters: - test: low_hitrate_text + test_ids: low_hitrate_text - name: jankbench id: jb_edit_text classifiers: - test: jb_edit_text + test_ids: jb_edit_text workload_parameters: - test: edit_text + test_ids: edit_text diff --git a/tools/wltests/agendas/sched-evaluation-full-traced.yaml b/tools/wltests/agendas/sched-evaluation-full-traced.yaml index 43c4ee26a..07dc4edfc 100644 --- a/tools/wltests/agendas/sched-evaluation-full-traced.yaml +++ b/tools/wltests/agendas/sched-evaluation-full-traced.yaml @@ -57,6 +57,7 @@ workloads: classifiers: tag: mov_720p_30s workload_parameters: + version: 2.4 format: "mov_720p" duration: 30 landscape: True @@ -67,6 +68,7 @@ workloads: classifiers: tag: ogg_128kbps_30s workload_parameters: + version: 2.4 format: "ogg_128kbps" duration: 30 @@ -78,6 +80,8 @@ workloads: classifiers: tag: single iterations: 10 + runtime_parameters: + airplane_mode: false ################################################################################ # Geekbench @@ -98,7 +102,7 @@ workloads: classifiers: tag: iter_30 workload_parameters: - test: list_view + test_ids: list_view iterations: 30 - name: jankbench @@ -106,7 +110,7 @@ workloads: classifiers: tag: iter_30 workload_parameters: - test: image_list_view + test_ids: image_list_view iterations: 30 - name: jankbench @@ -114,7 +118,7 @@ workloads: classifiers: tag: iter_30 workload_parameters: - test: shadow_grid + test_ids: shadow_grid iterations: 30 - name: jankbench @@ -122,7 +126,7 @@ workloads: classifiers: tag: iter_30 workload_parameters: - test: low_hitrate_text + test_ids: low_hitrate_text iterations: 30 - name: jankbench @@ -130,5 +134,5 @@ workloads: classifiers: tag: iter_30 workload_parameters: - test: edit_text + test_ids: edit_text iterations: 30 diff --git a/tools/wltests/agendas/sched-evaluation-full.yaml b/tools/wltests/agendas/sched-evaluation-full.yaml index 971ebd12f..e820fbc8a 100644 --- a/tools/wltests/agendas/sched-evaluation-full.yaml +++ b/tools/wltests/agendas/sched-evaluation-full.yaml @@ -80,6 +80,8 @@ workloads: classifiers: tag: single iterations: 10 + runtime_parameters: + airplane_mode: false ################################################################################ # Geekbench diff --git a/tools/wltests/agendas/sched-evaluation-small.yaml b/tools/wltests/agendas/sched-evaluation-small.yaml index 086fb94c6..3a46cd4ec 100644 --- a/tools/wltests/agendas/sched-evaluation-small.yaml +++ b/tools/wltests/agendas/sched-evaluation-small.yaml @@ -57,6 +57,7 @@ workloads: classifiers: tag: mov_720p_3s workload_parameters: + version: 2.4 format: "mov_720p" duration: 3 landscape: True @@ -67,6 +68,7 @@ workloads: classifiers: tag: ogg_128kbps_3s workload_parameters: + version: 2.4 format: "ogg_128kbps" duration: 3 @@ -78,6 +80,8 @@ workloads: classifiers: tag: iter_5 iterations: 5 + runtime_parameters: + airplane_mode: false ################################################################################ # Geekbench @@ -98,7 +102,7 @@ workloads: classifiers: tag: iter_3 workload_parameters: - test: list_view + test_ids: list_view iterations: 3 - name: jankbench @@ -106,7 +110,7 @@ workloads: classifiers: tag: iter_3 workload_parameters: - test: image_list_view + test_ids: image_list_view iterations: 3 - name: jankbench @@ -114,7 +118,7 @@ workloads: classifiers: tag: iter_3 workload_parameters: - test: shadow_grid + test_ids: shadow_grid iterations: 3 - name: jankbench @@ -122,7 +126,7 @@ workloads: classifiers: tag: iter_3 workload_parameters: - test: low_hitrate_text + test_ids: low_hitrate_text iterations: 3 - name: jankbench @@ -130,5 +134,5 @@ workloads: classifiers: tag: iter_3 workload_parameters: - test: edit_text + test_ids: edit_text iterations: 3 -- GitLab From 474bcf7061e4fca0f0aba951802c4283c5181ce5 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 29 Oct 2018 16:23:33 +0100 Subject: [PATCH 5/8] WaResultsCollector: fix no kernel path case If no kernel path has been defined, kernel_refs is empty and df['kernel_sha1'].replace(kernel_refs) fails Add sha1 as kernel ref in case no kernel path is provied Signed-off-by: Vincent Guittot (cherry picked from commit c3741945a101638960c01d0d4129afc444502c2d) [Minor merge conflict] Signed-off-by: Valentin Schneider --- lisa/wa_results_collector.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lisa/wa_results_collector.py b/lisa/wa_results_collector.py index b67c9e910..78842ff81 100644 --- a/lisa/wa_results_collector.py +++ b/lisa/wa_results_collector.py @@ -162,11 +162,11 @@ class WaResultsCollector(Loggable): df = df.append(df_list) kernel_refs = {} - if kernel_repo_path: - for sha1 in df['kernel_sha1'].unique(): - ref = find_shortest_symref(kernel_repo_path, sha1) - if ref: - kernel_refs[sha1] = ref + for sha1 in df['kernel_sha1'].unique(): + if kernel_repo_path: + kernel_refs[sha1] = find_shortest_symref(kernel_repo_path, sha1) or sha1 + else: + kernel_refs[sha1] = sha1 common_prefix = os.path.commonprefix(list(kernel_refs.values())) for sha1, ref in kernel_refs.items(): -- GitLab From 97c3ed408ac7e842a6c8df87df6c61e7f22ca32e Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Fri, 26 Oct 2018 15:51:10 +0200 Subject: [PATCH 6/8] lisa-ipython: shell support fr-FR language ifconfig returns inet adr:172.17.0.1 instead inet addr:172.17.0.1 when language is set to fr-FR update the sed command to support it Signed-off-by: Vincent Guittot (cherry-picked from commit 6e02465fb0b7644c933b221fb2ccc7f3b261ecaa) [Moved change to right file] Signed-off-by: Valentin Schneider --- shell/lisa_shell | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shell/lisa_shell b/shell/lisa_shell index cc1a949c2..0d373e6e4 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -258,9 +258,9 @@ function _lisa-jupyter-start { IPADDR= if [[ -x /sbin/ifconfig ]]; then - IPADDR=$(/sbin/ifconfig $NETIF 2>/dev/null | \ - awk '/inet / {print $2}' | \ - sed 's/addr://') + IPADDR=$(/sbin/ifconfig $NETIF 2>/dev/null | \ + awk '/inet / {print $2}' | \ + grep -Eo [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) fi if [[ -z "$IPADDR" && -x /sbin/ip ]]; then -- GitLab From 89381a53524c43694496b705981613a8b19e51a7 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 28 Dec 2018 15:30:32 +0000 Subject: [PATCH 7/8] lisa_shell: Cleanup lisa-wltest commands The test on the presence of realpath was inverted. When that failed, lisa-wltest-series would also fail and exit, which would close the shell and prevent the user from reading what went wrong. --- shell/lisa_shell | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell/lisa_shell b/shell/lisa_shell index 0d373e6e4..e1114954a 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -396,7 +396,7 @@ EOF # Check for required external dependencies - if which realpath &>/dev/null; then + if ! which realpath &>/dev/null; then cat < Date: Fri, 4 Jan 2019 17:57:44 +0000 Subject: [PATCH 8/8] lisa_shell: Don't re-activate the venv in lisa-wltest-init We're already guaranteed to have an active venv when sourcing init_env, so this is no longer needed. --- shell/lisa_shell | 1 - 1 file changed, 1 deletion(-) diff --git a/shell/lisa_shell b/shell/lisa_shell index e1114954a..785ceee55 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -380,7 +380,6 @@ function lisa-test { function lisa-wltest-init { export WLTEST_HOME="$LISA_HOME/tools/wltests" export WA_USER_DIRECTORY="$LISA_HOME/tools/wa_user_directory" - lisa-venv-activate # Check that the environment is properly configured if [[ -z "$ANDROID_HOME" ]]; then -- GitLab