From cc727d2c0f472883f28aa70ad352b6c59be5d486 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 15:29:59 +0000 Subject: [PATCH 1/8] shell: Fix LISA_USE_ENV=0 Avoid deactivating any venv if LISA is setup to *not* use its own venv. --- shell/lisa_shell | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shell/lisa_shell b/shell/lisa_shell index 8f610dcbb..3b5a2b451 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -83,11 +83,11 @@ export LISA_USE_VENV=${LISA_USE_VENV:-1} export LISA_VENV_PATH=${LISA_VENV_PATH:-"$LISA_HOME/.lisa-venv3"} function _lisa-venv-create { - # Make sure we don't nest venv, by calling deactivate if it exists, - # otherwise it will fail with a symlink levels errors - lisa-venv-deactivate || return 1 - if [[ "$LISA_USE_VENV" == 1 ]]; then + # Make sure we don't nest venv, by calling deactivate if it exists, + # otherwise it will fail with a symlink levels errors + lisa-venv-deactivate || return 1 + echo "Creating LISA venv from scratch ($LISA_VENV_PATH) ..." # With --clear, the folder is emptied to create a fresh environment python3 -m venv --clear "$LISA_VENV_PATH" -- GitLab From 1135af9a7fa1fa18c3172214db87aaaa98b1ba83 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 17:36:21 +0000 Subject: [PATCH 2/8] doc: add details on alternative install Document the way to install without a venv --- doc/overview.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/overview.rst b/doc/overview.rst index eba8cb2df..1177232d7 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -24,8 +24,20 @@ In case the venv becomes unusable for some reason, the ``lisa-install`` shell command available after sourcing ``init_env`` will allow to create a new clean venv from scratch. -Alternatively, ``lisa`` package is packaged according to the usual -Python practices, which includes a ``setup.py`` script, and a +Without automatic ``venv`` +-------------------------- + +Sometimes, LISA needs to operate in an environment setup for multiple tools. In +that case, it may be easier to manage manually a venv/virtualenv instead of +letting LISA create one for its shell. + +Setting ``export LISA_USE_VENV=0`` prior to ``source init_env`` will avoid the +creation and usage of the LISA-managed venv. ``lisa-install`` command can still +be used to install the necessary Python packages, which will honor any +venv-like system manually setup. + +Alternatively, ``lisa`` package is packaged according to the usual Python +practices, which includes a ``setup.py`` script, and a ``devmode_requirements.txt`` file that will install all the shipped packages in editable mode (including those that are not developped in that repository, but still included for convenience). -- GitLab From 92dbbd0bed61abcb2affd6cd503aefa4c2027cd0 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 17:34:34 +0000 Subject: [PATCH 3/8] lisa: Add systemcheck subcommand to setup.py Add a setup.py subcommand to check some prerequisites on the system, that cannot be satisfied using pip in a venv --- setup.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ shell/lisa_shell | 5 +++++ 2 files changed, 55 insertions(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index f723be382..550cd5373 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2018, Arm Limited and contributors. @@ -17,6 +18,50 @@ from setuptools import setup +import importlib +import distutils.cmd +import distutils.log + +class SystemCheckCommand(distutils.cmd.Command): + """A custom command to check some prerequisites on the system.""" + + description = 'check some requirements on the system, that cannot be satisfied by installing Python packages in a venv' + user_options = [ + # The format is (long option, short option, description). + ] + + def initialize_options(self): + """Set default values for options.""" + pass + + def finalize_options(self): + """Post-process options.""" + pass + + def run(self): + """Run command.""" + self.check_system_install() + + def check_system_install(self): + missing_pkg_msg = '\n\nThe following packages need to be installed using your Linux distribution package manager. On ubuntu, the following packages are needed:\n apt-get install {}\n\n' + distro_pkg_list = list() + def check_pkg(python_pkg, distro_pkg): + try: + importlib.import_module(python_pkg) + except ImportError: + distro_pkg_list.append(distro_pkg) + + # Some packages are not always present by default on Debian-based + # systems, even if there are part of the Python standard library + check_pkg('tkinter', 'python3-tk') + check_pkg('ensurepip', 'python3-venv') + + if distro_pkg_list: + self.announce( + missing_pkg_msg.format(' '.join(distro_pkg_list)), + level=distutils.log.INFO + ) + with open('README.md', 'r') as fh: long_description = fh.read() @@ -85,6 +130,11 @@ setup( "Topic :: Software Development :: Testing", "Intended Audience :: Developers", ], + + # Add extra subcommands to setup.py + cmdclass={ + 'systemcheck': SystemCheckCommand, + } ) # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab diff --git a/shell/lisa_shell b/shell/lisa_shell index 3b5a2b451..904aa863d 100755 --- a/shell/lisa_shell +++ b/shell/lisa_shell @@ -123,6 +123,10 @@ function _lisa-upgrade-pip { } function lisa-install { + # Check that some prerequisites are available on the system, since they + # cannot be installed using pip in a venv + python3 "$LISA_HOME/setup.py" systemcheck + _lisa-venv-create && _lisa-upgrade-pip || return 1 @@ -139,6 +143,7 @@ function lisa-install { echo echo "Installing LISA packages ..." echo + # Make sure we install all packages, even if they are satisfied by the # system's site-packages location. This ensures we use up to date packages, # and that the installation process will give the same result on all -- GitLab From 6b8038c36140d1df5dcb4f085543e38e23209e8e Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Wed, 21 Nov 2018 17:35:27 +0000 Subject: [PATCH 4/8] tools: Add shebang to setup.py --- tools/bisector/setup.py | 1 + tools/exekall/setup.py | 1 + 2 files changed, 2 insertions(+) mode change 100644 => 100755 tools/bisector/setup.py mode change 100644 => 100755 tools/exekall/setup.py diff --git a/tools/bisector/setup.py b/tools/bisector/setup.py old mode 100644 new mode 100755 index 75edcdf15..f6999e964 --- a/tools/bisector/setup.py +++ b/tools/bisector/setup.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2018, Arm Limited and contributors. diff --git a/tools/exekall/setup.py b/tools/exekall/setup.py old mode 100644 new mode 100755 index 0ce7da5d4..4bd38315a --- a/tools/exekall/setup.py +++ b/tools/exekall/setup.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2018, Arm Limited and contributors. -- GitLab From ceeb1dd32aed3d610b886826427a9f618649c3df Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Thu, 22 Nov 2018 11:29:22 +0000 Subject: [PATCH 5/8] shell: Add hash -r for bash/zsh Make sure the shell will pick up potential new paths for existing binaries, since we update the PATH --- init_env | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/init_env b/init_env index 0e1d22cdf..2e7be0462 100644 --- a/init_env +++ b/init_env @@ -26,6 +26,8 @@ if grep "bash" /proc/$$/cmdline &>/dev/null; then source "$(dirname "$BASH_SOURCE")/shell/lisa_shell" PS1="\[${LISASHELL_BLUE}\][LISAShell \[${LISASHELL_LCYAN}\]\W\[${LISASHELL_BLUE}\]] \> \[${LISASHELL_RESET}\]" + # Make sure that bash picks up new location for all binaries + hash -r # Running under ZSH elif grep "zsh" /proc/$$/cmdline &>/dev/null; then @@ -36,6 +38,8 @@ elif grep "zsh" /proc/$$/cmdline &>/dev/null; then # functions declared there to be executed in emulated mode, so they will # work as well emulate sh -c "source "$LISA_HOME/shell/lisa_shell"" + # Make sure that zsh picks up new location for all binaries + hash -r else echo "WARNING: Current shell is not a BASH" -- GitLab From 5929b85afae8b362a39ce54867c3cf1712a716f7 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Fri, 23 Nov 2018 17:12:31 +0000 Subject: [PATCH 6/8] bisector: minor cleanup --- tools/bisector/bisector/bisector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/bisector/bisector/bisector.py b/tools/bisector/bisector/bisector.py index 183dadc11..cab7dae29 100755 --- a/tools/bisector/bisector/bisector.py +++ b/tools/bisector/bisector/bisector.py @@ -2124,9 +2124,10 @@ class ExekallLISATestStep(ShellStep): stats = dict() testcase_stats[testcase_id] = stats - stats['total'] = iteration_n stats['iterations_summary'] = [entry['result'] for entry in entry_list] - stats['counters'] = dict() + stats['counters'] = { + 'total': iteration_n + } stats['events'] = dict() for issue, pretty_issue in ( ('passed', 'passed'), -- GitLab From a585f3d312fc4b7c74125791d4c4bd44d7032ea7 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Fri, 23 Nov 2018 18:07:36 +0000 Subject: [PATCH 7/8] shell: make init_env forward ret code of lisa_shell --- init_env | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/init_env b/init_env index 2e7be0462..e1d388e63 100644 --- a/init_env +++ b/init_env @@ -23,7 +23,7 @@ if grep "bash" /proc/$$/cmdline &>/dev/null; then # Get base installation path of LISA export LISA_HOME=$(readlink -f "$(dirname "$BASH_SOURCE")") - source "$(dirname "$BASH_SOURCE")/shell/lisa_shell" + source "$(dirname "$BASH_SOURCE")/shell/lisa_shell"; RET=$? PS1="\[${LISASHELL_BLUE}\][LISAShell \[${LISASHELL_LCYAN}\]\W\[${LISASHELL_BLUE}\]] \> \[${LISASHELL_RESET}\]" # Make sure that bash picks up new location for all binaries @@ -37,7 +37,7 @@ elif grep "zsh" /proc/$$/cmdline &>/dev/null; then # Source the script in "sh" emulation mode. This will also mark the # functions declared there to be executed in emulated mode, so they will # work as well - emulate sh -c "source "$LISA_HOME/shell/lisa_shell"" + emulate sh -c "source "$LISA_HOME/shell/lisa_shell""; RET=$? # Make sure that zsh picks up new location for all binaries hash -r @@ -46,7 +46,7 @@ else # Check if a bash shell is available if which bash &>/dev/null; then # Switch to a BASH shell - exec bash --init-file ./init_env + exec bash --init-file ./init_env; RET=$? else echo "ERROR: A BASH shell is not available in PATH" fi @@ -55,4 +55,6 @@ else echo "Please, source this configuration from a terminal running BASH." fi +exit $RET + # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab -- GitLab From 530f9c16fdae6c9ed358826e43baf99b4cb3de67 Mon Sep 17 00:00:00 2001 From: Douglas RAILLARD Date: Fri, 23 Nov 2018 17:37:13 +0000 Subject: [PATCH 8/8] bisector: fix xunit2json-analyze --- tools/bisector/bisector/xunit2json.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/bisector/bisector/xunit2json.py b/tools/bisector/bisector/xunit2json.py index dde94b60a..fc283cea5 100755 --- a/tools/bisector/bisector/xunit2json.py +++ b/tools/bisector/bisector/xunit2json.py @@ -56,27 +56,32 @@ def compare_df(json_df1, json_df2, alpha, alternative='two-sided', non_significa regression_map = collections.defaultdict(dict) for row in merged_df.itertuples(index=True): testcase = row[0] + failure_old = row.counters_old['failure'] + failure_new = row.counters_new['failure'] + passed_old = row.counters_old['passed'] + passed_new = row.counters_new['passed'] + odds_ratio, p_val = scipy.stats.fisher_exact( [ # Ignore errors and skipped tests - [row.failure_old, row.passed_old], - [row.failure_new, row.passed_new], + [failure_old, passed_old], + [failure_new, passed_new], ], alternative = alternative ) # Ignore errors and skipped tests - meaningful_total_old = row.failure_old + row.passed_old - meaningful_total_new = row.failure_new + row.passed_new + meaningful_total_old = failure_old + passed_old + meaningful_total_new = failure_new + passed_new # If no meaningful iterations are available, return NaN if not meaningful_total_old: failure_old_pc = float('Inf') else: - failure_old_pc = 100 * row.failure_old / (row.failure_old + row.passed_old) + failure_old_pc = 100 * failure_old / (failure_old + passed_old) if not meaningful_total_new: failure_new_pc = float('Inf') else: - failure_new_pc = 100 * row.failure_new / (row.failure_new + row.passed_new) + failure_new_pc = 100 * failure_new / (failure_new + passed_new) delta = failure_new_pc - failure_old_pc -- GitLab