diff --git a/.gitignore b/.gitignore index f5ab617ad5ed2ca6b7a0b112f12891b2f3774270..ed594580adf99a4c92b3d05ec94472c8541fb300 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ server.url /vagrant /tools/wa_user_directory/dependencies /src/buildroot +_build diff --git a/doc/bisector b/doc/bisector new file mode 120000 index 0000000000000000000000000000000000000000..463efb3266f799e25c0809159e53370db78383b9 --- /dev/null +++ b/doc/bisector @@ -0,0 +1 @@ +../tools/bisector/doc \ No newline at end of file diff --git a/doc/bisector.rst b/doc/bisector.rst index ca82e213be5b3af2354fcec1542239abf181879a..4e38d2cb87b9081fd71c093edcff0125b7621feb 100644 --- a/doc/bisector.rst +++ b/doc/bisector.rst @@ -1,7 +1,14 @@ + +.. This shim allows including the tool's documentation as one entity in LISA + documentation. A plain ".. include::" would break relative paths. + +.. _bisector-doc: + ******** Bisector ******** -.. TODO:: write me +.. toctree:: + :maxdepth: 2 -**(This page is being worked on, come back later!)** + bisector/index diff --git a/doc/conf.py b/doc/conf.py index a6528259efbfd6e5f175dde43c13b5f407479df0..8cd3eaed30bfed047d810db489c54f8fc3ddff09 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -12,14 +12,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import itertools import logging import os import re import subprocess import sys -import inspect -import tempfile import unittest from docutils import nodes @@ -33,6 +30,9 @@ sys.path.insert(0, os.path.abspath('../')) # Import our packages after modifying sys.path import lisa +from lisa.doc.helpers import ( + autodoc_process_test_method, autodoc_process_analysis_events +) # This ugly hack is required because by default TestCase.__module__ is # equal to 'case', so sphinx replaces all of our TestCase uses to @@ -356,111 +356,8 @@ autodoc_default_options = { } autodoc_inherit_docstrings = True -def is_test(method): - if not callable(method): - return False - - if method.__name__.startswith('test_'): - return True - - # Tests are methods with an annotated return type, with at least - # one base class with a name containing 'result' - try: - ret_type = method.__annotations__['return'] - base_cls_list = inspect.getmro(ret_type) - except (AttributeError, KeyError): - return False - else: - return any( - 'result' in cls.__qualname__.lower() - for cls in base_cls_list - ) - -def autodoc_process_test_method(app, what, name, obj, options, lines): - # Append the list of available test methods for all classes that appear to - # have some. - if what == 'class': - test_list = [ - member - for member_name, member in inspect.getmembers(obj, is_test) - ] - if test_list: - test_list_doc = '\n:Test methods:\n\n{}\n\n'.format('\n'.join( - ' * :meth:`~{}`'.format( - method.__module__ + '.' + method.__qualname__ - ) - for method in test_list - )) - - lines.extend(test_list_doc.splitlines()) - -def autodoc_process_analysis_events(app, what, name, obj, options, lines): - # Append the list of required trace events - if what != 'method' or not hasattr(obj, "used_events"): - return - - events_doc = "\n:Required trace events:\n\n{}\n\n".format(obj.used_events.doc_str()) - lines.extend(events_doc.splitlines()) - def setup(app): app.connect('autodoc-process-docstring', autodoc_process_test_method) app.connect('autodoc-process-docstring', autodoc_process_analysis_events) - -def get_test_id_list(): - import lisa.tests as test_package - rst_list = [] - for path in test_package.__path__: - rst_list.extend(subprocess.check_output(( - 'exekall', 'run', path, - '--rst-list', '--inject-empty-target-conf', - ), stderr=subprocess.STDOUT).decode('utf-8').splitlines() - ) - return rst_list - -def get_analysis_list(prefix): - from lisa.analysis.base import TraceAnalysisBase - from lisa.utils import get_subclasses - - rst_list = [] - - for subclass in get_subclasses(TraceAnalysisBase): - class_path = "{}.{}".format(subclass.__module__, subclass.__qualname__) - analysis = subclass.__module__.split(".")[-1] - meth_list = [name for name, member in inspect.getmembers(subclass, callable) - if name.startswith(prefix)] - - rst_list += [":class:`{0}<{1}>`::meth:`~{1}.{2}`".format(analysis, class_path, meth) - for meth in meth_list] - - return rst_list - -def get_analysis_df_list(): - return get_analysis_list("df_") - -def get_analysis_plot_list(): - return get_analysis_list("plot_") - -def create_doc_list_file(path, list_generator): - content = '\n'.join( - '* {}'.format(item) - for item in list_generator() - ) - - with open(path, 'wt') as f: - f.write(content + '\n') - -def create_test_list_file(path): - try: - create_doc_list_file(path, get_test_id_list) - except FileNotFoundError: - content = 'Please install exekall in order to generate the list of tests.' - print('WARNING: could not generate the list of test without exekall', file=sys.stderr) - with open(path, 'wt') as f: - f.write(content + '\n') - -create_test_list_file('test_list.rst') -create_doc_list_file('analysis_df_list.rst', get_analysis_df_list) -create_doc_list_file('analysis_plot_list.rst', get_analysis_plot_list) - # vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab: diff --git a/doc/exekall b/doc/exekall new file mode 120000 index 0000000000000000000000000000000000000000..aa5ea274a902b52bf7d8ece77cdf7bc28d34d75b --- /dev/null +++ b/doc/exekall @@ -0,0 +1 @@ +../tools/exekall/doc/ \ No newline at end of file diff --git a/doc/exekall.rst b/doc/exekall.rst index 05055200a01ca3f2738079e2cb2aff10c29dc98e..92d365e40c60ee712f3257d99ce91f33816fb0ed 100644 --- a/doc/exekall.rst +++ b/doc/exekall.rst @@ -1,7 +1,14 @@ + +.. This shim allows including the tool's documentation as one entity in LISA + documentation. A plain ".. include::" would break relative paths. + +.. _exekall-doc: + ******* Exekall ******* -.. TODO:: write me +.. toctree:: + :maxdepth: 2 -**(This page is being worked on, come back later!)** + exekall/index diff --git a/doc/index.rst b/doc/index.rst index 90b7356148256e258c3416c64cdaab2232e9f88c..8a1e71188f392b1e9a5a30b92ef7f261f9978644 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -25,8 +25,6 @@ via Github pull requests. Contents: -.. TODO: Move wiki to here, wirte a proper module doc, proove Riemann's hypothesis - .. toctree:: :maxdepth: 2 diff --git a/doc/kernel_tests.rst b/doc/kernel_tests.rst index c7ed538c12ec9975a666b46bd40892f0c4c91a28..c5f659ff32d2dc1cedd09a62c7448e27c5b76bfb 100644 --- a/doc/kernel_tests.rst +++ b/doc/kernel_tests.rst @@ -35,7 +35,10 @@ The following tests are available. They can be used as: * the individual classes/methods they are composed of can be used in custom scripts/jupyter notebooks (see ipynb/tests/synthetics_example.ipynb) -.. include:: test_list.rst +.. run-command:: + :capture-stderr: + + exekall run $LISA_HOME/lisa/tests --rst-list --inject-empty-target-conf Running tests ============= diff --git a/doc/man1/bisector.1 b/doc/man1/bisector.1 new file mode 100644 index 0000000000000000000000000000000000000000..151ec61833e10c9d46fbf32b2f95d3c394f59617 --- /dev/null +++ b/doc/man1/bisector.1 @@ -0,0 +1,1158 @@ +.\" Man page generated from reStructuredText. +. +.TH "BISECTOR" "1" "2019" "" "bisector" +.SH NAME +bisector \- command sequencer +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH DESCRIPTION +.sp +\fBbisector\fP is a \fBgit bisect run\fP [1] compatible tool used in LISA. Its goal is +to sequence commands to be repeated an arbitrary number of times and generates +a report. +.sp +For example, \fBbisector\fP can be used to compile a kernel, flash a board, +reboot it and run test suite. To achieve that, \fBbisector\fP runs a sequence of +steps, and saves a report file containing the result of each step. This step +sequence is run for a number of iterations, or for a given duration. That +allows tracking fluctuating bugs by repeating a test process many times. +.sp +The steps class can implement any run or report behavior, with the ability to +take parameters. They also decide what data is saved into the report, and +their contribution to the overall \fBgit bisect\fP result: +.INDENT 0.0 +.IP \(bu 2 +\fIgood\fP: the test steps passed +.IP \(bu 2 +\fIbad\fP: the test steps failed +.IP \(bu 2 +\fIuntestable\fP: the build or flash step failed +.IP \(bu 2 +\fIabort\fP: the reboot step failed, and the board is now unusable and requires +manual intervention +.IP \(bu 2 +\fINA\fP: the step will not impact the overall result +.UNINDENT +.sp +It is important to note that \fBbisector\fP step classes usually only invoke a +user\-specified command line tool and will use the return code to decide what +\fBgit bisect\fP result to return. They can also implement more specific result +reporting, and additional run behaviors. +.SH OPTIONS +.SS bisector +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +usage: bisector [\-h] [\-\-cli\-options CLI_OPTIONS] + {step\-help,run,report,edit,monitor\-server,monitor} ... + + Git\-bisect\-compatible command sequencer. + + bisector allows to run commands with timeout handling in predefined + sequences (steps). The output is saved in a report that can be later + analyzed. The exit status of all steps is merged to obtain a git bisect + compatible value which is returned. The error code 254 is used + when bisector itself encounters an error. + + SIGNAL HANDLING + + The script can be stopped with SIGINT (Ctrl\-C) or SIGTERM (kill or timeout + commands) or SIGHUP (when the terminal dies) and will save the completed + iterations to the report. The execution can later be resumed with \-\-resume. + + +optional arguments: + \-h, \-\-help show this help message and exit + \-\-cli\-options CLI_OPTIONS + YAML file containing command line option string in a + "cmd\-line" toplevel key. Therefore, the same file can be used + for \-\-steps. The options are inserted at the location of + \-\-cli\-options in the command line. That can be used to control what + can be overriden by configuration file and what is forced by the + command line + +subcommands: + {step\-help,run,report,edit,monitor\-server,monitor} + +.ft P +.fi +.UNINDENT +.UNINDENT +.SS bisector run +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +usage: bisector run [\-h] [\-\-cli\-options CLI_OPTIONS] [\-o OPTION] [\-\-debug] + [\-\-steps STEPS] [\-\-allowed\-bad ALLOWED_BAD] [\-\-skip SKIP] + [\-\-only ONLY] [\-\-git\-clean] [\-\-inline CLASS NAME] + [\-n ITERATIONS] [\-\-timeout TIMEOUT] [\-\-log LOG] + [\-\-report REPORT] [\-\-overwrite] [\-\-resume] [\-\-desc DESC] + [\-\-early\-bailout] [\-\-upload\-report] [\-\-no\-dbus] + +Run the given steps in a loop and record the result in a report file. The +report file can be inspected using the "report" subcommand. The exit status is +suitable for git bisect. + +optional arguments: + \-h, \-\-help show this help message and exit + \-\-cli\-options CLI_OPTIONS + YAML file containing command line option string in a + "cmd\-line" toplevel key. Therefore, the same file can + be used for \-\-steps. The options are inserted at the + location of \-\-cli\-options in the command line. That + can be used to control what can be overriden by + configuration file and what is forced by the command + line + \-o OPTION, \-\-option OPTION + Step\-specific options. Can be repeated. The format is + .