diff --git a/EBBR.yaml b/EBBR.yaml index e72ee54d147faa71fa569b3fdf65d11bad117fed..1f4378090096ccac703b4e15f19a2d0a151d47cd 100644 --- a/EBBR.yaml +++ b/EBBR.yaml @@ -1214,6 +1214,42 @@ update: result: IGNORED +- rule: Force false-positive simple network protocol failure as ignored. + A full implementation of the UEFI general purpose networking ABIs is not + required (EBBR Table 2.4 Notable Deviations from UEFI § 2.6.2) + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: 6880BD92-7004-41B8-9E43-7B271FD9AC2B + log: Status - Unsupported + name: EFI_SIMPLE_NETWORK_PROTOCOL.MCastIpToMac - Invoke MCastIpToMac() when + network interface not start. + result: FAILURE + revision: '0x00010001' + set guid: D924B177-7E44-445D-92AA-5E15C0FD2E27 + sub set: MCastIpToMac_Conf + test set: SimpleNetworkProtocolTest + update: + result: IGNORED + +- rule: Force false-positive simple network protocol failure (2) as ignored. + A full implementation of the UEFI general purpose networking ABIs is not + required (EBBR Table 2.4 Notable Deviations from UEFI § 2.6.2) + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: 544B08C0-1D26-4462-9207-DD7EB754DC9E + log: Status - Unsupported, MAC address - 000000000000 + name: EFI_SIMPLE_NETWORK_PROTOCOL.MCastIpToMac - Invoke MCastIpToMac() and + verify interface correctness within test case + result: FAILURE + revision: '0x00010001' + set guid: 3DE1C94C-4CBB-47C4-9407-0C0F20FA66E7 + sub set: MCastIpToMac_Func + test set: SimpleNetworkProtocolTest + update: + result: IGNORED + # We force the following dropped or skipped tests result as `IGNORED' because # they are run only when some platform features are present, which are not # mandated by EBBR. @@ -2339,3 +2375,67 @@ comments: Check logs for messages such as "No EFI system partition" or "Failed to persist EFI variables" and check that system has an EFI System Partition + +- rule: Add comments to failure due to ethernet cable unplugged + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: EA3773EA-0E0F-45A3-82A0-64D485A10B52 + log: Status - Not Ready + name: EFI_SIMPLE_NETWORK_PROTOCOL.Transmit - Invoke Transmit() with + HeaderSize is nonzero and not equal to MediaHeaderSize. + result: FAILURE + revision: '0x00010001' + set guid: 5DDF5C49-7ACA-4724-9553-614A53EE998A + sub set: Transmit_Conf + test set: SimpleNetworkProtocolTest + update: + comments: Check if ethernet cable is unplugged + +- rule: Add comments to failure (2) due to ethernet cable unplugged + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: DE544DE1-178E-4B5F-97D7-19119B1B7B18 + log: Status - Not Ready + name: EFI_SIMPLE_NETWORK_PROTOCOL.Transmit - Invoke Transmit() with + BufferSize is less than MediaHeaderSize. + result: FAILURE + revision: '0x00010001' + set guid: 5DDF5C49-7ACA-4724-9553-614A53EE998A + sub set: Transmit_Conf + test set: SimpleNetworkProtocolTest + update: + comments: Check if ethernet cable is unplugged + +- rule: Add comments to failure (3) due to ethernet cable unplugged + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: A449842C-F5F8-47E9-987B-4B6141AEBD45 + log: Status - Not Ready + name: EFI_SIMPLE_NETWORK_PROTOCOL.Transmit - Invoke Transmit() with + HeaderSize is nonzero and DestAddr is NULL. + result: FAILURE + revision: '0x00010001' + set guid: 5DDF5C49-7ACA-4724-9553-614A53EE998A + sub set: Transmit_Conf + test set: SimpleNetworkProtocolTest + update: + comments: Check if ethernet cable is unplugged + +- rule: Add comments to failure (4) due to ethernet cable unplugged + criteria: + descr: Simple Network Protocol Test + group: NetworkSupportTest + guid: 2E3DD087-DD0C-426E-85BA-65E58310B1DE + log: Status - Not Ready + name: EFI_SIMPLE_NETWORK_PROTOCOL.Transmit - Invoke Transmit() with + HeaderSize is nonzero and Protocol is NULL. + result: FAILURE + revision: '0x00010001' + set guid: 5DDF5C49-7ACA-4724-9553-614A53EE998A + sub set: Transmit_Conf + test set: SimpleNetworkProtocolTest + update: + comments: Check if ethernet cable is unplugged diff --git a/README.md b/README.md index 3fe31154d029a919d7a302426b403836f7ed6732..87630a546732146cbd1de84c27d6f9a600d27658 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,39 @@ It will proceed to generate a Markdown file listing number of failures, passes, [UEFI SCT]: https://uefi.org/testtools +## Dependencies + +You need to install the [PyYAML] module for the configuration file to be loaded +correctly. Depending on your Linux distribution, this might be available as the +`python3-yaml` package. +It is also recommended to install the [packaging] library for smooth version +detection. Depending on your Linux distribution, this might be available as the +`python3-packaging` package. +See [Configuration file]. + +If you want to generate the pdf version of this documentation or convert +markdown results to HTML, you need to install [pandoc]. See [Usage] and +[Documentation]. + +[PyYAML]: https://github.com/yaml/pyyaml +[packaging]: https://github.com/pypa/packaging +[pandoc]: https://pandoc.org + ## Quick Start If you're using this tool to analyze EBBR test results, use the following command. The parsed report can be found in `result.md`. ``` {.sh} -$ ./parser.py --config EBBR.yaml \ - \ +$ ./parser.py \ + \ contrib/v21.07_0.9_BETA/EBBR.seq INFO apply_rules: Updated 200 test(s) out of 12206 after applying 124 rule(s) INFO main: 0 dropped(s), 1 failure(s), 93 ignored(s), 106 known u-boot limitation(s), 12006 pass(s), 0 warning(s) ``` +(The `EBBR.yaml' configuration file is used to process results by default.) + ## Usage Usage to generate a `result.md` is such: @@ -38,12 +58,14 @@ $ ./parser.py --md out.md ... An online help is available with the `-h` option. -The generated `result md` can be easily converted to HTML using `pandoc` with: +The generated `result md` can be easily converted to HTML using [pandoc] with: ``` {.sh} $ pandoc -oresult.html result.md ``` +See [Dependencies]. + ### Custom search For a custom Key:value search, the next two arguments *MUST be included together.* The program will search and display files that met that constraint, without the crosscheck, and display the names, guid, and key:value to the command line. `python3 parser.py ` @@ -165,21 +187,18 @@ $ ./parser.py --input-md 'result.md' ... ## Configuration file -It is possible to use a configuration file with command line option `--config -`. -This configuration file describes operations to perform on the tests results, +By default, the `EBBR.yaml` configuration file is used to process results. It is +intended to help triaging failures when testing specifically for [EBBR] +compliance. It describes operations to perform on the tests results, such as marking tests as false positives or waiving failures. +It is possible to specify another configuration file with the command line +option `--config `. -Example command for [EBBR]: - -``` {.sh} -$ ./parser.py --config EBBR.yaml /path/to/Summary.ekl EBBR.seq ... -``` - -You need to install the [PyYAML] module for this to work. +You need to install the [PyYAML] module for the configuration file to be loaded +correctly, and installing the [packaging] library is recommended. See +[Dependencies]. [EBBR]: https://github.com/ARM-software/ebbr -[PyYAML]: https://github.com/yaml/pyyaml ### Configuration file format @@ -250,8 +269,9 @@ This generated configuration can then be further edited manually. ### EBBR configuration -The `EBBR.yaml` file is a configuration file meant for [EBBR] testing. It can -override the result of some tests with the following ones: +The `EBBR.yaml` file is the configuration file used by default. It is meant for +[EBBR] testing and can override the result of some tests with the following +ones: ------------------------------------------------------------------------------- Result Description @@ -265,13 +285,37 @@ override the result of some tests with the following ones: filesystem implementation limitations and they do not prevent an OS to boot. - `KNOWN ACS LIMITATION` Genuine bugs, which are fixed in a more recent version - of the ACS or which must ultimately be fixed and which - we know about. + `KNOWN ACS LIMITATION` Genuine bugs, which are fixed in a more recent + version of the ACS or which must ultimately be fixed + and which we know about. ------------------------------------------------------------------------------- Some of the rules just add a `comments` field with some help text. +Example command to see those comments: + +``` {.sh} +$ ./parser.py \ + --filter "x['result'] == 'FAILURE'" \ + --fields 'count,result,name,comments' --uniq --print ... +``` + +### Database of sequence files + +The `seq.db` file contains a list of known sequence files, which allows to +identify the input sequence file. + +This database file contains lines describing each known sequence file in turn, +in the following format: + +``` +sha256 description +``` + +Everything appearing after a '#' sign is treated as a comment and ignored. + +The database filename can be specified with the `--seq-db` option. + ## Notes ### Known Issues: * "comment" is currently not implemented, as formatting is not currently consistent, should reflect the comments from the test. @@ -281,8 +325,8 @@ Some of the rules just add a `comments` field with some help text. ### Documentation -It is possible to convert this `README.md` into `README.pdf` with pandoc using -`make doc`. See `make help`. +It is possible to convert this `README.md` into `README.pdf` with [pandoc] using +`make doc`. See `make help` and [Dependencies]. ### Sanity checks @@ -384,7 +428,10 @@ convenience: `v21.05_0.8_BETA-0/` EBBR sequence file from [ACS-IR v21.05_0.8_BETA-0]. `v21.07_0.9_BETA/` EBBR sequence files from [ACS-IR v21.07_0.9_BETA]. + + `v21.09_1.0` EBBR sequence files from [ACS-IR v21.09_1.0]. ------------------------------------------------------------------------------- [ACS-IR v21.05_0.8_BETA-0]: https://github.com/ARM-software/arm-systemready/tree/main/IR/prebuilt_images/v21.05_0.8_BETA-0 [ACS-IR v21.07_0.9_BETA]: https://github.com/ARM-software/arm-systemready/tree/main/IR/prebuilt_images/v21.07_0.9_BETA +[ACS-IR v21.09_1.0]: https://github.com/ARM-software/arm-systemready/tree/main/IR/prebuilt_images/v21.09_1.0 diff --git a/contrib/v21.09_1.0/EBBR.seq b/contrib/v21.09_1.0/EBBR.seq new file mode 100755 index 0000000000000000000000000000000000000000..9b531d9d9e494944a065958c578a1eea9cbaebf2 Binary files /dev/null and b/contrib/v21.09_1.0/EBBR.seq differ diff --git a/contrib/v21.09_1.0/EBBR_manual.seq b/contrib/v21.09_1.0/EBBR_manual.seq new file mode 100755 index 0000000000000000000000000000000000000000..243314f8cfc0183f6d1cdec4e6ae232f0b9a1518 Binary files /dev/null and b/contrib/v21.09_1.0/EBBR_manual.seq differ diff --git a/parser.py b/parser.py index dc2da208ca57f36c6b14e7381d7d516dbf843744..b1b153e3691acc8137beee7f673805b853bd2621 100755 --- a/parser.py +++ b/parser.py @@ -8,16 +8,20 @@ import csv import logging import json import re +import hashlib +import os try: from packaging import version except ImportError: - print('No packaging...') + print('No packaging. You should install python3-packaging...') try: import yaml except ImportError: - print('No yaml...') + print( + 'No yaml. You should install PyYAML/python3-yaml for configuration' + ' file support...') if 'yaml' in sys.modules: try: @@ -504,26 +508,29 @@ def gen_template(cross_check, filename): def do_print(cross_check, fields): logging.debug(f'Print (fields: {fields})') - # First pass to find the width for each field except the last one - fm1 = fields[:len(fields) - 1] + # First pass to find the width for each field w = {} - for f in fm1: + for f in fields: w[f] = len(f) for x in cross_check: - for f in fm1: + for f in fields: w[f] = max(w[f], len(str(x[f]) if f in x else '')) # Second pass where we print + fm1 = fields[:len(fields) - 1] lf = fields[len(fields) - 1] + sep = ' ' + + print(sep.join([ + *map(lambda f: f"{f.capitalize():{w[f]}}", fm1), + lf.capitalize()])) - print(' '.join([ - *map(lambda f: f"{f:{w[f]}}", fm1), - lf])) + print(sep.join([*map(lambda f: '-' * w[f], fields)])) for x in cross_check: - print(' '.join([ + print(sep.join([ *map(lambda f: f"{x[f] if f in x else '':{w[f]}}", fm1), x[lf] if lf in x else ''])) @@ -597,6 +604,50 @@ def combine_dbs(db1, db2): return cross_check +# Load the database of known sequence files. +def load_known_seq(seq_db): + known_seqs = {} + + with open(seq_db, 'r') as f: + for line in f: + line = line.rstrip() + line = re.sub(r'#.*', '', line) + m = re.match(r'\s*([0-9a-fA-F]+)\s+(.*)', line) + + if not m: + continue + + kh = m.group(1) + d = m.group(2) + assert(kh not in known_seqs) + logging.debug(f'{kh} {d}') + known_seqs[kh] = d + + logging.debug(f'{len(known_seqs)} known seq file(s)') + return known_seqs + + +# Try to identify the .seq file in a list of known versions using its sha256. +def ident_seq(seq_file, seq_db): + known_seqs = load_known_seq(seq_db) + + # Hash seq file + hm = 'sha256' + hl = hashlib.new(hm) + + with open(seq_file, 'rb') as f: + hl.update(f.read()) + + h = hl.hexdigest() + logging.debug(f'{hm} {h} {seq_file}') + + # Try to identify the seq file + if h in known_seqs: + logging.info(f"""Identified `{seq_file}' as "{known_seqs[h]}".""") + else: + logging.debug(f"Could not identify `{seq_file}'...") + + # Read the .ekl log file and the .seq file and combine them into a single # database, which we return. def read_log_and_seq(log_file, seq_file): @@ -714,6 +765,8 @@ def read_md(input_md): if __name__ == '__main__': + me = os.path.realpath(__file__) + here = os.path.dirname(me) parser = argparse.ArgumentParser( description='Process SCT results.' ' This program takes the SCT summary and sequence files,' @@ -740,6 +793,9 @@ if __name__ == '__main__': parser.add_argument( '--print', action='store_true', help='Print results to stdout') parser.add_argument('--input-md', help='Input .md filename') + parser.add_argument( + '--seq-db', help='Known sequence files database filename', + default=f'{here}/seq.db') parser.add_argument('log_file', help='Input .ekl filename') parser.add_argument('seq_file', help='Input .seq filename') parser.add_argument('find_key', nargs='?', help='Search key') @@ -749,7 +805,8 @@ if __name__ == '__main__': # could actually import yaml. if 'yaml' in sys.modules: parser.add_argument( - '--config', help='Input .yaml configuration filename') + '--config', help='Input .yaml configuration filename', + default=f'{here}/EBBR.yaml') parser.add_argument('--yaml', help='Output .yaml filename') parser.add_argument( '--template', help='Output .yaml config template filename') @@ -765,6 +822,10 @@ if __name__ == '__main__': else: # Command line argument 1 is the ekl file to open. # Command line argument 2 is the seq file to open. + + # Try to identify the sequence file + ident_seq(args.seq_file, args.seq_db) + # Read both and combine them into a single cross_check database. cross_check = read_log_and_seq(args.log_file, args.seq_file) diff --git a/seq.db b/seq.db new file mode 100644 index 0000000000000000000000000000000000000000..6c072cca7c14c922e9ed65558ef28900873c9ba8 --- /dev/null +++ b/seq.db @@ -0,0 +1,8 @@ +# List of known sequence files and their sha256 +6a381192057c511b2b69282c58d6107c1daeaf0b95038605d4c58383eb5cc88b Test sample.seq +6ae5a92f0b83b3a1469c89f6a02b0771ae3818a424e7118a28bba053be394078 Supplied SBBR.seq +6b83dbfbd1f07fc61a918297f02f449591a72131b64ac746f969a4210f97aee8 ACS-IR v21.05_0.8_BETA-0 EBBR.seq +c06684b3f8b35871e37b9447f609f9aab6070a7ca1c4ba63a52e029c018c9b73 ACS-IR v21.07_0.9_BETA EBBR.seq +d66485b5e436409ef8c0667baf5250e784cbf292f2b9ef1b3893d474a0585fae ACS-IR v21.09_1.0 EBBR.seq +ec730fd81eada415278b39533fe7cc21147b39183447dc11fa77ad4419d13969 Supplied EBBR.seq +f7793d53c10106c1c275a4992e1710ce9863e210dd07581a3d783c4f4cf2312b ACS-IR v21.07_0.9_BETA or v21.09_1.0 EBBR_manual.seq