diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index b50e4b14b65e9e154dad90f0a793cf8e4d528498..fc16e195c085fee6df0c60096611cc17ecf3ea66 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -229,6 +229,7 @@ port_regex string Regex to use to find the TCP port of the terminal when p type enum-string Terminal type. See below for options. no_color boolean Optional (defaults to false, only applies to ['stdout', 'stdinout'] types): If true, output from this terminal is not color-coded. If this terminal carries the interactive shell, it is advised to set this to true to prevent interferring with the shell's escape sequences. --no-color command line option causes this to behave as if set to true. no_escapes bool/string Optional (defaults to false, only applies to ['stdout', 'stdinout'] types): If true, strips any escape sequences from the output stream before forwarding to the terminal. If a string, behaves as if true until the string is found in the output, which sets it to false. Useful to expunge escape sequences from EDK2 during boot. +logfile string Optional (defaults to none, only applies to ['stdout', 'stdinout'] types): Specifies path to a log file where all output to the terminal will be duplicated. =========== =========== =========== Terminal types: diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index f86d4d604ce4f8b8c9b1acc1e052c0042a11b9a8..eb46e6d496153c99903b68a67dc37a4f5f58c5f0 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -748,16 +748,17 @@ Alternatively, you could have passed ``--dry-run`` to see the FVP invocation scr

Overlays are an important concept for Shrinkwrap. An overlay is a config -fragment (yaml file) that can be passed separately on the command line and forms -the top layer of the config. In this way, it can override or add any required -configuration. You could achive the same effect by creating a new config and -specifying the main config as a layer in that new config, but with an overlay, -you can apply a config fragment to many different existing configs without the -need to write a new config file each time. You can see overlays being using in -the above commands to target a specific Arm architecture revision (v9.3 in the -example). You can change the targetted architecture just by changing the -overlay. There are many other places where overlays come in handy. See -:ref:`userguide/recipes:Shrinkwrap Recipes` for more examples. +fragment (either a yaml file or a json-encoded string) that can be passed +separately on the command line and forms the top layer of the config. In this +way, it can override or add any required configuration. You could achive the +same effect by creating a new config and specifying the main config as a layer +in that new config, but with an overlay, you can apply a config fragment to many +different existing configs without the need to write a new config file each +time. You can see overlays being using in the above commands to target a +specific Arm architecture revision (v9.3 in the example). You can change the +targetted architecture just by changing the overlay. There are many other places +where overlays come in handy. See :ref:`userguide/recipes:Shrinkwrap Recipes` +for more examples. You will notice in the examples above, that only ``build`` commands include the overlay and ``run`` commands don't specify it. This is because the final config diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst index 15dec343ba9a89d8a6b17f1511fac08ecd1ea11b..c093d8edf38eb813e38079f0e5b5123eb96fbf18 100644 --- a/documentation/userguide/recipes.rst +++ b/documentation/userguide/recipes.rst @@ -243,9 +243,35 @@ comand line: shrinkwrap run ns-edk2.yaml --rtvar=KERNEL=path/to/Image --rtvar=CMDLINE="console=ttyAMA0 earlycon=pl011,0x1c090000 root=/dev/vda ip=dhcp acpi=force" -****************************************** -Example Linux Feature Development Use Case -****************************************** +************************************* +Pass Overlay Directly on Command Line +************************************* + +All of the previous examples that utilize overlays, put the overlay in a yaml +file and pass the file name on the command line. Here is an example that runs +the FVP as normal but saves all output from UART0 to uart0.log: + +Create a file called ``my-overlay.yaml``: + +.. code-block:: yaml + + run: + params: + -C bp.pl011_uart0.out_file: uart0.log + +Now run the FVP, passing in the overlay: + +.. code-block:: shell + + shrinkwrap run ns-edk2.yaml --rtvar=KERNEL=path/to/Image --overlay=my-overlay.yaml + +However, it is also possible to pass an overlay, encoded as json, directly on +the command line, without the need for a file. This is useful when the overlay +is small. JSON allows the entire content to be encoded on a single line without +having to care about whitespace, so is suited to this purpose. + +This is equivalent: + +.. code-block:: shell -.. todo:: - Add commentary on the config created to develop FEAT_LPA2. + shrinkwrap run ns-edk2.yaml --rtvar=KERNEL=path/to/Image --overlay='{"run":{"params":{"-C bp.pl011_uart0.out_file":"uart0.log"}}}' diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py index 3357dc82c07e89096269e244e4c1cf6569d744d2..7bc9522a2f3f695264378dcd4596c3e2a61e4d03 100644 --- a/shrinkwrap/commands/run.py +++ b/shrinkwrap/commands/run.py @@ -138,6 +138,13 @@ def dispatch(args): else: return False + def _logfile(terminal): + if terminal['type'] in ['stdinout', 'stdout']: + logfile = terminal.get('logfile') + return logfile if logfile else None + else: + return None + def _find_term_ports(pm, proc, data, streamid): """ Initial handler function called by ProcessManager. When the fvp @@ -179,13 +186,13 @@ def dispatch(args): cmd = f'nc localhost {port}' pm.add(process.Process(cmd, False, - (log.alloc_data(name, colorize, escape), k), + (log.alloc_data(name, colorize, escape, _logfile(t)), k), False)) if type in ['stdinout']: cmd = f'telnet localhost {port}' pm.add(process.Process(cmd, True, - (log.alloc_data(name, colorize, escape), k), + (log.alloc_data(name, colorize, escape, _logfile(t)), k), False)) t['strip'] = True strip = True @@ -199,8 +206,11 @@ def dispatch(args): print(f'To start {name} terminal, run:') print(f' telnet {ip} {port}') if wait: + # Temporarily restore sys.stdin for input(). + pm._stdin_deactivate() print() input("Press Enter to continue...") + pm._stdin_activate() if strip: pm.set_handler(_strip_telnet_header) @@ -208,6 +218,8 @@ def dispatch(args): pm.set_handler(log.log) def _complete(pm, proc, retcode): + log.free_data(proc.data[0]) + # If the FVP exits with non-zero exit code, we propagate that # error so that shrinkwrap also exits with non-zero exit code. if retcode not in [0, None] and proc.run_to_end: @@ -220,6 +232,12 @@ def dispatch(args): for rtvar in resolver['run']['rtvars'].values(): if rtvar['type'] == 'path': rt.add_volume(rtvar['value']) + for t in terminals.values(): + logfile = _logfile(t) + if logfile: + logdir = os.path.abspath(os.path.dirname(logfile)) + os.makedirs(logdir, exist_ok=True) + rt.add_volume(logdir) rt.add_volume(workspace.package) rt.start() diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 46e190762e919c62df2b0afc3c04cb7df8f0ea91..cbc27eee8e9f35890f3c57bd5ad08a125c76d1c9 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -3,6 +3,7 @@ import graphlib import io +import json import os import re import yaml @@ -355,7 +356,7 @@ def filename(name, rel=os.getcwd()): Given a config name, finds the path to the config on disk. If the config name exists relative to rel, we return that since it is a user config. Else, if the config name exists relative to the config store then we - return that. If neither exist, then we return the filepath option, since + return that. If neither exist, then we return name unmodified, since that will generate the most useful error. """ fpath = os.path.abspath(os.path.join(rel, name)) @@ -366,18 +367,25 @@ def filename(name, rel=os.getcwd()): elif cpath: return os.path.abspath(os.path.join(cpath, name)) else: - return fpath + return name def load(file_name, overlays=[], friendly=None): """ Load a config from disk and return it as a dictionary. The config is - fully normalized, validated and merged. + fully normalized, validated and merged. If file_name starts with '{' it + is treated as a json config instead of a file name and is parsed + directly. This allows passing json config snippets as overlays on the + command line. """ def _config_load(file_name): - with open(file_name) as file: - config = yaml.safe_load(file) - config_dir = os.path.dirname(file_name) + if file_name[0] == '{': + config = json.loads(file_name) + config_dir = os.getcwd() + else: + with open(file_name) as file: + config = yaml.safe_load(file) + config_dir = os.path.dirname(file_name) config = _config_normalize(config) _config_validate(config) diff --git a/shrinkwrap/utils/graph.py b/shrinkwrap/utils/graph.py index 3554eb15915c974adbc1093fd4a87046b26884f8..1d0e7c193dc753849e776d23aab3379da0fe3ea8 100644 --- a/shrinkwrap/utils/graph.py +++ b/shrinkwrap/utils/graph.py @@ -100,20 +100,10 @@ def _run_script(pm, data, script): with open(tmpfilename, 'w') as tmpfile: tmpfile.write(script.commands()) - if script.config and script.component: - logname = os.path.join(workspace.build, - 'log', - script.config, - f'{script.component}.log') - os.makedirs(os.path.dirname(logname), exist_ok=True) - logfile = open(logname, 'w', buffering=1) - else: - logfile = None - # Start the process asynchronously. pm.add(process.Process(f'bash {tmpfilename}', False, - (*data, script, tmpdir, logfile), + (*data, script, tmpdir), True)) @@ -132,47 +122,49 @@ def execute(graph, tasks, verbose=False, colorize=True): nonlocal log while len(queue) > 0 and active < tasks: frag = queue.pop() + logname = None + if frag.config and frag.component: + logname = os.path.join(workspace.build, + 'log', + frag.config, + f'{frag.component}.log') + os.makedirs(os.path.dirname(logname), exist_ok=True) _update_labels(labels, mask, frag.config, frag.component, frag.summary + '...') - data = (log.alloc_data(str(frag), colorize), []) + data = (log.alloc_data(str(frag), colorize, False, logname), []) _run_script(pm, data, frag) active += 1 def _should_log(proc, data, streamid): - if streamid == process.STDERR and \ + if verbose or \ + (streamid == process.STDERR and \ (not proc.data[2].stderrfilt or \ - 'warning' in data or 'error' in data): + 'warning' in data or 'error' in data)): return True return False def _log(pm, proc, data, streamid): - logfile = proc.data[4] - if logfile: - logfile.write(data) - if verbose: - log.log(pm, proc, data, streamid) - else: + logstd = _should_log(proc, data, streamid) + if not verbose: proc.data[1].append(data) - if _should_log(proc, data, streamid): - log.log(pm, proc, data, streamid) - lc.skip_overdraw_once() + log.log(pm, proc, data, streamid, logstd) + if logstd: + lc.skip_overdraw_once() def _complete(pm, proc, retcode): nonlocal queue nonlocal active nonlocal ts - data = proc.data[1] + data = proc.data[0] + err = proc.data[1] frag = proc.data[2] tmpdir = proc.data[3] - logfile = proc.data[4] - - if logfile: - logfile.close() + log.free_data(data) shutil.rmtree(tmpdir) if retcode is None: @@ -183,7 +175,7 @@ def execute(graph, tasks, verbose=False, colorize=True): if retcode: if not verbose: print('\n== error start ' + ('=' * 65)) - print(''.join(data)) + print(''.join(err)) print('== error end ' + ('=' * 67) + '\n') raise Exception(f"Failed to execute '{frag}'") diff --git a/shrinkwrap/utils/logger.py b/shrinkwrap/utils/logger.py index 233c4c254a211364c28ad7770ba67486f715b298..0892760e73caeeeeff45f8cd240e7a6974fd0002 100644 --- a/shrinkwrap/utils/logger.py +++ b/shrinkwrap/utils/logger.py @@ -15,7 +15,7 @@ def _import_termcolor(): _ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') _colors = ['blue', 'cyan', 'green', 'yellow', 'magenta'] -Data = namedtuple("Data", "id tag color noesc escbuf") +Data = namedtuple("Data", "id tag color noesc escbuf logfile") class MatchBuf: @@ -49,7 +49,7 @@ class Logger: self._prev_id = None self._prev_char = '\n' - def alloc_data(self, tag, colorize, no_escapes=False): + def alloc_data(self, tag, colorize, no_escapes=False, logname=None): """ Returns the object that should be stashed in proc.data[0] when log() is called. Includes the tag for the process and an @@ -73,9 +73,18 @@ class Logger: noesc = no_escapes escbuf = None - return Data(id, tag, color, [noesc], escbuf) + if logname: + logfile = open(logname, 'w', buffering=1) + else: + logfile = None + + return Data(id, tag, color, [noesc], escbuf, logfile) + + def free_data(self, data): + if data.logfile: + data.logfile.close() - def log(self, pm, proc, data, streamid): + def log(self, pm, proc, data, streamid, logstd=True): """ Logs text data from one of the processes (FVP or one of its uart terminals) to the terminal. Text is colored and a tag is added @@ -86,6 +95,15 @@ class Logger: color = proc.data[0].color noesc = proc.data[0].noesc escbuf = proc.data[0].escbuf + logfile = proc.data[0].logfile + + # Write out to file if requested. + if logfile: + logfile.write(data) + + # Write to stdout if requested. + if not logstd: + return # Remove any ansi escape sequences if requested. if noesc[0]: