From 54c923f716d5f06e4b75fbb7507bd36e932720c6 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Fri, 15 Mar 2024 12:19:49 +0000 Subject: [PATCH 1/4] build, clean: UI improvements to keep progress at bottom When doing a build or clean in interactive mode without --verbose, two categories of information are output to the terminal; progress of all the tasks and warnings/errors from the tasks. Perviously if any warning was output, this would be under the progress block and a new progress block would be drawn under that, the next time progress was updated. This is a bit ugly. So let's change this so that the progress block is always at the bottom and all warnings are output immediately before the progress block begins. And we draw a dashed separator between the two areas. This is much tidier. The implementation is not perfect; if outputting a warning, we erase the progress block, rewind the cursor, output the warning, then redraw the progress block. The erasing step means we get some flicker. This is not noticable when only a few warnings are output, which is the common case. The alternative is to figure out exactly which cells will be covered by the new output and only erase the uncovered cells. But this is much more complex and requires extra coupling between the logger and the label controller, which I want to avoid. Signed-off-by: Ryan Roberts --- shrinkwrap/utils/graph.py | 4 +- shrinkwrap/utils/label.py | 127 +++++++++++++++++++++++++++++--------- 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/shrinkwrap/utils/graph.py b/shrinkwrap/utils/graph.py index 1d0e7c1..7ffee42 100644 --- a/shrinkwrap/utils/graph.py +++ b/shrinkwrap/utils/graph.py @@ -150,9 +150,11 @@ def execute(graph, tasks, verbose=False, colorize=True): logstd = _should_log(proc, data, streamid) if not verbose: proc.data[1].append(data) + if logstd: + lc.erase() log.log(pm, proc, data, streamid, logstd) if logstd: - lc.skip_overdraw_once() + lc.update() def _complete(pm, proc, retcode): nonlocal queue diff --git a/shrinkwrap/utils/label.py b/shrinkwrap/utils/label.py index 84b3afa..aecd9dd 100644 --- a/shrinkwrap/utils/label.py +++ b/shrinkwrap/utils/label.py @@ -2,7 +2,9 @@ # SPDX-License-Identifier: MIT import os +import re import sys +import shrinkwrap.utils.tty as tty class Label: @@ -34,53 +36,118 @@ class LabelController: the terminal while the labels are active. If the provided file is not backed by a terminal, overdrawing is not performed. """ - def __init__(self, labels=[], file=sys.stdout, overdraw=True): + def __init__(self, labels=[], overdraw=True): self._labels = labels - self._file = file - self._overdraw = False - self._overdraw_usually = overdraw + self._overdraw = overdraw + self._drawn = False + self._cursor_col = 1 + self._separator = '' self._update_pending = True - try: - term_sz = os.get_terminal_size(file.fileno()) - self._term_lines = term_sz.lines - self._term_cols = term_sz.columns - except OSError: - self._overdraw_usually = False + + self._out = sys.stdout + self._in = sys.stdin + + if overdraw and not self._term_cols(): + self._overdraw = False for label in self._labels: label._lc = self - def _move_up(self, line_count): + def _term_cols(self): + try: + term_sz = os.get_terminal_size(self._out.fileno()) + return term_sz.columns + except OSError: + return None + + def cursor_pos(self): + orig = tty.configure(self._in) + if orig: + try: + pos = '' + self._out.write('\x1b[6n') + self._out.flush() + while not (pos := pos + self._in.read(1)).endswith('R'): + pass + res = re.match(r'.*\[(?P\d*);(?P\d*)R', pos) + finally: + tty.restore(self._in, orig) + if(res): + return (int(res.group("x")), int(res.group("y"))) + assert(False) + + def _move_up(self, line_count, column=1): assert(self._overdraw) - self._file.write('\033[F' * line_count) + self._out.write(f'\033[{line_count}F') + if column > 1: + self._out.write(f'\033[{column}G') - def _line_count(self, text): + def _line_count(self, text, term_cols): assert(self._overdraw) - return (len(text) + self._term_cols - 1) // self._term_cols + return (len(text) + term_cols - 1) // term_cols def update(self): if not self._update_pending: return - if self._overdraw: - line_count = 0 + term_cols = self._term_cols() + if not self._drawn: + self._cursor_col = self.cursor_pos()[0] + if self._cursor_col > 1: + self._out.write('\n') + self._separator = '-' * term_cols + self._out.write(self._separator) + self._out.write('\n') + for l in self._labels: + self._out.write(l.text) + self._out.write('\n') + l._prev_text = l.text + else: + erase_lines = 0 + for l in self._labels: + erase_lines += self._line_count(l._prev_text, term_cols) + self._move_up(erase_lines) + for l in self._labels: + cc = len(l.text) + pcc = len(l._prev_text) + self._out.write(l.text) + self._out.write(' ' * (pcc - cc)) + self._out.write('\n') + l._prev_text = l.text + else: for l in self._labels: - line_count += self._line_count(l.text) - self._move_up(line_count) + if l.text != l._prev_text: + self._out.write(l.text) + self._out.write('\n') + l._prev_text = l.text + self._out.flush() + self._drawn = True + self._update_pending = False + + def erase(self): + if not self._overdraw or not self._drawn: + return + term_cols = self._term_cols() + + erase_lines = self._line_count(self._separator, term_cols) for l in self._labels: - if self._overdraw_usually or l.text != l._prev_text: - cc = len(l.text) - pcc = len(l._prev_text) - self._file.write(l.text) - self._file.write(' ' * (pcc - cc)) - self._file.write('\n') - l._prev_text = l.text + erase_lines += self._line_count(l._prev_text, term_cols) - self._file.flush() + self._move_up(erase_lines) - self._overdraw = self._overdraw_usually - self._update_pending = False + self._out.write(' ' * len(self._separator)) + self._out.write('\n') - def skip_overdraw_once(self): - self._overdraw = False + for l in self._labels: + pcc = len(l._prev_text) + self._out.write(' ' * pcc) + self._out.write('\n') + + if self._cursor_col > 1: + erase_lines += 1 + self._move_up(erase_lines, self._cursor_col) + + self._out.flush() + self._drawn = False + self._update_pending = True -- GitLab From 7be0dacf20425c272a15ab3b23548147c2ea6623 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 18 Mar 2024 10:00:33 +0000 Subject: [PATCH 2/4] build: Split per-component artifact copy step Previously there was a single, global artifact copy step at the end of a build process, which copied all artifacts to the package. The step dependend on all component build steps to be complete. This meant that every component would display "waiting..." in the UI until all other components were built, then all would display the "copying artifacts..." message before all changing to "done". Let's split the artifact copying step so that it's per-component and therefore only needs to depend on the specific component. Then components can run to completion and display "done" when they are done rather than wait for the others. Signed-off-by: Ryan Roberts --- shrinkwrap/utils/config.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 211671e..8c7d95a 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -496,10 +496,11 @@ def resolveb(config, btvars={}, clivars={}): def _combine_full(config): artifact_map = {} - for desc in config['build'].values(): + for name, desc in config['build'].items(): locs = {key: { 'src': val, 'dst': os.path.join(config['name'], os.path.basename(val)), + 'component': name, } for key, val in desc['artifacts'].items()} artifact_map.update(locs) return artifact_map @@ -944,15 +945,16 @@ def build_graph(configs, echo, nosync): build_scripts[name] = b ts.done(name) - a = Script('Copying artifacts', config["name"], preamble=pre, final=True) - if len(config['artifacts']) > 0: - a.append(f'# Copy artifacts for config={config["name"]}.') - for artifact in config['artifacts'].values(): - src = artifact['src'] - dst = os.path.join(workspace.package, artifact['dst']) - a.append(f'cp -r {src} {dst}') - a.seal() - graph[a] = [gl2] + [s for s in build_scripts.values()] + a = Script('Copying artifacts', config["name"], name, preamble=pre, final=True) + artifacts = {k: v for k, v in config['artifacts'].items() if v['component'] == name} + if len(artifacts) > 0: + a.append(f'# Copy artifacts for config={config["name"]} component={name}.') + for artifact in artifacts.values(): + src = artifact['src'] + dst = os.path.join(workspace.package, artifact['dst']) + a.append(f'cp -r {src} {dst}') + a.seal() + graph[a] = [b] return graph @@ -983,7 +985,7 @@ def clean_graph(configs, echo): for name in ts.get_ready(): component = config['build'][name] - c = Script('Cleaning', config["name"], name, preamble=pre) + c = Script('Cleaning', config["name"], name, preamble=pre, final=True) c.append(f'# Clean for config={config["name"]} component={name}.') c.append(f'rm -rf {component["builddir"]} > /dev/null 2>&1 || true') if len(component['repo']) > 0: -- GitLab From e0de609b8d40f6bae4b176dd92143347323aa58e Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 18 Mar 2024 11:24:52 +0000 Subject: [PATCH 3/4] build, clean: Improve progress tag readability Reduce the config field size and increase the component field size. In most cases, this allows everything to fit without truncating. When things do need to be truncated, truncate the config since most users are only ever building a single config at a time. Signed-off-by: Ryan Roberts --- shrinkwrap/utils/graph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shrinkwrap/utils/graph.py b/shrinkwrap/utils/graph.py index 7ffee42..5ecff5e 100644 --- a/shrinkwrap/utils/graph.py +++ b/shrinkwrap/utils/graph.py @@ -65,9 +65,9 @@ def _mk_tag(config, component): config = '' if config is None else config component = '' if component is None else component - config = _clamp(config, 16) - component = _clamp(component, 8) - return '[ {:>16} : {:8} ]'.format(config, component) + config = _clamp(config, 10) + component = _clamp(component, 14) + return '[ {:>10} : {:14} ]'.format(config, component) def _update_labels(labels, mask, config, component, summary): -- GitLab From e07873fbb31189a2b0998468f6d21339f41afb73 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 18 Mar 2024 12:14:45 +0000 Subject: [PATCH 4/4] docs: Clarify comments in runtimes.rst Make it clear that a "-local" runtime is required when using a locally built container image. And remove out-of-date warning; aarch64 container images do now contain an FVP. Signed-off-by: Ryan Roberts --- documentation/userguide/runtimes.rst | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/documentation/userguide/runtimes.rst b/documentation/userguide/runtimes.rst index 86b0afa..6c18a37 100644 --- a/documentation/userguide/runtimes.rst +++ b/documentation/userguide/runtimes.rst @@ -47,14 +47,6 @@ platform. Images are automatically downloaded by shrinkwrap when the ``docker`` or ``podman`` runtime is selected. Images are available on Docker Hub and can be freely downloaded without the need for an account. -.. warning:: - - There is currently no FVP available for aarch64, so the current aarch64 arch - images do not include any FVP, and as a consequence, the ``run`` command will - not work. For the time being, when wanting to run on aarch64 you must install - your own FVP on your system and follow the recipe at - :ref:`userguide/recipes:Use a Custom FVP Version`. - =============================================== ==== image name description =============================================== ==== @@ -96,10 +88,14 @@ system: .. code-block:: shell - shrinkwrap --runtime= --image=shrinkwraptool/base-slim:local-x86_64 ... + shrinkwrap --runtime=-local --image=shrinkwraptool/base-slim:local-x86_64 ... Or like this if running on an aarch64 system: .. code-block:: shell - shrinkwrap --runtime= --image=shrinkwraptool/base-slim:local-aarch64 ... + shrinkwrap --runtime=-local --image=shrinkwraptool/base-slim:local-aarch64 ... + +where is either docker or podman. Note that because the image is not on +Docker Hub, the -local runtime is required to prevent Shrinkwrap from +erroneously trying to download an update. -- GitLab