diff --git a/config/qemu/cca.yaml b/config/qemu/cca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f61130e2cf420371ca83e19941fa4b399f35b694 --- /dev/null +++ b/config/qemu/cca.yaml @@ -0,0 +1,91 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Boot the cca-3world stack with QEMU. Use this as overlay with the cca-3world config. + +layers: + - qemu/host-base.yaml + - qemu/edk2-base.yaml + - qemu/tfa-base.yaml + - qemu/rmm-base.yaml + # After host-base.yaml, because it overrides the runner name + - qemu/host-build.yaml + +buildex: + runners: + QEMU: + tfa: + params: + ENABLE_RME: 1 + RMM: ${artifact:RMM} + RME_GPT_BITLOCK_BLOCK: 1 + + rmm: + repo: + remote: https://git.codelinaro.org/linaro/dcap/rmm + revision: cca/v8 + + edk2: + params: + -D: ENABLE_RME + + prebuild: + - export PLATFORM_NAME=SbsaQemuRme + + artifacts: + EDK2_FLASH0: ${param:sourcedir}/Build/SbsaQemuRme/RELEASE_GCC5/FV/SBSA_FLASH0.fd + EDK2_FLASH1: ${param:sourcedir}/Build/SbsaQemuRme/RELEASE_GCC5/FV/SBSA_FLASH1.fd + +run: + runners: + QEMU: + rtvars: + KERNEL: + type: path + value: ${artifact:KERNEL} + + ROOTFS: + type: path + value: ${artifact:BUILDROOT} + + SHARE: + type: path + value: ${param:packagedir} + + CMDLINE: + type: string + value: root=/dev/vda console=ttyAMA0 + + params: + - -nographic + - -nodefaults + - -M sbsa-ref + - -cpu max,x-rme=on,sme=off,pauth-impdef=on + - -drive format=raw,id=hd0,if=none,file='${rtvar:ROOTFS}' + - -device virtio-blk-pci,drive=hd0 + - -chardev socket,id=chr0,port=13557,to=13657,host=0.0.0.0,server=on,wait=on,telnet=on,mux=on + - -chardev socket,id=chr1,port=13558,to=13658,host=0.0.0.0,server=on,wait=on,telnet=on + - -serial chardev:chr0 + - -serial chardev:chr1 + - -mon chr0 + - -device virtio-9p-pci,fsdev=shr0,mount_tag=shr0 + - -fsdev local,security_model=none,path='${rtvar:SHARE}',id=shr0 + - -device virtio-net-pci,netdev=net0 + - -netdev user,id=net0 + + terminals: + # This one also provides the QEMU monitor, accessible with c + serial0: + friendly: '' + type: stdinout + no_color: true + port_regex: '-chardev socket,id=chr0.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on' + no_escapes: 'EFI stub: Booting Linux Kernel...' + + serial1: + friendly: 'Secure' + type: stdout + port_regex: '-chardev socket,id=chr1.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on' diff --git a/config/qemu/edk2-base.yaml b/config/qemu/edk2-base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32cd09a9d0802ace3b0e1333c730a089e6a0a898 --- /dev/null +++ b/config/qemu/edk2-base.yaml @@ -0,0 +1,85 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + EDK2 UEFI firmware implementation for QEMU SBSA. This depend on + Trusted-Firmware-A, and produces the flash images passed to QEMU. + +layers: + - ../edk2-base.yaml + +buildex: + runners: + QEMU: + edk2: + repo: + edk2: + remote: https://github.com/tianocore/edk2.git + revision: 336e7e06eb91fee6f87b2559772aed948fb7bbfe + edk2-platforms: + remote: https://github.com/tianocore/edk2-platforms.git + revision: 400b3e083d5353292def78ea52663c5717e679a2 + edk2-non-osi: + remote: https://github.com/tianocore/edk2-non-osi.git + revision: 45e337daa24b5595be4b81c9f6d379bb105dcc17 + + toolchain: aarch64-none-elf- + + stderrfilt: true + + prebuild: + - export WORKSPACE=${param:sourcedir} + - export GCC5_AARCH64_PREFIX=$$CROSS_COMPILE + - export PACKAGES_PATH=$$WORKSPACE/edk2:$$WORKSPACE/edk2-platforms:$$WORKSPACE/edk2-non-osi + - export IASL_PREFIX=${artifact:ACPICA}/ + - export PYTHON_COMMAND=/usr/bin/python3 + - export PLATFORM_NAME=SbsaQemu + + - cp ${artifact:BL1} ${param:sourcedir}/edk2-non-osi/Platform/Qemu/Sbsa/ + - cp ${artifact:FIP} ${param:sourcedir}/edk2-non-osi/Platform/Qemu/Sbsa/ + + params: + -a: AARCH64 + -t: GCC5 + -b: RELEASE + -p: edk2-platforms/Platform/Qemu/SbsaQemu/SbsaQemu.dsc + --pcd: PcdUefiShellDefaultBootEnable=1 + ' --pcd': PcdShellDefaultDelay=0 + + build: + - source edk2/edksetup.sh --reconfig + - make -j${param:jobs} -C edk2/BaseTools + - build -n ${param:jobs} ${param:join_space} + - truncate -s 256M ${param:sourcedir}/Build/$$PLATFORM_NAME/RELEASE_GCC5/FV/SBSA_FLASH0.fd + - truncate -s 256M ${param:sourcedir}/Build/$$PLATFORM_NAME/RELEASE_GCC5/FV/SBSA_FLASH1.fd + + artifacts: + EDK2_FLASH0: ${param:sourcedir}/Build/SbsaQemu/RELEASE_GCC5/FV/SBSA_FLASH0.fd + EDK2_FLASH1: ${param:sourcedir}/Build/SbsaQemu/RELEASE_GCC5/FV/SBSA_FLASH1.fd + +run: + runners: + QEMU: + prerun: + # Create the virtual boot disk containing the EFI shell. Wrap this up + # as a command in the startup.nsh along with the command line. UEFI + # will execute this when entering its shell. Using a unique temp + # directory means we can run multiple instances in parallel. + - BOOT_DISK=`mktemp -d` + - function finish { rm -rf $$BOOT_DISK; } + - trap finish EXIT + - mkdir -p $$BOOT_DISK/ + - cp ${rtvar:KERNEL} $$BOOT_DISK/ + - cat << EOF > $$BOOT_DISK/startup.nsh + - mode 100 31 + - pci + - fs0:\Image ${rtvar:CMDLINE} + - reset -c + - EOF + + params: + - -drive file=${artifact:EDK2_FLASH0},format=raw,if=pflash + - -drive file=${artifact:EDK2_FLASH1},format=raw,if=pflash + - -drive file=fat:rw:$$BOOT_DISK,format=raw diff --git a/config/qemu/host-base.yaml b/config/qemu/host-base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfc7e0b3ca79bd912a4ac4ff0b304347cb81cebd --- /dev/null +++ b/config/qemu/host-base.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Select QEMU as the runner, and set some basic parameters. + +run: + runner: QEMU + runners: + QEMU: + name: qemu-system-aarch64 + rtvars: + MEM_SIZE: + value: 4G + type: string + + # Number of vCPUs + SMP: + value: 8 + type: string + + params: + - -m ${rtvar:MEM_SIZE} + - -smp ${rtvar:SMP} diff --git a/config/qemu/host-build.yaml b/config/qemu/host-build.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c467088539d23f5a08398b6f114ca9daec9f1da --- /dev/null +++ b/config/qemu/host-build.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Build QEMU instead of using the one available in the container. + +build: + qemu: + repo: + remote: https://gitlab.com/qemu-project/qemu.git + revision: master + + build: + - cd ${param:builddir} + - ${param:sourcedir}/configure --target-list=aarch64-softmmu --prefix=${param:builddir}/install + - make -j${param:jobs} install + + artifacts: + QEMU: ${param:builddir}/install/bin/qemu-system-aarch64 + # The SBSA machine needs a ROM for bochs-display. virtio-net-pci needs + # one too. + QEMU_ROMS: ${param:builddir}/install/share/qemu + +run: + runners: + QEMU: + name: ${artifact:QEMU} + params: + - -L ${artifact:QEMU_ROMS} diff --git a/config/qemu/multi-term.yaml b/config/qemu/multi-term.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8784e3fcdab54186ddae65a6856f57a3a402d0cd --- /dev/null +++ b/config/qemu/multi-term.yaml @@ -0,0 +1,49 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Launch all terminals in separate consoles, and add a virtio-console port for + a guest. This should be used with a compatible overlay, like cca.yaml + +run: + runners: + QEMU: + params: + - -chardev socket,id=chr2,port=13559,to=13659,host=0.0.0.0,server=on,wait=on,telnet=on + - -device virtio-serial-pci + - -device virtconsole,chardev=chr2 + - -chardev socket,id=chr3,port=13560,to=13660,host=0.0.0.0,server=on,wait=on,telnet=on + - -device virtio-serial-pci + - -device virtconsole,chardev=chr3 + + rtvars: + # Use the first virtio console for the host + CMDLINE: + type: string + value: root=/dev/vda console=hvc0 rw + + terminals: + serial0: + friendly: 'Firmware' + type: term + command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port} + +# At the moment the secure console is unused and takes space. +# serial1: +# friendly: 'Secure' +# type: term +# command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port} + + serial2: + friendly: 'Host' + type: term + command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port} + port_regex: '-chardev socket,id=chr2.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on' + + serial3: + friendly: 'Guest' + type: term + command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port} + port_regex: '-chardev socket,id=chr3.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on' diff --git a/config/qemu/rmm-base.yaml b/config/qemu/rmm-base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57741dcaef9212c00a7d85aefed0acef5a83ede3 --- /dev/null +++ b/config/qemu/rmm-base.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +buildex: + runners: + QEMU: + rmm: + repo: + remote: https://git.trustedfirmware.org/TF-RMM/tf-rmm.git + revision: tf-rmm-v0.6.0 + + toolchain: aarch64-none-elf- + + params: + -DRMM_CONFIG: qemu_sbsa_defcfg + -DCMAKE_BUILD_TYPE: Debug + -DLOG_LEVEL: 40 + + build: + - cmake ${param:join_equal} -S . -B ${param:builddir} + - cmake --build ${param:builddir} -j ${param:jobs} + + artifacts: + RMM: ${param:builddir}/Debug/rmm.img diff --git a/config/qemu/tfa-base.yaml b/config/qemu/tfa-base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8da04d19a376e7372b26934a77ac3920ad12090e --- /dev/null +++ b/config/qemu/tfa-base.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2024, Linaro Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Trusted Firmware for QEMU virt. This provides a baseline configuration that + can be customized by higher layers. + +buildex: + runners: + QEMU: + tfa: + repo: + remote: https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git + revision: v2.13-rc0 + + toolchain: aarch64-none-elf- + + params: + PLAT: qemu_sbsa + DEBUG: 0 + LOG_LEVEL: 40 + + build: + # tfa has makefile dependency bug that makes parallel make for more than + # ~8 jobs unreliable, so limit it to 8. + - "make BUILD_BASE=${param:builddir} ${param:join_equal} -j$$(( ${param:jobs} < 8 ? ${param:jobs} : 8 )) all fip" + + artifacts: + BL1: ${param:builddir}/qemu_sbsa/release/bl1.bin + FIP: ${param:builddir}/qemu_sbsa/release/fip.bin diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index 93331eee0adc03c258b2bcfb2893ebf4b10c2823..cc9ab0f07e256b7acd94e3e7c98e6264c2ea4816 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -184,6 +184,7 @@ created. key type description =========== =========== =========== btvars dictionary Build-Time variables. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Build-Time variables can be overridden by the user at the command line. +runners dictionary Components built per runner. If a given runner is selected in the run section, components in its buildex section are built, and replace ones with the same name in the build section. =========== =========== =========== ~~~~~~~~~~~~~~~~~ @@ -220,10 +221,12 @@ run section key type description =========== =========== =========== name string Name of the FVP binary, which must be in $PATH. -rtvars dictionary Run-Time variables. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Run-Time variables can be overridden by the user at the command line. +rtvars dictionary Run-Time variables for the FVP. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Run-Time variables can be overridden by the user at the command line. params dictionary Dictionary of parameters to be passed to the FVP. Similar to the component's params, laying these out in a dictionary makes it easy for higher layers to override and add parameters. prerun list List of shell commands to be executed before the FVP is started. terminals dictionary Describes the set of UART terminals available for the FVP. key is the terminal parameter name known to the FVP (e.g. ``bp.terminal_0``) See below for format of the value. +runner string Type of runner to use, by default FVP. +runners dictionary Dictionary of runners other than the FVP, with their variables. See below for the format of the runners. =========== =========== =========== ~~~~~~~~~~~~~~~~ @@ -247,3 +250,21 @@ Terminal types: - **stdinout**: Mux output to stdout. Forward stdin to its input. Max of 1 of these types allowed. - **telnet**: Shrinkwrap will print out a telnet command to run in a separate terminal to get a unique interactive terminal. - **xterm**: Shrinkwrap will automatically launch xterm to provide a unique interactive terminal. Only works when runtime=null. + +~~~~~~~~~~~~~~~ +runners section +~~~~~~~~~~~~~~~ + +The *run* section contains parameters for the FVP. A *runners* subsection +allows specifying parameters for other runners. The format of each value is the +same as in the *run* section, except for *params*. + +=========== =========== =========== +key type description +=========== =========== =========== +name string Name or path to the runner binary +rtvars dictionary Run-Time variables. +params dictionary Parameters to be passed to the runner. Unlike the `params` value in the run section, this is a list of parameters, which gets joined by spaces. +prerun list List of shell commands to be executed before the runner is started. +terminals dictionary Describes the set of UART terminals available for the runner. +=========== =========== =========== diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index 557bdf90517737e3328e85e75fd559f494744361..2a68ac2a2c4fa5ae602a677e1ad24a0f35f7c797 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -629,17 +629,17 @@ interact directly with the FVP in a terminal without the need for a GUI setup: .. code-block:: none - [ fvp ] terminal_0: Listening for serial connection on port 5000 - [ fvp ] terminal_1: Listening for serial connection on port 5001 - [ fvp ] terminal_2: Listening for serial connection on port 5002 - [ fvp ] terminal_3: Listening for serial connection on port 5003 - [ fvp ] - [ fvp ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.flashloader0: FlashLoader: Loaded 100 kB from file '/package/ns-preload/fip.bin' - [ fvp ] - [ fvp ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.secureflashloader: FlashLoader: Loaded 30 kB from file '/package/ns-preload/bl1.bin' - [ fvp ] - [ fvp ] libdbus-1.so.3: cannot open shared object file: No such file or directory - [ fvp ] libdbus-1.so.3: cannot open shared object file: No such file or directory + [ FVP ] terminal_0: Listening for serial connection on port 5000 + [ FVP ] terminal_1: Listening for serial connection on port 5001 + [ FVP ] terminal_2: Listening for serial connection on port 5002 + [ FVP ] terminal_3: Listening for serial connection on port 5003 + [ FVP ] + [ FVP ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.flashloader0: FlashLoader: Loaded 100 kB from file '/package/ns-preload/fip.bin' + [ FVP ] + [ FVP ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.secureflashloader: FlashLoader: Loaded 30 kB from file '/package/ns-preload/bl1.bin' + [ FVP ] + [ FVP ] libdbus-1.so.3: cannot open shared object file: No such file or directory + [ FVP ] libdbus-1.so.3: cannot open shared object file: No such file or directory [ tfa+linux ] NOTICE: BL31: v2.7(release):v2.7.0-391-g9dedc1ab2 [ tfa+linux ] NOTICE: BL31: Built : 09:41:20, Sep 15 2022 [ tfa+linux ] INFO: GICv3 with legacy support detected. diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py index 699d8de718c60a4e48f04fb4aef394a1e47a81ad..0593e7666d621c5efc16ee1eba13bd0fe75f307f 100644 --- a/shrinkwrap/commands/run.py +++ b/shrinkwrap/commands/run.py @@ -23,7 +23,7 @@ def add_parser(parser, formatter): """ cmdp = parser.add_parser(cmd_name, formatter_class=formatter, - help="""Boot and run the FVP for the specified config.""", + help="""Boot and run the specified config.""", epilog="""FW is accessed from . defaults to '~/.shrinkwrap/package', but the user can override it by setting the environment @@ -87,24 +87,32 @@ def dispatch(args): resolveb = config.load(filename, overlays) rtvars_dict = vars.parse(args.rtvar, type='rt') resolver = config.resolver(resolveb, rtvars_dict) - cmds = _pretty_print_sh(resolver['run']) - # If dry run, just output the FVP command that we would have run. We + runner_name = resolver['run']['runner'] + if runner_name == 'FVP': + runner = resolver['run'] + else: + runner = resolver['run']['runners'].get(runner_name) + if runner is None: + raise Exception(f'invalid runner `{runner_name}`') + cmds = _pretty_print_sh(runner_name, runner) + + # If dry run, just output the command that we would have run. We # don't include the netcat magic to access the fvp terminals. if args.dry_run: print(cmds) return - # The FVP and any associated uart terminals are output to our terminal + # The runner and any associated uart terminals are output to our terminal # with a tag to indicate where each line originated. Figure out how big # that tag field needs to be so that everything remains aligned. max_name_field = 10 name_field = 0 - terminals = resolver['run']['terminals'] + terminals = runner['terminals'] terminals = dict(sorted(terminals.items())) for t in terminals.values(): t['port'] = None - t['strip'] = False + t['started'] = False if len(t['friendly']) > name_field: name_field = min(len(t['friendly']), max_name_field) @@ -116,19 +124,14 @@ def dispatch(args): few lines of output, which is output by telnet. This is confusing for users since telnet is an implementation detail. """ - + skip_line = True match = "Escape character is '^]'." - pdata = proc.data - for line in logger.splitlines(data): - if len(pdata) >= 2 and terminals[pdata[1]]['strip']: - if line.find(match) >= 0: - terminals[pdata[1]]['strip'] = False - if all([not t['strip'] \ - for t in terminals.values()]): - pm.set_handler(log.log) - else: + if not skip_line: log.log(pm, proc, line, streamid) + elif line.find(match) >= 0: + proc.set_handler(None) + skip_line = False def _colorize(global_no_color, terminal): if global_no_color: @@ -151,14 +154,56 @@ def dispatch(args): else: return None + def _launch_term(pm, logname, t): + name = t['friendly'] + type = t['type'] + port = t["port"] + colorize = _colorize(args.no_color, t) + escape = _escape(t) + + if type in ['stdout']: + cmd = f'nc localhost {port}' + pm.add(process.Process(cmd, + False, + (log.alloc_data(name, colorize, escape, _logfile(t)), logname), + False)) + if type in ['stdinout']: + cmd = f'telnet localhost {port}' + pm.add(process.Process(cmd, + True, + (log.alloc_data(name, colorize, escape, _logfile(t)), logname), + False, + # Initially use the special handler to skip telnet header + handler=_strip_telnet_header)) + if type in ['telnet']: + ip = runtime.get().ip_address() + print(f'To start {name} terminal, run:') + print(f' telnet {ip} {port}') + if type in ['term']: + lut = { + 'term': { + 'host': runtime.get().ip_address(), + 'port': port, + 'name': name, + } + } + cmd = t.get('command') + if cmd is None: + raise Exception(f'{logname}: `term` type requires a command') + cmd = config.string_substitute(cmd, lut) + pm.add(process.Process(cmd, + False, + (log.alloc_data(name, colorize, escape, _logfile(t)), logname), + False, + local=True)) + def _find_term_ports(pm, proc, data, streamid): """ - Initial handler function called by ProcessManager. When the fvp - starts, we must parse the output to determine the port numbers - to connect to with netcat to access the fvp uart terminals. We - look for all the ports, start the netcat instances, add them to - the process manager and finally switch the handler to the - standard logger. + Initial handler function called by ProcessManager. When the runner + starts, we must parse the output to determine the port numbers to + connect to with netcat to access the uart terminals. We look for all + the ports, start the netcat instances, add them to the process manager + and finally switch the handler to the standard logger. """ # First, forward to the standard log handler. log.log(pm, proc, data, streamid) @@ -168,49 +213,28 @@ def dispatch(args): # Iterate over the terminals dict from the config applying the # supplied regexes to try to find the ports for all the # terminals. - for t in terminals.values(): + for k, t in terminals.items(): + if t['started']: + continue + if t['port'] is None: res = re.search(t['port_regex'], data) if res: t['port'] = res.group(1) else: found_all_ports = False + continue + + # Not yet started but we found a port: launch the netcat process. + if t['port'] is not None: + _launch_term(pm, k, t); + + t['started'] = True - # Once all ports have been found, launch the netcat processes - # and change the handler so we never get called again. + # Once all ports have been found change the handler so we never get + # called again. if found_all_ports: - wait = False - strip = False - for k, t in terminals.items(): - name = t['friendly'] - type = t['type'] - port = t["port"] - colorize = _colorize(args.no_color, t) - escape = _escape(t) - - if type in ['stdout']: - cmd = f'nc localhost {port}' - pm.add(process.Process(cmd, - False, - (log.alloc_data(name, colorize, escape, _logfile(t)), k), - False)) - if type in ['stdinout']: - cmd = f'telnet localhost {port}' - pm.add(process.Process(cmd, - True, - (log.alloc_data(name, colorize, escape, _logfile(t)), k), - False)) - t['strip'] = True - strip = True - if type in ['xterm']: - # Nothing to do. The FVP will start this - # automatically. - pass - if type in ['telnet']: - wait = True - ip = runtime.get().ip_address() - print(f'To start {name} terminal, run:') - print(f' telnet {ip} {port}') + wait = any(t['type'] == 'telnet' for t in terminals.values()) if wait: # Temporarily restore sys.stdin for input(). pm._stdin_deactivate() @@ -218,18 +242,15 @@ def dispatch(args): input("Press Enter to continue...") pm._stdin_activate() - if strip: - pm.set_handler(_strip_telnet_header) - else: - pm.set_handler(log.log) + proc.set_handler(None) def _complete(pm, proc, retcode): log.free_data(proc.data[0]) - # If the FVP exits with non-zero exit code, we propagate that + # If the run command exits with non-zero exit code, we propagate that # error so that shrinkwrap also exits with non-zero exit code. if retcode not in [0, None] and proc.run_to_end: - raise Exception(f'FVP failed with {retcode}') + raise Exception(f'run command failed with {retcode}') # Run under a runtime environment, which may just run commands natively # on the host or may execute commands in a container, depending on what @@ -237,7 +258,7 @@ def dispatch(args): with runtime.Runtime(name=args.runtime, image=config.get_image([resolveb], args), ssh_agent_keys=args.ssh_agent_keys, timeout=args.timeout) as rt: - for rtvar in resolver['run']['rtvars'].values(): + for rtvar in runner['rtvars'].values(): if rtvar['type'] == 'path': rt.add_volume(rtvar['value']) for t in terminals.values(): @@ -258,17 +279,18 @@ def dispatch(args): with open(tmpfilename, 'w') as tmpfile: tmpfile.write(cmds) - # Create a process manager with 1 process; the fvp. As + # Create a process manager with 1 process; the runner. As # it boots _find_term_ports() will add the netcat # processes in parallel. It will exit once all processes - # have terminated. The fvp will terminate when its told + # have terminated. The runner will terminate when its told # to `poweroff` and netcat will terminate when it sees - # the fvp has gone. - pm = process.ProcessManager(_find_term_ports, _complete) + # the runner has gone. + pm = process.ProcessManager(log.log, _complete) pm.add(process.Process(f'bash {tmpfilename}', False, - (log.alloc_data('fvp', not args.no_color),), - True)) + (log.alloc_data(runner_name, not args.no_color),), + True, + handler=_find_term_ports)) rt_ip = runtime.get().ip_address() @@ -281,21 +303,18 @@ def dispatch(args): pm.run(forward_stdin=True) -def _pretty_print_sh(run): - prerun = run['prerun'] - run = run['run'] +def _pretty_print_sh(runner_name, runner): + prerun = runner['prerun'] + run = runner['run'] # This is a hack to improve the way the FVP arguments look. It tends to # be huge so attempt to make it more readable by putting each option on - # a separate line and sorting alphabetically. Only likely to work for - # FVP so try to infer its definitely the FVP. - if len(run) == 1: - prog = run[0].split(' ')[0].lower() - if prog.find('isim') >= 0 or prog.find('fvp') >= 0: - parts = run[0].split(' -') - prog = parts[0] - args = sorted(parts[1:]) - run = [' \\\n -'.join([prog] + args)] + # a separate line and sorting alphabetically. + if runner_name == 'FVP': + parts = run[0].split(' -') + prog = parts[0] + args = sorted(parts[1:]) + run = [' \\\n -'.join([prog] + args)] pre = config.script_preamble(False) script = config.Script('run model', preamble=pre) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index d1cfdf989c02ee15b9dff69bf67075cc010ded2d..7b8eec8c3ce0685381dcfb20b795aa246488719e 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -89,6 +89,18 @@ def _buildex_normalize(buildex): """ buildex.setdefault('btvars', {}) + for build in buildex.setdefault('runners', {}).values(): + for name, component in build.items(): + _component_normalize(component, name) + +def _runners_normalize(runners): + for runner in runners.values(): + runner.setdefault('name', None) + runner.setdefault('rtvars', {}) + runner.setdefault('params', []) + runner.setdefault('prerun', []) + runner.setdefault('run', []) + runner.setdefault('terminals', {}) def _run_normalize(run): """ @@ -100,6 +112,9 @@ def _run_normalize(run): run.setdefault('prerun', []) run.setdefault('run', []) run.setdefault('terminals', {}) + run.setdefault('runner', None) + run.setdefault('runners', {}) + _runners_normalize(run['runners']) def _config_normalize(config): @@ -172,7 +187,7 @@ def _run_sort(run): Sort the run section so that the keys are in a canonical order. This improves readability by humans. """ - lut = ['name', 'rtvars', 'params', 'prerun', 'run', 'terminals'] + lut = ['name', 'rtvars', 'params', 'prerun', 'run', 'terminals', 'runners', 'runner'] lut = {k: i for i, k in enumerate(lut)} return dict(sorted(run.items(), key=lambda x: lut[x[0]])) @@ -283,8 +298,7 @@ def _string_tokenize(string, escape=True): return tokens - -def _string_substitute(string, lut, final=True): +def string_substitute(string, lut, final=True): """ Takes a string containg macros and returns a string with the macros substituted for the values found in the lut. If final is False, any @@ -527,7 +541,7 @@ def resolveb(config, btvars={}, clivars={}): for desc in config['build'].values(): for v in desc['artifacts'].values(): - v['path'] = _string_substitute(v['path'], artifact_lut, False) + v['path'] = string_substitute(v['path'], artifact_lut, False) if artifact_nr > 0: artifact_lut = _combine(config) @@ -536,32 +550,40 @@ def resolveb(config, btvars={}, clivars={}): def _substitute_macros(config, lut, final): for desc in config['build'].values(): - desc['sourcedir'] = _string_substitute(desc['sourcedir'], lut, final) - desc['builddir'] = _string_substitute(desc['builddir'], lut, final) + desc['sourcedir'] = string_substitute(desc['sourcedir'], lut, final) + desc['builddir'] = string_substitute(desc['builddir'], lut, final) lut['param']['sourcedir'] = desc['sourcedir'] lut['param']['builddir'] = desc['builddir'] - desc['params'] = { k: _string_substitute(v, lut, final) for k, v in desc['params'].items() } + desc['params'] = { k: string_substitute(v, lut, final) for k, v in desc['params'].items() } lut['param']['join_equal'] = _mk_params(desc['params'], '=') lut['param']['join_space'] = _mk_params(desc['params'], ' ') for r in desc['repo'].values(): - r['remote'] = _string_substitute(r['remote'], lut, final) - r['revision'] = _string_substitute(r['revision'], lut, final) + r['remote'] = string_substitute(r['remote'], lut, final) + r['revision'] = string_substitute(r['revision'], lut, final) - desc['toolchain'] = _string_substitute(desc['toolchain'], lut, final) + desc['toolchain'] = string_substitute(desc['toolchain'], lut, final) for k in ( 'prebuild', 'build', 'postbuild', ): - desc[k] = [ _string_substitute(s, lut, final) for s in desc[k] ] + desc[k] = [ string_substitute(s, lut, final) for s in desc[k] ] for v in desc['artifacts'].values(): - v['path'] = _string_substitute(v['path'], lut, False) - v['base'] = _string_substitute(v['base'], lut, False) + v['path'] = string_substitute(v['path'], lut, False) + v['base'] = string_substitute(v['base'], lut, False) for v in config['buildex']['btvars'].values(): - v['value'] = _string_substitute(v['value'], lut, final) + v['value'] = string_substitute(v['value'], lut, final) + + # If the runner is different from the default, override some of the + # components by those defined in buildex for this runner. + runner = config['run']['runner'] + build_components_override = config['buildex']['runners'].get(runner, {}) + + for name, component in build_components_override.items(): + config['build'][name] = component # Compute the source and build directories for each component. If they # are already present, then don't override. This allows users to supply @@ -634,6 +656,17 @@ def resolveb(config, btvars={}, clivars={}): return _config_sort(config) +def _resolve_run(run, lut): + # Now create a lookup table with all the rtvars and resolve all the + # parameters. An exception will be thrown if there are any macros that + # we don't have values for. + lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()} + + for i, s, in enumerate(run['run']): + run['run'][i] = string_substitute(s, lut) + + for i, s in enumerate(run['prerun']): + run['prerun'][i] = string_substitute(s, lut) def resolver(config, rtvars={}, clivars={}): """ @@ -643,21 +676,25 @@ def resolver(config, rtvars={}, clivars={}): clivars = uclivars.get(**clivars) run = config['run'] - # Find the list of imported artifacts before any processing passes artifacts_imp = set() - _string_extract_artifacts(artifacts_imp, run['params'].values()) - _string_extract_artifacts(artifacts_imp, run['prerun']) - _string_extract_artifacts(artifacts_imp, run['run']) - _string_extract_artifacts(artifacts_imp, run['rtvars'].values()) - - #Override the rtvars with any values supplied by the user and check that - #all rtvars are defined. - for k, v in run['rtvars'].items(): - if k in rtvars: - v['value'] = rtvars[k] - if v['value'] is None: - raise Exception(f'{k} run-time variable not ' \ - 'set by user and no default available.') + all_runners = [run] + list(run['runners'].values()) + + for runner in all_runners: + params = runner['params'].values() if type(runner['params']) == dict else runner['params'] + # Find the list of imported artifacts before any processing passes + _string_extract_artifacts(artifacts_imp, params) + _string_extract_artifacts(artifacts_imp, runner['prerun']) + _string_extract_artifacts(artifacts_imp, runner['run']) + _string_extract_artifacts(artifacts_imp, runner['rtvars'].values()) + + # Override the rtvars with any values supplied by the user and check + # that all rtvars are defined. + for k, v in runner['rtvars'].items(): + if k in rtvars: + v['value'] = rtvars[k] + if v['value'] is None: + raise Exception(f'{k} run-time variable not ' \ + 'set by user and no default available.') # Update the artifacts so that the destination now points to an absolute # path rather than one that is implictly relative to SHRINKWRAP_PACKAGE. @@ -684,40 +721,49 @@ def resolver(config, rtvars={}, clivars={}): 'btvar': {k: v['value'] for k, v in config['buildex']['btvars'].items()}, } - for v in run['rtvars'].values(): - v['value'] = _string_substitute(str(v['value']), lut) - if v['type'] == 'path' and v['value']: - v['value'] = os.path.expanduser(v['value']) - v['value'] = os.path.abspath(v['value']) - - # Now create a lookup table with all the rtvars and resolve all the - # parameters. An exception will be thrown if there are any macros that - # we don't have values for. - lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()} - - run['params'] = { k: _string_substitute(v, lut) for k, v in run['params'].items() } - - # Assemble the final runtime command and stuff it into the config. - params = _mk_params(run['params'], '=') - - terms = [] - for param, terminal in run['terminals'].items(): - if terminal['type'] in ['stdout']: - terms.append(f'-C {param}.start_telnet=0') - terms.append(f'-C {param}.mode=raw') - if terminal['type'] in ['xterm']: - terms.append(f'-C {param}.start_telnet=1') - terms.append(f'-C {param}.mode=telnet') - if terminal['type'] in ['telnet', 'stdinout']: - terms.append(f'-C {param}.start_telnet=0') - terms.append(f'-C {param}.mode=telnet') - terms = ' '.join(terms) + for runner in all_runners: + for v in runner['rtvars'].values(): + v['value'] = string_substitute(str(v['value']), lut) + if v['type'] == 'path' and v['value']: + v['value'] = os.path.expanduser(v['value']) + v['value'] = os.path.abspath(v['value']) + + # Also check that the rtvar is well formed + t = v.get('type') + if t not in ('path', 'string'): + raise Exception(f'invalid type `{t}` for run-time variable {k}') + + if run['runner'] is None: + run['runner'] = 'FVP' + + # Assemble the final runtime commands and stuff them into the config. + # For backward compatibility, run['run'] will contain the FVP command-line, + # and run['runners'][r]['run'] contain the command-line of each of the + # other runners. if run["name"]: - run['run'] = [' '.join([run["name"], params, terms])] - - for i, s in enumerate(run['prerun']): - run['prerun'][i] = _string_substitute(s, lut) + terms = [] + params = _mk_params(run['params'], '=') + for param, terminal in run['terminals'].items(): + if terminal['type'] in ['stdout']: + terms.append(f'-C {param}.start_telnet=0') + terms.append(f'-C {param}.mode=raw') + if terminal['type'] in ['xterm']: + terms.append(f'-C {param}.start_telnet=1') + terms.append(f'-C {param}.mode=telnet') + if terminal['type'] in ['telnet', 'stdinout']: + terms.append(f'-C {param}.start_telnet=0') + terms.append(f'-C {param}.mode=telnet') + + run['run'] = [' '.join([run["name"], params] + terms)] + _resolve_run(run, lut) + + # Additional runners use a list of params rather than a dict + for runner in run['runners'].values(): + if runner['name']: + params = ' '.join(runner['params']) + runner['run'] = [' '.join([runner['name'], params])] + _resolve_run(runner, lut) return _config_sort(config) diff --git a/shrinkwrap/utils/logger.py b/shrinkwrap/utils/logger.py index 0892760e73caeeeeff45f8cd240e7a6974fd0002..6ca949d2179763c19b6ec3a8606af0f54d204597 100644 --- a/shrinkwrap/utils/logger.py +++ b/shrinkwrap/utils/logger.py @@ -86,7 +86,7 @@ class Logger: def log(self, pm, proc, data, streamid, logstd=True): """ - Logs text data from one of the processes (FVP or one of its uart + Logs text data from one of the processes (runner or one of its uart terminals) to the terminal. Text is colored and a tag is added on the left to identify the originating process. """ diff --git a/shrinkwrap/utils/process.py b/shrinkwrap/utils/process.py index d0d9d4f3da0ea9167b68270b81a881251770617e..10a9848e8a117d273fe1829fda456d104d0fec94 100644 --- a/shrinkwrap/utils/process.py +++ b/shrinkwrap/utils/process.py @@ -21,17 +21,29 @@ class Process: """ A wrapper to a process that should be managed by the ProcessManager. """ - def __init__(self, args, interactive, data, run_to_end): + def __init__(self, args, interactive, data, run_to_end, handler=None, local=None): + """ + @args: array containing the command and its parameters + @interactive: True if the process requires stdin + @data: tuple (log data, log name) + @run_to_end: exit shrinkwrap on error + @handler: optional handler function for stdout logs + @local: execute the command locally instead of in the runtime + """ self.args = shlex.split(args) self.interactive = interactive self.data = data self.run_to_end = run_to_end + self._handler = handler self._popen = None self._stdout = None self._stdin = None self._stderr = None self._active = 0 + self._local = local + def set_handler(self, handler): + self._handler = handler class ProcessManager: """ @@ -101,11 +113,16 @@ class ProcessManager: if data == '': self._proc_stream_deactivate(proc, key.fileobj, streamid) + elif proc._handler: + proc._handler(self, proc, data, streamid) elif self._handler: self._handler(self, proc, data, streamid) def _proc_activate(self, proc): - cmd = runtime.mkcmd(proc.args, proc.interactive) + if proc._local: + cmd = proc.args + else: + cmd = runtime.mkcmd(proc.args, proc.interactive) if proc.interactive: master, slave = pty.openpty()