From 79dfcb6fb6b278f5335faf3ae808840050184447 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Wed, 31 Jul 2024 18:29:39 +0100
Subject: [PATCH 01/12] run: Prepare for additional runners

Remove some of the built-in assumptions that we're running the FVP, so
other runners can be added.

Going forward, we'll have a 'runner_name' defaulting to 'FVP', hence the
change to 'FVP' in the stdout output.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/quickstart.rst | 22 +++++++++----------
 shrinkwrap/commands/run.py             | 29 +++++++++++++-------------
 shrinkwrap/utils/logger.py             |  2 +-
 3 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst
index 557bdf9..2a68ac2 100644
--- a/documentation/userguide/quickstart.rst
+++ b/documentation/userguide/quickstart.rst
@@ -629,17 +629,17 @@ interact directly with the FVP in a terminal without the need for a GUI setup:
 
 .. code-block:: none
 
-  [       fvp ] terminal_0: Listening for serial connection on port 5000
-  [       fvp ] terminal_1: Listening for serial connection on port 5001
-  [       fvp ] terminal_2: Listening for serial connection on port 5002
-  [       fvp ] terminal_3: Listening for serial connection on port 5003
-  [       fvp ]
-  [       fvp ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.flashloader0: FlashLoader: Loaded 100 kB from file '<root>/package/ns-preload/fip.bin'
-  [       fvp ]
-  [       fvp ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.secureflashloader: FlashLoader: Loaded 30 kB from file '<root>/package/ns-preload/bl1.bin'
-  [       fvp ]
-  [       fvp ] libdbus-1.so.3: cannot open shared object file: No such file or directory
-  [       fvp ] libdbus-1.so.3: cannot open shared object file: No such file or directory
+  [       FVP ] terminal_0: Listening for serial connection on port 5000
+  [       FVP ] terminal_1: Listening for serial connection on port 5001
+  [       FVP ] terminal_2: Listening for serial connection on port 5002
+  [       FVP ] terminal_3: Listening for serial connection on port 5003
+  [       FVP ]
+  [       FVP ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.flashloader0: FlashLoader: Loaded 100 kB from file '<root>/package/ns-preload/fip.bin'
+  [       FVP ]
+  [       FVP ] Info: FVP_Base_RevC_2xAEMvA: FVP_Base_RevC_2xAEMvA.bp.secureflashloader: FlashLoader: Loaded 30 kB from file '<root>/package/ns-preload/bl1.bin'
+  [       FVP ]
+  [       FVP ] libdbus-1.so.3: cannot open shared object file: No such file or directory
+  [       FVP ] libdbus-1.so.3: cannot open shared object file: No such file or directory
   [ tfa+linux ] NOTICE:  BL31: v2.7(release):v2.7.0-391-g9dedc1ab2
   [ tfa+linux ] NOTICE:  BL31: Built : 09:41:20, Sep 15 2022
   [ tfa+linux ] INFO:    GICv3 with legacy support detected.
diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py
index 699d8de..47a496c 100644
--- a/shrinkwrap/commands/run.py
+++ b/shrinkwrap/commands/run.py
@@ -23,7 +23,7 @@ def add_parser(parser, formatter):
 	"""
 	cmdp = parser.add_parser(cmd_name,
 		formatter_class=formatter,
-		help="""Boot and run the FVP for the specified config.""",
+		help="""Boot and run the specified config.""",
 		epilog="""FW is accessed from <SHRINKWRAP_PACKAGE>.
 		     <SHRINKWRAP_PACKAGE> defaults to '~/.shrinkwrap/package',
 		     but the user can override it by setting the environment
@@ -89,13 +89,13 @@ def dispatch(args):
 	resolver = config.resolver(resolveb, rtvars_dict)
 	cmds = _pretty_print_sh(resolver['run'])
 
-	# If dry run, just output the FVP command that we would have run. We
+	# If dry run, just output the command that we would have run. We
 	# don't include the netcat magic to access the fvp terminals.
 	if args.dry_run:
 		print(cmds)
 		return
 
-	# The FVP and any associated uart terminals are output to our terminal
+	# The runner and any associated uart terminals are output to our terminal
 	# with a tag to indicate where each line originated. Figure out how big
 	# that tag field needs to be so that everything remains aligned.
 	max_name_field = 10
@@ -153,12 +153,11 @@ def dispatch(args):
 
 	def _find_term_ports(pm, proc, data, streamid):
 		"""
-		Initial handler function called by ProcessManager. When the fvp
-		starts, we must parse the output to determine the port numbers
-		to connect to with netcat to access the fvp uart terminals. We
-		look for all the ports, start the netcat instances, add them to
-		the process manager and finally switch the handler to the
-		standard logger.
+		Initial handler function called by ProcessManager. When the runner
+		starts, we must parse the output to determine the port numbers to
+		connect to with netcat to access the uart terminals. We look for all
+		the ports, start the netcat instances, add them to the process manager
+		and finally switch the handler to the standard logger.
 		"""
 		# First, forward to the standard log handler.
 		log.log(pm, proc, data, streamid)
@@ -226,10 +225,10 @@ def dispatch(args):
 	def _complete(pm, proc, retcode):
 		log.free_data(proc.data[0])
 
-		# If the FVP exits with non-zero exit code, we propagate that
+		# If the run command exits with non-zero exit code, we propagate that
 		# error so that shrinkwrap also exits with non-zero exit code.
 		if retcode not in [0, None] and proc.run_to_end:
-			raise Exception(f'FVP failed with {retcode}')
+			raise Exception(f'run command failed with {retcode}')
 
 	# Run under a runtime environment, which may just run commands natively
 	# on the host or may execute commands in a container, depending on what
@@ -258,16 +257,16 @@ def dispatch(args):
 			with open(tmpfilename, 'w') as tmpfile:
 				tmpfile.write(cmds)
 
-			# Create a process manager with 1 process; the fvp. As
+			# Create a process manager with 1 process; the runner. As
 			# it boots _find_term_ports() will add the netcat
 			# processes in parallel. It will exit once all processes
-			# have terminated. The fvp will terminate when its told
+			# have terminated. The runner will terminate when its told
 			# to `poweroff` and netcat will terminate when it sees
-			# the fvp has gone.
+			# the runner has gone.
 			pm = process.ProcessManager(_find_term_ports, _complete)
 			pm.add(process.Process(f'bash {tmpfilename}',
 				False,
-				(log.alloc_data('fvp', not args.no_color),),
+				(log.alloc_data('FVP', not args.no_color),),
 				True))
 
 			rt_ip = runtime.get().ip_address()
diff --git a/shrinkwrap/utils/logger.py b/shrinkwrap/utils/logger.py
index 0892760..6ca949d 100644
--- a/shrinkwrap/utils/logger.py
+++ b/shrinkwrap/utils/logger.py
@@ -86,7 +86,7 @@ class Logger:
 
 	def log(self, pm, proc, data, streamid, logstd=True):
 		"""
-		Logs text data from one of the processes (FVP or one of its uart
+		Logs text data from one of the processes (runner or one of its uart
 		terminals) to the terminal. Text is colored and a tag is added
 		on the left to identify the originating process.
 		"""
-- 
GitLab


From 04ee60acd6a18ef85930bc8b9e8960259e8c9651 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Wed, 14 Aug 2024 14:46:13 +0100
Subject: [PATCH 02/12] run: Move term and parameters setting

The terminal parameters as well as 'params' are specific to FVP, and
only used when filling run['run']. Move them one level down to make
space for additional runners.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index d1cfdf9..84dc0f0 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -698,23 +698,22 @@ def resolver(config, rtvars={}, clivars={}):
 	run['params'] = { k: _string_substitute(v, lut) for k, v in run['params'].items() }
 
 	# Assemble the final runtime command and stuff it into the config.
-	params = _mk_params(run['params'], '=')
-
-	terms = []
-	for param, terminal in run['terminals'].items():
-		if terminal['type'] in ['stdout']:
-			terms.append(f'-C {param}.start_telnet=0')
-			terms.append(f'-C {param}.mode=raw')
-		if terminal['type'] in ['xterm']:
-			terms.append(f'-C {param}.start_telnet=1')
-			terms.append(f'-C {param}.mode=telnet')
-		if terminal['type'] in ['telnet', 'stdinout']:
-			terms.append(f'-C {param}.start_telnet=0')
-			terms.append(f'-C {param}.mode=telnet')
-	terms = ' '.join(terms)
 
 	if run["name"]:
-		run['run'] = [' '.join([run["name"], params, terms])]
+		terms = []
+		params = _mk_params(run['params'], '=')
+		for param, terminal in run['terminals'].items():
+			if terminal['type'] in ['stdout']:
+				terms.append(f'-C {param}.start_telnet=0')
+				terms.append(f'-C {param}.mode=raw')
+			if terminal['type'] in ['xterm']:
+				terms.append(f'-C {param}.start_telnet=1')
+				terms.append(f'-C {param}.mode=telnet')
+			if terminal['type'] in ['telnet', 'stdinout']:
+				terms.append(f'-C {param}.start_telnet=0')
+				terms.append(f'-C {param}.mode=telnet')
+
+		run['run'] = [' '.join([run["name"], params] + terms)]
 
 	for i, s in enumerate(run['prerun']):
 		run['prerun'][i] = _string_substitute(s, lut)
-- 
GitLab


From ab9ab9ab572b6a62364c9af0672567c00bf5a43c Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Wed, 31 Jul 2024 18:38:01 +0100
Subject: [PATCH 03/12] run: Substitute variables in 'run' rather than 'params'

Make variable substitution more generic by performing it on the final
'run' strings. run['run'] could be set directly by a config, so it makes
more sense to perform the substitution as late as possible. In addition,
this code will also be used for additional runners.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 84dc0f0..05a8127 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -695,8 +695,6 @@ def resolver(config, rtvars={}, clivars={}):
 	# we don't have values for.
 	lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()}
 
-	run['params'] = { k: _string_substitute(v, lut) for k, v in run['params'].items() }
-
 	# Assemble the final runtime command and stuff it into the config.
 
 	if run["name"]:
@@ -715,6 +713,9 @@ def resolver(config, rtvars={}, clivars={}):
 
 		run['run'] = [' '.join([run["name"], params] + terms)]
 
+	for i, s, in enumerate(run['run']):
+		run['run'][i] = _string_substitute(s, lut)
+
 	for i, s in enumerate(run['prerun']):
 		run['prerun'][i] = _string_substitute(s, lut)
 
-- 
GitLab


From cf7709ae9bd0d99fc6ff63fa808e527186eb0206 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 1 Aug 2024 18:00:51 +0100
Subject: [PATCH 04/12] run: Enable additional runners

So far we've only supported running with the FVP. Introduce the concept
of 'runner', which allows running the software stack in different
environments.

Add a 'runners' dictionary to the 'run' section, that contains
parameters and configuration for runners other than the 'FVP'. A single
config like cca-3world can thus allow running the same kernel and rootfs
in different emulation environments.

Different runners need different firmwares, so the build section
contains components common to all runners, and components specific to
one runner.  The 'artifacts' section is common and must use different
names to disambiguate the runners, for example 'BL1' for FVP and
'BL1_QEMU' for QEMU. The run-time vars are specific to each runner, and
can use overlapping names.

A runner is selected with the run['runner'] variable. The default is
'FVP', and can be overridden by an overlay.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/configmodel.rst | 22 +++++-
 shrinkwrap/commands/run.py              | 37 +++++-----
 shrinkwrap/utils/config.py              | 92 ++++++++++++++++---------
 3 files changed, 103 insertions(+), 48 deletions(-)

diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst
index 93331ee..b91ca3d 100644
--- a/documentation/userguide/configmodel.rst
+++ b/documentation/userguide/configmodel.rst
@@ -220,10 +220,12 @@ run section
 key         type        description
 =========== =========== ===========
 name        string      Name of the FVP binary, which must be in $PATH.
-rtvars      dictionary  Run-Time variables. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Run-Time variables can be overridden by the user at the command line.
+rtvars      dictionary  Run-Time variables for the FVP. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Run-Time variables can be overridden by the user at the command line.
 params      dictionary  Dictionary of parameters to be passed to the FVP. Similar to the component's params, laying these out in a dictionary makes it easy for higher layers to override and add parameters.
 prerun      list        List of shell commands to be executed before the FVP is started.
 terminals   dictionary  Describes the set of UART terminals available for the FVP. key is the terminal parameter name known to the FVP (e.g. ``bp.terminal_0``) See below for format of the value.
+runner      string      Type of runner to use, by default FVP.
+runners     dictionary  Dictionary of runners other than the FVP, with their variables. See below for the format of the runners.
 =========== =========== ===========
 
 ~~~~~~~~~~~~~~~~
@@ -247,3 +249,21 @@ Terminal types:
 - **stdinout**: Mux output to stdout. Forward stdin to its input. Max of 1 of these types allowed.
 - **telnet**: Shrinkwrap will print out a telnet command to run in a separate terminal to get a unique interactive terminal.
 - **xterm**: Shrinkwrap will automatically launch xterm to provide a unique interactive terminal. Only works when runtime=null.
+
+~~~~~~~~~~~~~~~
+runners section
+~~~~~~~~~~~~~~~
+
+The *run* section contains parameters for the FVP. A *runners* subsection
+allows specifying parameters for other runners. The format of each value is the
+same as in the *run* section, except for *params*.
+
+=========== =========== ===========
+key         type        description
+=========== =========== ===========
+name        string      Name or path to the runner binary
+rtvars      dictionary  Run-Time variables.
+params      dictionary  Parameters to be passed to the runner. Unlike the `params` value in the run section, this is a list of parameters, which gets joined by spaces.
+prerun      list        List of shell commands to be executed before the runner is started.
+terminals   dictionary  Describes the set of UART terminals available for the runner.
+=========== =========== ===========
diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py
index 47a496c..6c7daca 100644
--- a/shrinkwrap/commands/run.py
+++ b/shrinkwrap/commands/run.py
@@ -87,7 +87,15 @@ def dispatch(args):
 	resolveb = config.load(filename, overlays)
 	rtvars_dict = vars.parse(args.rtvar, type='rt')
 	resolver = config.resolver(resolveb, rtvars_dict)
-	cmds = _pretty_print_sh(resolver['run'])
+
+	runner_name = resolver['run']['runner']
+	if runner_name == 'FVP':
+		runner = resolver['run']
+	else:
+		runner = resolver['run']['runners'].get(runner_name)
+		if runner is None:
+			raise Exception(f'invalid runner `{runner_name}`')
+	cmds = _pretty_print_sh(runner_name, runner)
 
 	# If dry run, just output the command that we would have run. We
 	# don't include the netcat magic to access the fvp terminals.
@@ -100,7 +108,7 @@ def dispatch(args):
 	# that tag field needs to be so that everything remains aligned.
 	max_name_field = 10
 	name_field = 0
-	terminals = resolver['run']['terminals']
+	terminals = runner['terminals']
 	terminals = dict(sorted(terminals.items()))
 	for t in terminals.values():
 		t['port'] = None
@@ -236,7 +244,7 @@ def dispatch(args):
 	with runtime.Runtime(name=args.runtime, image=config.get_image([resolveb], args),
 		       		ssh_agent_keys=args.ssh_agent_keys,
 				timeout=args.timeout) as rt:
-		for rtvar in resolver['run']['rtvars'].values():
+		for rtvar in runner['rtvars'].values():
 			if rtvar['type'] == 'path':
 				rt.add_volume(rtvar['value'])
 		for t in terminals.values():
@@ -266,7 +274,7 @@ def dispatch(args):
 			pm = process.ProcessManager(_find_term_ports, _complete)
 			pm.add(process.Process(f'bash {tmpfilename}',
 				False,
-				(log.alloc_data('FVP', not args.no_color),),
+				(log.alloc_data(runner_name, not args.no_color),),
 				True))
 
 			rt_ip = runtime.get().ip_address()
@@ -280,21 +288,18 @@ def dispatch(args):
 			pm.run(forward_stdin=True)
 
 
-def _pretty_print_sh(run):
-	prerun = run['prerun']
-	run = run['run']
+def _pretty_print_sh(runner_name, runner):
+	prerun = runner['prerun']
+	run = runner['run']
 
 	# This is a hack to improve the way the FVP arguments look. It tends to
 	# be huge so attempt to make it more readable by putting each option on
-	# a separate line and sorting alphabetically. Only likely to work for
-	# FVP so try to infer its definitely the FVP.
-	if len(run) == 1:
-		prog = run[0].split(' ')[0].lower()
-		if prog.find('isim') >= 0 or prog.find('fvp') >= 0:
-			parts = run[0].split(' -')
-			prog = parts[0]
-			args = sorted(parts[1:])
-			run = [' \\\n    -'.join([prog] + args)]
+	# a separate line and sorting alphabetically.
+	if runner_name == 'FVP':
+		parts = run[0].split(' -')
+		prog = parts[0]
+		args = sorted(parts[1:])
+		run = [' \\\n    -'.join([prog] + args)]
 
 	pre = config.script_preamble(False)
 	script = config.Script('run model', preamble=pre)
diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 05a8127..579e9d5 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -89,6 +89,14 @@ def _buildex_normalize(buildex):
 	"""
 	buildex.setdefault('btvars', {})
 
+def _runners_normalize(runners):
+	for runner in runners.values():
+		runner.setdefault('name', None)
+		runner.setdefault('rtvars', {})
+		runner.setdefault('params', [])
+		runner.setdefault('prerun', [])
+		runner.setdefault('run', [])
+		runner.setdefault('terminals', {})
 
 def _run_normalize(run):
 	"""
@@ -100,6 +108,9 @@ def _run_normalize(run):
 	run.setdefault('prerun', [])
 	run.setdefault('run', [])
 	run.setdefault('terminals', {})
+	run.setdefault('runner', None)
+	run.setdefault('runners', {})
+	_runners_normalize(run['runners'])
 
 
 def _config_normalize(config):
@@ -172,7 +183,7 @@ def _run_sort(run):
 	Sort the run section so that the keys are in a canonical order. This
 	improves readability by humans.
 	"""
-	lut = ['name', 'rtvars', 'params', 'prerun', 'run', 'terminals']
+	lut = ['name', 'rtvars', 'params', 'prerun', 'run', 'terminals', 'runners', 'runner']
 	lut = {k: i for i, k in enumerate(lut)}
 	return dict(sorted(run.items(), key=lambda x: lut[x[0]]))
 
@@ -634,6 +645,17 @@ def resolveb(config, btvars={}, clivars={}):
 
 	return _config_sort(config)
 
+def _resolve_run(run, lut):
+	# Now create a lookup table with all the rtvars and resolve all the
+	# parameters. An exception will be thrown if there are any macros that
+	# we don't have values for.
+	lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()}
+
+	for i, s, in enumerate(run['run']):
+		run['run'][i] = _string_substitute(s, lut)
+
+	for i, s in enumerate(run['prerun']):
+		run['prerun'][i] = _string_substitute(s, lut)
 
 def resolver(config, rtvars={}, clivars={}):
 	"""
@@ -643,21 +665,25 @@ def resolver(config, rtvars={}, clivars={}):
 	clivars = uclivars.get(**clivars)
 	run = config['run']
 
-	# Find the list of imported artifacts before any processing passes
 	artifacts_imp = set()
-	_string_extract_artifacts(artifacts_imp, run['params'].values())
-	_string_extract_artifacts(artifacts_imp, run['prerun'])
-	_string_extract_artifacts(artifacts_imp, run['run'])
-	_string_extract_artifacts(artifacts_imp, run['rtvars'].values())
-
-	#Override the rtvars with any values supplied by the user and check that
-	#all rtvars are defined.
-	for k, v in run['rtvars'].items():
-		if k in rtvars:
-			v['value'] = rtvars[k]
-		if v['value'] is None:
-			raise Exception(f'{k} run-time variable not ' \
-					'set by user and no default available.')
+	all_runners = [run] + list(run['runners'].values())
+
+	for runner in all_runners:
+		params = runner['params'].values() if type(runner['params']) == dict else runner['params']
+		# Find the list of imported artifacts before any processing passes
+		_string_extract_artifacts(artifacts_imp, params)
+		_string_extract_artifacts(artifacts_imp, runner['prerun'])
+		_string_extract_artifacts(artifacts_imp, runner['run'])
+		_string_extract_artifacts(artifacts_imp, runner['rtvars'].values())
+
+		# Override the rtvars with any values supplied by the user and check
+		# that all rtvars are defined.
+		for k, v in runner['rtvars'].items():
+			if k in rtvars:
+				v['value'] = rtvars[k]
+			if v['value'] is None:
+				raise Exception(f'{k} run-time variable not ' \
+						'set by user and no default available.')
 
 	# Update the artifacts so that the destination now points to an absolute
 	# path rather than one that is implictly relative to SHRINKWRAP_PACKAGE.
@@ -684,19 +710,21 @@ def resolver(config, rtvars={}, clivars={}):
 		'btvar': {k: v['value']
 				for k, v in config['buildex']['btvars'].items()},
 	}
-	for v in run['rtvars'].values():
-		v['value'] = _string_substitute(str(v['value']), lut)
-		if v['type'] == 'path' and v['value']:
-			v['value'] = os.path.expanduser(v['value'])
-			v['value'] = os.path.abspath(v['value'])
 
-	# Now create a lookup table with all the rtvars and resolve all the
-	# parameters. An exception will be thrown if there are any macros that
-	# we don't have values for.
-	lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()}
+	for runner in all_runners:
+		for v in runner['rtvars'].values():
+			v['value'] = _string_substitute(str(v['value']), lut)
+			if v['type'] == 'path' and v['value']:
+				v['value'] = os.path.expanduser(v['value'])
+				v['value'] = os.path.abspath(v['value'])
 
-	# Assemble the final runtime command and stuff it into the config.
+	if run['runner'] is None:
+		run['runner'] = 'FVP'
 
+	# Assemble the final runtime commands and stuff them into the config.
+	# For backward compatibility, run['run'] will contain the FVP command-line,
+	# and run['runners'][r]['run'] contain the command-line of each of the
+	# other runners.
 	if run["name"]:
 		terms = []
 		params = _mk_params(run['params'], '=')
@@ -712,12 +740,14 @@ def resolver(config, rtvars={}, clivars={}):
 				terms.append(f'-C {param}.mode=telnet')
 
 		run['run'] = [' '.join([run["name"], params] + terms)]
-
-	for i, s, in enumerate(run['run']):
-		run['run'][i] = _string_substitute(s, lut)
-
-	for i, s in enumerate(run['prerun']):
-		run['prerun'][i] = _string_substitute(s, lut)
+	_resolve_run(run, lut)
+
+	# Additional runners use a list of params rather than a dict
+	for runner in run['runners'].values():
+		if runner['name']:
+			params = ' '.join(runner['params'])
+			runner['run'] = [' '.join([runner['name'], params])]
+		_resolve_run(runner, lut)
 
 	return _config_sort(config)
 
-- 
GitLab


From 086ca33ae50a037a7170b9d29e0a7d08451ce0cf Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 8 May 2025 09:44:03 +0100
Subject: [PATCH 05/12] config: Add buildex["runners"]

Different runners need different versions of a given component.
For example runner QEMU needs a different build of EDK2 than FVP. The
simplest solution is to have a new component "edk2-qemu", but this means
"edk2" for FVP will still be downloaded and built, which seems wasteful.
Overriding the "edk2" component directly isn't possible since parts of
the component will be concatenated rather than replaced.

Introduce a new "runners" section under buildex, which completely
overrides components, depending on the value of run["runner"].

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/configmodel.rst |  1 +
 shrinkwrap/utils/config.py              | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst
index b91ca3d..cc9ab0f 100644
--- a/documentation/userguide/configmodel.rst
+++ b/documentation/userguide/configmodel.rst
@@ -184,6 +184,7 @@ created.
 key         type        description
 =========== =========== ===========
 btvars      dictionary  Build-Time variables. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Build-Time variables can be overridden by the user at the command line.
+runners     dictionary  Components built per runner. If a given runner is selected in the run section, components in its buildex section are built, and replace ones with the same name in the build section.
 =========== =========== ===========
 
 ~~~~~~~~~~~~~~~~~
diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 579e9d5..dd62cec 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -89,6 +89,10 @@ def _buildex_normalize(buildex):
 	"""
 	buildex.setdefault('btvars', {})
 
+	for build in buildex.setdefault('runners', {}).values():
+		for name, component in build.items():
+			_component_normalize(component, name)
+
 def _runners_normalize(runners):
 	for runner in runners.values():
 		runner.setdefault('name', None)
@@ -574,6 +578,14 @@ def resolveb(config, btvars={}, clivars={}):
 		for v in config['buildex']['btvars'].values():
 			v['value'] = _string_substitute(v['value'], lut, final)
 
+	# If the runner is different from the default, override some of the
+	# components by those defined in buildex for this runner.
+	runner = config['run']['runner']
+	build_components_override = config['buildex']['runners'].get(runner, {})
+
+	for name, component in build_components_override.items():
+		config['build'][name] = component
+
 	# Compute the source and build directories for each component. If they
 	# are already present, then don't override. This allows users to supply
 	# their own source and build tree locations.
-- 
GitLab


From 329a922069f7bee42601732e173ca793bbecf6b7 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Wed, 14 Aug 2024 11:36:01 +0100
Subject: [PATCH 06/12] config: Validate rtvars type

Make sure the config contains a valid type for each rtvar. This allows
easier debugging of configuration and overlay, by stating which rtvar is
malformed.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index dd62cec..255e26e 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -730,6 +730,11 @@ def resolver(config, rtvars={}, clivars={}):
 				v['value'] = os.path.expanduser(v['value'])
 				v['value'] = os.path.abspath(v['value'])
 
+			# Also check that the rtvar is well formed
+			t = v.get('type')
+			if t not in ('path', 'string'):
+				raise Exception(f'invalid type `{t}` for run-time variable {k}')
+
 	if run['runner'] is None:
 		run['runner'] = 'FVP'
 
-- 
GitLab


From 2e7f673b7419e5e69cf456f587d2563f21236d53 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 15 Aug 2024 17:44:26 +0100
Subject: [PATCH 07/12] run: Add per-process stdout handlers

Simplify the telnet header stripping by registering a handler for each
process that needs it, instead of a global handler.

This will be necessary in order to set up the terminal processes
one by one: if the terminal starts logging before the handler is
switched to the telnet handler, then we may miss the header and never
output anything.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/commands/run.py  | 32 +++++++++++---------------------
 shrinkwrap/utils/process.py |  7 ++++++-
 2 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py
index 6c7daca..7c683f1 100644
--- a/shrinkwrap/commands/run.py
+++ b/shrinkwrap/commands/run.py
@@ -112,7 +112,6 @@ def dispatch(args):
 	terminals = dict(sorted(terminals.items()))
 	for t in terminals.values():
 		t['port'] = None
-		t['strip'] = False
 		if len(t['friendly']) > name_field:
 			name_field = min(len(t['friendly']), max_name_field)
 
@@ -124,19 +123,14 @@ def dispatch(args):
 		few lines of output, which is output by telnet. This is
 		confusing for users since telnet is an implementation detail.
 		"""
-
+		skip_line = True
 		match = "Escape character is '^]'."
-		pdata = proc.data
-
 		for line in logger.splitlines(data):
-			if len(pdata) >= 2 and terminals[pdata[1]]['strip']:
-				if line.find(match) >= 0:
-					terminals[pdata[1]]['strip'] = False
-					if all([not t['strip'] \
-						for t in terminals.values()]):
-						pm.set_handler(log.log)
-			else:
+			if not skip_line:
 				log.log(pm, proc, line, streamid)
+			elif line.find(match) >= 0:
+				proc.set_handler(None)
+				skip_line = False
 
 	def _colorize(global_no_color, terminal):
 		if global_no_color:
@@ -187,7 +181,6 @@ def dispatch(args):
 		# and change the handler so we never get called again.
 		if found_all_ports:
 			wait = False
-			strip = False
 			for k, t in terminals.items():
 				name = t['friendly']
 				type = t['type']
@@ -206,9 +199,8 @@ def dispatch(args):
 					pm.add(process.Process(cmd,
 						True,
 						(log.alloc_data(name, colorize, escape, _logfile(t)), k),
-						False))
-					t['strip'] = True
-					strip = True
+						False,
+						handler=_strip_telnet_header))
 				if type in ['xterm']:
 					# Nothing to do. The FVP will start this
 					# automatically.
@@ -225,10 +217,7 @@ def dispatch(args):
 				input("Press Enter to continue...")
 				pm._stdin_activate()
 
-			if strip:
-				pm.set_handler(_strip_telnet_header)
-			else:
-				pm.set_handler(log.log)
+			proc.set_handler(None)
 
 	def _complete(pm, proc, retcode):
 		log.free_data(proc.data[0])
@@ -271,11 +260,12 @@ def dispatch(args):
 			# have terminated. The runner will terminate when its told
 			# to `poweroff` and netcat will terminate when it sees
 			# the runner has gone.
-			pm = process.ProcessManager(_find_term_ports, _complete)
+			pm = process.ProcessManager(log.log, _complete)
 			pm.add(process.Process(f'bash {tmpfilename}',
 				False,
 				(log.alloc_data(runner_name, not args.no_color),),
-				True))
+				True,
+				handler=_find_term_ports))
 
 			rt_ip = runtime.get().ip_address()
 
diff --git a/shrinkwrap/utils/process.py b/shrinkwrap/utils/process.py
index d0d9d4f..1fd26da 100644
--- a/shrinkwrap/utils/process.py
+++ b/shrinkwrap/utils/process.py
@@ -21,17 +21,20 @@ class Process:
 	"""
 	A wrapper to a process that should be managed by the ProcessManager.
 	"""
-	def __init__(self, args, interactive, data, run_to_end):
+	def __init__(self, args, interactive, data, run_to_end, handler=None):
 		self.args = shlex.split(args)
 		self.interactive = interactive
 		self.data = data
 		self.run_to_end = run_to_end
+		self._handler = handler
 		self._popen = None
 		self._stdout = None
 		self._stdin = None
 		self._stderr = None
 		self._active = 0
 
+	def set_handler(self, handler):
+		self._handler = handler
 
 class ProcessManager:
 	"""
@@ -101,6 +104,8 @@ class ProcessManager:
 
 		if data == '':
 			self._proc_stream_deactivate(proc, key.fileobj, streamid)
+		elif proc._handler:
+			proc._handler(self, proc, data, streamid)
 		elif self._handler:
 			self._handler(self, proc, data, streamid)
 
-- 
GitLab


From 2a6f47de8ed877d6c5d12cc7dceb9baba8f143eb Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 15 Aug 2024 16:04:26 +0100
Subject: [PATCH 08/12] run: Connect to ports one by one

QEMU waits for a connection to each console port sequentially, but
shrinkwrap looks for all port information before connecting. To avoid
hanging at boot, connect to a port as soon as it is announced.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/commands/run.py | 74 +++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 33 deletions(-)

diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py
index 7c683f1..bc90f09 100644
--- a/shrinkwrap/commands/run.py
+++ b/shrinkwrap/commands/run.py
@@ -112,6 +112,7 @@ def dispatch(args):
 	terminals = dict(sorted(terminals.items()))
 	for t in terminals.values():
 		t['port'] = None
+		t['started'] = False
 		if len(t['friendly']) > name_field:
 			name_field = min(len(t['friendly']), max_name_field)
 
@@ -153,6 +154,32 @@ def dispatch(args):
 		else:
 			return None
 
+	def _launch_term(pm, logname, t):
+		name = t['friendly']
+		type = t['type']
+		port = t["port"]
+		colorize = _colorize(args.no_color, t)
+		escape = _escape(t)
+
+		if type in ['stdout']:
+			cmd = f'nc localhost {port}'
+			pm.add(process.Process(cmd,
+				False,
+				(log.alloc_data(name, colorize, escape, _logfile(t)), logname),
+				False))
+		if type in ['stdinout']:
+			cmd = f'telnet localhost {port}'
+			pm.add(process.Process(cmd,
+				True,
+				(log.alloc_data(name, colorize, escape, _logfile(t)), logname),
+				False,
+				# Initially use the special handler to skip telnet header
+				handler=_strip_telnet_header))
+		if type in ['telnet']:
+			ip = runtime.get().ip_address()
+			print(f'To start {name} terminal, run:')
+			print(f'    telnet {ip} {port}')
+
 	def _find_term_ports(pm, proc, data, streamid):
 		"""
 		Initial handler function called by ProcessManager. When the runner
@@ -169,47 +196,28 @@ def dispatch(args):
 		# Iterate over the terminals dict from the config applying the
 		# supplied regexes to try to find the ports for all the
 		# terminals.
-		for t in terminals.values():
+		for k, t in terminals.items():
+			if t['started']:
+				continue
+
 			if t['port'] is None:
 				res = re.search(t['port_regex'], data)
 				if res:
 					t['port'] = res.group(1)
 				else:
 					found_all_ports = False
+					continue
+
+			# Not yet started but we found a port: launch the netcat process.
+			if t['port'] is not None:
+				_launch_term(pm, k, t);
+
+			t['started'] = True
 
-		# Once all ports have been found, launch the netcat processes
-		# and change the handler so we never get called again.
+		# Once all ports have been found change the handler so we never get
+		# called again.
 		if found_all_ports:
-			wait = False
-			for k, t in terminals.items():
-				name = t['friendly']
-				type = t['type']
-				port = t["port"]
-				colorize = _colorize(args.no_color, t)
-				escape = _escape(t)
-
-				if type in ['stdout']:
-					cmd = f'nc localhost {port}'
-					pm.add(process.Process(cmd,
-						False,
-						(log.alloc_data(name, colorize, escape, _logfile(t)), k),
-						False))
-				if type in ['stdinout']:
-					cmd = f'telnet localhost {port}'
-					pm.add(process.Process(cmd,
-						True,
-						(log.alloc_data(name, colorize, escape, _logfile(t)), k),
-						False,
-						handler=_strip_telnet_header))
-				if type in ['xterm']:
-					# Nothing to do. The FVP will start this
-					# automatically.
-					pass
-				if type in ['telnet']:
-					wait = True
-					ip = runtime.get().ip_address()
-					print(f'To start {name} terminal, run:')
-					print(f'    telnet {ip} {port}')
+			wait = any(t['type'] == 'telnet' for t in terminals.values())
 			if wait:
 				# Temporarily restore sys.stdin for input().
 				pm._stdin_deactivate()
-- 
GitLab


From c4db6618aec68db0f7173ffed9dadb814a07b801 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Fri, 2 Aug 2024 15:46:12 +0100
Subject: [PATCH 09/12] configs: Add qemu/cca overlay

Using the cca-3world config with the qemu/cca.yaml overlay builds the
CCA stack and runs it on QEMU.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 config/qemu/cca.yaml        | 91 +++++++++++++++++++++++++++++++++++++
 config/qemu/edk2-base.yaml  | 85 ++++++++++++++++++++++++++++++++++
 config/qemu/host-base.yaml  | 26 +++++++++++
 config/qemu/host-build.yaml | 31 +++++++++++++
 config/qemu/rmm-base.yaml   | 26 +++++++++++
 config/qemu/tfa-base.yaml   | 32 +++++++++++++
 6 files changed, 291 insertions(+)
 create mode 100644 config/qemu/cca.yaml
 create mode 100644 config/qemu/edk2-base.yaml
 create mode 100644 config/qemu/host-base.yaml
 create mode 100644 config/qemu/host-build.yaml
 create mode 100644 config/qemu/rmm-base.yaml
 create mode 100644 config/qemu/tfa-base.yaml

diff --git a/config/qemu/cca.yaml b/config/qemu/cca.yaml
new file mode 100644
index 0000000..f61130e
--- /dev/null
+++ b/config/qemu/cca.yaml
@@ -0,0 +1,91 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  Boot the cca-3world stack with QEMU. Use this as overlay with the cca-3world config.
+
+layers:
+  - qemu/host-base.yaml
+  - qemu/edk2-base.yaml
+  - qemu/tfa-base.yaml
+  - qemu/rmm-base.yaml
+  # After host-base.yaml, because it overrides the runner name
+  - qemu/host-build.yaml
+
+buildex:
+  runners:
+    QEMU:
+      tfa:
+        params:
+          ENABLE_RME: 1
+          RMM: ${artifact:RMM}
+          RME_GPT_BITLOCK_BLOCK: 1
+
+      rmm:
+        repo:
+          remote: https://git.codelinaro.org/linaro/dcap/rmm
+          revision: cca/v8
+
+      edk2:
+        params:
+          -D: ENABLE_RME
+
+        prebuild:
+          - export PLATFORM_NAME=SbsaQemuRme
+
+        artifacts:
+          EDK2_FLASH0: ${param:sourcedir}/Build/SbsaQemuRme/RELEASE_GCC5/FV/SBSA_FLASH0.fd
+          EDK2_FLASH1: ${param:sourcedir}/Build/SbsaQemuRme/RELEASE_GCC5/FV/SBSA_FLASH1.fd
+
+run:
+  runners:
+    QEMU:
+      rtvars:
+        KERNEL:
+          type: path
+          value: ${artifact:KERNEL}
+
+        ROOTFS:
+          type: path
+          value: ${artifact:BUILDROOT}
+
+        SHARE:
+          type: path
+          value: ${param:packagedir}
+
+        CMDLINE:
+          type: string
+          value: root=/dev/vda console=ttyAMA0
+
+      params:
+        - -nographic
+        - -nodefaults
+        - -M sbsa-ref
+        - -cpu max,x-rme=on,sme=off,pauth-impdef=on
+        - -drive format=raw,id=hd0,if=none,file='${rtvar:ROOTFS}'
+        - -device virtio-blk-pci,drive=hd0
+        - -chardev socket,id=chr0,port=13557,to=13657,host=0.0.0.0,server=on,wait=on,telnet=on,mux=on
+        - -chardev socket,id=chr1,port=13558,to=13658,host=0.0.0.0,server=on,wait=on,telnet=on
+        - -serial chardev:chr0
+        - -serial chardev:chr1
+        - -mon chr0
+        - -device virtio-9p-pci,fsdev=shr0,mount_tag=shr0
+        - -fsdev local,security_model=none,path='${rtvar:SHARE}',id=shr0
+        - -device virtio-net-pci,netdev=net0
+        - -netdev user,id=net0
+
+      terminals:
+        # This one also provides the QEMU monitor, accessible with <ctrl-a>c
+        serial0:
+          friendly: ''
+          type: stdinout
+          no_color: true
+          port_regex: '-chardev socket,id=chr0.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on'
+          no_escapes: 'EFI stub: Booting Linux Kernel...'
+
+        serial1:
+          friendly: 'Secure'
+          type: stdout
+          port_regex: '-chardev socket,id=chr1.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on'
diff --git a/config/qemu/edk2-base.yaml b/config/qemu/edk2-base.yaml
new file mode 100644
index 0000000..32cd09a
--- /dev/null
+++ b/config/qemu/edk2-base.yaml
@@ -0,0 +1,85 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  EDK2 UEFI firmware implementation for QEMU SBSA. This depend on
+  Trusted-Firmware-A, and produces the flash images passed to QEMU.
+
+layers:
+  - ../edk2-base.yaml
+
+buildex:
+  runners:
+    QEMU:
+      edk2:
+        repo:
+          edk2:
+            remote: https://github.com/tianocore/edk2.git
+            revision: 336e7e06eb91fee6f87b2559772aed948fb7bbfe
+          edk2-platforms:
+            remote: https://github.com/tianocore/edk2-platforms.git
+            revision: 400b3e083d5353292def78ea52663c5717e679a2
+          edk2-non-osi:
+            remote: https://github.com/tianocore/edk2-non-osi.git
+            revision: 45e337daa24b5595be4b81c9f6d379bb105dcc17
+
+        toolchain: aarch64-none-elf-
+
+        stderrfilt: true
+
+        prebuild:
+          - export WORKSPACE=${param:sourcedir}
+          - export GCC5_AARCH64_PREFIX=$$CROSS_COMPILE
+          - export PACKAGES_PATH=$$WORKSPACE/edk2:$$WORKSPACE/edk2-platforms:$$WORKSPACE/edk2-non-osi
+          - export IASL_PREFIX=${artifact:ACPICA}/
+          - export PYTHON_COMMAND=/usr/bin/python3
+          - export PLATFORM_NAME=SbsaQemu
+
+          - cp ${artifact:BL1} ${param:sourcedir}/edk2-non-osi/Platform/Qemu/Sbsa/
+          - cp ${artifact:FIP} ${param:sourcedir}/edk2-non-osi/Platform/Qemu/Sbsa/
+
+        params:
+          -a: AARCH64
+          -t: GCC5
+          -b: RELEASE
+          -p: edk2-platforms/Platform/Qemu/SbsaQemu/SbsaQemu.dsc
+          --pcd: PcdUefiShellDefaultBootEnable=1
+          ' --pcd': PcdShellDefaultDelay=0
+
+        build:
+          - source edk2/edksetup.sh --reconfig
+          - make -j${param:jobs} -C edk2/BaseTools
+          - build -n ${param:jobs} ${param:join_space}
+          - truncate -s 256M ${param:sourcedir}/Build/$$PLATFORM_NAME/RELEASE_GCC5/FV/SBSA_FLASH0.fd
+          - truncate -s 256M ${param:sourcedir}/Build/$$PLATFORM_NAME/RELEASE_GCC5/FV/SBSA_FLASH1.fd
+
+        artifacts:
+          EDK2_FLASH0: ${param:sourcedir}/Build/SbsaQemu/RELEASE_GCC5/FV/SBSA_FLASH0.fd
+          EDK2_FLASH1: ${param:sourcedir}/Build/SbsaQemu/RELEASE_GCC5/FV/SBSA_FLASH1.fd
+
+run:
+  runners:
+    QEMU:
+      prerun:
+        # Create the virtual boot disk containing the EFI shell. Wrap this up
+        # as a command in the startup.nsh along with the command line. UEFI
+        # will execute this when entering its shell. Using a unique temp
+        # directory means we can run multiple instances in parallel.
+        - BOOT_DISK=`mktemp -d`
+        - function finish { rm -rf $$BOOT_DISK; }
+        - trap finish EXIT
+        - mkdir -p $$BOOT_DISK/
+        - cp ${rtvar:KERNEL} $$BOOT_DISK/
+        - cat << EOF > $$BOOT_DISK/startup.nsh
+        - mode 100 31
+        - pci
+        - fs0:\Image ${rtvar:CMDLINE}
+        - reset -c
+        - EOF
+
+      params:
+        - -drive file=${artifact:EDK2_FLASH0},format=raw,if=pflash
+        - -drive file=${artifact:EDK2_FLASH1},format=raw,if=pflash
+        - -drive file=fat:rw:$$BOOT_DISK,format=raw
diff --git a/config/qemu/host-base.yaml b/config/qemu/host-base.yaml
new file mode 100644
index 0000000..cfc7e0b
--- /dev/null
+++ b/config/qemu/host-base.yaml
@@ -0,0 +1,26 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  Select QEMU as the runner, and set some basic parameters.
+
+run:
+  runner: QEMU
+  runners:
+    QEMU:
+      name: qemu-system-aarch64
+      rtvars:
+        MEM_SIZE:
+          value: 4G
+          type: string
+
+        # Number of vCPUs
+        SMP:
+          value: 8
+          type: string
+
+      params:
+        - -m ${rtvar:MEM_SIZE}
+        - -smp ${rtvar:SMP}
diff --git a/config/qemu/host-build.yaml b/config/qemu/host-build.yaml
new file mode 100644
index 0000000..3c46708
--- /dev/null
+++ b/config/qemu/host-build.yaml
@@ -0,0 +1,31 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  Build QEMU instead of using the one available in the container.
+
+build:
+  qemu:
+    repo:
+      remote: https://gitlab.com/qemu-project/qemu.git
+      revision: master
+
+    build:
+      - cd ${param:builddir}
+      - ${param:sourcedir}/configure --target-list=aarch64-softmmu --prefix=${param:builddir}/install
+      - make -j${param:jobs} install
+
+    artifacts:
+      QEMU: ${param:builddir}/install/bin/qemu-system-aarch64
+      # The SBSA machine needs a ROM for bochs-display. virtio-net-pci needs
+      # one too.
+      QEMU_ROMS: ${param:builddir}/install/share/qemu
+
+run:
+  runners:
+    QEMU:
+      name: ${artifact:QEMU}
+      params:
+        - -L ${artifact:QEMU_ROMS}
diff --git a/config/qemu/rmm-base.yaml b/config/qemu/rmm-base.yaml
new file mode 100644
index 0000000..57741dc
--- /dev/null
+++ b/config/qemu/rmm-base.yaml
@@ -0,0 +1,26 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+buildex:
+  runners:
+    QEMU:
+      rmm:
+        repo:
+          remote: https://git.trustedfirmware.org/TF-RMM/tf-rmm.git
+          revision: tf-rmm-v0.6.0
+
+        toolchain: aarch64-none-elf-
+
+        params:
+          -DRMM_CONFIG: qemu_sbsa_defcfg
+          -DCMAKE_BUILD_TYPE: Debug
+          -DLOG_LEVEL: 40
+
+        build:
+          - cmake ${param:join_equal} -S . -B ${param:builddir}
+          - cmake --build ${param:builddir} -j ${param:jobs}
+
+        artifacts:
+          RMM: ${param:builddir}/Debug/rmm.img
diff --git a/config/qemu/tfa-base.yaml b/config/qemu/tfa-base.yaml
new file mode 100644
index 0000000..8da04d1
--- /dev/null
+++ b/config/qemu/tfa-base.yaml
@@ -0,0 +1,32 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  Trusted Firmware for QEMU virt. This provides a baseline configuration that
+  can be customized by higher layers.
+
+buildex:
+  runners:
+    QEMU:
+      tfa:
+        repo:
+          remote: https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git
+          revision: v2.13-rc0
+
+        toolchain: aarch64-none-elf-
+
+        params:
+          PLAT: qemu_sbsa
+          DEBUG: 0
+          LOG_LEVEL: 40
+
+        build:
+          # tfa has makefile dependency bug that makes parallel make for more than
+          # ~8 jobs unreliable, so limit it to 8.
+          - "make BUILD_BASE=${param:builddir} ${param:join_equal} -j$$(( ${param:jobs} < 8 ? ${param:jobs} : 8 )) all fip"
+
+        artifacts:
+          BL1: ${param:builddir}/qemu_sbsa/release/bl1.bin
+          FIP: ${param:builddir}/qemu_sbsa/release/fip.bin
-- 
GitLab


From afd11c1366c29b5ed2c4e3f0543047d538d1718f Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Fri, 16 Aug 2024 11:25:52 +0100
Subject: [PATCH 10/12] config: Export string_subsitute()

This function is useful for handling variables in any string. Make it
public.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 255e26e..7b8eec8 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -298,8 +298,7 @@ def _string_tokenize(string, escape=True):
 
 	return tokens
 
-
-def _string_substitute(string, lut, final=True):
+def string_substitute(string, lut, final=True):
 	"""
 	Takes a string containg macros and returns a string with the macros
 	substituted for the values found in the lut. If final is False, any
@@ -542,7 +541,7 @@ def resolveb(config, btvars={}, clivars={}):
 
 			for desc in config['build'].values():
 				for v in desc['artifacts'].values():
-					v['path'] = _string_substitute(v['path'], artifact_lut, False)
+					v['path'] = string_substitute(v['path'], artifact_lut, False)
 
 			if artifact_nr > 0:
 				artifact_lut = _combine(config)
@@ -551,32 +550,32 @@ def resolveb(config, btvars={}, clivars={}):
 
 	def _substitute_macros(config, lut, final):
 		for desc in config['build'].values():
-			desc['sourcedir'] = _string_substitute(desc['sourcedir'], lut, final)
-			desc['builddir'] = _string_substitute(desc['builddir'], lut, final)
+			desc['sourcedir'] = string_substitute(desc['sourcedir'], lut, final)
+			desc['builddir'] = string_substitute(desc['builddir'], lut, final)
 
 			lut['param']['sourcedir'] = desc['sourcedir']
 			lut['param']['builddir'] = desc['builddir']
 
-			desc['params'] = { k: _string_substitute(v, lut, final) for k, v in desc['params'].items() }
+			desc['params'] = { k: string_substitute(v, lut, final) for k, v in desc['params'].items() }
 
 			lut['param']['join_equal'] = _mk_params(desc['params'], '=')
 			lut['param']['join_space'] = _mk_params(desc['params'], ' ')
 
 			for r in desc['repo'].values():
-				r['remote'] = _string_substitute(r['remote'], lut, final)
-				r['revision'] = _string_substitute(r['revision'], lut, final)
+				r['remote'] = string_substitute(r['remote'], lut, final)
+				r['revision'] = string_substitute(r['revision'], lut, final)
 
-			desc['toolchain'] = _string_substitute(desc['toolchain'], lut, final)
+			desc['toolchain'] = string_substitute(desc['toolchain'], lut, final)
 
 			for k in ( 'prebuild', 'build', 'postbuild', ):
-				desc[k] = [ _string_substitute(s, lut, final) for s in desc[k] ]
+				desc[k] = [ string_substitute(s, lut, final) for s in desc[k] ]
 
 			for v in desc['artifacts'].values():
-				v['path'] = _string_substitute(v['path'], lut, False)
-				v['base'] = _string_substitute(v['base'], lut, False)
+				v['path'] = string_substitute(v['path'], lut, False)
+				v['base'] = string_substitute(v['base'], lut, False)
 
 		for v in config['buildex']['btvars'].values():
-			v['value'] = _string_substitute(v['value'], lut, final)
+			v['value'] = string_substitute(v['value'], lut, final)
 
 	# If the runner is different from the default, override some of the
 	# components by those defined in buildex for this runner.
@@ -664,10 +663,10 @@ def _resolve_run(run, lut):
 	lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()}
 
 	for i, s, in enumerate(run['run']):
-		run['run'][i] = _string_substitute(s, lut)
+		run['run'][i] = string_substitute(s, lut)
 
 	for i, s in enumerate(run['prerun']):
-		run['prerun'][i] = _string_substitute(s, lut)
+		run['prerun'][i] = string_substitute(s, lut)
 
 def resolver(config, rtvars={}, clivars={}):
 	"""
@@ -725,7 +724,7 @@ def resolver(config, rtvars={}, clivars={}):
 
 	for runner in all_runners:
 		for v in runner['rtvars'].values():
-			v['value'] = _string_substitute(str(v['value']), lut)
+			v['value'] = string_substitute(str(v['value']), lut)
 			if v['type'] == 'path' and v['value']:
 				v['value'] = os.path.expanduser(v['value'])
 				v['value'] = os.path.abspath(v['value'])
-- 
GitLab


From eb19ce6b687295274299a8310a3d32b943c81df9 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Fri, 16 Aug 2024 13:51:40 +0100
Subject: [PATCH 11/12] run: Support launching a terminal locally

It is convenient to automatically spawn xterm when running multiple
interactive software, for example a CCA host and guest. The FVP already
supports this (when using the 'xterm' terminal type), and it is possible
with the 'telnet' terminal type but requires manually launching the
process.

Add a new 'term' type which takes an additional 'command' parameter and
is launched locally when the run starts.

An important security consideration is that this new option allows
launching arbitrary commands locally rather than sandboxed in a
container. But that was already possible by adding 'terminal_command'
parameters to the FVP. The user must trust the source of its
configurations.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/commands/run.py  | 17 +++++++++++++++++
 shrinkwrap/utils/process.py | 16 ++++++++++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py
index bc90f09..0593e76 100644
--- a/shrinkwrap/commands/run.py
+++ b/shrinkwrap/commands/run.py
@@ -179,6 +179,23 @@ def dispatch(args):
 			ip = runtime.get().ip_address()
 			print(f'To start {name} terminal, run:')
 			print(f'    telnet {ip} {port}')
+		if type in ['term']:
+			lut = {
+					'term': {
+						'host': runtime.get().ip_address(),
+						'port': port,
+						'name': name,
+					}
+				}
+			cmd = t.get('command')
+			if cmd is None:
+				raise Exception(f'{logname}: `term` type requires a command')
+			cmd = config.string_substitute(cmd, lut)
+			pm.add(process.Process(cmd,
+					False,
+					(log.alloc_data(name, colorize, escape, _logfile(t)), logname),
+					False,
+					local=True))
 
 	def _find_term_ports(pm, proc, data, streamid):
 		"""
diff --git a/shrinkwrap/utils/process.py b/shrinkwrap/utils/process.py
index 1fd26da..10a9848 100644
--- a/shrinkwrap/utils/process.py
+++ b/shrinkwrap/utils/process.py
@@ -21,7 +21,15 @@ class Process:
 	"""
 	A wrapper to a process that should be managed by the ProcessManager.
 	"""
-	def __init__(self, args, interactive, data, run_to_end, handler=None):
+	def __init__(self, args, interactive, data, run_to_end, handler=None, local=None):
+		"""
+		@args: array containing the command and its parameters
+		@interactive: True if the process requires stdin
+		@data: tuple (log data, log name)
+		@run_to_end: exit shrinkwrap on error
+		@handler: optional handler function for stdout logs
+		@local: execute the command locally instead of in the runtime
+		"""
 		self.args = shlex.split(args)
 		self.interactive = interactive
 		self.data = data
@@ -32,6 +40,7 @@ class Process:
 		self._stdin = None
 		self._stderr = None
 		self._active = 0
+		self._local = local
 
 	def set_handler(self, handler):
 		self._handler = handler
@@ -110,7 +119,10 @@ class ProcessManager:
 			self._handler(self, proc, data, streamid)
 
 	def _proc_activate(self, proc):
-		cmd = runtime.mkcmd(proc.args, proc.interactive)
+		if proc._local:
+			cmd = proc.args
+		else:
+			cmd = runtime.mkcmd(proc.args, proc.interactive)
 
 		if proc.interactive:
 			master, slave = pty.openpty()
-- 
GitLab


From 4cee76cfea5d479871e3401077ed4b73f15d52b6 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Fri, 16 Aug 2024 14:20:09 +0100
Subject: [PATCH 12/12] configs/qemu: Add multi-term.yaml

Extend the qemu/cca.yaml overlay with support for spawning multiple
terminals, which is very useful to launch commands in both host and
guest. This is enabled by adding the overlay directly to the run
command (not the build command):

  shrinkwrap run cca-3world.yaml -o qemu/multi-term.yaml

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 config/qemu/multi-term.yaml | 49 +++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 config/qemu/multi-term.yaml

diff --git a/config/qemu/multi-term.yaml b/config/qemu/multi-term.yaml
new file mode 100644
index 0000000..8784e3f
--- /dev/null
+++ b/config/qemu/multi-term.yaml
@@ -0,0 +1,49 @@
+# Copyright (c) 2024, Linaro Limited.
+# SPDX-License-Identifier: MIT
+
+%YAML 1.2
+---
+description: >-
+  Launch all terminals in separate consoles, and add a virtio-console port for
+  a guest. This should be used with a compatible overlay, like cca.yaml
+
+run:
+  runners:
+    QEMU:
+      params:
+        - -chardev socket,id=chr2,port=13559,to=13659,host=0.0.0.0,server=on,wait=on,telnet=on
+        - -device virtio-serial-pci
+        - -device virtconsole,chardev=chr2
+        - -chardev socket,id=chr3,port=13560,to=13660,host=0.0.0.0,server=on,wait=on,telnet=on
+        - -device virtio-serial-pci
+        - -device virtconsole,chardev=chr3
+
+      rtvars:
+        # Use the first virtio console for the host
+        CMDLINE:
+          type: string
+          value: root=/dev/vda console=hvc0 rw
+
+      terminals:
+        serial0:
+          friendly: 'Firmware'
+          type: term
+          command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port}
+
+# At the moment the secure console is unused and takes space.
+#        serial1:
+#          friendly: 'Secure'
+#          type: term
+#          command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port}
+
+        serial2:
+          friendly: 'Host'
+          type: term
+          command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port}
+          port_regex: '-chardev socket,id=chr2.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on'
+
+        serial3:
+          friendly: 'Guest'
+          type: term
+          command: xterm -T "${term:name}" -e telnet ${term:host} ${term:port}
+          port_regex: '-chardev socket,id=chr3.*QEMU waiting for connection on: disconnected:telnet:0.0.0.0:(\d+),server=on'
-- 
GitLab