From 9843cdc7cbe163e42b23c2f6b2302e405504d52f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Tue, 25 Feb 2025 17:01:03 +0000 Subject: [PATCH 1/5] test: Add --dry-run With -n/--dry-run, display all the commands that would be run, without executing anything. Signed-off-by: Jean-Philippe Brucker --- test/test.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/test/test.py b/test/test.py index e39b59e..dcab6e0 100755 --- a/test/test.py +++ b/test/test.py @@ -18,6 +18,7 @@ import yaml RUNTIME = None IMAGE = None FVPJOBS = None +DRY_RUN = False SCRIPTDIR = os.path.dirname(os.path.abspath(__file__)) @@ -233,13 +234,17 @@ def print_results(junit=None): if r['status'] == 'pass': nr_pass += 1 - print(f'pass: {nr_pass}, fail: {len(results) - nr_pass}') + dry_run = "(dry run) " if DRY_RUN else "" + print(f'{dry_run}pass: {nr_pass}, fail: {len(results) - nr_pass}') return nr_pass == len(results) def run(cmd, timeout=None, expect=0, capture=False): print(f'+ {cmd}') + if DRY_RUN: + return "" + ret = subprocess.run(cmd, timeout=timeout, shell=True, stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None) @@ -348,6 +353,19 @@ def run_configs(configs, overlay=None, rtvarss=None): sys.stdout.write(stdout.decode()) +def run_repo_sync_test(args): + if DRY_RUN: + ret = 0 + print(f"+ {SYNCTEST}") + else: + ret = subprocess.run(SYNCTEST).returncode + results.append({ + 'type': 'repo-sync-behaviours', + 'status': 'pass' if ret == 0 else 'fail', + 'error': None, + }) + + def do_main(args): arch_configs = [c for c in CONFIGS if 'arch' in c] noarch_configs = [c for c in CONFIGS if 'arch' not in c] @@ -374,13 +392,7 @@ def do_main(args): build_configs(configs, btvarss=btvarss) run_configs(configs, rtvarss=rtvarss) - # Run repo sync tests. - ret = subprocess.run(SYNCTEST).returncode - results.append({ - 'type': 'repo-sync-behaviours', - 'status': 'pass' if ret == 0 else 'fail', - 'error': None, - }) + run_repo_sync_test(args) success = print_results(args.junit) exit(not success) @@ -416,6 +428,10 @@ def main(): metavar='count', required=False, default=1, type=int, help="""Maximum number of FVPs to run in parallel.""") + parser.add_argument('-n', '--dry-run', + required=False, default=False, action='store_true', + help="""Do not build or run anything, only print what would be done.""") + parser.add_argument('-s', '--smoke-test', required=False, default=False, action='store_true', help="""If specified, run a smaller selection of tests.""") @@ -425,9 +441,11 @@ def main(): global RUNTIME global IMAGE global FVPJOBS + global DRY_RUN RUNTIME = args.runtime IMAGE = args.image FVPJOBS = args.fvpjobs + DRY_RUN = args.dry_run do_main(args) -- GitLab From a1aba111f04bf37955c4fbcc65a2cd204610664d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Tue, 25 Feb 2025 17:07:02 +0000 Subject: [PATCH 2/5] test: Add --config to select a single config to test Allow selecting for example '-c cca-3world.yaml' to only test this config. Note that all tests are run for a given config, if it has multiple rtvars settings or multiple tests: ./test.py -c cca-edk2.yaml PASS: build:cca-edk2.yaml PASS: run:cca-edk2.yaml:dt PASS: run:cca-edk2.yaml:acpi Signed-off-by: Jean-Philippe Brucker --- test/test.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/test.py b/test/test.py index dcab6e0..e35ee57 100755 --- a/test/test.py +++ b/test/test.py @@ -354,6 +354,10 @@ def run_configs(configs, overlay=None, rtvarss=None): def run_repo_sync_test(args): + # Cannot select synctest with `-c`, but it's easy to run manually + if args.config is not None: + return + if DRY_RUN: ret = 0 print(f"+ {SYNCTEST}") @@ -367,8 +371,14 @@ def run_repo_sync_test(args): def do_main(args): - arch_configs = [c for c in CONFIGS if 'arch' in c] - noarch_configs = [c for c in CONFIGS if 'arch' not in c] + selected_configs = [c for c in CONFIGS + if args.config is None or args.config == c['config']] + if not selected_configs: + print(f"Unknown config {args.config}") + exit(1) + + arch_configs = [c for c in selected_configs if 'arch' in c] + noarch_configs = [c for c in selected_configs if 'arch' not in c] if args.smoke_test: arches = set([c['arch']['end'] for c in arch_configs]) @@ -420,6 +430,10 @@ def main(): help="""If using a container runtime, specifies the name of the image to use. Defaults to the official shrinkwrap image.""") + parser.add_argument('-c', '--config', + metavar='name', required=False, default=None, + help="""Only test the given config.""") + parser.add_argument('-j', '--junit', metavar='file', required=False, default=None, help="""Optionally output results in junit format to specified file.""") -- GitLab From a26498b4e5730dd80ebdb65fbfb1cea434f847e8 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Tue, 25 Feb 2025 17:10:08 +0000 Subject: [PATCH 3/5] test: Gather tests by overlay Since the clean and buildall commands work on multiple configurations and apply the given overlays to all configurations, gather the tests by overlay. This will allow using extra overlays for some tests. Signed-off-by: Jean-Philippe Brucker --- test/test.py | 72 +++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/test/test.py b/test/test.py index e35ee57..6c6b1de 100755 --- a/test/test.py +++ b/test/test.py @@ -254,22 +254,19 @@ def run(cmd, timeout=None, expect=0, capture=False): return ret.stdout -def build_configs(configs, overlay=None, btvarss=None): +def build_configs(configs, overlays): status = 'pass' error = None rt = f'-R {RUNTIME} -I {IMAGE}' - cleanargs = f'{" ".join(configs)} {f"-o {overlay}" if overlay else ""}' - - if btvarss is None: - btvarss = [{}] * len(configs) - - assert(len(configs) == len(btvarss)) + cfg_files = [c['config'] for c in configs] + overlay_args = ' '.join(f'-o {o}' for o in overlays) + cleanargs = f'{" ".join(cfg_files)} {overlay_args}' cfgs = [] - for c, b in zip(configs, btvarss): - cfgs.append({'config': c, 'btvars': b}) + for c in configs: + cfgs.append({'config': c['config'], 'btvars': c['btvars']}) with tempfile.TemporaryDirectory() as tmpdir: tmpfilename = os.path.join(tmpdir, 'configs.yaml') @@ -281,7 +278,7 @@ def build_configs(configs, overlay=None, btvarss=None): version=(1, 2)) with open(tmpfilename, 'r') as tmpfile: print(tmpfile.read()) - buildargs = f'{tmpfilename} {f"-o {overlay}" if overlay else ""}' + buildargs = f'{tmpfilename} {overlay_args}' try: run(f'shrinkwrap {rt} clean {cleanargs}', None) @@ -295,10 +292,10 @@ def build_configs(configs, overlay=None, btvarss=None): 'type': 'build', 'status': status, 'error': error, - 'config': config, - 'overlay': overlay, - 'btvars': btvars, - } for config, btvars in zip(configs, btvarss)] + 'config': c['config'], + 'overlays': ",".join(overlays), + 'btvars': c['btvars'], + } for c in configs] def run_config(config, overlay, rtvars, tag, capture): @@ -313,14 +310,14 @@ def run_config(config, overlay, rtvars, tag, capture): 'status': 'fail', 'error': None, 'config': config, - 'overlay': overlay, + 'overlays': ','.join(overlays), 'rtvars': rtvars, 'tag': tag, } rt = f'-R {RUNTIME} -I {IMAGE}' - overlay = f'-o {overlay}' if overlay else '' - args = f'{config} {overlay} {runargs}' + overlay_args = ' '.join(f"-o {o}" for o in overlays) + args = f'{config} {overlay_args} {runargs}' try: stdout = run(f'shrinkwrap {rt} run {args}', @@ -336,15 +333,11 @@ def run_config(config, overlay, rtvars, tag, capture): return result, stdout -def run_configs(configs, overlay=None, rtvarss=None): - - if rtvarss is None: - rtvarss = [{'default': {}}] * len(configs) - +def run_configs(configs, overlays): params = [] - for config, _rtvars in zip(configs, rtvarss): - for tag, rtvars in _rtvars.items(): - params.append((config, overlay, rtvars, tag, FVPJOBS > 1)) + for c in configs: + for tag, rtvars in c['rtvars'].items(): + params.append((c['config'], overlays, rtvars, tag, FVPJOBS > 1)) with mp.Pool(processes=FVPJOBS) as pool: for result, stdout in pool.starmap(run_config, params): @@ -385,22 +378,27 @@ def do_main(args): else: arches = list(arch_range('v8.0', ARCH_LATEST)) + # Gather configs that have the same overlays, and can be built together + test_batches = {} + # Configs that support an arch override. for arch in arches: - configs = [c['config'] for c in arch_configs if arch_in_range(arch, c['arch']['end'] if args.smoke_test else c['arch']['start'], c['arch']['end'])] - btvarss = [c['btvars'] for c in arch_configs if arch_in_range(arch, c['arch']['end'] if args.smoke_test else c['arch']['start'], c['arch']['end'])] - rtvarss = [c['rtvars'] for c in arch_configs if arch_in_range(arch, c['arch']['end'] if args.smoke_test else c['arch']['start'], c['arch']['end'])] - if len(configs) > 0: - build_configs(configs, f'arch/{arch}.yaml', btvarss=btvarss) - run_configs(configs, f'arch/{arch}.yaml', rtvarss=rtvarss) + for c in arch_configs: + first_arch = c['arch']['end'] if args.smoke_test else c['arch']['start'] + if not arch_in_range(arch, first_arch, c['arch']['end']): + continue + + overlays = (f'arch/{arch}.yaml',) + tuple(c.get('overlays', ())) + test_batches.setdefault(overlays, []).append(c) # Configs that don't support an arch override. - configs = [c['config'] for c in noarch_configs] - btvarss = [c['btvars'] for c in noarch_configs] - rtvarss = [c['rtvars'] for c in noarch_configs] - if len(configs) > 0: - build_configs(configs, btvarss=btvarss) - run_configs(configs, rtvarss=rtvarss) + for c in noarch_configs: + overlays = tuple(c.get('overlays', ())) + test_batches.setdefault(overlays, []).append(c) + + for overlays, configs in test_batches.items(): + build_configs(configs, overlays) + run_configs(configs, overlays) run_repo_sync_test(args) -- GitLab From 746c3b7778926127dc5f884ba2cb5725dfab7818 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 26 Feb 2025 17:44:52 +0000 Subject: [PATCH 4/5] test: Add an optional "timeout" config options Configs can specify a longer timeout if they have long-running tests, like spawning guest VMs. The presence of a "timeout" option also removes the test from smoke tests, since it will most likely be used for timeouts longer than the default. Signed-off-by: Jean-Philippe Brucker --- test/test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/test.py b/test/test.py index 6c6b1de..386ce09 100755 --- a/test/test.py +++ b/test/test.py @@ -298,7 +298,7 @@ def build_configs(configs, overlays): } for c in configs] -def run_config(config, overlay, rtvars, tag, capture): +def run_config(config, overlays, rtvars, tag, capture, timeout): def make_rtcmds(rtvars): return ' '.join([f'-r {k}={v}' for k, v in rtvars.items()]) @@ -321,7 +321,7 @@ def run_config(config, overlay, rtvars, tag, capture): try: stdout = run(f'shrinkwrap {rt} run {args}', - timeout=600, capture=capture) + timeout=timeout, capture=capture) result['status'] = 'pass' except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e: stdout = e.stdout @@ -337,7 +337,8 @@ def run_configs(configs, overlays): params = [] for c in configs: for tag, rtvars in c['rtvars'].items(): - params.append((c['config'], overlays, rtvars, tag, FVPJOBS > 1)) + timeout = c.get('timeout', 600) + params.append((c['config'], overlays, rtvars, tag, FVPJOBS > 1, timeout)) with mp.Pool(processes=FVPJOBS) as pool: for result, stdout in pool.starmap(run_config, params): @@ -370,6 +371,10 @@ def do_main(args): print(f"Unknown config {args.config}") exit(1) + if args.smoke_test: + # Assume tests with a special timeout are long-running ones + selected_configs = [c for c in selected_configs if not 'timeout' in c] + arch_configs = [c for c in selected_configs if 'arch' in c] noarch_configs = [c for c in selected_configs if 'arch' not in c] -- GitLab From 54ff494a11a4fb9e6a1b9048b5e49fc032bb4ca9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 26 Feb 2025 17:52:05 +0000 Subject: [PATCH 5/5] test: Add CCA boot test Launch a guest and check that it is running in a Realm Signed-off-by: Jean-Philippe Brucker --- config/test/cca.yaml | 34 ++++++++++++++++++++++++++++++++++ test/test.py | 9 +++++++++ 2 files changed, 43 insertions(+) create mode 100644 config/test/cca.yaml diff --git a/config/test/cca.yaml b/config/test/cca.yaml new file mode 100644 index 0000000..9b28968 --- /dev/null +++ b/config/test/cca.yaml @@ -0,0 +1,34 @@ +# Copyright (c) 2025, Arm Limited. +# SPDX-License-Identifier: MIT + +%YAML 1.2 +--- +description: >- + Add an initscript to buildroot to automatically execute tests. + The test launches a guest and checks that it is running in a Realm. + +build: + buildroot: + prebuild: + - mkdir -p ${param:builddir}/overlay/etc/init.d + - | + echo -e "#!/bin/sh + case \$$1 in + start|stop|restart|reload) + if [ -e /dev/kvm ]; then + # Host + # On failure, exit this script and wait for the test timeout + gen-run-vmm.sh --kvmtool || exit 1 + else + # Guest + cca-workload-attestation report | grep cca-realm-initial-measurement || exit 1 + fi + poweroff -f + ;; + *) + echo Usage \$$0 {start|stop|restart|reload} + exit 1 + ;; + esac" > ${param:builddir}/overlay/etc/init.d/S50test + - chmod +x ${param:builddir}/overlay/etc/init.d/S50test + - ${param:sourcedir}/utils/config --file ${param:builddir}/.config --set-val BR2_ROOTFS_OVERLAY \"${param:builddir}/overlay\" diff --git a/test/test.py b/test/test.py index 386ce09..6d56ae3 100755 --- a/test/test.py +++ b/test/test.py @@ -112,6 +112,15 @@ CONFIGS = [ 'btvars': {'GUEST_ROOTFS': ROOTFS}, 'rtvars': {'default': {'KERNEL': KERNEL, 'ROOTFS': ROOTFS}}, }, + { + 'config': 'cca-3world.yaml', + 'btvars': {}, + 'rtvars': {'realm': {}}, + 'overlays': ['buildroot-cca.yaml', 'test/cca.yaml'], + # Building the whole userspace and booting multiple guests takes a while! + # 30min timeout should be enough. + 'timeout': 1800, + }, { 'config': 'cca-4world.yaml', 'btvars': {'GUEST_ROOTFS': ROOTFS}, -- GitLab