From f0583af1503866bacd37de6f545d224de260d333 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 10:25:23 +0100 Subject: [PATCH 01/12] config: cca-3world.yaml: cleanup CPU_IDLE workaround Now that we are using the upstream DT and not the TF-A one, there is no need to apply the CPU_IDLE workaround to Linux, since the upstream DT does not expose CPU idle controls. This should have been removed when moving to the upstream DT. This also prepares the ground for booting the host with EDK2/ACPI. Signed-off-by: Ryan Roberts --- config/cca-3world.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/config/cca-3world.yaml b/config/cca-3world.yaml index e744fa2..15531f7 100644 --- a/config/cca-3world.yaml +++ b/config/cca-3world.yaml @@ -56,17 +56,6 @@ build: remote: https://git.gitlab.arm.com/linux-arm/linux-cca.git revision: cca-full/rfc-v1 - prebuild: - # Disable CPU_IDLE as a workaround to speed up the FVP. Since we are using - # the TF-A DT, which provides CPU idle state parameters, it otherwise - # causes Linux to constantly enter cpu idle, slowing the FVP down. We - # can't easily use the upstream DT right now, due to some RAM having been - # carved out for the RMM and this is not reflected in that DT. CPU_IDLE is - # selected by ACPI, so we have to disable that too to maintain a legal - # config. That's OK for now since we are using the DT. - - ./scripts/config --file ${param:builddir}/.config --disable CONFIG_ACPI - - ./scripts/config --file ${param:builddir}/.config --disable CONFIG_CPU_IDLE - kvmtool: repo: dtc: -- GitLab From 2c4ed72015135bda0b7afeeae82b4141170d57f8 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 12:10:52 +0100 Subject: [PATCH 02/12] config: Support artifacts that are directories Previously, all artifacts had to be files. With this change, an artifact can be a directory, in which case it is recursively copied. Signed-off-by: Ryan Roberts --- documentation/userguide/config.rst | 2 +- shrinkwrap/utils/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/userguide/config.rst b/documentation/userguide/config.rst index c75c4df..d15c352 100644 --- a/documentation/userguide/config.rst +++ b/documentation/userguide/config.rst @@ -182,7 +182,7 @@ prebuild list List of shell commands to be executed during component b build list List of shell commands to be executed during component build. postbuild list List of shell commands to be executed during component build after the ``build`` list. clean list List of shell commands to be executed during component clean. -artifacts dictionary Set of artifacts that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. +artifacts dictionary Set of artifacts (files and/or directories) that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. =========== =========== =========== ----------- diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 943138f..e2bd22f 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -835,7 +835,7 @@ def build_graph(configs, echo): for artifact in config['artifacts'].values(): src = artifact['src'] dst = os.path.join(workspace.package, artifact['dst']) - a.append(f'cp {src} {dst}') + a.append(f'cp -r {src} {dst}') a.seal() graph[a] = [gl2] + [s for s in build_scripts.values()] -- GitLab From 4211fa3db560a24b0d8999449714cc7e8b3d114d Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 12:18:45 +0100 Subject: [PATCH 03/12] config: edk2-base.yaml: Split acpica into its own component acpica is a completely separate component to edk2, so model it as such. Now that we support artifacts that are directories, it is easy enough to expose the built tools to edk2. This change will prevent duplication when we shortly introduce a new component for the edk2 cca guest fw, which also depends on acpica. Signed-off-by: Ryan Roberts --- config/edk2-base.yaml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/config/edk2-base.yaml b/config/edk2-base.yaml index c2603fd..7527aef 100644 --- a/config/edk2-base.yaml +++ b/config/edk2-base.yaml @@ -9,6 +9,22 @@ description: >- Builds acpica from source as part of the build process. build: + acpica: + repo: + remote: https://github.com/acpica/acpica.git + revision: R10_20_22 + + build: + - rm -rf ${param:sourcedir}/generate/unix/acpica + - make -j${param:jobs} + - mv ${param:sourcedir}/generate/unix/bin ${param:sourcedir}/generate/unix/acpica + + clean: + - make -j${param:jobs} clean + + artifacts: + ACPICA: ${param:sourcedir}/generate/unix/acpica + edk2: repo: edk2: @@ -17,9 +33,6 @@ build: edk2-platforms: remote: https://github.com/tianocore/edk2-platforms.git revision: 20e07099d8f11889d101dd710ca85001be20e179 - acpica: - remote: https://github.com/acpica/acpica.git - revision: R10_20_22 toolchain: aarch64-none-elf- @@ -27,7 +40,7 @@ build: - export WORKSPACE=${param:sourcedir} - export GCC5_AARCH64_PREFIX=$$CROSS_COMPILE - export PACKAGES_PATH=$$WORKSPACE/edk2:$$WORKSPACE/edk2-platforms - - export IASL_PREFIX=$$WORKSPACE/acpica/generate/unix/bin/ + - export IASL_PREFIX=${artifact:ACPICA}/ - export PYTHON_COMMAND=/usr/bin/python3 params: @@ -37,7 +50,6 @@ build: -b: RELEASE build: - - make -j${param:jobs} -C acpica - source edk2/edksetup.sh --reconfig - make -j${param:jobs} -C edk2/BaseTools - build -n ${param:jobs} -D EDK2_OUT_DIR=${param:builddir} ${param:join_space} -- GitLab From e9fba050ae4a51d70431829738937e8a7a9ce39c Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 14:19:56 +0100 Subject: [PATCH 04/12] build: Suppress some stderr output if requested in config Add an optional component property, stderrfilt, which if present and set to true, causes stderr emitted during a component build to be suppressed unless they contain the words 'error' or 'warning'. Only applies when --verbose is not in use. Used by EDK2, which has an extremely chatty build process. Signed-off-by: Ryan Roberts --- config/edk2-base.yaml | 2 ++ documentation/userguide/config.rst | 1 + shrinkwrap/utils/config.py | 11 ++++++++--- shrinkwrap/utils/graph.py | 9 ++++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/config/edk2-base.yaml b/config/edk2-base.yaml index 7527aef..fcddd91 100644 --- a/config/edk2-base.yaml +++ b/config/edk2-base.yaml @@ -36,6 +36,8 @@ build: toolchain: aarch64-none-elf- + stderrfilt: true + prebuild: - export WORKSPACE=${param:sourcedir} - export GCC5_AARCH64_PREFIX=$$CROSS_COMPILE diff --git a/documentation/userguide/config.rst b/documentation/userguide/config.rst index d15c352..0b1a2eb 100644 --- a/documentation/userguide/config.rst +++ b/documentation/userguide/config.rst @@ -177,6 +177,7 @@ repo dictionary Specifies information about the git repo(s) that must be sourcedir string If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists. builddir string If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD. toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild/clean commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. +stderrfilt bool Optional, defaults to false. When true, and --verbose is not specified, filters stderr of the component's build task so that only lines containing 'error' and 'warning' are output. Everything else is suppressed. Useful for EDK2 which is extremely chatty. params dictionary Optional set of key:value pairs. When building most components, they require a set of parameters to be passed. By setting them out as a dictionary, it is easy to override and add to them in higher layers. See ``${param:join_*}`` macros. prebuild list List of shell commands to be executed during component build before the ``build`` list. build list List of shell commands to be executed during component build. diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index e2bd22f..5a0dbe6 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -37,6 +37,9 @@ def _component_normalize(component, name): if 'toolchain' not in component: component['toolchain'] = None + if 'stderrfilt' not in component: + component['stderrfilt'] = None + if 'prebuild' not in component: component['prebuild'] = [] @@ -143,7 +146,7 @@ def _component_sort(component): Sort the component so that the keys are in a canonical order. This improves readability by humans. """ - lut = ['repo', 'sourcedir', 'builddir', 'toolchain', 'params', + lut = ['repo', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt', 'params', 'prebuild', 'build', 'postbuild', 'clean', 'artifacts'] lut = {k: i for i, k in enumerate(lut)} return dict(sorted(component.items(), key=lambda x: lut[x[0]])) @@ -665,11 +668,13 @@ class Script: config=None, component=None, preamble=None, - final=False): + final=False, + stderrfilt=None): self.summary = summary self.config = config self.component = component self.final = final + self.stderrfilt = stderrfilt self._cmds = '' self._sealed = False self._preamble = preamble @@ -809,7 +814,7 @@ def build_graph(configs, echo): g.seal() graph[g] = [gl2] - b = Script('Building', config["name"], name, preamble=pre) + b = Script('Building', config["name"], name, preamble=pre, stderrfilt=component['stderrfilt']) if len(component['prebuild']) + \ len(component['build']) + \ len(component['postbuild']) > 0: diff --git a/shrinkwrap/utils/graph.py b/shrinkwrap/utils/graph.py index b80f4fc..84e2f0d 100644 --- a/shrinkwrap/utils/graph.py +++ b/shrinkwrap/utils/graph.py @@ -131,12 +131,19 @@ def execute(graph, tasks, verbose=False, colorize=True): _run_script(pm, data, frag) active += 1 + def _should_log(proc, data, streamid): + if streamid == process.STDERR and \ + (not proc.data[2].stderrfilt or \ + 'warning' in data or 'error' in data): + return True + return False + def _log(pm, proc, data, streamid): if verbose: log.log(pm, proc, data, streamid) else: proc.data[1].append(data) - if streamid == process.STDERR: + if _should_log(proc, data, streamid): log.log(pm, proc, data, streamid) lc.skip_overdraw_once() -- GitLab From 6f3754eeb16e4485d31eedd2f2432af26b2966bf Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 18:09:25 +0100 Subject: [PATCH 05/12] inspect: Display null and empty rtvars more clearly. rtvars with no default are now displayed as , rather than None. rtvars with empty strings are now displayed as rather than as whitespace. Signed-off-by: Ryan Roberts --- shrinkwrap/commands/inspect.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shrinkwrap/commands/inspect.py b/shrinkwrap/commands/inspect.py index 78ba16a..b1dbf38 100644 --- a/shrinkwrap/commands/inspect.py +++ b/shrinkwrap/commands/inspect.py @@ -82,7 +82,8 @@ def dispatch(args): indent=indent, paraspace=1)) buf.write('\n') - rtvars = {k: v['value'] for k,v in c['run']['rtvars'].items()} + rtvars = {k: _var_value(v['value']) + for k,v in c['run']['rtvars'].items()} buf.write(_dict_wrap('run-time variables', rtvars, width=width, @@ -95,6 +96,12 @@ def dispatch(args): all = separator.join(descs) print(all) +def _var_value(value): + if value is None: + return '' + if value == '': + return '' + return str(value) def _text_wrap(tag, text, width=80, indent=0, paraspace=1, end='\n'): text = str(text) -- GitLab From fc5c36a626584891fd3475cc1093a4dc9503e269 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 18:21:01 +0100 Subject: [PATCH 06/12] utils: Rename rtvars module to vars This module will be reused for btvars (build-time variables) so let's rename the module to be more generic. Signed-off-by: Ryan Roberts --- shrinkwrap/commands/process.py | 4 ++-- shrinkwrap/commands/run.py | 4 ++-- shrinkwrap/utils/{rtvars.py => vars.py} | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) rename shrinkwrap/utils/{rtvars.py => vars.py} (57%) diff --git a/shrinkwrap/commands/process.py b/shrinkwrap/commands/process.py index 58631e7..6854e27 100644 --- a/shrinkwrap/commands/process.py +++ b/shrinkwrap/commands/process.py @@ -3,7 +3,7 @@ import os import shrinkwrap.utils.config as config -import shrinkwrap.utils.rtvars as rtvars +import shrinkwrap.utils.vars as vars cmd_name = os.path.splitext(os.path.basename(__file__))[0] @@ -84,7 +84,7 @@ def dispatch(args): if args.action == 'resolveb': print(config.dumps(resolveb)) else: - rtvars_dict = rtvars.parse(args.rtvar) + rtvars_dict = vars.parse(args.rtvar, type='rt') resolver = config.resolver(resolveb, rtvars_dict) if args.action == 'resolver': diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py index d81307b..3357dc8 100644 --- a/shrinkwrap/commands/run.py +++ b/shrinkwrap/commands/run.py @@ -7,7 +7,7 @@ import tempfile import shrinkwrap.utils.config as config import shrinkwrap.utils.logger as logger import shrinkwrap.utils.process as process -import shrinkwrap.utils.rtvars as rtvars +import shrinkwrap.utils.vars as vars import shrinkwrap.utils.runtime as runtime import shrinkwrap.utils.workspace as workspace @@ -79,7 +79,7 @@ def dispatch(args): filename = os.path.join(workspace.package, args.config) resolveb = config.load(filename, overlays) - rtvars_dict = rtvars.parse(args.rtvar) + rtvars_dict = vars.parse(args.rtvar, type='rt') resolver = config.resolver(resolveb, rtvars_dict) cmds = _pretty_print_sh(resolver['run']) diff --git a/shrinkwrap/utils/rtvars.py b/shrinkwrap/utils/vars.py similarity index 57% rename from shrinkwrap/utils/rtvars.py rename to shrinkwrap/utils/vars.py index 1f07be3..96ffc8d 100644 --- a/shrinkwrap/utils/rtvars.py +++ b/shrinkwrap/utils/vars.py @@ -1,14 +1,14 @@ # Copyright (c) 2022, Arm Limited. # SPDX-License-Identifier: MIT -def parse(args): - rtvars = {} +def parse(args, type): + vars = {} for pair in args: try: key, value = pair.split('=', maxsplit=1) - rtvars[key] = value + vars[key] = value except ValueError: - raise Exception(f'Invalid rtvar {pair}') + raise Exception(f'Invalid {type}var {pair}') - return rtvars + return vars -- GitLab From 61be1d6163bd5773397b78db7434d5aecb0a46d3 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 19:36:35 +0100 Subject: [PATCH 07/12] build: Split into separate build and buildall commands We are about to introduce build-time variables (btvars), which are a per-config set of variables that the user can override on the command line at build time. This means that it won't be possible to implicitly build multiple configs at the same time, because each config will need its own set of btvars. Our solution is to split 'build' into 2 commands; 'build' will take a single config and optionally btvars on the command line, and build that single config. 'buildall' will take a yaml file that declares a set of configs and the btvars for each config, and build them all concurrently. Start that change by defining the new command, buildall, and modify build to take a single config. Most of the code remains common and lives in buildall; We just need to create a list of one config and everything looks the same from then on. btvars are not yet supported with this change. Signed-off-by: Ryan Roberts --- documentation/userguide/commands.rst | 1 + shrinkwrap/commands/build.py | 121 ++----------------- shrinkwrap/commands/buildall.py | 171 +++++++++++++++++++++++++++ shrinkwrap/shrinkwrap.py | 2 + 4 files changed, 183 insertions(+), 112 deletions(-) create mode 100644 shrinkwrap/commands/buildall.py diff --git a/documentation/userguide/commands.rst b/documentation/userguide/commands.rst index a268d22..a1ee58e 100644 --- a/documentation/userguide/commands.rst +++ b/documentation/userguide/commands.rst @@ -22,6 +22,7 @@ For help on a specific command: shrinkwrap inspect --help shrinkwrap build --help + shrinkwrap buildall --help shrinkwrap clean --help shrinkwrap run --help shrinkwrap process --help diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py index ce8d1d1..fd885db 100644 --- a/shrinkwrap/commands/build.py +++ b/shrinkwrap/commands/build.py @@ -2,19 +2,12 @@ # SPDX-License-Identifier: MIT import os -import shrinkwrap.utils.config as config -import shrinkwrap.utils.graph as ugraph -import shrinkwrap.utils.runtime as runtime -import shrinkwrap.utils.workspace as workspace +import shrinkwrap.commands.buildall as buildall cmd_name = os.path.splitext(os.path.basename(__file__))[0] -def dflt_jobs(): - return min(os.cpu_count() // 2, 32) - - def add_parser(parser, formatter): """ Part of the command interface expected by shrinkwrap.py. Adds the @@ -23,8 +16,7 @@ def add_parser(parser, formatter): """ cmdp = parser.add_parser(cmd_name, formatter_class=formatter, - help="""Builds either all concrete standard configs or an - explicitly specified set of configs and packages them ready + help="""Builds a specified config and packages it ready to run.""", epilog="""Custom config store(s) can be defined at at as a colon-separated list of @@ -39,53 +31,13 @@ def add_parser(parser, formatter): and '~/.shrinkwrap/package'. The user can override them by setting the environment variables.""") - cmdp.add_argument('configs', - metavar='config', nargs='*', - help="""0 or more configs to build. If a config exists relative - to the current directory that config is used. Else if a - config exists relative to the config store then it is used. - If no configs are provided, all concrete configs in the - config store are built.""") - - cmdp.add_argument('-o', '--overlay', - metavar='cfgfile', required=False, default=[], - action='append', - help="""Optional config file overlay to override run-time and - build-time settings. Only entries within the "build" and - "run" sections are used. Applied to all configs being - built. Can be specified multiple times; left-most overlay - is the first overlay applied.""") - - cmdp.add_argument('-t', '--tasks', - required=False, default=dflt_jobs(), metavar='count', type=int, - help="""Maximum number of "high-level" tasks that will be - performed in parallel by Shrinkwrap. Tasks include syncing - git repositories, building components and copying - artifacts. Default={}""".format(dflt_jobs())) + cmdp.add_argument('config', + metavar='config', + help="""Config to build. If the config exists relative to the + current directory that config is used. Else if the config + exists relative to the config store then it is used.""") - cmdp.add_argument('-j', '--jobs', - required=False, default=dflt_jobs(), metavar='count', type=int, - help="""Maximum number of low-level jobs that will be - performed in parallel by each component build task. - Default={}""".format(dflt_jobs())) - - cmdp.add_argument('-v', '--verbose', - required=False, default=False, action='store_true', - help="""If specified, the output from all executed commands will - be displayed. It is advisable to set tasks to 1 when - this option is selected.""") - - cmdp.add_argument('-n', '--dry-run', - required=False, default=False, action='store_true', - help="""If specified, and - will not be touched and none of the - build commands will be executed. Instead the set of - commands that would have been executed are output to stdout - as a bash script.""") - - cmdp.add_argument('-c', '--no-color', - required=False, default=False, action='store_true', - help="""If specified, logs will not be colorized.""") + buildall.add_common_args(cmdp) return cmd_name @@ -96,59 +48,4 @@ def dispatch(args): execute the subcommand, with the arguments the user passed on the command line. The arguments comply with those requested in add_parser(). """ - clivars = {'jobs': args.jobs} - configs = config.load_resolveb_all(args.configs, args.overlay, clivars) - if len(args.configs) == 0: - configs = [c for c in configs if c['concrete']] - graph = config.build_graph(configs, args.verbose) - - if args.dry_run: - script = ugraph.make_script(graph) - print(script) - else: - if args.verbose: - workspace.dump() - - # Run under a runtime environment, which may just run commands - # natively on the host or may execute commands in a container, - # depending on what the user specified. - with runtime.Runtime(args.runtime, args.image) as rt: - def add_volume(path, levels_up=0): - while levels_up: - path = os.path.dirname(path) - levels_up -= 1 - os.makedirs(path, exist_ok=True) - rt.add_volume(path) - - add_volume(workspace.build) - add_volume(workspace.package) - for c in workspace.configs(): - add_volume(c) - - for conf in configs: - for comp in conf['build'].values(): - add_volume(comp['sourcedir'], 1) - add_volume(comp['builddir']) - - rt.start() - - ugraph.execute(graph, - args.tasks, - args.verbose, - not args.no_color) - - for c in configs: - # Dump the config. - cfg_name = os.path.join(workspace.package, - f'{c["name"]}.yaml') - with open(cfg_name, 'w') as cfg: - config.dump(c, cfg) - - # Dump the script to build the config. - graph = config.build_graph([c], args.verbose) - script = ugraph.make_script(graph) - build_name = os.path.join(workspace.package, - c['name'], - 'build.sh') - with open(build_name, 'w') as build: - build.write(script) + buildall.build([args.config], args) diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py new file mode 100644 index 0000000..3ed77d6 --- /dev/null +++ b/shrinkwrap/commands/buildall.py @@ -0,0 +1,171 @@ +# Copyright (c) 2022, Arm Limited. +# SPDX-License-Identifier: MIT + +import os +import yaml +import shrinkwrap.utils.config as config +import shrinkwrap.utils.graph as ugraph +import shrinkwrap.utils.runtime as runtime +import shrinkwrap.utils.workspace as workspace + + +cmd_name = os.path.splitext(os.path.basename(__file__))[0] + + +def dflt_jobs(): + return min(os.cpu_count() // 2, 32) + + +def add_parser(parser, formatter): + """ + Part of the command interface expected by shrinkwrap.py. Adds the + subcommand to the parser, along with all options and documentation. + Returns the subcommand name. + """ + cmdp = parser.add_parser(cmd_name, + formatter_class=formatter, + help="""Builds either all concrete standard configs or an + explicitly specified set of configs and packages them ready + to run.""", + epilog="""Custom config store(s) can be defined at at + as a colon-separated list of + directories. Building is done at and + output is saved to . The package + includes all FW binaries, a manifest and a build.sh script + containing all the commands that were executed per config. + Any pre-existing config package directory is first deleted. + Shrinkwrap will always search its default config store even + if is not defined. + and default to '~/.shrinkwrap/build' + and '~/.shrinkwrap/package'. The user can override them by + setting the environment variables.""") + + cmdp.add_argument('configs', + metavar='yamlfile', + help="""A yaml file containing all the configs to be built. The + top level dictionary contains a 'configs' key, whose value + is a list of dictionaries, each with a 'config' key, whose + value is a config filename.""") + + add_common_args(cmdp) + + return cmd_name + + +def add_common_args(cmdp): + """ + Common args shared between build and buildmulti. + """ + cmdp.add_argument('-o', '--overlay', + metavar='cfgfile', required=False, default=[], + action='append', + help="""Optional config file overlay to override run-time and + build-time settings. Only entries within the "build" and + "run" sections are used. Applied to all configs being + built. Can be specified multiple times; left-most overlay + is the first overlay applied.""") + + cmdp.add_argument('-t', '--tasks', + required=False, default=dflt_jobs(), metavar='count', type=int, + help="""Maximum number of "high-level" tasks that will be + performed in parallel by Shrinkwrap. Tasks include syncing + git repositories, building components and copying + artifacts. Default={}""".format(dflt_jobs())) + + cmdp.add_argument('-j', '--jobs', + required=False, default=dflt_jobs(), metavar='count', type=int, + help="""Maximum number of low-level jobs that will be + performed in parallel by each component build task. + Default={}""".format(dflt_jobs())) + + cmdp.add_argument('-v', '--verbose', + required=False, default=False, action='store_true', + help="""If specified, the output from all executed commands will + be displayed. It is advisable to set tasks to 1 when + this option is selected.""") + + cmdp.add_argument('-n', '--dry-run', + required=False, default=False, action='store_true', + help="""If specified, and + will not be touched and none of the + build commands will be executed. Instead the set of + commands that would have been executed are output to stdout + as a bash script.""") + + cmdp.add_argument('-c', '--no-color', + required=False, default=False, action='store_true', + help="""If specified, logs will not be colorized.""") + + +def dispatch(args): + """ + Part of the command interface expected by shrinkwrap.py. Called to + execute the subcommand, with the arguments the user passed on the + command line. The arguments comply with those requested in add_parser(). + """ + with open(args.configs) as file: + configs = yaml.safe_load(file) + + configs = [c['config'] for c in configs['configs']] + build(configs, args) + + +def build(configs, args): + """ + Concurrently builds a list of configs. Intended to be called as a common + handler for the build and buildmulti commands. + """ + clivars = {'jobs': args.jobs} + configs = config.load_resolveb_all(configs, args.overlay, clivars) + graph = config.build_graph(configs, args.verbose) + + if args.dry_run: + script = ugraph.make_script(graph) + print(script) + else: + if args.verbose: + workspace.dump() + + # Run under a runtime environment, which may just run commands + # natively on the host or may execute commands in a container, + # depending on what the user specified. + with runtime.Runtime(args.runtime, args.image) as rt: + def add_volume(path, levels_up=0): + while levels_up: + path = os.path.dirname(path) + levels_up -= 1 + os.makedirs(path, exist_ok=True) + rt.add_volume(path) + + add_volume(workspace.build) + add_volume(workspace.package) + for c in workspace.configs(): + add_volume(c) + + for conf in configs: + for comp in conf['build'].values(): + add_volume(comp['sourcedir'], 1) + add_volume(comp['builddir']) + + rt.start() + + ugraph.execute(graph, + args.tasks, + args.verbose, + not args.no_color) + + for c in configs: + # Dump the config. + cfg_name = os.path.join(workspace.package, + f'{c["name"]}.yaml') + with open(cfg_name, 'w') as cfg: + config.dump(c, cfg) + + # Dump the script to build the config. + graph = config.build_graph([c], args.verbose) + script = ugraph.make_script(graph) + build_name = os.path.join(workspace.package, + c['name'], + 'build.sh') + with open(build_name, 'w') as build: + build.write(script) diff --git a/shrinkwrap/shrinkwrap.py b/shrinkwrap/shrinkwrap.py index 60fc8eb..1b32e62 100755 --- a/shrinkwrap/shrinkwrap.py +++ b/shrinkwrap/shrinkwrap.py @@ -17,6 +17,7 @@ from shrinkwrap import __version__ from shrinkwrap.commands import build +from shrinkwrap.commands import buildall from shrinkwrap.commands import clean from shrinkwrap.commands import inspect from shrinkwrap.commands import process @@ -91,6 +92,7 @@ def main(): # Register all the commands. cmds = {} cmds[build.add_parser(subparsers, formatter)] = build + cmds[buildall.add_parser(subparsers, formatter)] = buildall cmds[clean.add_parser(subparsers, formatter)] = clean cmds[inspect.add_parser(subparsers, formatter)] = inspect cmds[process.add_parser(subparsers, formatter)] = process -- GitLab From f0a4fcecc76654fd5f68f2e58188fb92a5ccf806 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Wed, 5 Apr 2023 22:24:41 +0100 Subject: [PATCH 08/12] inspect: Don't resolveb configs As far as I can tell, there is no good reason to resolveb the configs for inspect. Loading/merging them should be sufficient. As we introduce btvars it would significantly complicate things to resolveb here anyway, so lets change it to just load and dodge the bullet. Signed-off-by: Ryan Roberts --- shrinkwrap/commands/inspect.py | 2 +- shrinkwrap/utils/config.py | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/shrinkwrap/commands/inspect.py b/shrinkwrap/commands/inspect.py index b1dbf38..ff8a7b8 100644 --- a/shrinkwrap/commands/inspect.py +++ b/shrinkwrap/commands/inspect.py @@ -51,7 +51,7 @@ def dispatch(args): execute the subcommand, with the arguments the user passed on the command line. The arguments comply with those requested in add_parser(). """ - configs = config.load_resolveb_all(args.configs) + configs = config.load_all(args.configs) width = 80 indent = 21 diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 5a0dbe6..f00fbac 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -625,11 +625,11 @@ def resolver(config, rtvars={}, clivars={}): return _config_sort(config) -def load_resolveb_all(names, overlaynames=[], clivars={}): +def load_all(names, overlaynames=[]): """ Takes a list of config names and returns a corresponding list of - resolved configs. If the input list is None or empty, all standard - configs are loaded and resolved. + loaded configs. If the input list is None or empty, all standard + configs are loaded. """ explicit = names is not None and len(names) != 0 configs = [] @@ -653,8 +653,7 @@ def load_resolveb_all(names, overlaynames=[], clivars={}): try: file = filename(name) merged = load(file, overlays, name) - resolved = resolveb(merged, clivars) - configs.append(resolved) + configs.append(merged) except Exception: if explicit: raise @@ -662,6 +661,23 @@ def load_resolveb_all(names, overlaynames=[], clivars={}): return configs +def load_resolveb_all(names, overlaynames=[], clivars={}): + """ + Takes a list of config names and returns a corresponding list of + resolved configs. If the input list is None or empty, all standard + configs are loaded and resolved. + """ + configs_m = load_all(names, overlaynames) + + configs_r = [] + + for merged in configs_m: + resolved = resolveb(merged, clivars) + configs_r.append(resolved) + + return configs_r + + class Script: def __init__(self, summary, -- GitLab From 4f1ceed7b44612099b47d6bcc31c3251926e310c Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 6 Apr 2023 14:50:20 +0100 Subject: [PATCH 09/12] config: Improve macro substitution Dramatically improve the macro substitution logic ahead of adding btvars support. It is now possible to have artifacts that refer to other artifacts and everything gets resolved correctly, as long as there are no circular dependencies. Once we add btvars, it will be possible to set those btvars to artifacts macros and everything will continue to work. The code is also significantly cleaned up to the point where it mostly makes sense, although still plently of room for improvement. Signed-off-by: Ryan Roberts --- documentation/userguide/config.rst | 4 +- shrinkwrap/utils/config.py | 167 +++++++++++++++++------------ 2 files changed, 98 insertions(+), 73 deletions(-) diff --git a/documentation/userguide/config.rst b/documentation/userguide/config.rst index 0b1a2eb..4f992c1 100644 --- a/documentation/userguide/config.rst +++ b/documentation/userguide/config.rst @@ -128,10 +128,10 @@ macro scope ``${param:sourcedir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component's source code is located. ``${param:builddir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component should be built, if the component's build system supports separation of source and build trees. ``${param:configdir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory containing the config store. This MUST only be used for resolving files that already exist in the store. -``${param:jobs}`` build..{params, prebuild, build, postbuild, clean} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. +``${param:jobs}`` build..{params, prebuild, build, postbuild, clean, artifacts} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. ``${param:join_equal}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key=value`` pairs. ``${param:join_space}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key value`` pairs. -``${artifact:}`` build..{params, prebuild, build, postbuild, clean} Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. +``${artifact:}`` build..{params, prebuild, build, postbuild, clean, artifacts} Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. Artifacts must not be circular. ``${artifact:}`` run.rtvars Package path of an artifact. ``${rtvar:}`` run.params Run-time variables. The variable names, along with default values are declared in run.rtvars, and the user may override the value on the command line. ======================= ========================================================================= ==== diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index f00fbac..d21ba1f 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -221,7 +221,7 @@ def _config_merge(base, new): return config -def _string_tokenize(string): +def _string_tokenize(string, escape=True): """ Returns ordered list of tokens, where each token has a 'type' and 'value'. If 'type' is 'literal', 'value' is the literal string. If @@ -255,9 +255,10 @@ def _string_tokenize(string): raise Exception(f"Macro at col {lit_end}" \ f" in '{string}' is invalid.") if m['escape'] is not None: + assert(m['escape'] == '$') tokens.append({ 'type': 'literal', - 'value': m['escape'], + 'value': '$' if escape else '$$', }) if m['type'] is not None: tokens.append({ @@ -278,18 +279,19 @@ def _string_tokenize(string): return tokens -def _string_substitute(string, lut, partial=False): +def _string_substitute(string, lut, final=True): """ Takes a string containg macros and returns a string with the macros - substituted for the values found in the lut. If partial is True, any + substituted for the values found in the lut. If final is False, any macro that does not have a value in the lut will be left as a macro in - the returned string. If partial is False, any macro that does not have a - value in the lut will cause an interrupt. + the returned string. If final is True, any macro that does not have a + value in the lut will cause an exception. Final also controls unescaping + on $. If False, $$ is left as is, otherwise they are replaced with $. """ calls = [] frags = [] frag = '' - tokens = _string_tokenize(string) + tokens = _string_tokenize(string, final) for t in tokens: if t['type'] == 'literal': @@ -304,12 +306,9 @@ def _string_substitute(string, lut, partial=False): frag = '' else: frag += lu - except KeyError: - if partial: - frag += f"${{{m['type']}:{m['name']}}}" - else: - raise - + except Exception: + macro = f"${{{m['type']}:{m['name']}}}" + frag += macro else: assert(False) @@ -424,21 +423,9 @@ def resolveb(config, clivars={}): def _importers_update(importers, name, component): artifacts = set() - macros = [] - - for s in component['params'].values(): - tokens = _string_tokenize(str(s)) - macros += [t['value'] for t in tokens if t['type'] == 'macro'] - for m in macros: - if m['type'] != 'artifact': - raise Exception(f"'{name}' uses macro of type '{m['type']}'. Components must only use 'artifact' macros.") - if m['name'] is None: - raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") - artifacts.add(m['name']) - - for scope in ['prebuild', 'build', 'postbuild', 'clean']: - for s in component[scope]: + def _find_artifacts(strings): + for s in strings: for t in _string_tokenize(str(s)): if t['type'] != 'macro': continue @@ -449,6 +436,13 @@ def resolveb(config, clivars={}): raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") artifacts.add(m['name']) + _find_artifacts(component['params'].values()) + _find_artifacts(component['prebuild']) + _find_artifacts(component['build']) + _find_artifacts(component['postbuild']) + _find_artifacts(component['clean']) + _find_artifacts(component['artifacts'].values()) + importers[name] = sorted(list(artifacts)) artifacts_exp = {} @@ -461,64 +455,73 @@ def resolveb(config, clivars={}): for depender, deps in artifacts_imp.items(): graph[depender] = [] for dep in deps: + if dep not in artifacts_exp: + raise Exception(f"Imported artifact '{dep}' not exported by any component.") dependee = artifacts_exp[dep] - graph[depender].append(dependee) + if depender != dependee: + graph[depender].append(dependee) return graph def _resolve_artifact_map(config): - - artifact_map = {} - - for desc in config['build'].values(): - lut = { - 'param': { - 'sourcedir': desc['sourcedir'], - 'builddir': desc['builddir'], - 'configdir': lambda x: workspace.config(x, False), - }, - } - - for key, val in desc['artifacts'].items(): - desc['artifacts'][key] = _string_substitute(val, lut) - - locs = {key: { - 'src': val, - 'dst': os.path.join(config['name'], os.path.basename(val)), - } for key, val in desc['artifacts'].items()} - - artifact_map.update(locs) - - return artifact_map - - def _substitute_macros(config, artifacts, clivars): + def _combine(config): + artifact_map = {} + for desc in config['build'].values(): + artifact_map.update(desc['artifacts'].items()) + return {'artifact': artifact_map} + + def _combine_full(config): + artifact_map = {} + for desc in config['build'].values(): + locs = {key: { + 'src': val, + 'dst': os.path.join(config['name'], os.path.basename(val)), + } for key, val in desc['artifacts'].items()} + artifact_map.update(locs) + return artifact_map + + # ${artifact:*} macros could refer to other ${artifact:*} + # macros, so iteratively substitute the maximum number of times, + # which would be once per entry in the pathalogical case. + + artifact_lut = _combine(config) + artifact_nr = len(artifact_lut['artifact']) + + while artifact_nr > 0: + artifact_nr -= 1 + + for desc in config['build'].values(): + for k, v in desc['artifacts'].items(): + desc['artifacts'][k] = _string_substitute(v, artifact_lut, False) + + if artifact_nr > 0: + artifact_lut = _combine(config) + + return _combine_full(config) + + def _substitute_macros(config, lut, final): for desc in config['build'].values(): - lut = { - 'artifact': artifacts, - 'param': { - **clivars, - 'sourcedir': desc['sourcedir'], - 'builddir': desc['builddir'], - 'configdir': lambda x: workspace.config(x, False), - }, - } + lut['param']['sourcedir'] = desc['sourcedir'] + lut['param']['builddir'] = desc['builddir'] - for k in desc['params']: - v = desc['params'][k] + for k, v in desc['params'].items(): if v: - desc['params'][k] = _string_substitute(str(v), lut) + desc['params'][k] = _string_substitute(str(v), lut, final) lut['param']['join_equal'] = _mk_params(desc['params'], '=') lut['param']['join_space'] = _mk_params(desc['params'], ' ') for i, s in enumerate(desc['prebuild']): - desc['prebuild'][i] = _string_substitute(s, lut) + desc['prebuild'][i] = _string_substitute(s, lut, final) for i, s in enumerate(desc['build']): - desc['build'][i] = _string_substitute(s, lut) + desc['build'][i] = _string_substitute(s, lut, final) for i, s in enumerate(desc['postbuild']): - desc['postbuild'][i] = _string_substitute(s, lut) + desc['postbuild'][i] = _string_substitute(s, lut, final) for i, s in enumerate(desc['clean']): - desc['clean'][i] = _string_substitute(s, lut) + desc['clean'][i] = _string_substitute(s, lut, final) + + for k, v in desc['artifacts'].items(): + desc['artifacts'][k] = _string_substitute(v, lut, final) # Compute the source and build directories for each component. If they # are already present, then don't override. This allows users to supply @@ -534,11 +537,33 @@ def resolveb(config, clivars={}): 'build', comp_dir) + macro_lut = { + 'param': { + **uclivars.get(**clivars), + 'configdir': lambda x: workspace.config(x, False), + }, + } + + # Do a first partial substitution, to resolve all macros except + # ${artifact:*}. These macros must remain in place in order to resolve + # the build graph. But its possible that ${artifact:*} resolve to other + # ${artifact:*} so we need to do the first pass prior to resolving the + # build graph. + _substitute_macros(config, macro_lut, False) + + # Now resolve the build graph, which finds ${artifact:*} users. graph = _resolve_build_graph(config) + + # At this point we should only have ${artifacts:*} macros remaining to + # resolve. But there may be some cases where ${artifacts:*} resolve to + # other ${artifacts:*}. So we need to iteratively resolve the + # artifact_map. artifact_map = _resolve_artifact_map(config) artifact_src_map = {k: v['src'] for k, v in artifact_map.items()} - clivars = uclivars.get(**clivars) - _substitute_macros(config, artifact_src_map, clivars) + macro_lut['artifact'] = artifact_src_map + + # Final check to ensure everything is resolved and to fix escaped $. + _substitute_macros(config, macro_lut, True) config['graph'] = graph config['artifacts'] = artifact_map -- GitLab From 4f56d55200d1e20216ad9b79d92b4503c7aef0dc Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 6 Apr 2023 16:47:46 +0100 Subject: [PATCH 10/12] config: Add btvars to yaml schema btvars are like rtvars, except they are used during build-time. They can be defined in the config (under the new 'buildex:btvars:' key), with an optional default value. Users can optionally override them on the command line when invoking a build. (or at least they will be able to once we enable that part in a future commit). btvars can be used in the config through the macro system. It is even possible to set a btvar to an ${artifact:*} macro, and things will resolve correctly. Signed-off-by: Ryan Roberts --- documentation/userguide/config.rst | 43 ++++++++++++++------ shrinkwrap/commands/inspect.py | 10 ++++- shrinkwrap/utils/config.py | 63 ++++++++++++++++++++++++++---- 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/documentation/userguide/config.rst b/documentation/userguide/config.rst index 4f992c1..1ae50ce 100644 --- a/documentation/userguide/config.rst +++ b/documentation/userguide/config.rst @@ -122,19 +122,20 @@ output to get a better feel for how they work. See Defined Macros -------------- -======================= ========================================================================= ==== -macro scope description -======================= ========================================================================= ==== -``${param:sourcedir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component's source code is located. -``${param:builddir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component should be built, if the component's build system supports separation of source and build trees. -``${param:configdir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory containing the config store. This MUST only be used for resolving files that already exist in the store. -``${param:jobs}`` build..{params, prebuild, build, postbuild, clean, artifacts} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. -``${param:join_equal}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key=value`` pairs. -``${param:join_space}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key value`` pairs. -``${artifact:}`` build..{params, prebuild, build, postbuild, clean, artifacts} Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. Artifacts must not be circular. -``${artifact:}`` run.rtvars Package path of an artifact. -``${rtvar:}`` run.params Run-time variables. The variable names, along with default values are declared in run.rtvars, and the user may override the value on the command line. -======================= ========================================================================= ==== +======================= ====================================================================================== ==== +macro scope description +======================= ====================================================================================== ==== +``${param:sourcedir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component's source code is located. +``${param:builddir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component should be built, if the component's build system supports separation of source and build trees. +``${param:configdir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory containing the config store. This MUST only be used for resolving files that already exist in the store. +``${param:jobs}`` build..{params, prebuild, build, postbuild, clean, artifacts} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. +``${btvar:}`` build..{params, prebuild, build, postbuild, clean, artifacts} Build-time variables. The variable names, along with default values are declared in buildex.btvars, and the user may override the value on the command line. +``${param:join_equal}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key=value`` pairs. +``${param:join_space}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key value`` pairs. +``${artifact:}`` build..{params, prebuild, build, postbuild, clean, artifacts}, build.btvars Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. +``${artifact:}`` run.rtvars Package path of an artifact. +``${rtvar:}`` run.params Run-time variables. The variable names, along with default values are declared in run.rtvars, and the user may override the value on the command line. +======================= ====================================================================================== ==== ****** Schema @@ -166,6 +167,22 @@ The build section, contains a dictionary of components that must be built. The keys are the component names and the values are themselves dictionaries, each containing the component meta data. +--------------- +buildex section +--------------- + +When the schema was originally created, we made a mistake. The components should +have been under ``build: components:``, allowing room for new build data to be +added under ``build:`` without being confused for components. In order to +retrofit a solution without breaking compatibility, the buildex section is +created. + +=========== =========== =========== +key type description +=========== =========== =========== +btvars dictionary Build-Time variables. Keys are the variable names and values are a dictionary with keys 'type' (which must be one of 'path' and 'string') and 'value' (which takes the default value). Build-Time variables can be overridden by the user at the command line. +=========== =========== =========== + ~~~~~~~~~~~~~~~~~ component section ~~~~~~~~~~~~~~~~~ diff --git a/shrinkwrap/commands/inspect.py b/shrinkwrap/commands/inspect.py index ff8a7b8..caca6d4 100644 --- a/shrinkwrap/commands/inspect.py +++ b/shrinkwrap/commands/inspect.py @@ -82,9 +82,17 @@ def dispatch(args): indent=indent, paraspace=1)) buf.write('\n') + btvars = {k: _var_value(v['value']) + for k,v in c['buildex']['btvars'].items()} + buf.write(_dict_wrap('build-time vars', + btvars, + width=width, + kindent=indent, + vindent=vindent)) + buf.write('\n') rtvars = {k: _var_value(v['value']) for k,v in c['run']['rtvars'].items()} - buf.write(_dict_wrap('run-time variables', + buf.write(_dict_wrap('run-time vars', rtvars, width=width, kindent=indent, diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index d21ba1f..31f5232 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -72,6 +72,14 @@ def _build_normalize(build): _component_normalize(component, name) +def _buildex_normalize(buildex): + """ + Fills in any missing lists or dictionaries with empty ones. + """ + if 'btvars' not in buildex: + buildex['btvars'] = {} + + def _run_normalize(run): """ Fills in any missing lists or dictionaries with empty ones. @@ -120,7 +128,11 @@ def _config_normalize(config): if 'build' not in config: config['build'] = {} + if 'buildex' not in config: + config['buildex'] = {} + _build_normalize(config['build']) + _buildex_normalize(config['buildex']) if 'artifacts' not in config: config['artifacts'] = {} @@ -181,7 +193,7 @@ def _config_sort(config): config['run'] = _run_sort(config['run']) lut = ['name', 'fullname', 'description', 'concrete', 'layers', - 'graph', 'build', 'artifacts', 'run'] + 'graph', 'build', 'buildex', 'artifacts', 'run'] lut = {k: i for i, k in enumerate(lut)} return dict(sorted(config.items(), key=lambda x: lut[x[0]])) @@ -327,6 +339,11 @@ def _string_substitute(string, lut, final=True): return final +def _string_has_macros(string): + tokens = _string_tokenize(string) + return any([True for t in tokens if t['type'] == 'macro']) + + def _mk_params(params, separator): pairs = [f'{k}' if v is None else f'{k}{separator}{v}' for k, v in params.items()] @@ -403,12 +420,15 @@ def dump(config, fileobj): version=(1, 2)) -def resolveb(config, clivars={}): +def resolveb(config, btvars={}, clivars={}): """ Resolves the build-time macros (params, artifacts, etc) and fixes up the config. Based on the artifact dependencies, the component build graph is determined and placed into the config along with the global artifact map. Expects a config that was previously loaded with load(). + btvars=None implies that it is OK not to resolve btvars whose default + value is None. type(btvars) == dict implies btvars values must all be + resolved. """ def _resolve_build_graph(config): def _exporters_update(exporters, name, component): @@ -523,6 +543,10 @@ def resolveb(config, clivars={}): for k, v in desc['artifacts'].items(): desc['artifacts'][k] = _string_substitute(v, lut, final) + for k, v in config['buildex']['btvars'].items(): + if v['value'] is not None: + v['value'] = _string_substitute(str(v['value']), lut, final) + # Compute the source and build directories for each component. If they # are already present, then don't override. This allows users to supply # their own source and build tree locations. @@ -544,11 +568,29 @@ def resolveb(config, clivars={}): }, } + # Override the btvars with any values supplied by the user and check + # that all btvars are defined. + final_btvars = config['buildex']['btvars'] + + for k, v in final_btvars.items(): + if v['value'] is None: + raise Exception(f'{k} build-time variable not ' \ + 'set by user and no default available.') + + if v['type'] == 'path' and \ + v['value'] and \ + not _string_has_macros(v['value']): + v['value'] = os.path.expanduser(v['value']) + v['value'] = os.path.abspath(v['value']) + + macro_lut['btvar'] = {k: v['value'] for k, v in final_btvars.items()} + # Do a first partial substitution, to resolve all macros except # ${artifact:*}. These macros must remain in place in order to resolve - # the build graph. But its possible that ${artifact:*} resolve to other - # ${artifact:*} so we need to do the first pass prior to resolving the - # build graph. + # the build graph. But its possible that btvars resolve to ${artifact:*} + # so we need to do the first pass prior to resolving the build graph. + # btvars are external to the component so they can't be used directly to + # build the graph. _substitute_macros(config, macro_lut, False) # Now resolve the build graph, which finds ${artifact:*} users. @@ -686,7 +728,7 @@ def load_all(names, overlaynames=[]): return configs -def load_resolveb_all(names, overlaynames=[], clivars={}): +def load_resolveb_all(names, overlaynames=[], clivars={}, btvarss=None): """ Takes a list of config names and returns a corresponding list of resolved configs. If the input list is None or empty, all standard @@ -694,10 +736,15 @@ def load_resolveb_all(names, overlaynames=[], clivars={}): """ configs_m = load_all(names, overlaynames) + if btvarss is None: + btvarss = [None] * len(configs_m) + + assert(len(configs_m) == len(btvarss)) + configs_r = [] - for merged in configs_m: - resolved = resolveb(merged, clivars) + for merged, btvars in zip(configs_m, btvarss): + resolved = resolveb(merged, btvars, clivars) configs_r.append(resolved) return configs_r -- GitLab From 088f571e5aecccffc04a07e2f8849e9ee39e3249 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 6 Apr 2023 16:51:47 +0100 Subject: [PATCH 11/12] build, buildall, process: Handle btvars overrides Enable passing btvars overrides to build and process, using similar notation as for rtvars (--btvar =). For buildall, they are passed in the configs.yaml. Signed-off-by: Ryan Roberts --- shrinkwrap/commands/build.py | 12 +++++++++++- shrinkwrap/commands/buildall.py | 15 ++++++++++----- shrinkwrap/commands/process.py | 12 +++++++++++- shrinkwrap/utils/config.py | 10 +++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py index fd885db..046788f 100644 --- a/shrinkwrap/commands/build.py +++ b/shrinkwrap/commands/build.py @@ -3,6 +3,7 @@ import os import shrinkwrap.commands.buildall as buildall +import shrinkwrap.utils.vars as vars cmd_name = os.path.splitext(os.path.basename(__file__))[0] @@ -37,6 +38,14 @@ def add_parser(parser, formatter): current directory that config is used. Else if the config exists relative to the config store then it is used.""") + cmdp.add_argument('-b', '--btvar', + metavar='key=value', required=False, default=[], + action='append', + help="""Override value for a single build-time variable defined + by the config. Specify option multiple times for multiple + variables. Overrides for variables that have a default + specified by the config are optional.""") + buildall.add_common_args(cmdp) return cmd_name @@ -48,4 +57,5 @@ def dispatch(args): execute the subcommand, with the arguments the user passed on the command line. The arguments comply with those requested in add_parser(). """ - buildall.build([args.config], args) + btvars = vars.parse(args.btvar, type='bt') + buildall.build([args.config], [btvars], args) diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index 3ed77d6..a7677d0 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -104,19 +104,20 @@ def dispatch(args): command line. The arguments comply with those requested in add_parser(). """ with open(args.configs) as file: - configs = yaml.safe_load(file) + cfgs = yaml.safe_load(file) - configs = [c['config'] for c in configs['configs']] - build(configs, args) + configs = [c['config'] for c in cfgs['configs']] + btvarss = [c['btvars'] for c in cfgs['configs']] + build(configs, btvarss, args) -def build(configs, args): +def build(configs, btvarss, args): """ Concurrently builds a list of configs. Intended to be called as a common handler for the build and buildmulti commands. """ clivars = {'jobs': args.jobs} - configs = config.load_resolveb_all(configs, args.overlay, clivars) + configs = config.load_resolveb_all(configs, args.overlay, clivars, btvarss) graph = config.build_graph(configs, args.verbose) if args.dry_run: @@ -147,6 +148,10 @@ def build(configs, args): add_volume(comp['sourcedir'], 1) add_volume(comp['builddir']) + for btvar in conf['buildex']['btvars'].values(): + if btvar['type'] == 'path': + rt.add_volume(btvar['value']) + rt.start() ugraph.execute(graph, diff --git a/shrinkwrap/commands/process.py b/shrinkwrap/commands/process.py index 6854e27..046829a 100644 --- a/shrinkwrap/commands/process.py +++ b/shrinkwrap/commands/process.py @@ -48,6 +48,15 @@ def add_parser(parser, formatter): "run" sections are used. Can be specified multiple times; left-most overlay is the first overlay applied.""") + cmdp.add_argument('-b', '--btvar', + metavar='key=value', required=False, default=[], + action='append', + help="""Override value for a single build-time variable defined + by the config. Specify option multiple times for multiple + variables. Overrides for variables that have a default + specified by the config are optional. Only used if action + is "resolveb" or "resolver".""") + cmdp.add_argument('-r', '--rtvar', metavar='key=value', required=False, default=[], action='append', @@ -79,7 +88,8 @@ def dispatch(args): if args.action == 'merge': print(config.dumps(merged)) else: - resolveb = config.resolveb(merged) + btvars = vars.parse(args.btvar, type='bt') + resolveb = config.resolveb(merged, btvars) if args.action == 'resolveb': print(config.dumps(resolveb)) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 31f5232..0de5ff3 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -573,9 +573,13 @@ def resolveb(config, btvars={}, clivars={}): final_btvars = config['buildex']['btvars'] for k, v in final_btvars.items(): - if v['value'] is None: - raise Exception(f'{k} build-time variable not ' \ - 'set by user and no default available.') + if btvars is not None: + if k in btvars: + v['value'] = btvars[k] + if v['value'] is None: + raise Exception(f'{k} build-time variable ' \ + 'not set by user and no ' \ + 'default available.') if v['type'] == 'path' and \ v['value'] and \ -- GitLab From a1aec8685cee5f5deb81cfdc239694896b643cc0 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Sun, 9 Apr 2023 17:56:45 +0100 Subject: [PATCH 12/12] test: Rework to use new buildall command Now that we have separated build and buildall, rework the tests to use buildall by generating a temporary configs.yaml. This allows us to continue to benefit from building multiple configs in parallel. While we are at it, add a mechanism to easily specify config-specifc btvars, and update the cca-3world.yaml test to specify the GUEST_ROOTFS btvar. Signed-off-by: Ryan Roberts --- test/test.py | 67 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/test/test.py b/test/test.py index b9c6a77..fb26163 100755 --- a/test/test.py +++ b/test/test.py @@ -7,6 +7,8 @@ import argparse import json import os import subprocess +import tempfile +import yaml RUNTIME = None @@ -20,9 +22,21 @@ ROOTFS = os.path.join(ASSETS, 'rootfs.ext4') CONFIGS = [ - ('ns-preload.yaml', {}), - ('ns-edk2.yaml', {}), - ('ns-edk2.yaml', {'CMDLINE': '\"console=ttyAMA0 earlycon=pl011,0x1c090000 root=/dev/vda ip=dhcp acpi=force\"'}), + { + 'config': 'ns-preload.yaml', + 'btvars': {}, + 'rtvars': {}, + }, + { + 'config': 'ns-edk2.yaml', + 'btvars': {}, + 'rtvars': {}, + }, + { + 'config': 'ns-edk2.yaml', + 'btvars': {}, + 'rtvars': {'CMDLINE': '\"console=ttyAMA0 earlycon=pl011,0x1c090000 root=/dev/vda ip=dhcp acpi=force\"'}, + }, ] @@ -99,25 +113,47 @@ def run(cmd, timeout=None, expect=0): raise WrongExit(ret) -def build_configs(configs, overlay=None): +def build_configs(configs, overlay=None, btvarss=None): result = { 'type': 'build', 'status': 'fail', 'error': None, 'configs': configs, 'overlay': overlay, + 'btvarss': btvarss, } rt = f'-R {RUNTIME} -I {IMAGE}' overlay = f'-o {overlay}' if overlay else '' - args = f'{" ".join(configs)} {overlay}' - - try: - run(f'shrinkwrap {rt} clean {args} -d', None) - run(f'shrinkwrap {rt} build {args}', None) - result['status'] = 'pass' - except Exception as e: - result['error'] = str(e) + cleanargs = f'{" ".join(configs)} {overlay}' + + if btvarss is None: + btvarss = [{}] * len(configs) + + assert(len(configs) == len(btvarss)) + + cfgs = [] + for c, b in zip(configs, btvarss): + cfgs.append({'config': c, 'btvars': b}) + + with tempfile.TemporaryDirectory() as tmpdir: + tmpfilename = os.path.join(tmpdir, 'configs.yaml') + with open(tmpfilename, 'w') as tmpfile: + yaml.safe_dump({'configs': cfgs}, + tmpfile, + explicit_start=True, + sort_keys=False, + version=(1, 2)) + with open(tmpfilename, 'r') as tmpfile: + print(tmpfile.read()) + buildargs = f'{tmpfilename} {overlay}' + + try: + run(f'shrinkwrap {rt} clean {cleanargs} -d', None) + run(f'shrinkwrap {rt} buildall {buildargs}', None) + result['status'] = 'pass' + except Exception as e: + result['error'] = str(e) results.append(result) @@ -168,8 +204,11 @@ def do_main(smoke_test): arches = [ARCHES[-1]] if smoke_test else ARCHES for arch in arches: - build_configs([c for c, r in CONFIGS], arch) - for config, rtvars in CONFIGS: + configs = [c['config'] for c in CONFIGS] + btvarss = [c['btvars'] for c in CONFIGS] + rtvarss = [c['rtvars'] for c in CONFIGS] + build_configs(configs, arch, btvarss=btvarss) + for config, rtvars in zip(configs, rtvarss): run_config_kern(config, KERNEL, ROOTFS, arch, rtvars=rtvars) for arch in arches: -- GitLab