From 2ba8c3f7fd0ff6f65e19ed0bd5eb5487f23412cc Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 30 Mar 2023 16:50:21 +0100 Subject: [PATCH 1/2] cmds: Permit multiple --overlays Extend commands that support passing an --overlay (build, clean, process, run) to permit multiple. The left-most --overlay is the lowest overlay layer, and the right-most is the highest. This allows this sort of thing: shrinkwrap build ns-edk2.yaml -o arch/v9.3.yaml -o myete.yaml -o mycustom.yaml Signed-off-by: Ryan Roberts --- shrinkwrap/commands/build.py | 6 ++++-- shrinkwrap/commands/clean.py | 6 ++++-- shrinkwrap/commands/process.py | 15 +++++++++------ shrinkwrap/commands/run.py | 17 ++++++++++------- shrinkwrap/utils/config.py | 13 +++++++------ 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py index 22f3bd2..ce8d1d1 100644 --- a/shrinkwrap/commands/build.py +++ b/shrinkwrap/commands/build.py @@ -48,11 +48,13 @@ def add_parser(parser, formatter): config store are built.""") cmdp.add_argument('-o', '--overlay', - metavar='cfgfile', required=False, + metavar='cfgfile', required=False, default=[], + action='append', help="""Optional config file overlay to override run-time and build-time settings. Only entries within the "build" and "run" sections are used. Applied to all configs being - built.""") + built. Can be specified multiple times; left-most overlay + is the first overlay applied.""") cmdp.add_argument('-t', '--tasks', required=False, default=dflt_jobs(), metavar='count', type=int, diff --git a/shrinkwrap/commands/clean.py b/shrinkwrap/commands/clean.py index 8225328..6dd2172 100644 --- a/shrinkwrap/commands/clean.py +++ b/shrinkwrap/commands/clean.py @@ -36,11 +36,13 @@ def add_parser(parser, formatter): config store are cleaned.""") cmdp.add_argument('-o', '--overlay', - metavar='cfgfile', required=False, + metavar='cfgfile', required=False, default=[], + action='append', help="""Optional config file overlay to override run-time and build-time settings. Only entries within the "build" and "run" sections are used. Applied to all configs being - built.""") + built. Can be specified multiple times; left-most overlay + is the first overlay applied.""") cmdp.add_argument('-t', '--tasks', required=False, default=dflt_jobs(), metavar='count', type=int, diff --git a/shrinkwrap/commands/process.py b/shrinkwrap/commands/process.py index 0f14e77..58631e7 100644 --- a/shrinkwrap/commands/process.py +++ b/shrinkwrap/commands/process.py @@ -41,10 +41,12 @@ def add_parser(parser, formatter): within the build-time resolved config).""") cmdp.add_argument('-o', '--overlay', - metavar='cfgfile', required=False, + metavar='cfgfile', required=False, default=[], + action='append', help="""Optional config file overlay to override run-time and build-time settings. Only entries within the "build" and - "run" sections are used.""") + "run" sections are used. Can be specified multiple times; + left-most overlay is the first overlay applied.""") cmdp.add_argument('-r', '--rtvar', metavar='key=value', required=False, default=[], @@ -64,14 +66,15 @@ def dispatch(args): execute the subcommand, with the arguments the user passed on the command line. The arguments comply with those requested in add_parser(). """ - overlay = None - if args.overlay: - overlay = config.filename(args.overlay) + overlays = [] + for overlayname in args.overlay: + overlay = config.filename(overlayname) overlay = config.load(overlay) overlay = {'build': overlay['build'], 'run': overlay['run']} + overlays.append(overlay) filename = config.filename(args.config) - merged = config.load(filename, overlay) + merged = config.load(filename, overlays) if args.action == 'merge': print(config.dumps(merged)) diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py index 92f4e71..d81307b 100644 --- a/shrinkwrap/commands/run.py +++ b/shrinkwrap/commands/run.py @@ -35,11 +35,13 @@ def add_parser(parser, formatter): .""") cmdp.add_argument('-o', '--overlay', - metavar='cfgfile', required=False, + metavar='cfgfile', required=False, default=[], + action='append', help="""Optional config file overlay to override run-time settings. Only entries within the "run" section are used. - This is in addition to any overlay passed at - build-time.""") + This is in addition to any overlay passed at build-time. + Can be specified multiple times; left-most overlay is the + first overlay applied.""") cmdp.add_argument('-r', '--rtvar', metavar='key=value', required=False, default=[], @@ -68,14 +70,15 @@ def dispatch(args): execute the subcommand, with the arguments the user passed on the command line. The arguments comply with those requested in add_parser(). """ - overlay = None - if args.overlay: - overlay = config.filename(args.overlay) + overlays = [] + for overlayname in args.overlay: + overlay = config.filename(overlayname) overlay = config.load(overlay) overlay = {'run': overlay['run']} + overlays.append(overlay) filename = os.path.join(workspace.package, args.config) - resolveb = config.load(filename, overlay) + resolveb = config.load(filename, overlays) rtvars_dict = rtvars.parse(args.rtvar) resolver = config.resolver(resolveb, rtvars_dict) cmds = _pretty_print_sh(resolver['run']) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 6a67a8e..943138f 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -350,7 +350,7 @@ def filename(name, rel=os.getcwd()): return fpath -def load(file_name, overlay=None, friendly=None): +def load(file_name, overlays=[], friendly=None): """ Load a config from disk and return it as a dictionary. The config is fully normalized, validated and merged. @@ -375,7 +375,7 @@ def load(file_name, overlay=None, friendly=None): config = _config_load(file_name) - if overlay: + for overlay in overlays: config = _config_merge(config, overlay) # Now that the config is fully merged, we don't need the layers @@ -622,7 +622,7 @@ def resolver(config, rtvars={}, clivars={}): return _config_sort(config) -def load_resolveb_all(names, overlayname=None, clivars={}): +def load_resolveb_all(names, overlaynames=[], clivars={}): """ Takes a list of config names and returns a corresponding list of resolved configs. If the input list is None or empty, all standard @@ -639,16 +639,17 @@ def load_resolveb_all(names, overlayname=None, clivars={}): os.path.join(root, f), p) for f in files] - overlay = None - if overlayname: + overlays = [] + for overlayname in overlaynames: overlay = filename(overlayname) overlay = load(overlay) overlay = {'build': overlay['build'], 'run': overlay['run']} + overlays.append(overlay) for name in names: try: file = filename(name) - merged = load(file, overlay, name) + merged = load(file, overlays, name) resolved = resolveb(merged, clivars) configs.append(resolved) except Exception: -- GitLab From c0263deb446a3e483a3593e3d0922aed2310d187 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Fri, 31 Mar 2023 11:25:26 +0100 Subject: [PATCH 2/2] config: edk2: run edk2setup.sh with --reconfig flag For Sami Mujawar: when edk2setup.sh is run for the first time it copies these from edk2/Basetools/Conf to edk2/Conf but if the files under edk2/Basetools/Conf are updated, then running edk2setup.sh does not update edk2/Conf. The solution is to run edk2setup.sh --reconfig. I've tested this with incremental build and it does not impact build time. Signed-off-by: Ryan Roberts --- config/edk2-base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/edk2-base.yaml b/config/edk2-base.yaml index daf779a..c2603fd 100644 --- a/config/edk2-base.yaml +++ b/config/edk2-base.yaml @@ -38,7 +38,7 @@ build: build: - make -j${param:jobs} -C acpica - - source edk2/edksetup.sh + - source edk2/edksetup.sh --reconfig - make -j${param:jobs} -C edk2/BaseTools - build -n ${param:jobs} -D EDK2_OUT_DIR=${param:builddir} ${param:join_space} -- GitLab