From 8a368fb5ac585d8fd65a332183a68571e7dd7c2b Mon Sep 17 00:00:00 2001 From: Yuliang Wang Date: Thu, 7 Nov 2024 16:35:54 +0000 Subject: [PATCH 1/3] config: Allow artifacts to be omitted from final copy stage Create an alternative dictionary format for artifact fields, where the new keys are: - "path": location of the artifact - "export": true (default) if the artifact should be copied, else false - "rename": (optional) rename the artifact while copying Flag unexported dependencies in the run section instead of skipping them during string substitution. Signed-off-by: Yuliang Wang --- documentation/userguide/configmodel.rst | 4 +- shrinkwrap/utils/config.py | 86 +++++++++++++++++-------- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index bf881fd..827b5c8 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -157,7 +157,7 @@ description string A human-readable description of what the config contains image string An optional field to require a specific runtime image to build / run this config. If not defined, the default image used by ``shrinkwrap`` will be used. This can be overriden with the ``--image`` command line option to ``shrinkwrap``. concrete boolean true if the config is intended to be directly built and run, or false if it is intended as a fragment to be included in other configs. build dictionary Contains all the components to be built. The key is the component name and the value is a dictionary. -run dictionary Contains all the information about how to run the built artifacts on the FVP. +run dictionary Contains all the information about how to run the exported artifacts on the FVP. =========== ========== =========== ------------- @@ -200,7 +200,7 @@ params dictionary Optional set of key:value pairs. When building most comp prebuild list List of shell commands to be executed during component build before the ``build`` list. build list List of shell commands to be executed during component build. postbuild list List of shell commands to be executed during component build after the ``build`` list. -artifacts dictionary Set of artifacts (files and/or directories) that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. +artifacts dictionary Set of artifacts (files and/or directories) that the component exports. Key is artifact name. Value is a dictionary where the key ``path`` points to the built artifact, ``export`` is a boolean determining whether it will be copied to the package directory, and ``rename`` is an optional string to which the artifact will be renamed during export; for brevity, value can instead be the path itself, in which case ``export`` is implied. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. Note that an artifact can only be used by the ``run`` section if it was exported by the ``build`` section. sync enum-string Specifies how shrinkwrap should synchronize the repository. See below for options. =========== =========== =========== diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index d23333c..7a9df4d 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -57,13 +57,18 @@ def _component_normalize(component, name): component.setdefault('build', []) component.setdefault('postbuild', []) component.setdefault('params', {}) - component.setdefault('artifacts', {}) component.setdefault('sync', None) if component['sync'] == False: component['sync'] = 'false' elif component['sync'] == True: component['sync'] = 'true' + component['artifacts'] = { key: + { 'path': val, 'base': None, 'export': True } if type(val) == str else + { 'path': val['path'], 'base': val.get('rename'), + 'export': val.get('export', True) } if type(val) == dict else + val for key, val in component.get('artifacts', {}).items() } + return component @@ -288,6 +293,10 @@ def _string_substitute(string, lut, final=True): value in the lut will cause an exception. Final also controls unescaping on $. If False, $$ is left as is, otherwise they are replaced with $. """ + # Skip substitution if not a string or is empty + if type(string) != str or not string: + return string + calls = [] frags = [] frag = '' @@ -415,6 +424,19 @@ def dump(config, fileobj): version=(1, 2)) +def _string_extract_artifacts(artifacts, strings): + for s in strings: + for t in _string_tokenize(str(s)): + if t['type'] != 'macro': + continue + m = t['value'] + if m['type'] != 'artifact': + continue + if m['name'] is None: + raise KeyError('name') + artifacts.add(m['name']) + + def resolveb(config, btvars={}, clivars={}): """ Resolves the build-time macros (params, artifacts, etc) and fixes up the @@ -440,16 +462,11 @@ def resolveb(config, btvars={}, clivars={}): artifacts = set() def _find_artifacts(strings): - for s in strings: - for t in _string_tokenize(str(s)): - if t['type'] != 'macro': - continue - m = t['value'] - if m['type'] != 'artifact': - continue - if m['name'] is None: - raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") - artifacts.add(m['name']) + try: + return _string_extract_artifacts(artifacts, strings) + except KeyError as e: + if e.args[0] != 'name': raise + raise Exception(f"'{name}' uses unnamed 'artifact' macro. 'artifact' macros must be named.") _find_artifacts(component['params'].values()) _find_artifacts(component['prebuild']) @@ -484,12 +501,16 @@ def resolveb(config, btvars={}, clivars={}): artifact_map.update(desc['artifacts'].items()) return {'artifact': artifact_map} + def _normalize_basename(path): + return os.path.basename(os.path.normpath(path)) + def _combine_full(config): artifact_map = {} for name, desc in config['build'].items(): locs = {key: { - 'src': val, - 'dst': os.path.join(config['name'], os.path.basename(val)), + 'src': os.path.normpath(val['path']), + 'dst': os.path.join(config['name'], _normalize_basename(val['base'] or val['path'])) + if val['export'] == True else None, 'component': name, } for key, val in desc['artifacts'].items()} artifact_map.update(locs) @@ -507,7 +528,8 @@ def resolveb(config, btvars={}, clivars={}): for desc in config['build'].values(): for k, v in desc['artifacts'].items(): - desc['artifacts'][k] = _string_substitute(v, artifact_lut, False) + v['path'] = _string_substitute(v['path'], artifact_lut, False) + v['base'] = _string_substitute(v['base'], artifact_lut, False) if artifact_nr > 0: artifact_lut = _combine(config) @@ -523,20 +545,16 @@ def resolveb(config, btvars={}, clivars={}): lut['param']['builddir'] = desc['builddir'] for k, v in desc['params'].items(): - if v: - desc['params'][k] = _string_substitute(str(v), lut, final) + desc['params'][k] = _string_substitute(v, lut, final) lut['param']['join_equal'] = _mk_params(desc['params'], '=') lut['param']['join_space'] = _mk_params(desc['params'], ' ') for r in desc['repo'].values(): - if r['remote']: - r['remote'] = _string_substitute(r['remote'], lut, final) - if r['revision']: - r['revision'] = _string_substitute(r['revision'], lut, final) + r['remote'] = _string_substitute(r['remote'], lut, final) + r['revision'] = _string_substitute(r['revision'], lut, final) - if desc['toolchain']: - desc['toolchain'] = _string_substitute(desc['toolchain'], lut, final) + desc['toolchain'] = _string_substitute(desc['toolchain'], lut, final) for i, s in enumerate(desc['prebuild']): desc['prebuild'][i] = _string_substitute(s, lut, final) @@ -545,11 +563,11 @@ def resolveb(config, btvars={}, clivars={}): for i, s in enumerate(desc['postbuild']): desc['postbuild'][i] = _string_substitute(s, lut, final) for k, v in desc['artifacts'].items(): - desc['artifacts'][k] = _string_substitute(v, lut, final) + v['path'] = _string_substitute(v['path'], lut, False) + v['base'] = _string_substitute(v['base'], lut, False) for k, v in config['buildex']['btvars'].items(): - if v['value'] is not None: - v['value'] = _string_substitute(str(v['value']), lut, final) + v['value'] = _string_substitute(v['value'], lut, final) # Compute the source and build directories for each component. If they # are already present, then don't override. This allows users to supply @@ -631,6 +649,13 @@ def resolver(config, rtvars={}, clivars={}): clivars = uclivars.get(**clivars) run = config['run'] + # Find the list of imported artifacts before any processing passes + artifacts_imp = set() + _string_extract_artifacts(artifacts_imp, run['params'].values()) + _string_extract_artifacts(artifacts_imp, run['prerun']) + _string_extract_artifacts(artifacts_imp, run['run']) + _string_extract_artifacts(artifacts_imp, run['rtvars'].values()) + #Override the rtvars with any values supplied by the user and check that #all rtvars are defined. for k in run['rtvars']: @@ -647,6 +672,10 @@ def resolver(config, rtvars={}, clivars={}): # will be located at run-time. for k in config['artifacts']: v = config['artifacts'][k] + if v['dst'] is None: + if k not in artifacts_imp: + continue + raise Exception(f"Artifact '{k}' is required at run-time but not exported by any component") v['dst'] = os.path.join(workspace.package, v['dst']) # Create a lookup table with all the artifacts in their package @@ -676,8 +705,7 @@ def resolver(config, rtvars={}, clivars={}): for k in run['params']: v = run['params'][k] - if v: - run['params'][k] = _string_substitute(str(v), lut) + run['params'][k] = _string_substitute(v, lut) # Assemble the final runtime command and stuff it into the config. params = _mk_params(run['params'], '=') @@ -876,6 +904,8 @@ def build_graph(configs, echo, nosync, force_sync): gl2.append(f'mkdir -p {dir}') dirs.add(dir) for artifact in config['artifacts'].values(): + if artifact['dst'] is None: + continue dst = os.path.join(workspace.package, artifact['dst']) dir = os.path.dirname(dst) if dir not in dirs: @@ -995,6 +1025,8 @@ def build_graph(configs, echo, nosync, force_sync): if len(artifacts) > 0: a.append(f'# Copy artifacts for config={config["name"]} component={name}.') for artifact in artifacts.values(): + if artifact['dst'] is None: + continue src = artifact['src'] dst = os.path.join(workspace.package, artifact['dst']) a.append(f'cp -r {src} {dst}') -- GitLab From b9ee1693aa5a90561d5e7664ca5c18537f3a5155 Mon Sep 17 00:00:00 2001 From: Yuliang Wang Date: Thu, 7 Nov 2024 16:42:21 +0000 Subject: [PATCH 2/3] config: Don't export ACPICA Signed-off-by: Yuliang Wang --- config/edk2-base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/edk2-base.yaml b/config/edk2-base.yaml index f7c9da2..dc3ea48 100644 --- a/config/edk2-base.yaml +++ b/config/edk2-base.yaml @@ -15,12 +15,12 @@ build: revision: R09_27_24 build: - - rm -rf ${param:sourcedir}/generate/unix/acpica - make -j${param:jobs} - - mv ${param:sourcedir}/generate/unix/bin ${param:sourcedir}/generate/unix/acpica artifacts: - ACPICA: ${param:sourcedir}/generate/unix/acpica + ACPICA: + path: ${param:sourcedir}/generate/unix/bin + export: false edk2: repo: -- GitLab From 7e3b7113ee32af1c32ff365fa1113c8eb7f43f2d Mon Sep 17 00:00:00 2001 From: Yuliang Wang Date: Fri, 8 Nov 2024 13:47:48 +0000 Subject: [PATCH 3/3] config: Use comprehensions and element references where possible Signed-off-by: Yuliang Wang --- shrinkwrap/utils/config.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 7a9df4d..1d044af 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -527,7 +527,7 @@ def resolveb(config, btvars={}, clivars={}): artifact_nr -= 1 for desc in config['build'].values(): - for k, v in desc['artifacts'].items(): + for v in desc['artifacts'].values(): v['path'] = _string_substitute(v['path'], artifact_lut, False) v['base'] = _string_substitute(v['base'], artifact_lut, False) @@ -544,8 +544,7 @@ def resolveb(config, btvars={}, clivars={}): lut['param']['sourcedir'] = desc['sourcedir'] lut['param']['builddir'] = desc['builddir'] - for k, v in desc['params'].items(): - desc['params'][k] = _string_substitute(v, lut, final) + desc['params'] = { k: _string_substitute(v, lut, final) for k, v in desc['params'].items() } lut['param']['join_equal'] = _mk_params(desc['params'], '=') lut['param']['join_space'] = _mk_params(desc['params'], ' ') @@ -556,17 +555,14 @@ def resolveb(config, btvars={}, clivars={}): desc['toolchain'] = _string_substitute(desc['toolchain'], lut, final) - for i, s in enumerate(desc['prebuild']): - desc['prebuild'][i] = _string_substitute(s, lut, final) - for i, s in enumerate(desc['build']): - desc['build'][i] = _string_substitute(s, lut, final) - for i, s in enumerate(desc['postbuild']): - desc['postbuild'][i] = _string_substitute(s, lut, final) - for k, v in desc['artifacts'].items(): + for k in ( 'prebuild', 'build', 'postbuild', ): + desc[k] = [ _string_substitute(s, lut, final) for s in desc[k] ] + + for v in desc['artifacts'].values(): v['path'] = _string_substitute(v['path'], lut, False) v['base'] = _string_substitute(v['base'], lut, False) - for k, v in config['buildex']['btvars'].items(): + for v in config['buildex']['btvars'].values(): v['value'] = _string_substitute(v['value'], lut, final) # Compute the source and build directories for each component. If they @@ -658,10 +654,9 @@ def resolver(config, rtvars={}, clivars={}): #Override the rtvars with any values supplied by the user and check that #all rtvars are defined. - for k in run['rtvars']: - if k in rtvars: - run['rtvars'][k]['value'] = rtvars[k] for k, v in run['rtvars'].items(): + if k in rtvars: + v['value'] = rtvars[k] if v['value'] is None: raise Exception(f'{k} run-time variable not ' \ 'set by user and no default available.') @@ -670,8 +665,7 @@ def resolver(config, rtvars={}, clivars={}): # path rather than one that is implictly relative to SHRINKWRAP_PACKAGE. # We can't do this at build-time because we don't know where the package # will be located at run-time. - for k in config['artifacts']: - v = config['artifacts'][k] + for k, v in config['artifacts'].items(): if v['dst'] is None: if k not in artifacts_imp: continue @@ -691,8 +685,7 @@ def resolver(config, rtvars={}, clivars={}): 'btvar': {k: v['value'] for k, v in config['buildex']['btvars'].items()}, } - for k in run['rtvars']: - v = run['rtvars'][k] + for v in run['rtvars'].values(): v['value'] = _string_substitute(str(v['value']), lut) if v['type'] == 'path' and v['value']: v['value'] = os.path.expanduser(v['value']) @@ -703,9 +696,7 @@ def resolver(config, rtvars={}, clivars={}): # we don't have values for. lut['rtvar'] = {k: v['value'] for k, v in run['rtvars'].items()} - for k in run['params']: - v = run['params'][k] - run['params'][k] = _string_substitute(v, lut) + run['params'] = { k: _string_substitute(v, lut) for k, v in run['params'].items() } # Assemble the final runtime command and stuff it into the config. params = _mk_params(run['params'], '=') -- GitLab