From b903fde410e2b8c636e72db6e3a352426d058c79 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 14 Mar 2024 16:29:37 +0000 Subject: [PATCH 1/3] build: Always sync repos to prescribed revision Previously, Shrinkwrap would only sync a component's repos to the prescribed revision when the repo was first cloned. Any subsequent invocation would leave the repos as they were; the user was free to checkout a different revision or even dirty the working directory. This behaviour was by design, and intended to enable the development use case, where a developer is actively working on the codebase and wants to do incremental builds. However, this behaviour has led to a lot of user confusion. Most critically, when upgrading Shrinkwrap and getting updated configs with new specified versions, Shrinkwrap would not update the repo revision but apply new build directives, which may not match. Chaos. So let's change the behaviour. From now on, Shrinkwrap will explicitly sync all repos to the specified version on all build invocations. In a future commit, a per-component opt-out option will be added to re-enable the developer use case. We first try to checkout the specified revision. If that fails, we fetch from the remote, then try checking out again. This means we avoid the fetch, which is slow, in the common case. Checkout's --force ensures that any dirty files are reverted. Finally, the submodules voodoo ensures that if the repo has submodules, they are all correct too. Signed-off-by: Ryan Roberts --- documentation/userguide/configmodel.rst | 2 +- documentation/userguide/recipes.rst | 15 ++------------- shrinkwrap/utils/config.py | 7 +++++++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index fc16e19..b16765b 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -190,7 +190,7 @@ component section =========== =========== =========== key type description =========== =========== =========== -repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. Shrinkwrap will only sync the git repo if it does not already exist. If it exists, it leaves it in whatever state the user left it in and attempts to build it. Not required if ``sourcedir`` is provided. +repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. Shrinkwrap syncs the git repo to the specified revision when building. Not required if ``sourcedir`` is provided. sourcedir string If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists. builddir string If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD. toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild/clean commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst index c093d8e..8c39cc8 100644 --- a/documentation/userguide/recipes.rst +++ b/documentation/userguide/recipes.rst @@ -19,16 +19,6 @@ an existing config. Here we modify the revision and remote repository of the TF-A component from its default (defined in tfa-base.yaml). You could also specify the revision as a SHA or branch. -.. warning:: - - If you have previously built this config, shrinkwrap will skip syncing the git - repos since they will already exist and it doesn't want to trample any user - changes. So you will need to force shrinkwrap to re-sync. One approach is to - delete the following directories: - - - ``/source/ns-preload`` - - ``/build/ns-preload`` - Create a file called ``my-overlay.yaml``: .. code-block:: yaml @@ -37,7 +27,7 @@ Create a file called ``my-overlay.yaml``: tfa: repo: remote: https://github.com/ARM-software/arm-trusted-firmware.git - revision: v2.6 + revision: v2.9 Optionally, you can view the final, merged config as follows: @@ -51,8 +41,7 @@ Now do a build, passing in the overlay: shrinkwrap build --overlay=my-overlay.yaml ns-preload.yaml -Finally, boot the config. Here, were are providing a custom kernel command line. -But you could omit the command line and a sensible default would be used. +Finally, boot the config: .. code-block:: shell diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index cbc27ee..8df7552 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -914,6 +914,13 @@ def build_graph(configs, echo): g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') g.append(f'\tpopd') g.append(f'\trm {sync}') + g.append(f'else') + g.append(f'\tpushd {gitlocal}') + g.append(f'\tgit checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (') + g.append(f'\t\tgit fetch {gitargs}--prune --prune-tags {gitremote} &&') + g.append(f'\t\tgit checkout {gitargs}--force {gitrev})') + g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') + g.append(f'\tpopd') g.append(f'fi') g.append(f'popd') -- GitLab From 2595c3550fc62fe175a56943a002c8ff087a3c33 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 14 Mar 2024 17:15:37 +0000 Subject: [PATCH 2/3] build: --no-sync to opt-out of syncing repo(s) to specified revision Add optional --no-sync build option to specify any components whose git repos should not be synced. For all other components, Shrinkwrap ensures that all repos are clean and checked out at the correct revision. Option can be specified multiple times. If specified without an argument, behaves as if --no-sync was specified for every component in the config. Signed-off-by: Ryan Roberts --- documentation/userguide/configmodel.rst | 2 +- documentation/userguide/quickstart.rst | 8 +++ shrinkwrap/commands/build.py | 14 ++++- shrinkwrap/commands/buildall.py | 8 +-- shrinkwrap/utils/config.py | 74 +++++++++++++------------ 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index b16765b..cd6a04c 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -190,7 +190,7 @@ component section =========== =========== =========== key type description =========== =========== =========== -repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. Shrinkwrap syncs the git repo to the specified revision when building. Not required if ``sourcedir`` is provided. +repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync`` can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided. sourcedir string If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists. builddir string If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD. toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild/clean commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index eb46e6d..a9e0d34 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -311,6 +311,14 @@ the output from the component build systems). This will sync all the required repos, build the components and package the artifacts. +.. warning:: + + By default, Shrinkwrap will sync all component repos to the revision specified + in the config on every build invocation. If you have made changes in the + working directory, your CHANGES WILL BE LOST! You can override this behaviour + so that Shrinkwrap just builds whatever is in the working directory by adding + ``--no-sync []`` to the command line. + Alternatively, pass ``--dry-run`` to view the shell script that would have been run: diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py index 046788f..39180b4 100644 --- a/shrinkwrap/commands/build.py +++ b/shrinkwrap/commands/build.py @@ -46,6 +46,16 @@ def add_parser(parser, formatter): variables. Overrides for variables that have a default specified by the config are optional.""") + cmdp.add_argument('-s', '--no-sync', + metavar='component', required=False, default=[], + action='append', const=True, nargs='?', + help="""Optionally specify any components whose git repos should + not be synced. For all other components, Shrinkwrap ensures + that all repos are clean and checked out at the correct + revision. Option can be specified multiple times. If + specified without an argument, behaves as if --no-sync was + specified for every component in the config.""") + buildall.add_common_args(cmdp) return cmd_name @@ -58,4 +68,6 @@ def dispatch(args): command line. The arguments comply with those requested in add_parser(). """ btvars = vars.parse(args.btvar, type='bt') - buildall.build([args.config], [btvars], args) + if any([c == True for c in args.no_sync]): + args.no_sync = True + buildall.build([args.config], [btvars], args.no_sync, args) diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index d5219cc..916a4ac 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -108,17 +108,17 @@ def dispatch(args): configs = [c['config'] for c in cfgs['configs']] btvarss = [c['btvars'] for c in cfgs['configs']] - build(configs, btvarss, args) + build(configs, btvarss, [], args) -def build(configs, btvarss, args): +def build(configs, btvarss, nosync, args): """ Concurrently builds a list of configs. Intended to be called as a common handler for the build and buildmulti commands. """ clivars = {'jobs': args.jobs} configs = config.load_resolveb_all(configs, args.overlay, clivars, btvarss) - graph = config.build_graph(configs, args.verbose) + graph = config.build_graph(configs, args.verbose, nosync) if args.dry_run: script = ugraph.make_script(graph) @@ -167,7 +167,7 @@ def build(configs, btvarss, args): config.dump(c, cfg) # Dump the script to build the config. - graph = config.build_graph([c], args.verbose) + graph = config.build_graph([c], args.verbose, nosync) script = ugraph.make_script(graph) build_name = os.path.join(workspace.package, c['name'], diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 8df7552..8450643 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -841,7 +841,7 @@ def script_preamble(echo): return pre.commands(False) -def build_graph(configs, echo): +def build_graph(configs, echo, nosync): """ Returns a graph of scripts where the edges represent dependencies. The scripts should be executed according to the graph in order to correctly @@ -891,41 +891,45 @@ def build_graph(configs, echo): for name in ts.get_ready(): component = config['build'][name] - g = Script('Syncing git repo', config["name"], name, preamble=pre) - if len(component['repo']) > 0: - g.append(f'# Sync git repo for config={config["name"]} component={name}.') - g.append(f'pushd {os.path.dirname(component["sourcedir"])}') - - for gitlocal, repo in component['repo'].items(): - parent = os.path.basename(component["sourcedir"]) - gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) - gitremote = repo['remote'] - gitrev = repo['revision'] - basedir = os.path.normpath(os.path.join(gitlocal, '..')) - sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') - - g.append(f'if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then') - g.append(f'\trm -rf {gitlocal} > /dev/null 2>&1 || true') - g.append(f'\tmkdir -p {basedir}') - g.append(f'\ttouch {sync}') - g.append(f'\tgit clone {gitargs}{gitremote} {gitlocal}') - g.append(f'\tpushd {gitlocal}') - g.append(f'\tgit checkout {gitargs}--force {gitrev}') - g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') - g.append(f'\tpopd') - g.append(f'\trm {sync}') - g.append(f'else') - g.append(f'\tpushd {gitlocal}') - g.append(f'\tgit checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (') - g.append(f'\t\tgit fetch {gitargs}--prune --prune-tags {gitremote} &&') - g.append(f'\t\tgit checkout {gitargs}--force {gitrev})') - g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') - g.append(f'\tpopd') - g.append(f'fi') + if (type(nosync) == list and name not in nosync) or \ + (type(nosync) != list and not nosync): + g = Script('Syncing git repo', config["name"], name, preamble=pre) + if len(component['repo']) > 0: + g.append(f'# Sync git repo for config={config["name"]} component={name}.') + g.append(f'pushd {os.path.dirname(component["sourcedir"])}') + + for gitlocal, repo in component['repo'].items(): + parent = os.path.basename(component["sourcedir"]) + gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) + gitremote = repo['remote'] + gitrev = repo['revision'] + basedir = os.path.normpath(os.path.join(gitlocal, '..')) + sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') - g.append(f'popd') - g.seal() - graph[g] = [gl2] + g.append(f'if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then') + g.append(f'\trm -rf {gitlocal} > /dev/null 2>&1 || true') + g.append(f'\tmkdir -p {basedir}') + g.append(f'\ttouch {sync}') + g.append(f'\tgit clone {gitargs}{gitremote} {gitlocal}') + g.append(f'\tpushd {gitlocal}') + g.append(f'\tgit checkout {gitargs}--force {gitrev}') + g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') + g.append(f'\tpopd') + g.append(f'\trm {sync}') + g.append(f'else') + g.append(f'\tpushd {gitlocal}') + g.append(f'\tgit checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (') + g.append(f'\t\tgit fetch {gitargs}--prune --prune-tags {gitremote} &&') + g.append(f'\t\tgit checkout {gitargs}--force {gitrev})') + g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') + g.append(f'\tpopd') + g.append(f'fi') + + g.append(f'popd') + g.seal() + graph[g] = [gl2] + else: + g = gl2 b = Script('Building', config["name"], name, preamble=pre, stderrfilt=component['stderrfilt']) if len(component['prebuild']) + \ -- GitLab From 79456a1854e23cfec7fd338f2a2785fd50544b83 Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 14 Mar 2024 16:39:07 +0000 Subject: [PATCH 3/3] clean: Simplify approach to cleaning Each component has 2 directories which matter; source and build. We delete the build directory and `git clean --xdff` any repos within the source directory. Previously, we were trying to be too clever with `git reset`. But now that `shrinkwrap build` ensures the repos are synced, there is no need for this in clean. So let's simplify; get rid of --deep and always do the `git clean`. Since we are always doing this, there is no need for a component to be able to specify custom clean commands - they are just adding latency when cleaning. There is one corner case, which I'm choosing to ignore. If a component doesn't have any repos defined, but does have a custom sourcedir, there is no way to clean that custom source dir. But this is not really a valid approach. All components should have a repo if they have sources. And users should only be overriding sourcedir to point to a local git repo. And everything works in that case. Signed-off-by: Ryan Roberts --- config/cca-3world.yaml | 3 -- config/dt-base.yaml | 2 - config/edk2-base.yaml | 3 -- config/kvmtool-base.yaml | 4 -- config/linux-base.yaml | 4 -- config/tfa-base.yaml | 3 -- config/tftf-base.yaml | 3 -- documentation/userguide/configmodel.rst | 31 ++++++------- documentation/userguide/quickstart.rst | 6 --- shrinkwrap/commands/clean.py | 8 +--- shrinkwrap/utils/config.py | 62 +++++++++---------------- test/test.py | 2 +- 12 files changed, 38 insertions(+), 93 deletions(-) diff --git a/config/cca-3world.yaml b/config/cca-3world.yaml index ef58fe8..ea555e6 100644 --- a/config/cca-3world.yaml +++ b/config/cca-3world.yaml @@ -131,9 +131,6 @@ build: - make -j${param:jobs} - tar -caf ${param:builddir}/kvm-unit-tests.tgz -C ${param:sourcedir} . - clean: - - make -j${param:jobs} clean - artifacts: KVM_UNIT_TESTS: ${param:builddir}/kvm-unit-tests.tgz diff --git a/config/dt-base.yaml b/config/dt-base.yaml index f2fadbf..3d5cc98 100644 --- a/config/dt-base.yaml +++ b/config/dt-base.yaml @@ -110,8 +110,6 @@ build: };" - ( dtc -q -O dts -I dtb $${DTB_FINAL} ; echo -e "$${OVERLAY}" ) | dtc -q -O dtb -o $${DTB_FINAL} - fi - clean: - - make CPP=$${CROSS_COMPILE}cpp -j${param:jobs} clean artifacts: DTB: ${param:builddir}/dt_bootargs.dtb diff --git a/config/edk2-base.yaml b/config/edk2-base.yaml index 1fda602..b127752 100644 --- a/config/edk2-base.yaml +++ b/config/edk2-base.yaml @@ -19,9 +19,6 @@ build: - make -j${param:jobs} - mv ${param:sourcedir}/generate/unix/bin ${param:sourcedir}/generate/unix/acpica - clean: - - make -j${param:jobs} clean - artifacts: ACPICA: ${param:sourcedir}/generate/unix/acpica diff --git a/config/kvmtool-base.yaml b/config/kvmtool-base.yaml index 24de55b..2a414a6 100644 --- a/config/kvmtool-base.yaml +++ b/config/kvmtool-base.yaml @@ -29,9 +29,5 @@ build: - make -j${param:jobs} -C dtc libfdt - make -j${param:jobs} -C kvmtool - clean: - - make -j${param:jobs} -C dtc clean - - make -j${param:jobs} -C kvmtool clean - artifacts: LKVM: ${param:sourcedir}/kvmtool/lkvm \ No newline at end of file diff --git a/config/linux-base.yaml b/config/linux-base.yaml index b21b13d..e026400 100644 --- a/config/linux-base.yaml +++ b/config/linux-base.yaml @@ -69,10 +69,6 @@ build: - touch ${param:builddir}/kselftests.tgz - fi - clean: - - export ARCH=arm64 - - make -j${param:jobs} O=${param:builddir} clean - artifacts: KERNEL: ${param:builddir}/arch/arm64/boot/Image KMODULES: ${param:builddir}/modules.tgz diff --git a/config/tfa-base.yaml b/config/tfa-base.yaml index 404b404..ea9e714 100644 --- a/config/tfa-base.yaml +++ b/config/tfa-base.yaml @@ -27,9 +27,6 @@ build: # ~8 jobs unreliable, so limit it to 8. - "make BUILD_BASE=${param:builddir} ${param:join_equal} -j$$(( ${param:jobs} < 8 ? ${param:jobs} : 8 )) all fip" - clean: - - make BUILD_BASE=${param:builddir} realclean - artifacts: BL1: ${param:builddir}/fvp/release/bl1.bin BL2: ${param:builddir}/fvp/release/bl2.bin diff --git a/config/tftf-base.yaml b/config/tftf-base.yaml index 6822524..55e1831 100644 --- a/config/tftf-base.yaml +++ b/config/tftf-base.yaml @@ -25,9 +25,6 @@ build: # than ~8 jobs unreliable, so limit it to 8. - "make BUILD_BASE=${param:builddir} ${param:join_equal} -j$$(( ${param:jobs} < 8 ? ${param:jobs} : 8 )) all" - clean: - - make BUILD_BASE=${param:builddir} realclean - artifacts: SP_LAYOUT: ${param:builddir}/fvp/release/sp_layout.json diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index cd6a04c..c22a521 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -122,20 +122,20 @@ output to get a better feel for how they work. See Defined Macros -------------- -======================= ====================================================================================== ==== -macro scope description -======================= ====================================================================================== ==== -``${param:sourcedir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component's source code is located. -``${param:builddir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory in which the component should be built, if the component's build system supports separation of source and build trees. -``${param:configdir}`` build..{params, prebuild, build, postbuild, clean, artifacts} Directory containing the config store. This MUST only be used for resolving files that already exist in the store. -``${param:jobs}`` build..{params, prebuild, build, postbuild, clean, artifacts} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. -``${btvar:}`` build..{params, prebuild, build, postbuild, clean, artifacts}, run.rtvars Build-time variables. The variable names, along with default values are declared in buildex.btvars, and the user may override the value on the command line. -``${param:join_equal}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key=value`` pairs. -``${param:join_space}`` build..{prebuild, build, postbuild, clean} String containing all of the component's parameters (from its params dictionary), concatenated as ``key value`` pairs. -``${artifact:}`` build..{params, prebuild, build, postbuild, clean, artifacts}, build.btvars Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. -``${artifact:}`` run.rtvars Package path of an artifact. -``${rtvar:}`` run.params Run-time variables. The variable names, along with default values are declared in run.rtvars, and the user may override the value on the command line. -======================= ====================================================================================== ==== +======================= =============================================================================== ==== +macro scope description +======================= =============================================================================== ==== +``${param:sourcedir}`` build..{params, prebuild, build, postbuild, artifacts} Directory in which the component's source code is located. +``${param:builddir}`` build..{params, prebuild, build, postbuild, artifacts} Directory in which the component should be built, if the component's build system supports separation of source and build trees. +``${param:configdir}`` build..{params, prebuild, build, postbuild, artifacts} Directory containing the config store. This MUST only be used for resolving files that already exist in the store. +``${param:jobs}`` build..{params, prebuild, build, postbuild, artifacts} Maximum number of low level parallel jobs specified on the command line. To be passed to (e.g.) make as ``-j${param:jobs}``. +``${btvar:}`` build..{params, prebuild, build, postbuild, artifacts}, run.rtvars Build-time variables. The variable names, along with default values are declared in buildex.btvars, and the user may override the value on the command line. +``${param:join_equal}`` build..{prebuild, build, postbuild} String containing all of the component's parameters (from its params dictionary), concatenated as ``key=value`` pairs. +``${param:join_space}`` build..{prebuild, build, postbuild} String containing all of the component's parameters (from its params dictionary), concatenated as ``key value`` pairs. +``${artifact:}`` build..{params, prebuild, build, postbuild, artifacts}, build.btvars Build path of an artifact declared by another component. Usage of these macros determine the component build dependency graph. +``${artifact:}`` run.rtvars Package path of an artifact. +``${rtvar:}`` run.params Run-time variables. The variable names, along with default values are declared in run.rtvars, and the user may override the value on the command line. +======================= =============================================================================== ==== ****** Schema @@ -193,13 +193,12 @@ key type description repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync`` can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided. sourcedir string If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists. builddir string If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD. -toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild/clean commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. +toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. stderrfilt bool Optional, defaults to false. When true, and --verbose is not specified, filters stderr of the component's build task so that only lines containing 'error' and 'warning' are output. Everything else is suppressed. Useful for EDK2 which is extremely chatty. params dictionary Optional set of key:value pairs. When building most components, they require a set of parameters to be passed. By setting them out as a dictionary, it is easy to override and add to them in higher layers. See ``${param:join_*}`` macros. prebuild list List of shell commands to be executed during component build before the ``build`` list. build list List of shell commands to be executed during component build. postbuild list List of shell commands to be executed during component build after the ``build`` list. -clean list List of shell commands to be executed during component clean. artifacts dictionary Set of artifacts (files and/or directories) that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. =========== =========== =========== diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index a9e0d34..87f012f 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -874,8 +874,6 @@ command: -o $${DTB_FINAL} - fi postbuild: [] - clean: - - make CPP=$${CROSS_COMPILE}cpp -j${param:jobs} clean artifacts: DTB: ${param:builddir}/dt_bootargs.dtb edk2: @@ -909,7 +907,6 @@ command: - make -j${param:jobs} -C edk2/BaseTools - build -n ${param:jobs} -D EDK2_OUT_DIR=${param:builddir} ${param:join_space} postbuild: [] - clean: [] artifacts: EDK2: ${param:builddir}/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd edk2flash: @@ -921,7 +918,6 @@ command: prebuild: [] build: [] postbuild: [] - clean: [] artifacts: EDK2FLASH: ${param:configdir}/edk2-flash.img tfa: @@ -953,8 +949,6 @@ command: build: - make BUILD_BASE=${param:builddir} ${param:join_equal} all fip postbuild: [] - clean: - - make BUILD_BASE=${param:builddir} realclean artifacts: BL1: ${param:builddir}/fvp/release/bl1.bin BL2: ${param:builddir}/fvp/release/bl2.bin diff --git a/shrinkwrap/commands/clean.py b/shrinkwrap/commands/clean.py index 8089c07..0ffafa1 100644 --- a/shrinkwrap/commands/clean.py +++ b/shrinkwrap/commands/clean.py @@ -72,12 +72,6 @@ def add_parser(parser, formatter): required=False, default=False, action='store_true', help="""If specified, logs will not be colorized.""") - cmdp.add_argument('-d', '--deep', - required=False, default=False, action='store_true', - help="""A shallow clean removes the build directory and executes - any clean commands specified by the component. A deep clean - also cleans and resets the component's repository.""") - cmdp.add_argument('-f', '--filter', metavar='[config.]component', required=False, default=[], action='append', @@ -106,7 +100,7 @@ def dispatch(args): conf['graph'], args.filter) - graph = config.clean_graph(configs, args.verbose, args.deep) + graph = config.clean_graph(configs, args.verbose) if args.dry_run: script = ugraph.make_script(graph) diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 8450643..211671e 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -50,9 +50,6 @@ def _component_normalize(component, name): if 'postbuild' not in component: component['postbuild'] = [] - if 'clean' not in component: - component['clean'] = [] - if 'params' not in component: component['params'] = {} @@ -160,7 +157,7 @@ def _component_sort(component): improves readability by humans. """ lut = ['repo', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt', 'params', - 'prebuild', 'build', 'postbuild', 'clean', 'artifacts'] + 'prebuild', 'build', 'postbuild', 'artifacts'] lut = {k: i for i, k in enumerate(lut)} return dict(sorted(component.items(), key=lambda x: lut[x[0]])) @@ -468,7 +465,6 @@ def resolveb(config, btvars={}, clivars={}): _find_artifacts(component['prebuild']) _find_artifacts(component['build']) _find_artifacts(component['postbuild']) - _find_artifacts(component['clean']) _find_artifacts(component['artifacts'].values()) importers[name] = sorted(list(artifacts)) @@ -545,9 +541,6 @@ def resolveb(config, btvars={}, clivars={}): desc['build'][i] = _string_substitute(s, lut, final) for i, s in enumerate(desc['postbuild']): desc['postbuild'][i] = _string_substitute(s, lut, final) - for i, s in enumerate(desc['clean']): - desc['clean'][i] = _string_substitute(s, lut, final) - for k, v in desc['artifacts'].items(): desc['artifacts'][k] = _string_substitute(v, lut, final) @@ -964,7 +957,7 @@ def build_graph(configs, echo, nosync): return graph -def clean_graph(configs, echo, clean_repo): +def clean_graph(configs, echo): """ Returns a graph of scripts where the edges represent dependencies. The scripts should be executed according to the graph in order to correctly @@ -992,43 +985,30 @@ def clean_graph(configs, echo, clean_repo): c = Script('Cleaning', config["name"], name, preamble=pre) c.append(f'# Clean for config={config["name"]} component={name}.') - if len(component['clean']) > 0: - c.append(f'export CROSS_COMPILE={component["toolchain"] if component["toolchain"] else ""}') - c.append(f'if [ -d "{component["sourcedir"]}" ]; then') - c.append(f'\tpushd {component["sourcedir"]}') - for cmd in component['clean']: - c.append(f'\t{cmd}') + c.append(f'rm -rf {component["builddir"]} > /dev/null 2>&1 || true') + if len(component['repo']) > 0: + c.append(f'if [ -d "{os.path.dirname(component["sourcedir"])}" ]; then') + c.append(f'\tpushd {os.path.dirname(component["sourcedir"])}') + + for gitlocal, repo in component['repo'].items(): + parent = os.path.basename(component["sourcedir"]) + gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) + basedir = os.path.normpath(os.path.join(gitlocal, '..')) + sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') + + c.append(f'\tif [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then') + c.append(f'\t\tpushd {gitlocal}') + c.append(f'\t\tgit clean {gitargs}-xdff') + c.append(f'\t\tpopd') + c.append(f'\telse') + c.append(f'\t\trm -rf {gitlocal} {sync} > /dev/null 2>&1 || true') + c.append(f'\tfi') + c.append(f'\tpopd') c.append(f'fi') - c.append(f'rm -rf {component["builddir"]} > /dev/null 2>&1 || true') c.seal() graph[c] = [gl1] - if clean_repo: - g = Script('Cleaning git repo', config["name"], name, preamble=pre) - if len(component['repo']) > 0: - g.append(f'# Clean git repo for config={config["name"]} component={name}.') - g.append(f'if [ -d "{os.path.dirname(component["sourcedir"])}" ]; then') - g.append(f'\tpushd {os.path.dirname(component["sourcedir"])}') - - for gitlocal, repo in component['repo'].items(): - parent = os.path.basename(component["sourcedir"]) - gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) - basedir = os.path.normpath(os.path.join(gitlocal, '..')) - sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') - - g.append(f'\tif [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then') - g.append(f'\t\tpushd {gitlocal}') - g.append(f'\t\tgit clean {gitargs}-xdff') - g.append(f'\t\tgit reset {gitargs}--hard') - g.append(f'\t\tpopd') - g.append(f'\tfi') - - g.append(f'\tpopd') - g.append(f'fi') - g.seal() - graph[g] = [c] - ts.done(name) return graph diff --git a/test/test.py b/test/test.py index 8d7c8e3..54ef6a0 100755 --- a/test/test.py +++ b/test/test.py @@ -198,7 +198,7 @@ def build_configs(configs, overlay=None, btvarss=None): buildargs = f'{tmpfilename} {overlay}' try: - run(f'shrinkwrap {rt} clean {cleanargs} -d', None) + run(f'shrinkwrap {rt} clean {cleanargs}', None) run(f'shrinkwrap {rt} buildall {buildargs}', None) result['status'] = 'pass' except Exception as e: -- GitLab