From e8beab0692ffb8b394601a909d58c8072991e224 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Tue, 1 Oct 2024 12:27:44 +0100
Subject: [PATCH 1/7] config: Beautify multi-line git instructions

Use multi-line strings for long git instructions, making them easier to
read and update. No behavior change.

Remove one level of indent by folding the 'repo' check: if the config
doesn't have a 'repo', don't create a script.

textwrap.dedent() removes the trailing tabs so we can align the string
content within the current python context.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 89 ++++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 41 deletions(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 239cf8e..8d195dd 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -6,6 +6,7 @@ import io
 import json
 import os
 import re
+import textwrap
 import yaml
 import shrinkwrap.utils.clivars as uclivars
 import shrinkwrap.utils.workspace as workspace
@@ -825,6 +826,10 @@ class Script:
 
 		self._cmds += buf.getvalue()
 
+	def append_multiline(self, text, *args, indent='', **kwargs):
+		text = textwrap.indent(textwrap.dedent(text).strip(), indent)
+		self.append(text, *args, **kwargs)
+
 	def seal(self):
 		assert(not self._sealed)
 		self._sealed = True
@@ -923,41 +928,42 @@ def build_graph(configs, echo, nosync):
 			for name in ts.get_ready():
 				component = config['build'][name]
 
-				if (type(nosync) == list and name not in nosync) or \
-				   (type(nosync) != list and not nosync):
+				if ((type(nosync) == list and name not in nosync) or
+				    (type(nosync) != list and not nosync)) and \
+				   len(component['repo']) > 0:
 					g = Script('Syncing git repo', config["name"], name, preamble=pre)
-					if len(component['repo']) > 0:
-						g.append(f'# Sync git repo for config={config["name"]} component={name}.')
-						g.append(f'pushd {os.path.dirname(component["sourcedir"])}')
-
-						for gitlocal, repo in component['repo'].items():
-							parent = os.path.basename(component["sourcedir"])
-							gitlocal = os.path.normpath(os.path.join(parent, gitlocal))
-							gitremote = repo['remote']
-							gitrev = repo['revision']
-							basedir = os.path.normpath(os.path.join(gitlocal, '..'))
-							sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync')
-
-							g.append(f'if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then')
-							g.append(f'\trm -rf {gitlocal} > /dev/null 2>&1 || true')
-							g.append(f'\tmkdir -p {basedir}')
-							g.append(f'\ttouch {sync}')
-							g.append(f'\tgit clone {gitargs}{gitremote} {gitlocal}')
-							g.append(f'\tpushd {gitlocal}')
-							g.append(f'\tgit checkout {gitargs}--force {gitrev}')
-							g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force')
-							g.append(f'\tpopd')
-							g.append(f'\trm {sync}')
-							g.append(f'else')
-							g.append(f'\tpushd {gitlocal}')
-							g.append(f'\tgit checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (')
-							g.append(f'\t\tgit fetch {gitargs}--prune --prune-tags {gitremote} &&')
-							g.append(f'\t\tgit checkout {gitargs}--force {gitrev})')
-							g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force')
-							g.append(f'\tpopd')
-							g.append(f'fi')
-
-						g.append(f'popd')
+					g.append(f'# Sync git repo for config={config["name"]} component={name}.')
+					g.append(f'pushd {os.path.dirname(component["sourcedir"])}')
+
+					for gitlocal, repo in component['repo'].items():
+						parent = os.path.basename(component["sourcedir"])
+						gitlocal = os.path.normpath(os.path.join(parent, gitlocal))
+						gitremote = repo['remote']
+						gitrev = repo['revision']
+						basedir = os.path.normpath(os.path.join(gitlocal, '..'))
+						sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync')
+
+						g.append_multiline(f'''
+						if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then
+							rm -rf {gitlocal} > /dev/null 2>&1 || true
+							mkdir -p {basedir}
+							touch {sync}
+							git clone {gitargs}{gitremote} {gitlocal}
+							pushd {gitlocal}
+							git checkout {gitargs}--force {gitrev}
+							git submodule {gitargs}update --init --checkout --recursive --force
+							popd
+							rm {sync}
+						else
+							pushd {gitlocal}
+							git checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (
+								git fetch {gitargs}--prune --prune-tags {gitremote} &&
+								git checkout {gitargs}--force {gitrev})
+							git submodule {gitargs}update --init --checkout --recursive --force
+							popd
+						fi''')
+
+					g.append(f'popd')
 					g.seal()
 					graph[g] = [gl2]
 				else:
@@ -1036,13 +1042,14 @@ def clean_graph(configs, echo):
 						basedir = os.path.normpath(os.path.join(gitlocal, '..'))
 						sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync')
 
-						c.append(f'\tif [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then')
-						c.append(f'\t\tpushd {gitlocal}')
-						c.append(f'\t\tgit clean {gitargs}-xdff')
-						c.append(f'\t\tpopd')
-						c.append(f'\telse')
-						c.append(f'\t\trm -rf {gitlocal} {sync} > /dev/null 2>&1 || true')
-						c.append(f'\tfi')
+						c.append_multiline(f'''
+						if [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then
+							pushd {gitlocal}
+							git clean {gitargs}-xdff
+							popd
+						else
+							rm -rf {gitlocal} {sync} > /dev/null 2>&1 || true
+						fi''', indent='\t')
 
 					c.append(f'\tpopd')
 					c.append(f'fi')
-- 
GitLab


From c51bc391a20bad543929da31e50b63cddd961883 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Tue, 1 Oct 2024 12:27:44 +0100
Subject: [PATCH 2/7] build: Avoid overriding user source directories

When synchronizing repos, avoid overriding unsaved changes. We retain
most of the existing behavior by default:

1. Try to checkout the given revision. If it's already checked out, do
   nothing.

2. Otherwise try to fetch it from the remote and check it out.

3. But if the checkout would override unsaved changes in the local
   source dir, refuse the update the source directory and display an
   error:

      error: The following untracked working tree files would be overwritten by checkout:
      	some-file
      Please move or remove them before you switch branches.
      note: use --force-sync=tfa to override any change

With --force-sync=<component> or --force-sync-all, we fetch the given
revision from the remote, and force apply it, discarding any unsaved
changes. This also allows to fetch updates from the remote branch.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/quickstart.rst |  9 +++--
 shrinkwrap/commands/build.py           | 16 ++++++++-
 shrinkwrap/commands/buildall.py        |  8 ++---
 shrinkwrap/utils/config.py             | 50 +++++++++++++++++++++-----
 4 files changed, 66 insertions(+), 17 deletions(-)

diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst
index e1d5aac..3b4807b 100644
--- a/documentation/userguide/quickstart.rst
+++ b/documentation/userguide/quickstart.rst
@@ -314,9 +314,12 @@ artifacts.
 
   By default, Shrinkwrap will sync all component repos to the revision specified
   in the config on every build invocation. If you have made changes in the
-  working directory, your CHANGES WILL BE LOST! You can override this behaviour
-  so that Shrinkwrap just builds whatever is in the working directory by adding
-  ``--no-sync <component>`` or ``--no-sync-all`` to the command line.
+  working directory, shrinkwrap refuses to sync and displays an error. You can
+  override this behaviour so that Shrinkwrap just builds whatever is in the
+  working directory by adding ``--no-sync <component>`` or ``--no-sync-all`` to
+  the command line. Alternatively you can force shrinkwrap to override your
+  changes by adding ``--force-sync <component>`` or ``--force-sync-all`` to the
+  command line.
 
 Alternatively, pass ``--dry-run`` to view the shell script that would have been
 run:
diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py
index 564a8d2..36dc3cd 100644
--- a/shrinkwrap/commands/build.py
+++ b/shrinkwrap/commands/build.py
@@ -59,6 +59,18 @@ def add_parser(parser, formatter):
 		help="""Do not sync repos for any component, as if --no-sync was
 		     specified for every component in the config.""")
 
+	cmdp.add_argument('--force-sync',
+		metavar='component', required=False, default=[], action='append',
+		help="""Synchronize the repo of the given component even if the local
+		     source directory contains unsaved changes. YOUR CHANGES WILL BE
+		     LOST! In addition, download updates from remote branches.""")
+
+	cmdp.add_argument('--force-sync-all',
+		required=False, default=False, action='store_true',
+		help="""Synchronize all components even if the local source directories
+		     contain unsaved changes. YOUR CHANGES WILL BE LOST! In addition,
+		     download all remote branch updates.""")
+
 	buildall.add_common_args(cmdp)
 
 	return cmd_name
@@ -73,4 +85,6 @@ def dispatch(args):
 	btvars = vars.parse(args.btvar, type='bt')
 	if args.no_sync_all:
 		args.no_sync = True
-	buildall.build([args.config], [btvars], args.no_sync, args)
+	if args.force_sync_all:
+		args.force_sync = True
+	buildall.build([args.config], [btvars], args.no_sync, args.force_sync, args)
diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py
index 39104ab..41eecaf 100644
--- a/shrinkwrap/commands/buildall.py
+++ b/shrinkwrap/commands/buildall.py
@@ -108,17 +108,17 @@ def dispatch(args):
 
 	configs = [c['config'] for c in cfgs['configs']]
 	btvarss = [c['btvars'] for c in cfgs['configs']]
-	build(configs, btvarss, [], args)
+	build(configs, btvarss, [], [], args)
 
 
-def build(configs, btvarss, nosync, args):
+def build(configs, btvarss, nosync, force_sync, args):
 	"""
 	Concurrently builds a list of configs. Intended to be called as a common
 	handler for the build and buildmulti commands.
 	"""
 	clivars = {'jobs': args.jobs}
 	configs = config.load_resolveb_all(configs, args.overlay, clivars, btvarss)
-	graph = config.build_graph(configs, args.verbose, nosync)
+	graph = config.build_graph(configs, args.verbose, nosync, force_sync)
 
 	if args.dry_run:
 		script = ugraph.make_script(graph)
@@ -168,7 +168,7 @@ def build(configs, btvarss, nosync, args):
 				config.dump(c, cfg)
 
 			# Dump the script to build the config.
-			graph = config.build_graph([c], args.verbose, nosync)
+			graph = config.build_graph([c], args.verbose, nosync, force_sync)
 			script = ugraph.make_script(graph)
 			build_name = os.path.join(workspace.package,
 						  c['name'],
diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 8d195dd..ae422f3 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -878,7 +878,7 @@ def script_preamble(echo):
 	return pre.commands(False)
 
 
-def build_graph(configs, echo, nosync):
+def build_graph(configs, echo, nosync, force_sync):
 	"""
 	Returns a graph of scripts where the edges represent dependencies. The
 	scripts should be executed according to the graph in order to correctly
@@ -919,18 +919,28 @@ def build_graph(configs, echo, nosync):
 	gl2.seal()
 	graph[gl2] = [gl1]
 
+	force_sync_all = type(force_sync) != list
+	sync_none = type(nosync) != list
+
 	for config in configs:
 		build_scripts = {}
 
+		if force_sync_all:
+			force_sync = list(config['build'].keys())
+		if sync_none:
+			nosync = list(config['build'].keys())
+
+		invalid_sync_cfg = set(force_sync).intersection(nosync)
+		if invalid_sync_cfg:
+			raise Exception(f'Conflicting sync configuration for {invalid_sync_cfg}')
+
 		ts = graphlib.TopologicalSorter(config['graph'])
 		ts.prepare()
 		while ts.is_active():
 			for name in ts.get_ready():
 				component = config['build'][name]
 
-				if ((type(nosync) == list and name not in nosync) or
-				    (type(nosync) != list and not nosync)) and \
-				   len(component['repo']) > 0:
+				if name not in nosync and len(component['repo']) > 0:
 					g = Script('Syncing git repo', config["name"], name, preamble=pre)
 					g.append(f'# Sync git repo for config={config["name"]} component={name}.')
 					g.append(f'pushd {os.path.dirname(component["sourcedir"])}')
@@ -943,6 +953,32 @@ def build_graph(configs, echo, nosync):
 						basedir = os.path.normpath(os.path.join(gitlocal, '..'))
 						sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync')
 
+						if name in force_sync:
+							# We don't update any submodule before `git submodule sync`,
+							# to handle the case where a remote changes the submodule's URL.
+							# `git checkout` handles most cases, but doesn't update a local
+							# branch. So if gitrev is not a tag, do a `git reset`.
+							sync_cmd_when_exists = f'''
+							git remote set-url origin {gitremote}
+							git fetch {gitargs}--prune --prune-tags --force --recurse-submodules=off origin
+							git checkout {gitargs}--force {gitrev}
+							[ $(git tag -l {gitrev}) ] || git reset {gitargs}--hard origin/{gitrev}
+							git submodule {gitargs}sync --recursive
+							git submodule {gitargs}update --init --checkout --recursive --force
+							'''.strip()
+						else:
+							sync_cmd_when_exists = f'''
+							if ! git checkout {gitargs} {gitrev} > /dev/null 2>&1 &&
+							   ! ( git remote set-url origin {gitremote} &&
+							       git fetch {gitargs}--prune --tags origin &&
+							       git checkout {gitargs} {gitrev}) ||
+							   ! git submodule {gitargs}update --init --checkout --recursive
+							then
+								echo "note: use --force-sync={name} to override any change"
+								exit 1
+							fi
+							'''.strip()
+
 						g.append_multiline(f'''
 						if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then
 							rm -rf {gitlocal} > /dev/null 2>&1 || true
@@ -956,13 +992,9 @@ def build_graph(configs, echo, nosync):
 							rm {sync}
 						else
 							pushd {gitlocal}
-							git checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (
-								git fetch {gitargs}--prune --prune-tags {gitremote} &&
-								git checkout {gitargs}--force {gitrev})
-							git submodule {gitargs}update --init --checkout --recursive --force
+							{sync_cmd_when_exists}
 							popd
 						fi''')
-
 					g.append(f'popd')
 					g.seal()
 					graph[g] = [gl2]
-- 
GitLab


From 5bdf7c10bf1814edfddaba0756cdb9e1f4a2dc21 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 1 Aug 2024 09:55:52 +0100
Subject: [PATCH 3/7] config: Use setdefault() method

Use dict.setdefault() to assign default values, making the normalize
function more compact and easier to extend.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 115 ++++++++++---------------------------
 1 file changed, 31 insertions(+), 84 deletions(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index ae422f3..82b2ca0 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -39,46 +39,25 @@ def _component_normalize(component, name):
 	"""
 	Fills in any missing lists or dictionaries with empty ones.
 	"""
-	if 'repo' not in component:
-		component['repo'] = {}
+	component.setdefault('repo', {})
 
 	if len(component['repo']) > 0 and \
 		all(type(v) != dict for v in component['repo'].values()):
 		component['repo'] = {'.': component['repo']}
 
 	for repo in component['repo'].values():
-		if 'remote' not in repo:
-			repo['remote'] = None
-
-		if 'revision' not in repo:
-			repo['revision'] = None
-
-	if 'sourcedir' not in component:
-		component['sourcedir'] = None
-
-	if 'builddir' not in component:
-		component['builddir'] = None
-
-	if 'toolchain' not in component:
-		component['toolchain'] = None
-
-	if 'stderrfilt' not in component:
-		component['stderrfilt'] = None
-
-	if 'prebuild' not in component:
-		component['prebuild'] = []
-
-	if 'build' not in component:
-		component['build'] = []
-
-	if 'postbuild' not in component:
-		component['postbuild'] = []
-
-	if 'params' not in component:
-		component['params'] = {}
-
-	if 'artifacts' not in component:
-		component['artifacts'] = {}
+		repo.setdefault('remote', None)
+		repo.setdefault('revision', None)
+
+	component.setdefault('sourcedir', None)
+	component.setdefault('builddir', None)
+	component.setdefault('toolchain', None)
+	component.setdefault('stderrfilt', None)
+	component.setdefault('prebuild', [])
+	component.setdefault('build', [])
+	component.setdefault('postbuild', [])
+	component.setdefault('params', {})
+	component.setdefault('artifacts', {})
 
 	return component
 
@@ -98,72 +77,40 @@ def _buildex_normalize(buildex):
 	"""
 	Fills in any missing lists or dictionaries with empty ones.
 	"""
-	if 'btvars' not in buildex:
-		buildex['btvars'] = {}
+	buildex.setdefault('btvars', {})
 
 
 def _run_normalize(run):
 	"""
 	Fills in any missing lists or dictionaries with empty ones.
 	"""
-	if 'name' not in run:
-		run['name'] = None
-
-	if 'rtvars' not in run:
-		run['rtvars'] = {}
-
-	if 'params' not in run:
-		run['params'] = {}
-
-	if 'prerun' not in run:
-		run['prerun'] = []
-
-	if 'run' not in run:
-		run['run'] = []
-
-	if 'terminals' not in run:
-		run['terminals'] = {}
+	run.setdefault('name', None)
+	run.setdefault('rtvars', {})
+	run.setdefault('params', {})
+	run.setdefault('prerun', [])
+	run.setdefault('run', [])
+	run.setdefault('terminals', {})
 
 
 def _config_normalize(config):
 	"""
 	Fills in any missing lists or dictionaries with empty ones.
 	"""
-	if 'name' not in config:
-		config['name'] = None
-
-	if 'fullname' not in config:
-		config['fullname'] = None
-
-	if 'description' not in config:
-		config['description'] = None
-
-	if 'image' not in config:
-		config['image'] = None
-
-	if 'concrete' not in config:
-		config['concrete'] = False
-
-	if 'layers' not in config:
-		config['layers'] = []
-
-	if 'graph' not in config:
-		config['graph'] = {}
-
-	if 'build' not in config:
-		config['build'] = {}
-
-	if 'buildex' not in config:
-		config['buildex'] = {}
+	config.setdefault('name', None)
+	config.setdefault('fullname', None)
+	config.setdefault('description', None)
+	config.setdefault('image', None)
+	config.setdefault('concrete', False)
+	config.setdefault('layers', [])
+	config.setdefault('graph', {})
+	config.setdefault('build', {})
+	config.setdefault('buildex', {})
 
 	_build_normalize(config['build'])
 	_buildex_normalize(config['buildex'])
 
-	if 'artifacts' not in config:
-		config['artifacts'] = {}
-
-	if 'run' not in config:
-		config['run'] = {}
+	config.setdefault('artifacts', {})
+	config.setdefault('run', {})
 
 	_run_normalize(config['run'])
 
-- 
GitLab


From fef62657d065cbb3df8632847ac4cf3958480b85 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Thu, 12 Sep 2024 15:35:16 +0100
Subject: [PATCH 4/7] build: Add 'sync' config option for each build component

By default all components are synchronized. Passing --no-sync=component
or --no-sync-all on the command-line disables syncing the specified
components. When using a lot of custom user source directories, passing
a selection of --no-sync parameters can be tedious. Add a 'sync' config
option which defaults to true, and can be overridden by a user config.
For example a local.yaml config that describes local repositories could
contain:

	build:
	  component_1:
	    sourcedir: /path/to/component_1
	    sync: false
	  component_2:
	    sourcedir: /path/to/component_2
	    sync: false

sync is a tristate: true, false, force. Since yaml automatically
translates 'true' and 'false' to python bools, internal representation
is a bit awkward but not problematic.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/configmodel.rst |  9 ++++++-
 documentation/userguide/recipes.rst     |  1 +
 shrinkwrap/commands/build.py            |  3 ++-
 shrinkwrap/utils/config.py              | 36 ++++++++++++++++++++-----
 4 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst
index 7785669..bf881fd 100644
--- a/documentation/userguide/configmodel.rst
+++ b/documentation/userguide/configmodel.rst
@@ -191,7 +191,7 @@ component section
 =========== =========== ===========
 key         type        description
 =========== =========== ===========
-repo        dictionary  Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync`` or ``--no-sync-all`` can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided.
+repo        dictionary  Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync``, ``--no-sync-all`` or the ``sync`` parameter can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided.
 sourcedir   string      If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists.
 builddir    string      If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD.
 toolchain   string      Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``.
@@ -201,8 +201,15 @@ prebuild    list        List of shell commands to be executed during component b
 build       list        List of shell commands to be executed during component build.
 postbuild   list        List of shell commands to be executed during component build after the ``build`` list.
 artifacts   dictionary  Set of artifacts (files and/or directories) that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:<name>}`` macros. Used to determine build dependencies.
+sync        enum-string Specifies how shrinkwrap should synchronize the repository. See below for options.
 =========== =========== ===========
 
+Sync mode:
+
+- **true**: synchronize the source directory. Do not overwrite user modifications or download updates.
+- **false**: do not synchronize the source directory.
+- **force**: synchronize the source directory. Overwrite any user modification and download branch updates.
+
 -----------
 run section
 -----------
diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst
index 2135f44..c71bbae 100644
--- a/documentation/userguide/recipes.rst
+++ b/documentation/userguide/recipes.rst
@@ -73,6 +73,7 @@ Create a file called ``my-overlay.yaml``:
   build:
     tfa:
       sourcedir: /path/to/my/tfa/git/repo
+      sync: false
 
 Now do a build, passing in the overlay:
 
diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py
index 36dc3cd..073ac51 100644
--- a/shrinkwrap/commands/build.py
+++ b/shrinkwrap/commands/build.py
@@ -69,7 +69,8 @@ def add_parser(parser, formatter):
 		required=False, default=False, action='store_true',
 		help="""Synchronize all components even if the local source directories
 		     contain unsaved changes. YOUR CHANGES WILL BE LOST! In addition,
-		     download all remote branch updates.""")
+		     download all remote branch updates. Note that this will override
+		     any 'sync: false' config option.""")
 
 	buildall.add_common_args(cmdp)
 
diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index 82b2ca0..aa7c010 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -58,6 +58,11 @@ def _component_normalize(component, name):
 	component.setdefault('postbuild', [])
 	component.setdefault('params', {})
 	component.setdefault('artifacts', {})
+	component.setdefault('sync', None)
+	if component['sync'] == False:
+		component['sync'] = 'false'
+	elif component['sync'] == True:
+		component['sync'] = 'true'
 
 	return component
 
@@ -116,6 +121,14 @@ def _config_normalize(config):
 
 	return config
 
+def _component_validate(component):
+	sync = component.get('sync')
+	if sync not in (None, 'true', 'false', 'force'):
+		raise Exception(f'invalid "sync" value "{sync}"')
+
+def _build_validate(build):
+	for component in build.values():
+		_component_validate(component)
 
 def _config_validate(config):
 	"""
@@ -124,14 +137,17 @@ def _config_validate(config):
 	"""
 	# TODO:
 
+	if 'build' in config:
+		_build_validate(config['build'])
+
 
 def _component_sort(component):
 	"""
 	Sort the component so that the keys are in a canonical order. This
 	improves readability by humans.
 	"""
-	lut = ['repo', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt', 'params',
-			'prebuild', 'build', 'postbuild', 'artifacts']
+	lut = ['repo', 'sync', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt',
+			'params', 'prebuild', 'build', 'postbuild', 'artifacts']
 	lut = {k: i for i, k in enumerate(lut)}
 	return dict(sorted(component.items(), key=lambda x: lut[x[0]]))
 
@@ -548,6 +564,8 @@ def resolveb(config, btvars={}, clivars={}):
 			desc['builddir'] = os.path.join(workspace.build,
 							'build',
 							comp_dir)
+		if desc['sync'] is None:
+			desc['sync'] = 'true'
 
 	macro_lut = {
 		'param': {
@@ -872,14 +890,18 @@ def build_graph(configs, echo, nosync, force_sync):
 	for config in configs:
 		build_scripts = {}
 
-		if force_sync_all:
-			force_sync = list(config['build'].keys())
-		if sync_none:
-			nosync = list(config['build'].keys())
+		# Make copies of nosync and force_sync that we can modify below
+		force_sync = set(config['build'].keys() if force_sync_all else force_sync)
+		nosync = set(config['build'].keys() if sync_none else nosync)
 
-		invalid_sync_cfg = set(force_sync).intersection(nosync)
+		invalid_sync_cfg = force_sync.intersection(nosync)
 		if invalid_sync_cfg:
 			raise Exception(f'Conflicting sync configuration for {invalid_sync_cfg}')
+		for name, component in config['build'].items():
+			if component['sync'] == 'false' and name not in force_sync:
+				nosync.add(name)
+			elif component['sync'] == 'force' and name not in nosync:
+				force_sync.add(name)
 
 		ts = graphlib.TopologicalSorter(config['graph'])
 		ts.prepare()
-- 
GitLab


From c43f2fb7d62a3653b075b8e318fa4a540c88daa5 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Fri, 20 Sep 2024 11:54:22 +0100
Subject: [PATCH 5/7] build: Don't override git worktrees

With git-worktree, source directories contain a '.git' text file that
contains a path to the main worktree. Before rewriting the whole
directory, check if .git exists instead of checking if it is a
directory.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 shrinkwrap/utils/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py
index aa7c010..2e03147 100644
--- a/shrinkwrap/utils/config.py
+++ b/shrinkwrap/utils/config.py
@@ -949,7 +949,7 @@ def build_graph(configs, echo, nosync, force_sync):
 							'''.strip()
 
 						g.append_multiline(f'''
-						if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then
+						if [ ! -e "{gitlocal}/.git" ] || [ -f "{sync}" ]; then
 							rm -rf {gitlocal} > /dev/null 2>&1 || true
 							mkdir -p {basedir}
 							touch {sync}
-- 
GitLab


From 68f135103e01daeaabe20a3a0d10ca866a51d195 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Tue, 1 Oct 2024 11:14:12 +0100
Subject: [PATCH 6/7] doc: Update the generated script example

Update the generated script contained in the doc

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 documentation/userguide/quickstart.rst | 175 ++++++++++++++++++-------
 1 file changed, 126 insertions(+), 49 deletions(-)

diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst
index 3b4807b..848aea0 100644
--- a/documentation/userguide/quickstart.rst
+++ b/documentation/userguide/quickstart.rst
@@ -343,93 +343,165 @@ run:
   set -e
 
   # Remove old package.
-  rm -rf /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2.yaml > /dev/null 2>&1 || true
-  rm -rf /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2 > /dev/null 2>&1 || true
+  rm -rf package/ns-edk2.yaml > /dev/null 2>&1 || true
+  rm -rf package/ns-edk2 > /dev/null 2>&1 || true
 
   # Create directory structure.
-  mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/dt
-  mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2
-  mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/tfa
-  mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2
+  mkdir -p source/ns-edk2/acpica
+  mkdir -p source/ns-edk2/dt
+  mkdir -p source/ns-edk2/edk2
+  mkdir -p source/ns-edk2/tfa
+  mkdir -p package/ns-edk2
+
+  # Sync git repo for config=ns-edk2 component=acpica.
+  pushd source/ns-edk2
+  if [ ! -e "acpica/.git" ] || [ -f "./.acpica_sync" ]; then
+  	rm -rf acpica > /dev/null 2>&1 || true
+  	mkdir -p .
+  	touch ./.acpica_sync
+  	git clone --quiet https://github.com/acpica/acpica.git acpica
+  	pushd acpica
+  	git checkout --quiet --force R06_28_23
+  	git submodule --quiet update --init --checkout --recursive --force
+  	popd
+  	rm ./.acpica_sync
+  else
+  	pushd acpica
+  	if ! git checkout --quiet  R06_28_23 > /dev/null 2>&1 &&
+  	   ! ( git remote set-url origin https://github.com/acpica/acpica.git &&
+  	       git fetch --quiet --prune --tags origin &&
+  	       git checkout --quiet  R06_28_23) ||
+  	   ! git submodule --quiet update --init --checkout --recursive
+  	then
+  		echo "note: use --force-sync=acpica to override any change"
+  		exit 1
+  	fi
+  	popd
+  fi
+  popd
 
   # Sync git repo for config=ns-edk2 component=dt.
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2
-  if [ ! -d "dt/.git" ] || [ -f "./.dt_sync" ]; then
+  pushd source/ns-edk2
+  if [ ! -e "dt/.git" ] || [ -f "./.dt_sync" ]; then
   	rm -rf dt > /dev/null 2>&1 || true
   	mkdir -p .
   	touch ./.dt_sync
-  	git clone --quiet git://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git dt
+  	git clone --quiet https://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git dt
   	pushd dt
-  	git checkout --quiet --force v6.1-dts
+  	git checkout --quiet --force v6.6-dts
   	git submodule --quiet update --init --checkout --recursive --force
   	popd
   	rm ./.dt_sync
+  else
+  	pushd dt
+  	if ! git checkout --quiet  v6.6-dts > /dev/null 2>&1 &&
+  	   ! ( git remote set-url origin https://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git &&
+  	       git fetch --quiet --prune --tags origin &&
+  	       git checkout --quiet  v6.6-dts) ||
+  	   ! git submodule --quiet update --init --checkout --recursive
+  	then
+  		echo "note: use --force-sync=dt to override any change"
+  		exit 1
+  	fi
+  	popd
   fi
   popd
 
   # Sync git repo for config=ns-edk2 component=edk2.
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2
-  if [ ! -d "edk2/edk2/.git" ] || [ -f "edk2/.edk2_sync" ]; then
+  pushd source/ns-edk2
+  if [ ! -e "edk2/edk2/.git" ] || [ -f "edk2/.edk2_sync" ]; then
   	rm -rf edk2/edk2 > /dev/null 2>&1 || true
   	mkdir -p edk2
   	touch edk2/.edk2_sync
   	git clone --quiet https://github.com/tianocore/edk2.git edk2/edk2
   	pushd edk2/edk2
-  	git checkout --quiet --force edk2-stable202211
+  	git checkout --quiet --force edk2-stable202311
   	git submodule --quiet update --init --checkout --recursive --force
   	popd
   	rm edk2/.edk2_sync
+  else
+  	pushd edk2/edk2
+  	if ! git checkout --quiet  edk2-stable202311 > /dev/null 2>&1 &&
+  	   ! ( git remote set-url origin https://github.com/tianocore/edk2.git &&
+  	       git fetch --quiet --prune --tags origin &&
+  	       git checkout --quiet  edk2-stable202311) ||
+  	   ! git submodule --quiet update --init --checkout --recursive
+  	then
+  		echo "note: use --force-sync=edk2 to override any change"
+  		exit 1
+  	fi
+  	popd
   fi
-  if [ ! -d "edk2/edk2-platforms/.git" ] || [ -f "edk2/.edk2-platforms_sync" ]; then
+  if [ ! -e "edk2/edk2-platforms/.git" ] || [ -f "edk2/.edk2-platforms_sync" ]; then
   	rm -rf edk2/edk2-platforms > /dev/null 2>&1 || true
   	mkdir -p edk2
   	touch edk2/.edk2-platforms_sync
   	git clone --quiet https://github.com/tianocore/edk2-platforms.git edk2/edk2-platforms
   	pushd edk2/edk2-platforms
-  	git checkout --quiet --force 20e07099d8f11889d101dd710ca85001be20e179
+  	git checkout --quiet --force 4b07df2e6f3813c6e955197dacb2cdfbe3471caa
   	git submodule --quiet update --init --checkout --recursive --force
   	popd
   	rm edk2/.edk2-platforms_sync
-  fi
-  if [ ! -d "edk2/acpica/.git" ] || [ -f "edk2/.acpica_sync" ]; then
-  	rm -rf edk2/acpica > /dev/null 2>&1 || true
-  	mkdir -p edk2
-  	touch edk2/.acpica_sync
-  	git clone --quiet https://github.com/acpica/acpica.git edk2/acpica
-  	pushd edk2/acpica
-  	git checkout --quiet --force R10_20_22
-  	git submodule --quiet update --init --checkout --recursive --force
+  else
+  	pushd edk2/edk2-platforms
+  	if ! git checkout --quiet  4b07df2e6f3813c6e955197dacb2cdfbe3471caa > /dev/null 2>&1 &&
+  	   ! ( git remote set-url origin https://github.com/tianocore/edk2-platforms.git &&
+  	       git fetch --quiet --prune --tags origin &&
+  	       git checkout --quiet  4b07df2e6f3813c6e955197dacb2cdfbe3471caa) ||
+  	   ! git submodule --quiet update --init --checkout --recursive
+  	then
+  		echo "note: use --force-sync=edk2 to override any change"
+  		exit 1
+  	fi
   	popd
-  	rm edk2/.acpica_sync
   fi
   popd
 
-
   # Sync git repo for config=ns-edk2 component=tfa.
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2
-  if [ ! -d "tfa/.git" ] || [ -f "./.tfa_sync" ]; then
+  pushd source/ns-edk2
+  if [ ! -e "tfa/.git" ] || [ -f "./.tfa_sync" ]; then
   	rm -rf tfa > /dev/null 2>&1 || true
   	mkdir -p .
   	touch ./.tfa_sync
   	git clone --quiet https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git tfa
   	pushd tfa
-  	git checkout --quiet --force v2.8.0
+  	git checkout --quiet --force v2.11
   	git submodule --quiet update --init --checkout --recursive --force
   	popd
   	rm ./.tfa_sync
+  else
+  	pushd tfa
+  	if ! git checkout --quiet  v2.11 > /dev/null 2>&1 &&
+  	   ! ( git remote set-url origin https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git &&
+  	       git fetch --quiet --prune --tags origin &&
+  	       git checkout --quiet  v2.11) ||
+  	   ! git submodule --quiet update --init --checkout --recursive
+  	then
+  		echo "note: use --force-sync=tfa to override any change"
+  		exit 1
+  	fi
+  	popd
   fi
   popd
 
+  # Build for config=ns-edk2 component=acpica.
+  export CROSS_COMPILE=
+  pushd source/ns-edk2/acpica
+  rm -rf source/ns-edk2/acpica/generate/unix/acpica
+  make -j4
+  mv source/ns-edk2/acpica/generate/unix/bin source/ns-edk2/acpica/generate/unix/acpica
+  popd
+
   # Build for config=ns-edk2 component=dt.
   export CROSS_COMPILE=aarch64-none-elf-
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/dt
+  pushd source/ns-edk2/dt
   DTS=fvp-base-revc.dts
   INITRD_START=
   INITRD_END=
   DT_BASENAME=$(basename ${DTS} .dts)
   DTB_INTER=src/arm64/arm/${DT_BASENAME}.dtb
-  DTB_FINAL=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/dt/dt_bootargs.dtb
-  make CPP=${CROSS_COMPILE}cpp -j28 ${DTB_INTER}
+  DTB_FINAL=build/ns-edk2/dt/dt_bootargs.dtb
+  make CPP=${CROSS_COMPILE}cpp -j4 ${DTB_INTER}
   CHOSEN=
   if [ ! -z "" ]; then
   CHOSEN="${CHOSEN}bootargs = \"\";\n"
@@ -492,35 +564,40 @@ run:
   fi
   popd
 
+  # Copy artifacts for config=ns-edk2 component=acpica.
+  cp -r source/ns-edk2/acpica/generate/unix/acpica package/ns-edk2/acpica
+
   # Build for config=ns-edk2 component=edk2.
   export CROSS_COMPILE=aarch64-none-elf-
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2
-  export WORKSPACE=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2
+  pushd source/ns-edk2/edk2
+  export WORKSPACE=source/ns-edk2/edk2
   export GCC5_AARCH64_PREFIX=$CROSS_COMPILE
   export PACKAGES_PATH=$WORKSPACE/edk2:$WORKSPACE/edk2-platforms
-  export IASL_PREFIX=$WORKSPACE/acpica/generate/unix/bin/
+  export IASL_PREFIX=source/ns-edk2/acpica/generate/unix/acpica/
   export PYTHON_COMMAND=/usr/bin/python3
-  make -j28 -C acpica
-  source edk2/edksetup.sh
-  make -j28 -C edk2/BaseTools
-  build -n 28 -D EDK2_OUT_DIR=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2 -a AARCH64 -t GCC5 -p Platform/ARM/VExpressPkg/ArmVExpress-FVP-AArch64.dsc -b RELEASE
+  source edk2/edksetup.sh --reconfig
+  make -j4 -C edk2/BaseTools
+  build -n 4 -D EDK2_OUT_DIR=build/ns-edk2/edk2 -a AARCH64 -t GCC5 -p Platform/ARM/VExpressPkg/ArmVExpress-FVP-AArch64.dsc -b RELEASE --pcd PcdShellDefaultDelay=0  --pcd PcdUefiShellDefaultBootEnable=1
   popd
 
+  # Copy artifacts for config=ns-edk2 component=dt.
+  cp -r build/ns-edk2/dt/dt_bootargs.dtb package/ns-edk2/dt_bootargs.dtb
+
+  # Copy artifacts for config=ns-edk2 component=edk2.
+  cp -r build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd package/ns-edk2/FVP_AARCH64_EFI.fd
 
   # Build for config=ns-edk2 component=tfa.
   export CROSS_COMPILE=aarch64-none-elf-
-  pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/tfa
-  make BUILD_BASE=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa PLAT=fvp DEBUG=0 LOG_LEVEL=40 ARM_DISABLE_TRUSTED_WDOG=1 FVP_HW_CONFIG_DTS=fdts/fvp-base-gicv3-psci-1t.dts BL33=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd ARM_ARCH_MINOR=5 ENABLE_SVE_FOR_NS=1 ENABLE_SVE_FOR_SWD=1 CTX_INCLUDE_PAUTH_REGS=1 BRANCH_PROTECTION=1 CTX_INCLUDE_MTE_REGS=1 ENABLE_FEAT_HCX=1 CTX_INCLUDE_AARCH32_REGS=0 ENABLE_SME_FOR_NS=1 ENABLE_SME_FOR_SWD=1 all fip
+  pushd source/ns-edk2/tfa
+  make BUILD_BASE=build/ns-edk2/tfa LOG_LEVEL=40 ARM_ARCH_MINOR=2 DEBUG=0 ARM_DISABLE_TRUSTED_WDOG=1 CTX_INCLUDE_AARCH32_REGS=0 BRANCH_PROTECTION=1 ARM_ARCH_MAJOR=9 PLAT=fvp BL33=build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd FVP_HW_CONFIG_DTS=fdts/fvp-base-gicv3-psci-1t.dts -j$(( 4 < 8 ? 4 : 8 )) all fip
   popd
 
-  # Copy artifacts for config=ns-edk2.
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/dt/dt_bootargs.dtb /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/dt_bootargs.dtb
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/FVP_AARCH64_EFI.fd
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/shrinkwrap/config/edk2-flash.img /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/edk2-flash.img
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl1.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl1.bin
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl2.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl2.bin
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl31.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl31.bin
-  cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/fip.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/fip.bin
+  # Copy artifacts for config=ns-edk2 component=tfa.
+  cp -r build/ns-edk2/tfa/fvp/release/bl31.bin package/ns-edk2/bl31.bin
+  cp -r build/ns-edk2/tfa/fvp/release/bl1.bin package/ns-edk2/bl1.bin
+  cp -r build/ns-edk2/tfa/fvp/release/fip.bin package/ns-edk2/fip.bin
+  cp -r build/ns-edk2/tfa/fvp/release/bl2.bin package/ns-edk2/bl2.bin
+
 
 .. raw:: html
 
-- 
GitLab


From 51006d66ac7f621956ae41b885e2140aba5003bb Mon Sep 17 00:00:00 2001
From: Jean-Philippe Brucker <jean-philippe@linaro.org>
Date: Mon, 14 Oct 2024 18:35:15 +0100
Subject: [PATCH 7/7] test: Add test-sync.sh

Add a script to test the synchronization feature of shrinkwrap. Test
various scenarios including repo and module update.

This gives us a quick command to check for regressions when updating
shrinkwrap's synchronization code.

Signed-off-by: Jean-Philippe Brucker <jean-philippe@linaro.org>
---
 test/test-sync.sh | 542 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 542 insertions(+)
 create mode 100755 test/test-sync.sh

diff --git a/test/test-sync.sh b/test/test-sync.sh
new file mode 100755
index 0000000..6de6563
--- /dev/null
+++ b/test/test-sync.sh
@@ -0,0 +1,542 @@
+#!/bin/bash
+# Create temporary git repositories and test shrinkwrap's synchronization feature
+
+# Run on the host, so we don't need to deal with remote git URLs.
+SHRINKWRAP="shrinkwrap -R null"
+
+set -eu
+
+LOG=/tmp/shrinkwrap-sync-test.log
+echo > $LOG
+echo "Logging to file $LOG"
+
+set -x
+exec 2>>$LOG
+
+T=$(mktemp -d)
+# paranoid check: ensure we won't remove existing files
+[ -z "$T" -o -n "$(ls -A $T)" ] && exit 1
+trap "rm -rf $T" EXIT
+
+export SHRINKWRAP_BUILD=$T/build
+export SHRINKWRAP_PACKAGE=$T/package
+
+# Test $1 must succeed
+OK () {
+    local name="$1"
+    shift
+    $SHRINKWRAP build -v $T/base.yaml $@ >>$LOG 2>&1 ||
+        { echo "FAIL  $name" && exit 1; }
+    echo "PASS  $name"
+}
+
+# Test $1 must fail
+FAIL () {
+    local name="$1"
+    shift
+    $SHRINKWRAP build -v $T/base.yaml $@ >>$LOG 2>&1 &&
+        echo "FAIL  $name" && exit 1
+    echo "PASS  $name"
+}
+
+# File $1 content must be equal to $2
+CONTENT () {
+    local file="$1"
+    local nice="${file##$T}" # remove prefix
+    local content="$2"
+
+    if [ -e "$file" ] && [ "$(cat $file)" = "$content" ]; then
+        echo "PASS    $nice"
+    else
+        echo "FAIL    $nice"
+        exit 1
+    fi
+}
+
+TESTS () {
+    echo "$@"
+    echo "#### $@" >> $LOG
+}
+
+
+# Create the initial repos and config
+{
+    git init $T/repo1 -b main
+    git init $T/module1 -b main
+
+    pushd $T/module1
+    echo -n "module 1 commit 0" > README
+    git add README
+    git commit -m "commit 0"
+    popd
+
+    pushd $T/repo1
+    echo -n "repo 1 commit 0" > README
+    git add README
+    git commit -m "commit 0"
+
+    git submodule add ssh://localhost:$T/module1 module
+    git commit -a -m "Add module 1"
+    popd
+} >> $LOG
+
+cat << EOF > $T/base.yaml
+%YAML 1.2
+---
+concrete: true
+
+build:
+  test1:
+    repo:
+      remote: file://$T/repo1
+      revision: main
+    build:
+      - cat module/README
+EOF
+
+
+################################################################################
+TESTS   Basic clone and update
+
+OK "initial clone"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0"
+
+OK "second sync is nop"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0"
+
+# Update the repo1 README
+{
+    pushd $T/repo1
+    echo -n "repo 1 commit 1" > README
+    git commit -a -m "commit 1"
+    popd
+} >> $LOG
+
+OK "sync=on does not update"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0"
+
+OK "sync=force updates" --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0"
+
+
+################################################################################
+TESTS   Module update
+
+# Update the module1 README
+{
+    pushd $T/module1
+    echo -n "module 1 commit 1" > README
+    git commit -a -m "commit 1"
+    popd
+
+    pushd $T/repo1
+    git submodule update --remote
+    git commit -a -m "Update submodule 1"
+    popd
+} >> $LOG
+
+OK "sync=on does not update"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0"
+
+OK "sync=force updates module" --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 1"
+
+
+################################################################################
+TESTS   Tag and branch update
+
+# Update repo1 with a tag
+{
+    pushd $T/repo1
+    echo -n "repo 1 commit 2" > README
+    git commit -a -m "commit 1"
+    git tag v0.1
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.1
+EOF
+
+OK "update new tag" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+
+OK "rollback to main without overlay"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1"
+
+# Update repo1 with a branch
+{
+    pushd $T/repo1
+    git checkout -b topic1
+    echo -n "repo 1 commit topic.1" > README
+    git commit -a -m "commit 1 on topic"
+    git checkout -
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: topic1
+EOF
+
+
+OK "update new branch" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit topic.1"
+
+OK "rollback to main without overlay"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1"
+
+OK "force update" --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+
+
+################################################################################
+TESTS   Tagged module update
+
+# Update the module1 README
+{
+    pushd $T/module1
+    echo -n "module 1 commit 2" > README
+    git commit -a -m "commit 2"
+    popd
+
+    pushd $T/repo1
+    git submodule update --remote
+    git commit -a -m "Update submodule 1"
+    git tag v0.1.1
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.1.1
+EOF
+
+OK "tagged module update" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 2"
+
+
+################################################################################
+TESTS   Tag and branch udpates with --force-sync
+
+# Update repo1 with a tag
+{
+    pushd $T/repo1
+    echo -n "repo 1 commit 3" > README
+    git commit -a -m "commit 1"
+    git tag v0.2
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.2
+EOF
+
+OK "tag update with sync=force" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3"
+
+OK "back to main (overriden by force)"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+
+# Update repo1 with a branch
+{
+    pushd $T/repo1
+    git checkout -b topic2
+    echo -n "repo 1 commit topic.2" > README
+    git commit -a -m "commit 1 on topic 2"
+    git checkout -
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: topic2
+EOF
+
+OK "branch update with sync=force" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit topic.2"
+
+OK "back to main"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+
+
+
+################################################################################
+TESTS   Overwrite file
+
+# Add a file to repo1
+{
+    pushd $T/repo1
+    echo "int main() {}" > main.c
+    git add main.c
+    git commit -a -m "add main"
+    git tag v0.3
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.3
+EOF
+
+# Also create the file locally
+{
+    echo "int foo() {}" > $SHRINKWRAP_BUILD/source/base/test1/main.c
+} >> $LOG
+
+FAIL "refusing to overwrite" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/main.c "int foo() {}"
+
+OK "force overwrite" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/main.c "int main() {}"
+
+
+################################################################################
+TESTS   Overwrite module file
+
+# Add a file to module1, in a new branch
+{
+    pushd $T/module1
+    echo "int main() {}" > main.c
+    git add main.c
+    git commit -a -m "add main"
+    popd
+
+    pushd $T/repo1
+    git submodule update --remote --merge
+    git commit -a -m "Update submodule 2"
+    git tag v0.4
+    popd
+} >> $LOG
+
+# Update revision with an overlay
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.4
+EOF
+
+# Also create the file locally
+{
+    echo "int foo() {}" > $SHRINKWRAP_BUILD/source/base/test1/module/main.c
+} >> $LOG
+
+FAIL "refusing to overwrite submodule" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/main.c "int foo() {}"
+
+OK "force overwrite submodule" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/main.c "int main() {}"
+
+
+################################################################################
+TESTS   User source dir
+
+# Use a local source directory
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    sourcedir: $T/my-source/
+EOF
+
+OK "create local sourcedir" -o $T/overlay.yaml
+CONTENT $T/my-source/README "repo 1 commit 3"
+
+rm -rf $T/my-source/
+mkdir $T/my-source/
+
+OK "use empty local sourcedir" -o $T/overlay.yaml
+CONTENT $T/my-source/README "repo 1 commit 3"
+
+rm -rf $T/my-source/
+mkdir $T/my-source/
+echo GPL > $T/my-source/LICENSE
+
+# fatal: destination path '.' already exists and is not an empty directory.
+# --force-sync won't work in this case because it's the clone that fails
+FAIL "clone fail in non-empty sourcedir" -o $T/overlay.yaml
+CONTENT $T/my-source/LICENSE "GPL"
+
+rm -rf $T/my-source/
+
+
+################################################################################
+TESTS   Override branch
+
+# Reuse an existing branch name in a different repo
+{
+    git init $T/repo2 -b main
+    git init $T/module2 -b main
+
+    pushd $T/module2
+    echo -n "module 2 commit 0" > README
+    git add README
+    git commit -m "commit 0"
+    popd
+
+    pushd $T/repo2
+    echo repo 2 commit 0 > README
+    git add README
+    git commit -a -m "commit 0"
+
+    git submodule add ssh://localhost:$T/module2 module
+    git commit -a -m "Add module 2"
+    popd
+} >> $LOG
+
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      remote: file://$T/repo2
+EOF
+
+OK "sync=on use old main branch" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2"
+
+OK "sync=force use new main branch" -o $T/overlay.yaml --force-sync-all
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 0"
+
+
+################################################################################
+TESTS   Update module branch
+
+# Update repo2 module branch
+{
+    pushd $T/module2
+    git checkout -b branch2
+    echo -n "module 2 commit 1" > README
+    git commit -a -m "commit 1"
+    popd
+
+    pushd $T/repo2
+    git submodule set-branch -b branch2 module
+    git submodule update --remote
+    git commit -a -m "Update module branch"
+    popd
+} >> $LOG
+
+OK "sync=force update module branch" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 1"
+
+
+################################################################################
+TESTS   Update module URL
+
+# Update repo2 module URL
+{
+    pushd $T/repo2
+    git submodule set-branch -b main module
+    git submodule set-url module ssh://localhost:$T/module1
+    git submodule update --remote
+    git commit -a -m "Update module URL"
+    popd
+} >> $LOG
+
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      remote: file://$T/repo2
+EOF
+
+OK "sync=true doesn't update module URL" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 1"
+
+OK "sync=force updates module URL" -o $T/overlay.yaml --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0"
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 2"
+
+
+OK "rollback to repo1" --force-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3"
+
+
+################################################################################
+TESTS   nosync config
+
+# Update the repo1 README
+{
+    pushd $T/repo1
+    echo -n "repo 1 commit 4" > README
+    git commit -a -m "commit 4"
+    git tag v0.5
+    popd
+} >> $LOG
+
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    repo:
+      revision: v0.5
+    sync: false
+EOF
+
+OK "config sync=false does not update" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3"
+
+OK "config sync=false updates with --force-sync-all" -o $T/overlay.yaml --force-sync-all
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4"
+
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    sync: hello
+EOF
+
+FAIL "invalid config" -o $T/overlay.yaml
+
+# Update the repo1 README
+{
+    pushd $T/repo1
+    echo -n "repo 1 commit 5" > README
+    git commit -a -m "commit 5"
+    popd
+} >> $LOG
+
+cat << EOF > $T/overlay.yaml
+build:
+  test1:
+    sync: force
+EOF
+
+OK "config sync=force does not update with --no-sync" -o $T/overlay.yaml --no-sync=test1
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4"
+
+OK "config sync=force does not update with --no-sync-all" -o $T/overlay.yaml --no-sync-all
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4"
+
+OK "config sync=force updates" -o $T/overlay.yaml
+CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 5"
+
+
+echo ALL TESTS PASS
-- 
GitLab