diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index 93331eee0adc03c258b2bcfb2893ebf4b10c2823..3d8ed945b2cb2de780fdc4a4b47738cbf1ec0e21 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -212,6 +212,17 @@ Sync mode: - **false**: do not synchronize the source directory. - **force**: synchronize the source directory. Overwrite any user modification and download branch updates. +------------ +repo section +------------ + +=========== =========== =========== +key type description +=========== =========== =========== +remote string Address of the remote repository +revision string A git revision (branch name, tag, hash...) +project string Optional. Name of the project corresponding to this repository, used to retrieve a cached repository in SHRINKWRAP_PROJECT_CACHE. By default, the project name is the base name contained in the remote address. + ----------- run section ----------- diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index 557bdf90517737e3328e85e75fd559f494744361..f21b6bc918d183d45827e3aaa516b2985bf2971a 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -72,13 +72,14 @@ Optional Environment Variables Shrinkwrap consumes the following set of optional environment variables: -================== ===================== ==== -name default description -================== ===================== ==== -SHRINKWRAP_CONFIG Colon-separated list of paths to config stores. Configs are searched for relative to the current directory as well as relative to these paths. -SHRINKWRAP_BUILD ~/.shrinkwrap/build Location where config builds are performed. Each config has its own subdirectory, with further subdirectories for each of its components. -SHRINKWRAP_PACKAGE ~/.shrinkwrap/package Location where config builds are packaged to. When running a config, it is done from the package location. -================== ===================== ==== +========================= ===================== ==== +name default description +========================= ===================== ==== +SHRINKWRAP_CONFIG Colon-separated list of paths to config stores. Configs are searched for relative to the current directory as well as relative to these paths. +SHRINKWRAP_BUILD ~/.shrinkwrap/build Location where config builds are performed. Each config has its own subdirectory, with further subdirectories for each of its components. +SHRINKWRAP_PACKAGE ~/.shrinkwrap/package Location where config builds are packaged to. When running a config, it is done from the package location. +SHRINKWRAP_PROJECT_CACHE Location where cache repositories are stored. This directory contains git trees used as reference when downloading components, allowing to reduce network usage and local storage if the same project is used multiple times. Cache directories ending in ".git" are bare repositories, and ones without the suffix are full repositories. By default no cache is used. +======================== ===================== ==== *************************************************** Guided Tour: Configure a platform and boot a kernel diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index 41eecafd141ca379c07b9f689b863bc8ee8d07a0..3379c077cc9f516967eeadac8362b16935bedf48 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -153,6 +153,9 @@ def build(configs, btvarss, nosync, force_sync, args): if btvar['type'] == 'path': rt.add_volume(btvar['value']) + if workspace.project_cache: + add_volume(workspace.project_cache) + rt.start() ugraph.execute(graph, diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index d1cfdf989c02ee15b9dff69bf67075cc010ded2d..1c9d09622429b19efcf333a45a92ea51e17bfb61 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -10,6 +10,7 @@ import textwrap import yaml import shrinkwrap.utils.clivars as uclivars import shrinkwrap.utils.workspace as workspace +from urllib.parse import urlparse default_image = 'docker.io/shrinkwraptool/base-slim:latest' @@ -48,6 +49,7 @@ def _component_normalize(component, name): for repo in component['repo'].values(): repo.setdefault('remote', None) repo.setdefault('revision', None) + repo.setdefault('project', None) component.setdefault('sourcedir', None) component.setdefault('builddir', None) @@ -858,6 +860,40 @@ def script_preamble(echo): else: pre.append(f'# Exit on error.') pre.append(f'set -e') + + gitargs = '"--quiet "' if not echo else '""' + pre.append(f'gitargs={gitargs}') + pre.append_multiline(''' + # Function to update submodules without recursion + update_submodules() { + local repo_path="$1" + local reference_path="$2" + + local cwd=$(pwd) + cd $repo_path + # Check if .gitmodules file exists + local gitmodules_file=".gitmodules" + if [ -f "$gitmodules_file" ]; then + # Extract submodule paths from .gitmodules + git config --file "$gitmodules_file" --get-regexp path | while read -r path_key submodule_path; do + local git_submodule_reference="" + local submodule_reference="$reference_path/$submodule_path" + + # Check if the submodule exists in the project cache + if [[ -n "$reference_path" && -e "$submodule_reference/.git" ]]; then + git_submodule_reference="--reference $submodule_reference" + fi + + if [ -d "$submodule_path" ]; then + # Manually update nested submodules + git submodule $gitargs update --init --checkout --force $git_submodule_reference $submodule_path + # Recursively process the submodule + update_submodules "$submodule_path" "$submodule_reference" + fi + done + fi + cd $cwd + }''') return pre.commands(False) @@ -939,9 +975,30 @@ def build_graph(configs, echo, nosync, force_sync): gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) gitremote = repo['remote'] gitrev = repo['revision'] + git_project_cache = repo['project'] basedir = os.path.normpath(os.path.join(gitlocal, '..')) sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') + project_cache_dir = workspace.project_cache + git_local_reference=" " + if project_cache_dir: + if not git_project_cache: + parsed_url = urlparse(gitremote) + git_project_cache = os.path.basename(parsed_url.path) + + # Lets start searching for non bare repo first + git_project_cache = os.path.join(project_cache_dir, git_project_cache.removesuffix('.git')) + # Search for non bare repo + if os.path.isdir(os.path.join(git_project_cache, '.git')): + git_local_reference = f"--reference-if-able {os.path.join(git_project_cache, '.git')} " + # Search for a bare repo + elif os.path.isdir(f"{git_project_cache}.git"): + git_local_reference=f"--reference-if-able {git_project_cache}.git " + else: + git_project_cache = "" + else: + git_project_cache = "" + if name in force_sync: # We don't update any submodule before `git submodule sync`, # to handle the case where a remote changes the submodule's URL. @@ -973,10 +1030,11 @@ def build_graph(configs, echo, nosync, force_sync): rm -rf {gitlocal} > /dev/null 2>&1 || true mkdir -p {basedir} touch {sync} - git clone {gitargs}{gitremote} {gitlocal} + git clone {gitargs}{git_local_reference}{gitremote} {gitlocal} pushd {gitlocal} git checkout {gitargs}--force {gitrev} - git submodule {gitargs}update --init --checkout --recursive --force + # run with --reference + update_submodules "$(pwd)" "{git_project_cache}" popd rm {sync} else diff --git a/shrinkwrap/utils/workspace.py b/shrinkwrap/utils/workspace.py index bde46bc60a94fc8b81726b8d6de5ce2c4a0cf87e..6b8a23046db1364bf3d25d5a169cb446ccc8cd8d 100644 --- a/shrinkwrap/utils/workspace.py +++ b/shrinkwrap/utils/workspace.py @@ -6,14 +6,17 @@ import os _code_root = os.path.dirname(os.path.dirname(__file__)) _data_root = os.path.expanduser('~/.shrinkwrap') -def _get_loc(var, default): +def _get_loc(var, default=None): path = os.environ.get(var, default) + if not path: + return None path = os.path.abspath(path) os.makedirs(path, exist_ok=True) return path build = _get_loc('SHRINKWRAP_BUILD', os.path.join(_data_root, 'build')) package = _get_loc('SHRINKWRAP_PACKAGE', os.path.join(_data_root, 'package')) +project_cache = _get_loc("SHRINKWRAP_PROJECT_CACHE") _configs = None @@ -44,6 +47,8 @@ def config(path, join=True): return None def dump(): + print(f'workspace.project_cache:') + print(f' {project_cache}') print(f'workspace.build:') print(f' {build}') print(f'workspace.package:')