From 2392554669cf712e5a9cdab6b0539887e2460fde Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V (Arm)" Date: Thu, 20 Feb 2025 08:52:27 +0530 Subject: [PATCH 1/2] git: Add support for project cache directory This update introduces support for setting up a git clone alternate reference location. The SHRINKWRAP_PROJECT_CACHE environment variable specifies the mirror directory location. Both a bare clone and a full clone can be used as a reference repository. Additionally, a new component attribute, "project", allows naming the project cache directory. For example: kvmtool: project: kvmtool remote: https://git.kernel.org/pub/scm/linux/kernel/git/will/kvmtool.git revision: master Signed-off-by: Aneesh Kumar K.V (Arm) --- documentation/userguide/quickstart.rst | 15 ++++++++------- shrinkwrap/commands/buildall.py | 3 +++ shrinkwrap/utils/config.py | 25 ++++++++++++++++++++++++- shrinkwrap/utils/workspace.py | 7 ++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index 557bdf9..f21b6bc 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -72,13 +72,14 @@ Optional Environment Variables Shrinkwrap consumes the following set of optional environment variables: -================== ===================== ==== -name default description -================== ===================== ==== -SHRINKWRAP_CONFIG Colon-separated list of paths to config stores. Configs are searched for relative to the current directory as well as relative to these paths. -SHRINKWRAP_BUILD ~/.shrinkwrap/build Location where config builds are performed. Each config has its own subdirectory, with further subdirectories for each of its components. -SHRINKWRAP_PACKAGE ~/.shrinkwrap/package Location where config builds are packaged to. When running a config, it is done from the package location. -================== ===================== ==== +========================= ===================== ==== +name default description +========================= ===================== ==== +SHRINKWRAP_CONFIG Colon-separated list of paths to config stores. Configs are searched for relative to the current directory as well as relative to these paths. +SHRINKWRAP_BUILD ~/.shrinkwrap/build Location where config builds are performed. Each config has its own subdirectory, with further subdirectories for each of its components. +SHRINKWRAP_PACKAGE ~/.shrinkwrap/package Location where config builds are packaged to. When running a config, it is done from the package location. +SHRINKWRAP_PROJECT_CACHE Location where cache repositories are stored. This directory contains git trees used as reference when downloading components, allowing to reduce network usage and local storage if the same project is used multiple times. Cache directories ending in ".git" are bare repositories, and ones without the suffix are full repositories. By default no cache is used. +======================== ===================== ==== *************************************************** Guided Tour: Configure a platform and boot a kernel diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index 41eecaf..3379c07 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -153,6 +153,9 @@ def build(configs, btvarss, nosync, force_sync, args): if btvar['type'] == 'path': rt.add_volume(btvar['value']) + if workspace.project_cache: + add_volume(workspace.project_cache) + rt.start() ugraph.execute(graph, diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index d1cfdf9..48ec846 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -10,6 +10,7 @@ import textwrap import yaml import shrinkwrap.utils.clivars as uclivars import shrinkwrap.utils.workspace as workspace +from urllib.parse import urlparse default_image = 'docker.io/shrinkwraptool/base-slim:latest' @@ -48,6 +49,7 @@ def _component_normalize(component, name): for repo in component['repo'].values(): repo.setdefault('remote', None) repo.setdefault('revision', None) + repo.setdefault('project', None) component.setdefault('sourcedir', None) component.setdefault('builddir', None) @@ -939,9 +941,30 @@ def build_graph(configs, echo, nosync, force_sync): gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) gitremote = repo['remote'] gitrev = repo['revision'] + git_project_cache = repo['project'] basedir = os.path.normpath(os.path.join(gitlocal, '..')) sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') + project_cache_dir = workspace.project_cache + git_local_reference=" " + if project_cache_dir: + if not git_project_cache: + parsed_url = urlparse(gitremote) + git_project_cache = os.path.basename(parsed_url.path) + + # Lets start searching for non bare repo first + git_project_cache = os.path.join(project_cache_dir, git_project_cache.removesuffix('.git')) + # Search for non bare repo + if os.path.isdir(os.path.join(git_project_cache, '.git')): + git_local_reference = f"--reference-if-able {os.path.join(git_project_cache, '.git')} " + # Search for a bare repo + elif os.path.isdir(f"{git_project_cache}.git"): + git_local_reference=f"--reference-if-able {git_project_cache}.git " + else: + git_project_cache = "" + else: + git_project_cache = "" + if name in force_sync: # We don't update any submodule before `git submodule sync`, # to handle the case where a remote changes the submodule's URL. @@ -973,7 +996,7 @@ def build_graph(configs, echo, nosync, force_sync): rm -rf {gitlocal} > /dev/null 2>&1 || true mkdir -p {basedir} touch {sync} - git clone {gitargs}{gitremote} {gitlocal} + git clone {gitargs}{git_local_reference}{gitremote} {gitlocal} pushd {gitlocal} git checkout {gitargs}--force {gitrev} git submodule {gitargs}update --init --checkout --recursive --force diff --git a/shrinkwrap/utils/workspace.py b/shrinkwrap/utils/workspace.py index bde46bc..6b8a230 100644 --- a/shrinkwrap/utils/workspace.py +++ b/shrinkwrap/utils/workspace.py @@ -6,14 +6,17 @@ import os _code_root = os.path.dirname(os.path.dirname(__file__)) _data_root = os.path.expanduser('~/.shrinkwrap') -def _get_loc(var, default): +def _get_loc(var, default=None): path = os.environ.get(var, default) + if not path: + return None path = os.path.abspath(path) os.makedirs(path, exist_ok=True) return path build = _get_loc('SHRINKWRAP_BUILD', os.path.join(_data_root, 'build')) package = _get_loc('SHRINKWRAP_PACKAGE', os.path.join(_data_root, 'package')) +project_cache = _get_loc("SHRINKWRAP_PROJECT_CACHE") _configs = None @@ -44,6 +47,8 @@ def config(path, join=True): return None def dump(): + print(f'workspace.project_cache:') + print(f' {project_cache}') print(f'workspace.build:') print(f' {build}') print(f'workspace.package:') -- GitLab From 22e540d9ad43e00b6d433c8b2f394a478d5d9560 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V (Arm)" Date: Thu, 20 Feb 2025 21:27:20 +0530 Subject: [PATCH 2/2] submodules: Support for --reference when cloning submodules This update extends project cache directory support to submodules. All submodules can be updated in the project cache by running the following command from the top-level cache directory: git submodule update --init --checkout --recursive --force or by cloning with git clone --recurse-submodules Signed-off-by: Jean-Philippe Brucker Signed-off-by: Aneesh Kumar K.V (Arm) --- documentation/userguide/configmodel.rst | 11 ++++++++ shrinkwrap/utils/config.py | 37 ++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index 93331ee..3d8ed94 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -212,6 +212,17 @@ Sync mode: - **false**: do not synchronize the source directory. - **force**: synchronize the source directory. Overwrite any user modification and download branch updates. +------------ +repo section +------------ + +=========== =========== =========== +key type description +=========== =========== =========== +remote string Address of the remote repository +revision string A git revision (branch name, tag, hash...) +project string Optional. Name of the project corresponding to this repository, used to retrieve a cached repository in SHRINKWRAP_PROJECT_CACHE. By default, the project name is the base name contained in the remote address. + ----------- run section ----------- diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 48ec846..1c9d096 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -860,6 +860,40 @@ def script_preamble(echo): else: pre.append(f'# Exit on error.') pre.append(f'set -e') + + gitargs = '"--quiet "' if not echo else '""' + pre.append(f'gitargs={gitargs}') + pre.append_multiline(''' + # Function to update submodules without recursion + update_submodules() { + local repo_path="$1" + local reference_path="$2" + + local cwd=$(pwd) + cd $repo_path + # Check if .gitmodules file exists + local gitmodules_file=".gitmodules" + if [ -f "$gitmodules_file" ]; then + # Extract submodule paths from .gitmodules + git config --file "$gitmodules_file" --get-regexp path | while read -r path_key submodule_path; do + local git_submodule_reference="" + local submodule_reference="$reference_path/$submodule_path" + + # Check if the submodule exists in the project cache + if [[ -n "$reference_path" && -e "$submodule_reference/.git" ]]; then + git_submodule_reference="--reference $submodule_reference" + fi + + if [ -d "$submodule_path" ]; then + # Manually update nested submodules + git submodule $gitargs update --init --checkout --force $git_submodule_reference $submodule_path + # Recursively process the submodule + update_submodules "$submodule_path" "$submodule_reference" + fi + done + fi + cd $cwd + }''') return pre.commands(False) @@ -999,7 +1033,8 @@ def build_graph(configs, echo, nosync, force_sync): git clone {gitargs}{git_local_reference}{gitremote} {gitlocal} pushd {gitlocal} git checkout {gitargs}--force {gitrev} - git submodule {gitargs}update --init --checkout --recursive --force + # run with --reference + update_submodules "$(pwd)" "{git_project_cache}" popd rm {sync} else -- GitLab