diff --git a/documentation/userguide/configmodel.rst b/documentation/userguide/configmodel.rst index 77856691808f6a4d502afe40bf92e3c49be7cacd..bf881fd373523390b16bdfff91851c1077f53cc4 100644 --- a/documentation/userguide/configmodel.rst +++ b/documentation/userguide/configmodel.rst @@ -191,7 +191,7 @@ component section =========== =========== =========== key type description =========== =========== =========== -repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync`` or ``--no-sync-all`` can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided. +repo dictionary Specifies information about the git repo(s) that must be cloned and checked out. By default, Shrinkwrap syncs the git repo to the specified revision when building. ``--no-sync``, ``--no-sync-all`` or the ``sync`` parameter can be used to tell Shrinkwrap to build it in whatever state the user left it in. Not required if ``sourcedir`` is provided. sourcedir string If specified, points to the path on disk where the source repo can be found. Useful for developer use cases where a local repo already exists. builddir string If specified, the location where the component will be built. If not specified, shrinkwrap allocates its own location based on SHRINKWRAP_BUILD. toolchain string Defines the toolchain to be used for compilation. Value is set as CROSS_COMPILE environment variable before invoking any prebuild/build/postbuild commands. When using the standard image with a container runtime, the options are: ``aarch64-none-elf-``, ``arm-none-eabi-``, ``aarch64-linux-gnu-``, or ``arm-linux-gnueabihf-``. @@ -201,8 +201,15 @@ prebuild list List of shell commands to be executed during component b build list List of shell commands to be executed during component build. postbuild list List of shell commands to be executed during component build after the ``build`` list. artifacts dictionary Set of artifacts (files and/or directories) that the component exports. Key is artifact name and value is path to built artifact. Other components can reference them with the ``${artifact:}`` macros. Used to determine build dependencies. +sync enum-string Specifies how shrinkwrap should synchronize the repository. See below for options. =========== =========== =========== +Sync mode: + +- **true**: synchronize the source directory. Do not overwrite user modifications or download updates. +- **false**: do not synchronize the source directory. +- **force**: synchronize the source directory. Overwrite any user modification and download branch updates. + ----------- run section ----------- diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index e1d5aac7cfed12f4d34aa30246f4af4d2eee7960..848aea0486906777fec7caff8ad3f959e34f4c0c 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -314,9 +314,12 @@ artifacts. By default, Shrinkwrap will sync all component repos to the revision specified in the config on every build invocation. If you have made changes in the - working directory, your CHANGES WILL BE LOST! You can override this behaviour - so that Shrinkwrap just builds whatever is in the working directory by adding - ``--no-sync `` or ``--no-sync-all`` to the command line. + working directory, shrinkwrap refuses to sync and displays an error. You can + override this behaviour so that Shrinkwrap just builds whatever is in the + working directory by adding ``--no-sync `` or ``--no-sync-all`` to + the command line. Alternatively you can force shrinkwrap to override your + changes by adding ``--force-sync `` or ``--force-sync-all`` to the + command line. Alternatively, pass ``--dry-run`` to view the shell script that would have been run: @@ -340,93 +343,165 @@ run: set -e # Remove old package. - rm -rf /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2.yaml > /dev/null 2>&1 || true - rm -rf /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2 > /dev/null 2>&1 || true + rm -rf package/ns-edk2.yaml > /dev/null 2>&1 || true + rm -rf package/ns-edk2 > /dev/null 2>&1 || true # Create directory structure. - mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/dt - mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2 - mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/tfa - mkdir -p /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2 + mkdir -p source/ns-edk2/acpica + mkdir -p source/ns-edk2/dt + mkdir -p source/ns-edk2/edk2 + mkdir -p source/ns-edk2/tfa + mkdir -p package/ns-edk2 + + # Sync git repo for config=ns-edk2 component=acpica. + pushd source/ns-edk2 + if [ ! -e "acpica/.git" ] || [ -f "./.acpica_sync" ]; then + rm -rf acpica > /dev/null 2>&1 || true + mkdir -p . + touch ./.acpica_sync + git clone --quiet https://github.com/acpica/acpica.git acpica + pushd acpica + git checkout --quiet --force R06_28_23 + git submodule --quiet update --init --checkout --recursive --force + popd + rm ./.acpica_sync + else + pushd acpica + if ! git checkout --quiet R06_28_23 > /dev/null 2>&1 && + ! ( git remote set-url origin https://github.com/acpica/acpica.git && + git fetch --quiet --prune --tags origin && + git checkout --quiet R06_28_23) || + ! git submodule --quiet update --init --checkout --recursive + then + echo "note: use --force-sync=acpica to override any change" + exit 1 + fi + popd + fi + popd # Sync git repo for config=ns-edk2 component=dt. - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2 - if [ ! -d "dt/.git" ] || [ -f "./.dt_sync" ]; then + pushd source/ns-edk2 + if [ ! -e "dt/.git" ] || [ -f "./.dt_sync" ]; then rm -rf dt > /dev/null 2>&1 || true mkdir -p . touch ./.dt_sync - git clone --quiet git://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git dt + git clone --quiet https://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git dt pushd dt - git checkout --quiet --force v6.1-dts + git checkout --quiet --force v6.6-dts git submodule --quiet update --init --checkout --recursive --force popd rm ./.dt_sync + else + pushd dt + if ! git checkout --quiet v6.6-dts > /dev/null 2>&1 && + ! ( git remote set-url origin https://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git && + git fetch --quiet --prune --tags origin && + git checkout --quiet v6.6-dts) || + ! git submodule --quiet update --init --checkout --recursive + then + echo "note: use --force-sync=dt to override any change" + exit 1 + fi + popd fi popd # Sync git repo for config=ns-edk2 component=edk2. - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2 - if [ ! -d "edk2/edk2/.git" ] || [ -f "edk2/.edk2_sync" ]; then + pushd source/ns-edk2 + if [ ! -e "edk2/edk2/.git" ] || [ -f "edk2/.edk2_sync" ]; then rm -rf edk2/edk2 > /dev/null 2>&1 || true mkdir -p edk2 touch edk2/.edk2_sync git clone --quiet https://github.com/tianocore/edk2.git edk2/edk2 pushd edk2/edk2 - git checkout --quiet --force edk2-stable202211 + git checkout --quiet --force edk2-stable202311 git submodule --quiet update --init --checkout --recursive --force popd rm edk2/.edk2_sync + else + pushd edk2/edk2 + if ! git checkout --quiet edk2-stable202311 > /dev/null 2>&1 && + ! ( git remote set-url origin https://github.com/tianocore/edk2.git && + git fetch --quiet --prune --tags origin && + git checkout --quiet edk2-stable202311) || + ! git submodule --quiet update --init --checkout --recursive + then + echo "note: use --force-sync=edk2 to override any change" + exit 1 + fi + popd fi - if [ ! -d "edk2/edk2-platforms/.git" ] || [ -f "edk2/.edk2-platforms_sync" ]; then + if [ ! -e "edk2/edk2-platforms/.git" ] || [ -f "edk2/.edk2-platforms_sync" ]; then rm -rf edk2/edk2-platforms > /dev/null 2>&1 || true mkdir -p edk2 touch edk2/.edk2-platforms_sync git clone --quiet https://github.com/tianocore/edk2-platforms.git edk2/edk2-platforms pushd edk2/edk2-platforms - git checkout --quiet --force 20e07099d8f11889d101dd710ca85001be20e179 + git checkout --quiet --force 4b07df2e6f3813c6e955197dacb2cdfbe3471caa git submodule --quiet update --init --checkout --recursive --force popd rm edk2/.edk2-platforms_sync - fi - if [ ! -d "edk2/acpica/.git" ] || [ -f "edk2/.acpica_sync" ]; then - rm -rf edk2/acpica > /dev/null 2>&1 || true - mkdir -p edk2 - touch edk2/.acpica_sync - git clone --quiet https://github.com/acpica/acpica.git edk2/acpica - pushd edk2/acpica - git checkout --quiet --force R10_20_22 - git submodule --quiet update --init --checkout --recursive --force + else + pushd edk2/edk2-platforms + if ! git checkout --quiet 4b07df2e6f3813c6e955197dacb2cdfbe3471caa > /dev/null 2>&1 && + ! ( git remote set-url origin https://github.com/tianocore/edk2-platforms.git && + git fetch --quiet --prune --tags origin && + git checkout --quiet 4b07df2e6f3813c6e955197dacb2cdfbe3471caa) || + ! git submodule --quiet update --init --checkout --recursive + then + echo "note: use --force-sync=edk2 to override any change" + exit 1 + fi popd - rm edk2/.acpica_sync fi popd - # Sync git repo for config=ns-edk2 component=tfa. - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2 - if [ ! -d "tfa/.git" ] || [ -f "./.tfa_sync" ]; then + pushd source/ns-edk2 + if [ ! -e "tfa/.git" ] || [ -f "./.tfa_sync" ]; then rm -rf tfa > /dev/null 2>&1 || true mkdir -p . touch ./.tfa_sync git clone --quiet https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git tfa pushd tfa - git checkout --quiet --force v2.8.0 + git checkout --quiet --force v2.11 git submodule --quiet update --init --checkout --recursive --force popd rm ./.tfa_sync + else + pushd tfa + if ! git checkout --quiet v2.11 > /dev/null 2>&1 && + ! ( git remote set-url origin https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git && + git fetch --quiet --prune --tags origin && + git checkout --quiet v2.11) || + ! git submodule --quiet update --init --checkout --recursive + then + echo "note: use --force-sync=tfa to override any change" + exit 1 + fi + popd fi popd + # Build for config=ns-edk2 component=acpica. + export CROSS_COMPILE= + pushd source/ns-edk2/acpica + rm -rf source/ns-edk2/acpica/generate/unix/acpica + make -j4 + mv source/ns-edk2/acpica/generate/unix/bin source/ns-edk2/acpica/generate/unix/acpica + popd + # Build for config=ns-edk2 component=dt. export CROSS_COMPILE=aarch64-none-elf- - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/dt + pushd source/ns-edk2/dt DTS=fvp-base-revc.dts INITRD_START= INITRD_END= DT_BASENAME=$(basename ${DTS} .dts) DTB_INTER=src/arm64/arm/${DT_BASENAME}.dtb - DTB_FINAL=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/dt/dt_bootargs.dtb - make CPP=${CROSS_COMPILE}cpp -j28 ${DTB_INTER} + DTB_FINAL=build/ns-edk2/dt/dt_bootargs.dtb + make CPP=${CROSS_COMPILE}cpp -j4 ${DTB_INTER} CHOSEN= if [ ! -z "" ]; then CHOSEN="${CHOSEN}bootargs = \"\";\n" @@ -489,35 +564,40 @@ run: fi popd + # Copy artifacts for config=ns-edk2 component=acpica. + cp -r source/ns-edk2/acpica/generate/unix/acpica package/ns-edk2/acpica + # Build for config=ns-edk2 component=edk2. export CROSS_COMPILE=aarch64-none-elf- - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2 - export WORKSPACE=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/edk2 + pushd source/ns-edk2/edk2 + export WORKSPACE=source/ns-edk2/edk2 export GCC5_AARCH64_PREFIX=$CROSS_COMPILE export PACKAGES_PATH=$WORKSPACE/edk2:$WORKSPACE/edk2-platforms - export IASL_PREFIX=$WORKSPACE/acpica/generate/unix/bin/ + export IASL_PREFIX=source/ns-edk2/acpica/generate/unix/acpica/ export PYTHON_COMMAND=/usr/bin/python3 - make -j28 -C acpica - source edk2/edksetup.sh - make -j28 -C edk2/BaseTools - build -n 28 -D EDK2_OUT_DIR=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2 -a AARCH64 -t GCC5 -p Platform/ARM/VExpressPkg/ArmVExpress-FVP-AArch64.dsc -b RELEASE + source edk2/edksetup.sh --reconfig + make -j4 -C edk2/BaseTools + build -n 4 -D EDK2_OUT_DIR=build/ns-edk2/edk2 -a AARCH64 -t GCC5 -p Platform/ARM/VExpressPkg/ArmVExpress-FVP-AArch64.dsc -b RELEASE --pcd PcdShellDefaultDelay=0 --pcd PcdUefiShellDefaultBootEnable=1 popd + # Copy artifacts for config=ns-edk2 component=dt. + cp -r build/ns-edk2/dt/dt_bootargs.dtb package/ns-edk2/dt_bootargs.dtb + + # Copy artifacts for config=ns-edk2 component=edk2. + cp -r build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd package/ns-edk2/FVP_AARCH64_EFI.fd # Build for config=ns-edk2 component=tfa. export CROSS_COMPILE=aarch64-none-elf- - pushd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/source/ns-edk2/tfa - make BUILD_BASE=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa PLAT=fvp DEBUG=0 LOG_LEVEL=40 ARM_DISABLE_TRUSTED_WDOG=1 FVP_HW_CONFIG_DTS=fdts/fvp-base-gicv3-psci-1t.dts BL33=/data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd ARM_ARCH_MINOR=5 ENABLE_SVE_FOR_NS=1 ENABLE_SVE_FOR_SWD=1 CTX_INCLUDE_PAUTH_REGS=1 BRANCH_PROTECTION=1 CTX_INCLUDE_MTE_REGS=1 ENABLE_FEAT_HCX=1 CTX_INCLUDE_AARCH32_REGS=0 ENABLE_SME_FOR_NS=1 ENABLE_SME_FOR_SWD=1 all fip + pushd source/ns-edk2/tfa + make BUILD_BASE=build/ns-edk2/tfa LOG_LEVEL=40 ARM_ARCH_MINOR=2 DEBUG=0 ARM_DISABLE_TRUSTED_WDOG=1 CTX_INCLUDE_AARCH32_REGS=0 BRANCH_PROTECTION=1 ARM_ARCH_MAJOR=9 PLAT=fvp BL33=build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd FVP_HW_CONFIG_DTS=fdts/fvp-base-gicv3-psci-1t.dts -j$(( 4 < 8 ? 4 : 8 )) all fip popd - # Copy artifacts for config=ns-edk2. - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/dt/dt_bootargs.dtb /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/dt_bootargs.dtb - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/edk2/RELEASE_GCC5/FV/FVP_AARCH64_EFI.fd /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/FVP_AARCH64_EFI.fd - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/shrinkwrap/config/edk2-flash.img /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/edk2-flash.img - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl1.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl1.bin - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl2.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl2.bin - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/bl31.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/bl31.bin - cp /data_nvme0n1/ryarob01/shrinkwrap_demo/live/build/build/ns-edk2/tfa/fvp/release/fip.bin /data_nvme0n1/ryarob01/shrinkwrap_demo/live/package/ns-edk2/fip.bin + # Copy artifacts for config=ns-edk2 component=tfa. + cp -r build/ns-edk2/tfa/fvp/release/bl31.bin package/ns-edk2/bl31.bin + cp -r build/ns-edk2/tfa/fvp/release/bl1.bin package/ns-edk2/bl1.bin + cp -r build/ns-edk2/tfa/fvp/release/fip.bin package/ns-edk2/fip.bin + cp -r build/ns-edk2/tfa/fvp/release/bl2.bin package/ns-edk2/bl2.bin + .. raw:: html diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst index 2135f44ccd6bc62fd6ae388a0874749166979fcb..c71bbae4eca46d56d76472f1fc5fbce21de90e2b 100644 --- a/documentation/userguide/recipes.rst +++ b/documentation/userguide/recipes.rst @@ -73,6 +73,7 @@ Create a file called ``my-overlay.yaml``: build: tfa: sourcedir: /path/to/my/tfa/git/repo + sync: false Now do a build, passing in the overlay: diff --git a/shrinkwrap/commands/build.py b/shrinkwrap/commands/build.py index 564a8d2778885c4f3b7b4ccd7e422cdd6c204b1d..073ac5144078abcde7b9be69ed7f81aea4ab11ca 100644 --- a/shrinkwrap/commands/build.py +++ b/shrinkwrap/commands/build.py @@ -59,6 +59,19 @@ def add_parser(parser, formatter): help="""Do not sync repos for any component, as if --no-sync was specified for every component in the config.""") + cmdp.add_argument('--force-sync', + metavar='component', required=False, default=[], action='append', + help="""Synchronize the repo of the given component even if the local + source directory contains unsaved changes. YOUR CHANGES WILL BE + LOST! In addition, download updates from remote branches.""") + + cmdp.add_argument('--force-sync-all', + required=False, default=False, action='store_true', + help="""Synchronize all components even if the local source directories + contain unsaved changes. YOUR CHANGES WILL BE LOST! In addition, + download all remote branch updates. Note that this will override + any 'sync: false' config option.""") + buildall.add_common_args(cmdp) return cmd_name @@ -73,4 +86,6 @@ def dispatch(args): btvars = vars.parse(args.btvar, type='bt') if args.no_sync_all: args.no_sync = True - buildall.build([args.config], [btvars], args.no_sync, args) + if args.force_sync_all: + args.force_sync = True + buildall.build([args.config], [btvars], args.no_sync, args.force_sync, args) diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index 39104ab426f9400c65d8be2fba292aa7e7ab1d68..41eecafd141ca379c07b9f689b863bc8ee8d07a0 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -108,17 +108,17 @@ def dispatch(args): configs = [c['config'] for c in cfgs['configs']] btvarss = [c['btvars'] for c in cfgs['configs']] - build(configs, btvarss, [], args) + build(configs, btvarss, [], [], args) -def build(configs, btvarss, nosync, args): +def build(configs, btvarss, nosync, force_sync, args): """ Concurrently builds a list of configs. Intended to be called as a common handler for the build and buildmulti commands. """ clivars = {'jobs': args.jobs} configs = config.load_resolveb_all(configs, args.overlay, clivars, btvarss) - graph = config.build_graph(configs, args.verbose, nosync) + graph = config.build_graph(configs, args.verbose, nosync, force_sync) if args.dry_run: script = ugraph.make_script(graph) @@ -168,7 +168,7 @@ def build(configs, btvarss, nosync, args): config.dump(c, cfg) # Dump the script to build the config. - graph = config.build_graph([c], args.verbose, nosync) + graph = config.build_graph([c], args.verbose, nosync, force_sync) script = ugraph.make_script(graph) build_name = os.path.join(workspace.package, c['name'], diff --git a/shrinkwrap/utils/config.py b/shrinkwrap/utils/config.py index 239cf8ec2b4fe1625cb2d1743886bae3c00cbd83..2e03147beae0e7dbab43d3a0712971affe724564 100644 --- a/shrinkwrap/utils/config.py +++ b/shrinkwrap/utils/config.py @@ -6,6 +6,7 @@ import io import json import os import re +import textwrap import yaml import shrinkwrap.utils.clivars as uclivars import shrinkwrap.utils.workspace as workspace @@ -38,46 +39,30 @@ def _component_normalize(component, name): """ Fills in any missing lists or dictionaries with empty ones. """ - if 'repo' not in component: - component['repo'] = {} + component.setdefault('repo', {}) if len(component['repo']) > 0 and \ all(type(v) != dict for v in component['repo'].values()): component['repo'] = {'.': component['repo']} for repo in component['repo'].values(): - if 'remote' not in repo: - repo['remote'] = None - - if 'revision' not in repo: - repo['revision'] = None - - if 'sourcedir' not in component: - component['sourcedir'] = None - - if 'builddir' not in component: - component['builddir'] = None - - if 'toolchain' not in component: - component['toolchain'] = None - - if 'stderrfilt' not in component: - component['stderrfilt'] = None - - if 'prebuild' not in component: - component['prebuild'] = [] - - if 'build' not in component: - component['build'] = [] - - if 'postbuild' not in component: - component['postbuild'] = [] - - if 'params' not in component: - component['params'] = {} - - if 'artifacts' not in component: - component['artifacts'] = {} + repo.setdefault('remote', None) + repo.setdefault('revision', None) + + component.setdefault('sourcedir', None) + component.setdefault('builddir', None) + component.setdefault('toolchain', None) + component.setdefault('stderrfilt', None) + component.setdefault('prebuild', []) + component.setdefault('build', []) + component.setdefault('postbuild', []) + component.setdefault('params', {}) + component.setdefault('artifacts', {}) + component.setdefault('sync', None) + if component['sync'] == False: + component['sync'] = 'false' + elif component['sync'] == True: + component['sync'] = 'true' return component @@ -97,77 +82,53 @@ def _buildex_normalize(buildex): """ Fills in any missing lists or dictionaries with empty ones. """ - if 'btvars' not in buildex: - buildex['btvars'] = {} + buildex.setdefault('btvars', {}) def _run_normalize(run): """ Fills in any missing lists or dictionaries with empty ones. """ - if 'name' not in run: - run['name'] = None - - if 'rtvars' not in run: - run['rtvars'] = {} - - if 'params' not in run: - run['params'] = {} - - if 'prerun' not in run: - run['prerun'] = [] - - if 'run' not in run: - run['run'] = [] - - if 'terminals' not in run: - run['terminals'] = {} + run.setdefault('name', None) + run.setdefault('rtvars', {}) + run.setdefault('params', {}) + run.setdefault('prerun', []) + run.setdefault('run', []) + run.setdefault('terminals', {}) def _config_normalize(config): """ Fills in any missing lists or dictionaries with empty ones. """ - if 'name' not in config: - config['name'] = None - - if 'fullname' not in config: - config['fullname'] = None - - if 'description' not in config: - config['description'] = None - - if 'image' not in config: - config['image'] = None - - if 'concrete' not in config: - config['concrete'] = False - - if 'layers' not in config: - config['layers'] = [] - - if 'graph' not in config: - config['graph'] = {} - - if 'build' not in config: - config['build'] = {} - - if 'buildex' not in config: - config['buildex'] = {} + config.setdefault('name', None) + config.setdefault('fullname', None) + config.setdefault('description', None) + config.setdefault('image', None) + config.setdefault('concrete', False) + config.setdefault('layers', []) + config.setdefault('graph', {}) + config.setdefault('build', {}) + config.setdefault('buildex', {}) _build_normalize(config['build']) _buildex_normalize(config['buildex']) - if 'artifacts' not in config: - config['artifacts'] = {} - - if 'run' not in config: - config['run'] = {} + config.setdefault('artifacts', {}) + config.setdefault('run', {}) _run_normalize(config['run']) return config +def _component_validate(component): + sync = component.get('sync') + if sync not in (None, 'true', 'false', 'force'): + raise Exception(f'invalid "sync" value "{sync}"') + +def _build_validate(build): + for component in build.values(): + _component_validate(component) def _config_validate(config): """ @@ -176,14 +137,17 @@ def _config_validate(config): """ # TODO: + if 'build' in config: + _build_validate(config['build']) + def _component_sort(component): """ Sort the component so that the keys are in a canonical order. This improves readability by humans. """ - lut = ['repo', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt', 'params', - 'prebuild', 'build', 'postbuild', 'artifacts'] + lut = ['repo', 'sync', 'sourcedir', 'builddir', 'toolchain', 'stderrfilt', + 'params', 'prebuild', 'build', 'postbuild', 'artifacts'] lut = {k: i for i, k in enumerate(lut)} return dict(sorted(component.items(), key=lambda x: lut[x[0]])) @@ -600,6 +564,8 @@ def resolveb(config, btvars={}, clivars={}): desc['builddir'] = os.path.join(workspace.build, 'build', comp_dir) + if desc['sync'] is None: + desc['sync'] = 'true' macro_lut = { 'param': { @@ -825,6 +791,10 @@ class Script: self._cmds += buf.getvalue() + def append_multiline(self, text, *args, indent='', **kwargs): + text = textwrap.indent(textwrap.dedent(text).strip(), indent) + self.append(text, *args, **kwargs) + def seal(self): assert(not self._sealed) self._sealed = True @@ -873,7 +843,7 @@ def script_preamble(echo): return pre.commands(False) -def build_graph(configs, echo, nosync): +def build_graph(configs, echo, nosync, force_sync): """ Returns a graph of scripts where the edges represent dependencies. The scripts should be executed according to the graph in order to correctly @@ -914,50 +884,87 @@ def build_graph(configs, echo, nosync): gl2.seal() graph[gl2] = [gl1] + force_sync_all = type(force_sync) != list + sync_none = type(nosync) != list + for config in configs: build_scripts = {} + # Make copies of nosync and force_sync that we can modify below + force_sync = set(config['build'].keys() if force_sync_all else force_sync) + nosync = set(config['build'].keys() if sync_none else nosync) + + invalid_sync_cfg = force_sync.intersection(nosync) + if invalid_sync_cfg: + raise Exception(f'Conflicting sync configuration for {invalid_sync_cfg}') + for name, component in config['build'].items(): + if component['sync'] == 'false' and name not in force_sync: + nosync.add(name) + elif component['sync'] == 'force' and name not in nosync: + force_sync.add(name) + ts = graphlib.TopologicalSorter(config['graph']) ts.prepare() while ts.is_active(): for name in ts.get_ready(): component = config['build'][name] - if (type(nosync) == list and name not in nosync) or \ - (type(nosync) != list and not nosync): + if name not in nosync and len(component['repo']) > 0: g = Script('Syncing git repo', config["name"], name, preamble=pre) - if len(component['repo']) > 0: - g.append(f'# Sync git repo for config={config["name"]} component={name}.') - g.append(f'pushd {os.path.dirname(component["sourcedir"])}') - - for gitlocal, repo in component['repo'].items(): - parent = os.path.basename(component["sourcedir"]) - gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) - gitremote = repo['remote'] - gitrev = repo['revision'] - basedir = os.path.normpath(os.path.join(gitlocal, '..')) - sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') - - g.append(f'if [ ! -d "{gitlocal}/.git" ] || [ -f "{sync}" ]; then') - g.append(f'\trm -rf {gitlocal} > /dev/null 2>&1 || true') - g.append(f'\tmkdir -p {basedir}') - g.append(f'\ttouch {sync}') - g.append(f'\tgit clone {gitargs}{gitremote} {gitlocal}') - g.append(f'\tpushd {gitlocal}') - g.append(f'\tgit checkout {gitargs}--force {gitrev}') - g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') - g.append(f'\tpopd') - g.append(f'\trm {sync}') - g.append(f'else') - g.append(f'\tpushd {gitlocal}') - g.append(f'\tgit checkout {gitargs}--force {gitrev} > /dev/null 2>&1 || (') - g.append(f'\t\tgit fetch {gitargs}--prune --prune-tags {gitremote} &&') - g.append(f'\t\tgit checkout {gitargs}--force {gitrev})') - g.append(f'\tgit submodule {gitargs}update --init --checkout --recursive --force') - g.append(f'\tpopd') - g.append(f'fi') - - g.append(f'popd') + g.append(f'# Sync git repo for config={config["name"]} component={name}.') + g.append(f'pushd {os.path.dirname(component["sourcedir"])}') + + for gitlocal, repo in component['repo'].items(): + parent = os.path.basename(component["sourcedir"]) + gitlocal = os.path.normpath(os.path.join(parent, gitlocal)) + gitremote = repo['remote'] + gitrev = repo['revision'] + basedir = os.path.normpath(os.path.join(gitlocal, '..')) + sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') + + if name in force_sync: + # We don't update any submodule before `git submodule sync`, + # to handle the case where a remote changes the submodule's URL. + # `git checkout` handles most cases, but doesn't update a local + # branch. So if gitrev is not a tag, do a `git reset`. + sync_cmd_when_exists = f''' + git remote set-url origin {gitremote} + git fetch {gitargs}--prune --prune-tags --force --recurse-submodules=off origin + git checkout {gitargs}--force {gitrev} + [ $(git tag -l {gitrev}) ] || git reset {gitargs}--hard origin/{gitrev} + git submodule {gitargs}sync --recursive + git submodule {gitargs}update --init --checkout --recursive --force + '''.strip() + else: + sync_cmd_when_exists = f''' + if ! git checkout {gitargs} {gitrev} > /dev/null 2>&1 && + ! ( git remote set-url origin {gitremote} && + git fetch {gitargs}--prune --tags origin && + git checkout {gitargs} {gitrev}) || + ! git submodule {gitargs}update --init --checkout --recursive + then + echo "note: use --force-sync={name} to override any change" + exit 1 + fi + '''.strip() + + g.append_multiline(f''' + if [ ! -e "{gitlocal}/.git" ] || [ -f "{sync}" ]; then + rm -rf {gitlocal} > /dev/null 2>&1 || true + mkdir -p {basedir} + touch {sync} + git clone {gitargs}{gitremote} {gitlocal} + pushd {gitlocal} + git checkout {gitargs}--force {gitrev} + git submodule {gitargs}update --init --checkout --recursive --force + popd + rm {sync} + else + pushd {gitlocal} + {sync_cmd_when_exists} + popd + fi''') + g.append(f'popd') g.seal() graph[g] = [gl2] else: @@ -1036,13 +1043,14 @@ def clean_graph(configs, echo): basedir = os.path.normpath(os.path.join(gitlocal, '..')) sync = os.path.join(basedir, f'.{os.path.basename(gitlocal)}_sync') - c.append(f'\tif [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then') - c.append(f'\t\tpushd {gitlocal}') - c.append(f'\t\tgit clean {gitargs}-xdff') - c.append(f'\t\tpopd') - c.append(f'\telse') - c.append(f'\t\trm -rf {gitlocal} {sync} > /dev/null 2>&1 || true') - c.append(f'\tfi') + c.append_multiline(f''' + if [ -d "{gitlocal}/.git" ] && [ ! -f "{sync}" ]; then + pushd {gitlocal} + git clean {gitargs}-xdff + popd + else + rm -rf {gitlocal} {sync} > /dev/null 2>&1 || true + fi''', indent='\t') c.append(f'\tpopd') c.append(f'fi') diff --git a/test/test-sync.sh b/test/test-sync.sh new file mode 100755 index 0000000000000000000000000000000000000000..6de6563b5db4d604cf76c311866fd68998fcb812 --- /dev/null +++ b/test/test-sync.sh @@ -0,0 +1,542 @@ +#!/bin/bash +# Create temporary git repositories and test shrinkwrap's synchronization feature + +# Run on the host, so we don't need to deal with remote git URLs. +SHRINKWRAP="shrinkwrap -R null" + +set -eu + +LOG=/tmp/shrinkwrap-sync-test.log +echo > $LOG +echo "Logging to file $LOG" + +set -x +exec 2>>$LOG + +T=$(mktemp -d) +# paranoid check: ensure we won't remove existing files +[ -z "$T" -o -n "$(ls -A $T)" ] && exit 1 +trap "rm -rf $T" EXIT + +export SHRINKWRAP_BUILD=$T/build +export SHRINKWRAP_PACKAGE=$T/package + +# Test $1 must succeed +OK () { + local name="$1" + shift + $SHRINKWRAP build -v $T/base.yaml $@ >>$LOG 2>&1 || + { echo "FAIL $name" && exit 1; } + echo "PASS $name" +} + +# Test $1 must fail +FAIL () { + local name="$1" + shift + $SHRINKWRAP build -v $T/base.yaml $@ >>$LOG 2>&1 && + echo "FAIL $name" && exit 1 + echo "PASS $name" +} + +# File $1 content must be equal to $2 +CONTENT () { + local file="$1" + local nice="${file##$T}" # remove prefix + local content="$2" + + if [ -e "$file" ] && [ "$(cat $file)" = "$content" ]; then + echo "PASS $nice" + else + echo "FAIL $nice" + exit 1 + fi +} + +TESTS () { + echo "$@" + echo "#### $@" >> $LOG +} + + +# Create the initial repos and config +{ + git init $T/repo1 -b main + git init $T/module1 -b main + + pushd $T/module1 + echo -n "module 1 commit 0" > README + git add README + git commit -m "commit 0" + popd + + pushd $T/repo1 + echo -n "repo 1 commit 0" > README + git add README + git commit -m "commit 0" + + git submodule add ssh://localhost:$T/module1 module + git commit -a -m "Add module 1" + popd +} >> $LOG + +cat << EOF > $T/base.yaml +%YAML 1.2 +--- +concrete: true + +build: + test1: + repo: + remote: file://$T/repo1 + revision: main + build: + - cat module/README +EOF + + +################################################################################ +TESTS Basic clone and update + +OK "initial clone" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0" + +OK "second sync is nop" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0" + +# Update the repo1 README +{ + pushd $T/repo1 + echo -n "repo 1 commit 1" > README + git commit -a -m "commit 1" + popd +} >> $LOG + +OK "sync=on does not update" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0" + +OK "sync=force updates" --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0" + + +################################################################################ +TESTS Module update + +# Update the module1 README +{ + pushd $T/module1 + echo -n "module 1 commit 1" > README + git commit -a -m "commit 1" + popd + + pushd $T/repo1 + git submodule update --remote + git commit -a -m "Update submodule 1" + popd +} >> $LOG + +OK "sync=on does not update" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 0" + +OK "sync=force updates module" --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 1" + + +################################################################################ +TESTS Tag and branch update + +# Update repo1 with a tag +{ + pushd $T/repo1 + echo -n "repo 1 commit 2" > README + git commit -a -m "commit 1" + git tag v0.1 + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.1 +EOF + +OK "update new tag" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" + +OK "rollback to main without overlay" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1" + +# Update repo1 with a branch +{ + pushd $T/repo1 + git checkout -b topic1 + echo -n "repo 1 commit topic.1" > README + git commit -a -m "commit 1 on topic" + git checkout - + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: topic1 +EOF + + +OK "update new branch" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit topic.1" + +OK "rollback to main without overlay" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 1" + +OK "force update" --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" + + +################################################################################ +TESTS Tagged module update + +# Update the module1 README +{ + pushd $T/module1 + echo -n "module 1 commit 2" > README + git commit -a -m "commit 2" + popd + + pushd $T/repo1 + git submodule update --remote + git commit -a -m "Update submodule 1" + git tag v0.1.1 + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.1.1 +EOF + +OK "tagged module update" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 2" + + +################################################################################ +TESTS Tag and branch udpates with --force-sync + +# Update repo1 with a tag +{ + pushd $T/repo1 + echo -n "repo 1 commit 3" > README + git commit -a -m "commit 1" + git tag v0.2 + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.2 +EOF + +OK "tag update with sync=force" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3" + +OK "back to main (overriden by force)" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" + +# Update repo1 with a branch +{ + pushd $T/repo1 + git checkout -b topic2 + echo -n "repo 1 commit topic.2" > README + git commit -a -m "commit 1 on topic 2" + git checkout - + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: topic2 +EOF + +OK "branch update with sync=force" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit topic.2" + +OK "back to main" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" + + + +################################################################################ +TESTS Overwrite file + +# Add a file to repo1 +{ + pushd $T/repo1 + echo "int main() {}" > main.c + git add main.c + git commit -a -m "add main" + git tag v0.3 + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.3 +EOF + +# Also create the file locally +{ + echo "int foo() {}" > $SHRINKWRAP_BUILD/source/base/test1/main.c +} >> $LOG + +FAIL "refusing to overwrite" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/main.c "int foo() {}" + +OK "force overwrite" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/main.c "int main() {}" + + +################################################################################ +TESTS Overwrite module file + +# Add a file to module1, in a new branch +{ + pushd $T/module1 + echo "int main() {}" > main.c + git add main.c + git commit -a -m "add main" + popd + + pushd $T/repo1 + git submodule update --remote --merge + git commit -a -m "Update submodule 2" + git tag v0.4 + popd +} >> $LOG + +# Update revision with an overlay +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.4 +EOF + +# Also create the file locally +{ + echo "int foo() {}" > $SHRINKWRAP_BUILD/source/base/test1/module/main.c +} >> $LOG + +FAIL "refusing to overwrite submodule" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/main.c "int foo() {}" + +OK "force overwrite submodule" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/main.c "int main() {}" + + +################################################################################ +TESTS User source dir + +# Use a local source directory +cat << EOF > $T/overlay.yaml +build: + test1: + sourcedir: $T/my-source/ +EOF + +OK "create local sourcedir" -o $T/overlay.yaml +CONTENT $T/my-source/README "repo 1 commit 3" + +rm -rf $T/my-source/ +mkdir $T/my-source/ + +OK "use empty local sourcedir" -o $T/overlay.yaml +CONTENT $T/my-source/README "repo 1 commit 3" + +rm -rf $T/my-source/ +mkdir $T/my-source/ +echo GPL > $T/my-source/LICENSE + +# fatal: destination path '.' already exists and is not an empty directory. +# --force-sync won't work in this case because it's the clone that fails +FAIL "clone fail in non-empty sourcedir" -o $T/overlay.yaml +CONTENT $T/my-source/LICENSE "GPL" + +rm -rf $T/my-source/ + + +################################################################################ +TESTS Override branch + +# Reuse an existing branch name in a different repo +{ + git init $T/repo2 -b main + git init $T/module2 -b main + + pushd $T/module2 + echo -n "module 2 commit 0" > README + git add README + git commit -m "commit 0" + popd + + pushd $T/repo2 + echo repo 2 commit 0 > README + git add README + git commit -a -m "commit 0" + + git submodule add ssh://localhost:$T/module2 module + git commit -a -m "Add module 2" + popd +} >> $LOG + +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + remote: file://$T/repo2 +EOF + +OK "sync=on use old main branch" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 2" + +OK "sync=force use new main branch" -o $T/overlay.yaml --force-sync-all +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 0" + + +################################################################################ +TESTS Update module branch + +# Update repo2 module branch +{ + pushd $T/module2 + git checkout -b branch2 + echo -n "module 2 commit 1" > README + git commit -a -m "commit 1" + popd + + pushd $T/repo2 + git submodule set-branch -b branch2 module + git submodule update --remote + git commit -a -m "Update module branch" + popd +} >> $LOG + +OK "sync=force update module branch" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 1" + + +################################################################################ +TESTS Update module URL + +# Update repo2 module URL +{ + pushd $T/repo2 + git submodule set-branch -b main module + git submodule set-url module ssh://localhost:$T/module1 + git submodule update --remote + git commit -a -m "Update module URL" + popd +} >> $LOG + +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + remote: file://$T/repo2 +EOF + +OK "sync=true doesn't update module URL" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 2 commit 1" + +OK "sync=force updates module URL" -o $T/overlay.yaml --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 2 commit 0" +CONTENT $SHRINKWRAP_BUILD/source/base/test1/module/README "module 1 commit 2" + + +OK "rollback to repo1" --force-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3" + + +################################################################################ +TESTS nosync config + +# Update the repo1 README +{ + pushd $T/repo1 + echo -n "repo 1 commit 4" > README + git commit -a -m "commit 4" + git tag v0.5 + popd +} >> $LOG + +cat << EOF > $T/overlay.yaml +build: + test1: + repo: + revision: v0.5 + sync: false +EOF + +OK "config sync=false does not update" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 3" + +OK "config sync=false updates with --force-sync-all" -o $T/overlay.yaml --force-sync-all +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4" + +cat << EOF > $T/overlay.yaml +build: + test1: + sync: hello +EOF + +FAIL "invalid config" -o $T/overlay.yaml + +# Update the repo1 README +{ + pushd $T/repo1 + echo -n "repo 1 commit 5" > README + git commit -a -m "commit 5" + popd +} >> $LOG + +cat << EOF > $T/overlay.yaml +build: + test1: + sync: force +EOF + +OK "config sync=force does not update with --no-sync" -o $T/overlay.yaml --no-sync=test1 +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4" + +OK "config sync=force does not update with --no-sync-all" -o $T/overlay.yaml --no-sync-all +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 4" + +OK "config sync=force updates" -o $T/overlay.yaml +CONTENT $SHRINKWRAP_BUILD/source/base/test1/README "repo 1 commit 5" + + +echo ALL TESTS PASS