diff --git a/docker/Dockerfile.slim b/docker/Dockerfile.slim index 6e6a5ca0cb0b538c2dd5637384b138d0017f0724..c6bf3f686c971dba6e6678dc66f21459625f86d4 100644 --- a/docker/Dockerfile.slim +++ b/docker/Dockerfile.slim @@ -29,6 +29,7 @@ ENV PATH="/pyvenv/bin:${PATH}" # Install packages that Shrinkwrap relies upon. RUN apt-get install --assume-yes --no-install-recommends --option=debug::pkgProblemResolver=yes \ netcat-openbsd \ + openssh-client \ python3 \ python3-pip \ telnet \ @@ -184,5 +185,11 @@ RUN cd /tools \ ENV TCH_PATH_AARCH64="/tools/${TCH_PATH_AARCH64}" ENV PATH="${TCH_PATH_AARCH64}:${PATH}" +# Set ssh options used by git. +ENV GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +# Set ssh-agent socket path. +ENV SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock + # Create a user. RUN useradd --create-home shrinkwrap diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst index 8c39cc83199c4f2f0414203deffafcdfea35c368..78fd8573d83e4d4f59d951be73d53bf1bf52cf25 100644 --- a/documentation/userguide/recipes.rst +++ b/documentation/userguide/recipes.rst @@ -264,3 +264,29 @@ This is equivalent: .. code-block:: shell shrinkwrap run ns-edk2.yaml --rtvar=KERNEL=path/to/Image --overlay='{"run":{"params":{"-C bp.pl011_uart0.out_file":"uart0.log"}}}' + +**************** +Provide SSH keys +**************** + +The GIT subprocess used by Shrinkwrap to synchronise source code may require +access to SSH keys. This is supported, via ``ssh-agent``. + +If ``ssh-agent`` is already running, it will be automatically detected (via +the ``SSH_AUTH_SOCK`` environment variable) and used by Shrinkwrap. + +Alternatively, the ``--ssh-agent`` option can be used to request Shrinkwrap +to start an ``ssh-agent`` subprocess and add default keys. + +.. code-block:: shell + + shrinkwrap --ssh-agent build ... + +In order to add only specified keys, the ``--ssh-agent-key`` can be used. + +.. code-block:: shell + + shrinkwrap \ + --ssh-agent-key ~/.ssh/my-first-key \ + --ssh-agent-key ~/.ssh/my-second-key \ + build ... diff --git a/shrinkwrap/commands/buildall.py b/shrinkwrap/commands/buildall.py index 916a4ac9434701394805631ebdfed0e5452babc3..012cf695851e1db1bd454db8105ab00fd29f9a3f 100644 --- a/shrinkwrap/commands/buildall.py +++ b/shrinkwrap/commands/buildall.py @@ -130,7 +130,8 @@ def build(configs, btvarss, nosync, args): # Run under a runtime environment, which may just run commands # natively on the host or may execute commands in a container, # depending on what the user specified. - with runtime.Runtime(args.runtime, args.image) as rt: + with runtime.Runtime(name=args.runtime, image=args.image, + ssh_agent_keys=args.ssh_agent_keys) as rt: def add_volume(path, levels_up=0): while levels_up: path = os.path.dirname(path) diff --git a/shrinkwrap/commands/clean.py b/shrinkwrap/commands/clean.py index 0ffafa13a38f2ce070da71a70a8ba89a7f3c0540..cc50b68ea7ad2207c4231abbec91e76216778dcb 100644 --- a/shrinkwrap/commands/clean.py +++ b/shrinkwrap/commands/clean.py @@ -112,7 +112,8 @@ def dispatch(args): # Run under a runtime environment, which may just run commands # natively on the host or may execute commands in a container, # depending on what the user specified. - with runtime.Runtime(args.runtime, args.image) as rt: + with runtime.Runtime(name=args.runtime, image=args.image, + ssh_agent_keys=args.ssh_agent_keys) as rt: def add_volume(path, levels_up=0): while levels_up: path = os.path.dirname(path) diff --git a/shrinkwrap/commands/run.py b/shrinkwrap/commands/run.py index 7bc9522a2f3f695264378dcd4596c3e2a61e4d03..9c566ec3b04d5d5e8c01a8ea5cc6cb91dc564a3a 100644 --- a/shrinkwrap/commands/run.py +++ b/shrinkwrap/commands/run.py @@ -228,7 +228,8 @@ def dispatch(args): # Run under a runtime environment, which may just run commands natively # on the host or may execute commands in a container, depending on what # the user specified. - with runtime.Runtime(args.runtime, args.image) as rt: + with runtime.Runtime(name=args.runtime, image=args.image, + ssh_agent_keys=args.ssh_agent_keys) as rt: for rtvar in resolver['run']['rtvars'].values(): if rtvar['type'] == 'path': rt.add_volume(rtvar['value']) diff --git a/shrinkwrap/shrinkwrap.py b/shrinkwrap/shrinkwrap.py index 1b32e624d707291946182eb55d671d3db0a655e4..dbfcf74814f05ac302b60f4111b6b323a91a5286 100755 --- a/shrinkwrap/shrinkwrap.py +++ b/shrinkwrap/shrinkwrap.py @@ -83,6 +83,21 @@ def main(): help="""If using a container runtime, specifies the name of the image to use. Defaults to the official shrinkwrap image.""") + parser.add_argument('--ssh-agent', + default=False, + action='store_true', + required=False, + help="""Start ssh-agent and add default keys.""") + + parser.add_argument('--ssh-agent-key', + dest='ssh_agent_keys', + default=[], + metavar='key', + action='append', + type=str, + required=False, + help="""Start ssh-agent and add specified key.""") + subparsers = parser.add_subparsers(dest='command', metavar='', title=f'Supported commands (run ' @@ -102,6 +117,9 @@ def main(): args = parser.parse_args() config_verbose_flag(args) + if args.ssh_agent: + args.ssh_agent_keys.append(None) + # Dispatch to the correct command. if args.command in cmds: cmds[args.command].dispatch(args) diff --git a/shrinkwrap/utils/runtime.py b/shrinkwrap/utils/runtime.py index da43bd48f464a16dd3f3696de5f6492b2f0346ce..9b564c021515c96c3afd8a571876a3751884a19d 100644 --- a/shrinkwrap/utils/runtime.py +++ b/shrinkwrap/utils/runtime.py @@ -5,9 +5,18 @@ import os import subprocess import sys import tuxmake.runtime +import types +import shrinkwrap.utils.ssh_agent as ssh_agent_lib +_SSH_AUTH_SOCK = '/run/host-services/ssh-auth.sock' +"""Path to ssh-agent socket.""" -_stack = [] + +_instance = None + + +def get_null_user_opts(self): + return [] class Runtime: @@ -18,25 +27,46 @@ class Runtime: host. The 'docker', 'docker-local', 'podman' and 'podman-local' runtimes execute the commands in a container. """ - def __init__(self, name, image=None, modal=True): - self._modal = modal + def __init__(self, *, name, image=None, ssh_agent_keys=None): self._rt = None self._mountpoints = set() self._rt = tuxmake.runtime.Runtime.get(name) self._rt.set_image(image) - if not sys.platform.startswith('darwin'): - # Macos uses GIDs that overlap with already defined GIDs - # in the container so this fails. However, it appears - # that on macos, if we run as root in the container, any - # generated files on the host filesystem are still owned - # my the real macos user, so it seems we don't need this - # UID/GID fixup in the first place on macos. + + is_mac = sys.platform.startswith('darwin') + is_docker = name.startswith('docker') + + # MacOS uses GIDs that overlap with already defined GIDs in the + # container so we can't just bind the macos host UID/GID to the + # shrinkwrap user in the container. This concept doesn't really + # work anyway, because on MacOS the container is running on a + # completely separate (linux) kernel in a VM. Fortunately docker + # maps the VM to the current MacOS user when touching mapped + # volumes so it all works out. So on MacOS run as root. + # Unfortunately, tuxmake tries to be too clever (it assumes a + # linux host) and tries to map the in-container user to the host + # UID/GID. This fails when the in-container user is root. So we + # have this ugly workaround to override the user-opts with + # nothing. By passing nothing, we implicitly run as root and + # tuxmake doesn't try to run usermod. + if is_mac and is_docker: + self._rt.get_user_opts = \ + types.MethodType(get_null_user_opts, self._rt) + else: self._rt.set_user('shrinkwrap') self._rt.set_group('shrinkwrap') - if self._modal: - _stack.append(self) + for key in ssh_agent_keys: + ssh_agent_lib.add(key) + + socket = ssh_agent_lib.socket() + + if name != 'null' and socket is not None: + if is_mac: + socket = _SSH_AUTH_SOCK + + self._rt.add_volume(socket, _SSH_AUTH_SOCK) def start(self): for mp in self._mountpoints: @@ -101,29 +131,26 @@ print(ip) if res.returncode == 0: return res.stdout.strip() return '127.0.0.1' - - def cleanup(self): - if self._rt: - self._rt.cleanup() - self._rt = None - if self._modal: - s = _stack.pop() - assert(s == self) - + def __enter__(self): + global _instance + assert _instance is None + _instance = self return self - + def __exit__(self, exc_type, exc_val, exc_tb): - self.cleanup() + global _instance + assert _instance == self + _instance = None def get(): """ - Returns the current modal Runtime instance. At least one Runtime - instance must be living that was created with modal=True. + Returns the current Runtime instance. """ - assert(len(_stack) > 0) - return _stack[-1] + global _instance + assert _instance is not None + return _instance def mkcmd(cmd, interactive=False): diff --git a/shrinkwrap/utils/ssh_agent.py b/shrinkwrap/utils/ssh_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..422228e4eb1029d585381b7811aa07335a62e0e0 --- /dev/null +++ b/shrinkwrap/utils/ssh_agent.py @@ -0,0 +1,98 @@ +# Copyright (c) 2024, Arm Limited. +# SPDX-License-Identifier: MIT + +import atexit +import os +import re +import subprocess +from typing import Optional + +import shrinkwrap.utils.logger as logger + + +_logger = logger.Logger(27) +"""Logger used within this module.""" + + +def _log(msg): + """Print a message to the console.""" + _logger.print(msg, tag='ssh-agent', cont=False) + + +_proc = None +"""ssh-agent process started by this module.""" + + +def _start(): + """Start an ssh-agent process.""" + global _proc + + assert _proc is None + _proc = subprocess.run( + ['ssh-agent', '-s'], stdout=subprocess.PIPE, text=True) + + # Extract socket path and PID from ssh-agent output + output = _proc.stdout + match = re.search( + r'SSH_AUTH_SOCK=(?P[^;]+).*SSH_AGENT_PID=(?P[^;]+)', + output, + re.DOTALL | re.MULTILINE) + + if not match: + raise Exception(f'Failed to parse ssh-agent output: {output}') + + values = match.groupdict() + + os.environ['SSH_AUTH_SOCK'] = values['sock'] + os.environ['SSH_AGENT_PID'] = values['pid'] + + _log(f'Started ssh-agent pid {values["pid"]}') + + # Register a handler to kill the ssh-agent process + atexit.register(_stop) + + +def _stop(): + """Stop the ssh-agent process.""" + global _proc + + assert _proc is not None + + _log('Stopping ssh-agent') + + subprocess.check_call(['ssh-agent', '-k'], stdout=subprocess.DEVNULL) + + del os.environ['SSH_AUTH_SOCK'] + del os.environ['SSH_AGENT_PID'] + + _proc = None + + +def socket() -> Optional[str]: + """Return path to socket.""" + return os.environ.get('SSH_AUTH_SOCK') + + +def add(key: Optional[str]=None) -> None: + """ + Add specified key, or add default keys if key is None. + + If no ssh-agent process is running, one is started. + """ + if socket() is None: + _start() + else: + global _proc + if _proc is None: + raise Exception('Adding a key to an ssh-agent not owned by shrinkwrap is not supported.') + + if key is None: + _log('Adding default keys') + else: + _log(f'Adding key {key}') + + args = ['ssh-add'] + if key is not None: + args.append(key) + + subprocess.check_call(args)