From bf8ca5ebc4830259b31caacb5b0c1b3d058c1c6b Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Tue, 20 Dec 2022 13:41:29 +0000 Subject: [PATCH] runtime: Add support for podman containers As an alternative to Docker, users can now choose to use --runtime=podman. It uses the same container images (from Docker hub) but some people prefer podman because it doesn't require the use of a damemon with root privilege. Tuxmake provides most of the infrastructure so we just wire it up to the --runtime cli option and fix a few issues where podman is more picky (mount points and fully qualified container image names). Documentation is also updated to describe how to set up for podman. Signed-off-by: Ryan Roberts --- documentation/overview.rst | 7 ++-- documentation/userguide/quickstart.rst | 53 +++++++++++++++++++------- documentation/userguide/recipes.rst | 18 ++++----- documentation/userguide/runtimes.rst | 28 +++++++------- shrinkwrap/shrinkwrap.py | 10 +++-- shrinkwrap/utils/runtime.py | 33 +++++++++++++++- test/test.py | 10 +++-- 7 files changed, 110 insertions(+), 49 deletions(-) diff --git a/documentation/overview.rst b/documentation/overview.rst index e4a559d..fdc80dd 100644 --- a/documentation/overview.rst +++ b/documentation/overview.rst @@ -44,7 +44,8 @@ Features - Introspect and use any of the supplied the out-of-box configurations - Create your own configurations by composing with and extending others -- Choose teh Docker runtime backend or run everything natively if you prefer +- Choose from Docker or Podman runtime backends or run everything natively if + you prefer - Ensure Reproducible builds with supplied runtime container images - Transparently view the generated bash commands for a given config build or run - Parallelize builds to make best use of available resources @@ -59,8 +60,8 @@ Shrinkwrap is implemented in Python and has a command line interface similar to git, with sub-commands that take options. The Python code parses the supplied config(s) to generate shell commands that are executed in a backend runtime. The runtime is specified by the user and may be ``null`` (executed natively on the -user's system), or a container runtime such as ``docker``. For the container -runtimes, a standard image is provided with all tools preinstalled. +user's system), or a container runtime such as ``docker`` or ``podman``. For the +container runtimes, a standard image is provided with all tools preinstalled. ******************** Repository Structure diff --git a/documentation/userguide/quickstart.rst b/documentation/userguide/quickstart.rst index c139f4b..7cc547b 100644 --- a/documentation/userguide/quickstart.rst +++ b/documentation/userguide/quickstart.rst @@ -23,28 +23,52 @@ not tested. .. code-block:: shell - sudo apt-get install docker.io git netcat-openbsd python3 python3-pip telnet + sudo apt-get install git netcat-openbsd python3 python3-pip telnet sudo pip3 install pyyaml termcolor tuxmake git clone https://git.gitlab.arm.com/tooling/shrinkwrap.git export PATH=$PWD/shrinkwrap/shrinkwrap:$PATH -If Docker was not previously set up on your system, you will need to create a -'docker' group and add your user to it. This allows shrinkwrap to interact with -docker without needing sudo. For more information see `docker linux-postinstall -`_. +If using a Python version older than 3.9, you will also need to install the +``graphlib-backport`` pip package: + +.. code-block:: shell + + sudo pip3 install graphlib-backport + +------------------------------- +If using Docker Runtime Backend +------------------------------- + +If Docker was not previously set up on your system, you will need to install the +package, create a 'docker' group and add your user to it. This allows shrinkwrap +to interact with docker without needing sudo. For more information see `docker +linux-postinstall `_. .. code-block:: shell + sudo apt-get install docker.io sudo groupadd docker sudo usermod -aG docker $USER # Log out/log in for change to take effect -If using a Python version older than 3.9, you will also need to install the -``graphlib-backport`` pip package: +------------------------------- +If using Podman Runtime Backend +------------------------------- + +.. note:: + + Podman is only available within Ubuntu repositories from Ubuntu 20.10 and + newer. See `podman installation instructions + `_ for installation methods + for other distributions. .. code-block:: shell - sudo pip3 install graphlib-backport + sudo apt-get install podman + +------------------------------ +Optional Environment Variables +------------------------------ Shrinkwrap consumes the following set of optional environment variables: @@ -66,12 +90,13 @@ kernel. This example uses EDK2 (UEFI) but many other options are available. .. note:: - By default, the below commands will automatically download and use the - appropriate container image from Docker Hub. Alternatively, you can choose to - run with the ``null`` runtime by providing ``--runtime=null`` (between - ``shrinkwrap`` and the sub-command). This will cause all commands to be - executed on the native system. Users are responsible for setting up the - environment in this case. + By default, the below commands will use the docker runtime and automatically + download and use the appropriate container image from Docker Hub. + Alternatively, you can choose to run with the ``null`` runtime by providing + ``--runtime=null`` (between ``shrinkwrap`` and the sub-command). This will + cause all commands to be executed on the native system. Users are responsible + for setting up the environment in this case. Or if you have chosen to use + Podman as the runtime backend, add ``--runtime=podman``. First invoke the tool to view help: diff --git a/documentation/userguide/recipes.rst b/documentation/userguide/recipes.rst index 7251010..7d57bb3 100644 --- a/documentation/userguide/recipes.rst +++ b/documentation/userguide/recipes.rst @@ -221,15 +221,15 @@ Add the following to a higher layer of the config: Note that dt-base.yaml only accepts names of dts files that already exist in the device tree repo. -*********************************************** -Accessing the FVP over Nework when using Docker -*********************************************** - -When using the docker runtime, the FVP runs inside a container. This has a -different IP address to the host system. Shrinkwrap helpfully prints out the -runtime environment's IP address when starting the FVP. This is the IP address -you need to use to (e.g.) connect the debugger or to SSH into the hosted Linux -system. +******************************************************* +Accessing the FVP over Network when using Docker/Podman +******************************************************* + +When using the docker or podman runtimes, the FVP runs inside a container. This +has a different IP address to the host system. Shrinkwrap helpfully prints out +the runtime environment's IP address when starting the FVP. This is the IP +address you need to use to (e.g.) connect the debugger or to SSH into the hosted +Linux system. ****************************************** Example Linux Feature Development Use Case diff --git a/documentation/userguide/runtimes.rst b/documentation/userguide/runtimes.rst index f83b2d0..86b0afa 100644 --- a/documentation/userguide/runtimes.rst +++ b/documentation/userguide/runtimes.rst @@ -19,6 +19,8 @@ runtime description null Shell commands are executed natively on the user's system. The user is responsible for ensuring the the required toolchain, environment variables and any other dependencies are set up. docker (default). Shell commands are executed in a docker container. By default, the official shrinkwrap image will be pulled and used, which contains all dependencies already setup. docker-local Like docker, but will only look for the container image on the local system. Will not attempt to pull over the network. +podman Shell commands are executed in a podman container. By default, the official shrinkwrap image will be pulled and used, which contains all dependencies already setup. +podman-local Like podman, but will only look for the container image on the local system. Will not attempt to pull over the network. ============ ==== The desired runtime can be specified using the ``--runtime`` option, which is a @@ -35,15 +37,15 @@ optionally be specified. If omitted, the official shrinkwrap image is used: shrinkwrap --runtime= --image= ... -********************* -Docker Image Variants -********************* +************************ +Container Image Variants +************************ Shrinkwrap runs on both x86_64 and aarch64 architectures, and provides multiarch container images so that the correct variant is automatically selected for your platform. Images are automatically downloaded by shrinkwrap when the ``docker`` -runtime is selected. Images are available on Docker Hub and can be freely -downloaded without the need for an account. +or ``podman`` runtime is selected. Images are available on Docker Hub and can be +freely downloaded without the need for an account. .. warning:: @@ -53,14 +55,14 @@ downloaded without the need for an account. your own FVP on your system and follow the recipe at :ref:`userguide/recipes:Use a Custom FVP Version`. -===================================== ==== -image name description -===================================== ==== -shrinkwraptool/base-slim-nofvp:latest Contains all toolchains and other dependencies required to build all standard configs. Can be used as a base to create an image with a custom FVP. -shrinkwraptool/base-slim:latest (default). As per ``shrinkwraptool/base-slim-nofvp:latest`` but also contains the Base_RevC-2xAEMvA FVP. This is suffcient for most use cases and is much smaller than the ``full`` variant. -shrinkwraptool/base-full-nofvp:latest Builds upon ``shrinkwraptool/base-slim:latest``, adding aarch32 toolchains (both arm-none-eabi and arm-linux-gnueabihf). These are not needed for standard configs, but will be required if creating a custom config that includes (e.g.) SCP FW. Separated out due to big size increase. -shrinkwraptool/base-full:latest As per ``shrinkwraptool/base-full-nofvp:latest`` but also contains the Base_RevC-2xAEMvA FVP. -===================================== ==== +=============================================== ==== +image name description +=============================================== ==== +docker.io/shrinkwraptool/base-slim-nofvp:latest Contains all toolchains and other dependencies required to build all standard configs. Can be used as a base to create an image with a custom FVP. +docker.io/shrinkwraptool/base-slim:latest (default). As per ``shrinkwraptool/base-slim-nofvp:latest`` but also contains the Base_RevC-2xAEMvA FVP. This is suffcient for most use cases and is much smaller than the ``full`` variant. +docker.io/shrinkwraptool/base-full-nofvp:latest Builds upon ``shrinkwraptool/base-slim:latest``, adding aarch32 toolchains (both arm-none-eabi and arm-linux-gnueabihf). These are not needed for standard configs, but will be required if creating a custom config that includes (e.g.) SCP FW. Separated out due to big size increase. +docker.io/shrinkwraptool/base-full:latest As per ``shrinkwraptool/base-full-nofvp:latest`` but also contains the Base_RevC-2xAEMvA FVP. +=============================================== ==== ******************** Runtime Requirements diff --git a/shrinkwrap/shrinkwrap.py b/shrinkwrap/shrinkwrap.py index 5406f3e..60fc8eb 100755 --- a/shrinkwrap/shrinkwrap.py +++ b/shrinkwrap/shrinkwrap.py @@ -65,18 +65,20 @@ def main(): parser.add_argument('-R', '--runtime', metavar='engine', required=False, default='docker', - choices=['null', 'docker', 'docker-local'], + choices=['null', 'docker', 'docker-local', 'podman', 'podman-local'], help="""Specifies the environment in which to execute build and run commands. If 'null', executes natively on the host. 'docker' attempts to download the image from dockerhub and execute the commands in a container. 'docker-local' is like - 'docker' but will only look for the image locally. Defaults - to 'docker'.""") + 'docker' but will only look for the image locally. 'podman' + and 'podman-local' are like 'docker' and 'docker-local' + except podman is used as the runtime instead of docker. + Defaults to 'docker'.""") parser.add_argument('-I', '--image', metavar='name', required=False, - default='shrinkwraptool/base-slim:latest', + default='docker.io/shrinkwraptool/base-slim:latest', help="""If using a container runtime, specifies the name of the image to use. Defaults to the official shrinkwrap image.""") diff --git a/shrinkwrap/utils/runtime.py b/shrinkwrap/utils/runtime.py index 872a4a0..da43bd4 100644 --- a/shrinkwrap/utils/runtime.py +++ b/shrinkwrap/utils/runtime.py @@ -1,6 +1,7 @@ # Copyright (c) 2022, Arm Limited. # SPDX-License-Identifier: MIT +import os import subprocess import sys import tuxmake.runtime @@ -20,6 +21,7 @@ class Runtime: def __init__(self, name, image=None, modal=True): self._modal = modal self._rt = None + self._mountpoints = set() self._rt = tuxmake.runtime.Runtime.get(name) self._rt.set_image(image) @@ -37,10 +39,37 @@ class Runtime: _stack.append(self) def start(self): + for mp in self._mountpoints: + self._rt.add_volume(mp) + self._rt.prepare() - def add_volume(self, src, dst=None): - self._rt.add_volume(src, dst) + def add_volume(self, src): + # Podman can't deal with duplicate mount points, so filter out + # duplicates here, including mount-points that are children of + # other mount points. Then defer registering the actual final + # volumes until start() time. + + if not src: + return + + mountpoints = set() + + for mp in self._mountpoints: + common = os.path.commonpath([src, mp]) + if common == mp: + # mp is parent (or duplicate) of src, so src + # already covered. + return + elif common != src: + # src is not a parent of mp. So include mp in + # filtered set of mount points. + mountpoints.add(mp) + + # If we got here, then src is a unique mountpoint. Add it, and + # commit the filtered set. + mountpoints.add(src) + self._mountpoints = mountpoints def mkcmd(self, cmd, interactive=False): return self._rt.get_command_line(cmd, interactive, False) diff --git a/test/test.py b/test/test.py index 8d8f677..b2b4a49 100755 --- a/test/test.py +++ b/test/test.py @@ -177,18 +177,20 @@ def main(): parser.add_argument('-R', '--runtime', metavar='engine', required=False, default='docker', - choices=['null', 'docker', 'docker-local'], + choices=['null', 'docker', 'docker-local', 'podman', 'podman-local'], help="""Specifies the environment in which to execute build and run commands. If 'null', executes natively on the host. 'docker' attempts to download the image from dockerhub and execute the commands in a container. 'docker-local' is like - 'docker' but will only look for the image locally. Defaults - to 'docker'.""") + 'docker' but will only look for the image locally. 'podman' + and 'podman-local' are like 'docker' and 'docker-local' + except podman is used as the runtime instead of docker. + Defaults to 'docker'.""") parser.add_argument('-I', '--image', metavar='name', required=False, - default='shrinkwraptool/base-slim:latest', + default='docker.io/shrinkwraptool/base-slim:latest', help="""If using a container runtime, specifies the name of the image to use. Defaults to the official shrinkwrap image.""") -- GitLab