From 0b9b6b9a2965df87b53a2bd205dddbe652cee95e Mon Sep 17 00:00:00 2001 From: David Brown Date: Thu, 14 Nov 2024 08:48:56 +0000 Subject: [PATCH 1/4] feat(seed): give labgrid-user sudo access --- qemu/seed/image/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qemu/seed/image/main.py b/qemu/seed/image/main.py index 46fdd622..3f8dbc6e 100755 --- a/qemu/seed/image/main.py +++ b/qemu/seed/image/main.py @@ -21,6 +21,7 @@ def main(): #cloud-config users: - name: labgrid-user + sudo: ALL=(ALL) NOPASSWD:ALL chpasswd: expire: false users: -- GitLab From 4887cc4fbc3fb4680ecce31ac8788768c6456f11 Mon Sep 17 00:00:00 2001 From: David Brown Date: Fri, 15 Nov 2024 09:49:20 +0000 Subject: [PATCH 2/4] feat(strategy): Add an SSH-based strategy for QEMU --- bazel/labgrid/strategy/qemu.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/bazel/labgrid/strategy/qemu.py b/bazel/labgrid/strategy/qemu.py index 04d4166c..48c40518 100644 --- a/bazel/labgrid/strategy/qemu.py +++ b/bazel/labgrid/strategy/qemu.py @@ -4,6 +4,7 @@ import attr from labgrid.driver.qemudriver import QEMUDriver from labgrid.driver.shelldriver import ShellDriver +from labgrid.driver.sshdriver import SSHDriver from labgrid.factory import target_factory from labgrid.step import step from labgrid.strategy import Strategy, StrategyError @@ -49,3 +50,37 @@ class QEMUStrategy(Strategy): else: raise StrategyError(f"no transition found from {self.status} to {status}") self.status = status + +@target_factory.reg_driver +class QEMUSSHStrategy(Strategy): + bindings = {"qemu": QEMUDriver, "ssh": SSHDriver} + + status = attr.ib(default=Status.unknown) + + @step() + def transition(self, status, *, step): + if not isinstance(status, Status): + status = Status[status] + if status == Status.unknown: + # Unknown is the starting state + raise StrategyError(f"can not transition to {status}") + elif status == self.status: + step.skip("nothing to do") + return + elif status == Status.off: + # Resets ssh drivers status + self.target.deactivate(self.ssh) + self.target.activate(self.qemu) + # Stop the QEMU instance + self.qemu.off() + elif status == Status.shell: + # Resets ssh drivers status + self.target.deactivate(self.ssh) + self.target.activate(self.qemu) + # Instantiate QEMU instance + self.qemu.on() + # Wait until booted + self.target.activate(self.ssh) + else: + raise StrategyError(f"no transition found from {self.status} to {status}") + self.status = status \ No newline at end of file -- GitLab From d3dd8e53080de21f04ce6862c4aad82d667dca5e Mon Sep 17 00:00:00 2001 From: David Brown Date: Fri, 15 Nov 2024 09:51:03 +0000 Subject: [PATCH 3/4] feat(constraint): add BSP boot status constraight --- constraint/bsp/BUILD.bazel | 0 constraint/bsp/status/BUILD.bazel | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 constraint/bsp/BUILD.bazel create mode 100644 constraint/bsp/status/BUILD.bazel diff --git a/constraint/bsp/BUILD.bazel b/constraint/bsp/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/constraint/bsp/status/BUILD.bazel b/constraint/bsp/status/BUILD.bazel new file mode 100644 index 00000000..ad573b5c --- /dev/null +++ b/constraint/bsp/status/BUILD.bazel @@ -0,0 +1,19 @@ +constraint_setting(name = "status") + +constraint_value( + name = "preinstall", + constraint_setting = ":status", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "postinstall", + constraint_setting = ":status", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "booted", + constraint_setting = ":status", + visibility = ["//visibility:public"], +) \ No newline at end of file -- GitLab From bf1dd547f9ddd3c38496c0a7edf99e3465678ab2 Mon Sep 17 00:00:00 2001 From: David Brown Date: Wed, 13 Nov 2024 13:37:19 +0000 Subject: [PATCH 4/4] feat(savevm): demonstrator of QEMU saving VM state --- bazel/labgrid/strategy/qemu.py | 3 +- constraint/bsp/status/BUILD.bazel | 2 +- debian/amd64/qemu-utils/BUILD.bazel | 7 ++ debian/arm64/qemu-utils/BUILD.bazel | 7 ++ e2e/qemu/BUILD.bazel | 106 ++++++++++++++++++ e2e/qemu/boot_and_save_image.py | 46 ++++++++ e2e/qemu/config-booted.yaml | 22 ++++ e2e/qemu/config-preinstall.yaml | 27 +++++ e2e/qemu/config.yaml | 27 +++++ e2e/qemu/execute_with_writable_image.sh | 14 +++ .../toolchain/config/qemu/amd64/BUILD.bazel | 21 ++++ .../config/qemu/amd64/boot_and_save_image.py | 46 ++++++++ .../config/qemu/amd64/config-booted.yaml | 22 ++++ 13 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 e2e/qemu/boot_and_save_image.py create mode 100644 e2e/qemu/config-booted.yaml create mode 100644 e2e/qemu/config-preinstall.yaml create mode 100644 e2e/qemu/config.yaml create mode 100755 e2e/qemu/execute_with_writable_image.sh create mode 100644 labgrid/toolchain/config/qemu/amd64/boot_and_save_image.py create mode 100644 labgrid/toolchain/config/qemu/amd64/config-booted.yaml diff --git a/bazel/labgrid/strategy/qemu.py b/bazel/labgrid/strategy/qemu.py index 48c40518..0e36e2cf 100644 --- a/bazel/labgrid/strategy/qemu.py +++ b/bazel/labgrid/strategy/qemu.py @@ -51,6 +51,7 @@ class QEMUStrategy(Strategy): raise StrategyError(f"no transition found from {self.status} to {status}") self.status = status + @target_factory.reg_driver class QEMUSSHStrategy(Strategy): bindings = {"qemu": QEMUDriver, "ssh": SSHDriver} @@ -83,4 +84,4 @@ class QEMUSSHStrategy(Strategy): self.target.activate(self.ssh) else: raise StrategyError(f"no transition found from {self.status} to {status}") - self.status = status \ No newline at end of file + self.status = status diff --git a/constraint/bsp/status/BUILD.bazel b/constraint/bsp/status/BUILD.bazel index ad573b5c..906360f5 100644 --- a/constraint/bsp/status/BUILD.bazel +++ b/constraint/bsp/status/BUILD.bazel @@ -16,4 +16,4 @@ constraint_value( name = "booted", constraint_setting = ":status", visibility = ["//visibility:public"], -) \ No newline at end of file +) diff --git a/debian/amd64/qemu-utils/BUILD.bazel b/debian/amd64/qemu-utils/BUILD.bazel index 211e8536..8a20642e 100644 --- a/debian/amd64/qemu-utils/BUILD.bazel +++ b/debian/amd64/qemu-utils/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//debian/launcher:defs.bzl", "debian_launcher") load("//debian/patchelf:defs.bzl", "debian_patchelf") load(":srcs.bzl", "SRCS") @@ -12,3 +13,9 @@ debian_launcher( src = ":patched", visibility = ["//:__subpackages__"], ) + +bzl_library( + name = "srcs", + srcs = ["srcs.bzl"], + visibility = ["//visibility:public"], +) diff --git a/debian/arm64/qemu-utils/BUILD.bazel b/debian/arm64/qemu-utils/BUILD.bazel index 211e8536..8a20642e 100644 --- a/debian/arm64/qemu-utils/BUILD.bazel +++ b/debian/arm64/qemu-utils/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//debian/launcher:defs.bzl", "debian_launcher") load("//debian/patchelf:defs.bzl", "debian_patchelf") load(":srcs.bzl", "SRCS") @@ -12,3 +13,9 @@ debian_launcher( src = ":patched", visibility = ["//:__subpackages__"], ) + +bzl_library( + name = "srcs", + srcs = ["srcs.bzl"], + visibility = ["//visibility:public"], +) diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 8cf836ca..94123dd3 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -1,5 +1,8 @@ load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") +load("@rules_labgrid//labgrid/config/toolchain:defs.bzl", "labgrid_config_toolchain") +load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") +load("@rules_python//python:defs.bzl", "py_binary") CPUS = ("amd64",) @@ -28,3 +31,106 @@ test_suite( name = "run", tests = ["run-{}".format(cpu) for cpu in CPUS], ) + +platform( + name = "platform-preinstall", + constraint_values = [ + "@rules_labgrid//constraint/bsp/status:preinstall", + ], + parents = ["@rules_labgrid//platform:qemu-amd64-linux"], +) + +platform( + name = "platform-booted", + constraint_values = [ + "@rules_labgrid//constraint/bsp/status:booted", + ], + parents = ["@rules_labgrid//platform:qemu-amd64-linux"], +) + +labgrid_config_toolchain( + name = "amd64-preinstall", + src = "config-preinstall.yaml", + data = [ + "@rules_labgrid//qemu/debian/image", + "@rules_labgrid//qemu/seed/image", + ], + env = { + "LG_QEMU_SYSTEM_BIN": "$(QEMU_SYSTEM)", + "LG_QEMU_QCOW2_IMAGE": "$(location @rules_labgrid//qemu/debian/image)", + "LG_QEMU_VIRT_IMAGE": "$(location @rules_labgrid//qemu/seed/image)", + }, + target_compatible_with = [ + "@rules_labgrid//constraint/bsp/status:preinstall", + "@rules_labgrid//constraint/emulator:qemu", + "@toolchain_utils//toolchain/constraint/os:linux", + "@toolchain_utils//toolchain/constraint/cpu:amd64", + ], + toolchains = ["@rules_labgrid//labgrid/toolchain/qemu-system:resolved"], + deps = ["@rules_labgrid//bazel/labgrid/strategy"], +) + +labgrid_config_toolchain( + name = "amd64-booted", + src = "config-booted.yaml", + data = [ + ":booted_image", + ], + env = { + "LG_QEMU_SYSTEM_BIN": "$(QEMU_SYSTEM)", + "LG_QEMU_QCOW2_IMAGE": "$(location :booted_image)", + }, + target_compatible_with = [ + "@rules_labgrid//constraint/bsp/status:booted", + "@rules_labgrid//constraint/emulator:qemu", + "@toolchain_utils//toolchain/constraint/os:linux", + "@toolchain_utils//toolchain/constraint/cpu:amd64", + ], + toolchains = ["@rules_labgrid//labgrid/toolchain/qemu-system:resolved"], + deps = ["@rules_labgrid//bazel/labgrid/strategy"], +) + +sh_binary( + name = "execute_with_writable_image", + srcs = ["execute_with_writable_image.sh"], +) + +py_binary( + name = "boot_and_save_image", + srcs = ["boot_and_save_image.py"], + deps = [ + "@rules_labgrid//bazel/labgrid/strategy", + "@rules_labgrid//labgrid:pkg", + ], +) + +labgrid_genrule( + name = "booted_image", + outs = ["booted-vm.qcow2"], + cmd = "$(locations :boot_and_save_image) $@", + platform = ":platform-preinstall", + tools = [ + ":boot_and_save_image", + ], +) + +labgrid_genrule( + name = "os-release-booted", + srcs = [ + ":execute_with_writable_image", + "@ape//ape:cat", + ], + outs = ["stdout-booted.log"], + cmd = "$(locations :execute_with_writable_image) $(location @rules_labgrid//labgrid/run) $(location @ape//ape:cat) /etc/os-release > $@", + platform = ":platform-booted", + tools = [ + "@rules_labgrid//labgrid/run", + ], +) + +diff_file_test( + name = "run-booted", + size = "small", + a = ":release.txt", + b = ":stdout-booted.log", +) diff --git a/e2e/qemu/boot_and_save_image.py b/e2e/qemu/boot_and_save_image.py new file mode 100644 index 00000000..ecd04776 --- /dev/null +++ b/e2e/qemu/boot_and_save_image.py @@ -0,0 +1,46 @@ +from os import environ, linesep, chmod +from labgrid import Environment +from contextlib import contextmanager +from sys import stderr, stdout, argv +from pathlib import Path +from shutil import copy +import stat + +from bazel.labgrid.strategy import transition + +config = environ["LG_ENV"] +initial = environ.get("LG_INITIAL_STATE", None) +desired = environ["LG_STATE"] +final = environ.get("BZL_LG_FINAL_STATE", None) + +try: + # FIXME: Convert into context manager + runfiles_dir = environ["RUNFILES_DIR"] + del environ["RUNFILES_DIR"] +except KeyError: + pass + +# Copy the read-only disk image, make it writable, and override the environment variable to point labgrid at it +readonly_disk = Path(environ["LG_QEMU_QCOW2_IMAGE"]) +writable_target = argv[1] +copy(readonly_disk, writable_target) +chmod(writable_target, 0o666) +environ["LG_QEMU_QCOW2_IMAGE"] = writable_target + +# Now, boot-up the Qemu machine as normal +target = Environment(str(environ["LG_ENV"])).get_target() +strategy = target.get_driver("Strategy") + +with transition(strategy, initial, desired, final): + shell = target.get_driver("CommandProtocol") + qemu = target.get_driver("QEMUDriver") + + # Wait until the initial OS setup is complete + out, err, code = shell.run("cloud-init status --wait") + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + + # Issue a command to QEMU to save the VM state into the disk image + qemu.monitor_command("human-monitor-command", {"command-line": "savevm boottag"}) diff --git a/e2e/qemu/config-booted.yaml b/e2e/qemu/config-booted.yaml new file mode 100644 index 00000000..cabbd10c --- /dev/null +++ b/e2e/qemu/config-booted.yaml @@ -0,0 +1,22 @@ +targets: + main: + resources: + NetworkService: + address: "localhost" + port: 2222 + username: "labgrid-user" + password: "labgrid" + drivers: + QEMUDriver: + qemu_bin: "qemu" + machine: "pc" + cpu: "max" + memory: "2G" + nic: "user,model=e1000,hostfwd=tcp::2222-:22" + extra_args: !template "-loadvm boottag -drive file=$LG_QEMU_QCOW2_IMAGE,format=qcow2" + SSHDriver: {} + QEMUSSHStrategy: {} +tools: + qemu: !template "$LG_QEMU_SYSTEM_BIN" +imports: + - bazel.labgrid.strategy diff --git a/e2e/qemu/config-preinstall.yaml b/e2e/qemu/config-preinstall.yaml new file mode 100644 index 00000000..80687dec --- /dev/null +++ b/e2e/qemu/config-preinstall.yaml @@ -0,0 +1,27 @@ +targets: + main: + resources: + NetworkService: + address: "localhost" + port: 2222 + username: "labgrid-user" + password: "labgrid" + drivers: + QEMUDriver: + qemu_bin: "qemu" + machine: "pc" + cpu: "max" + memory: "2G" + nic: "user,model=e1000,hostfwd=tcp::2222-:22" + extra_args: !template "-drive file=$LG_QEMU_QCOW2_IMAGE,format=qcow2 -cdrom $LG_QEMU_VIRT_IMAGE" + ShellDriver: + prompt: " login: " + login_prompt: " login: " + username: "labgrid-user" + login_timeout: 600 + SSHDriver: {} + QEMUStrategy: {} +tools: + qemu: !template "$LG_QEMU_SYSTEM_BIN" +imports: + - bazel.labgrid.strategy diff --git a/e2e/qemu/config.yaml b/e2e/qemu/config.yaml new file mode 100644 index 00000000..ce01b979 --- /dev/null +++ b/e2e/qemu/config.yaml @@ -0,0 +1,27 @@ +targets: + main: + resources: + NetworkService: + address: "localhost" + port: 2222 + username: "labgrid-user" + password: "labgrid" + drivers: + QEMUDriver: + qemu_bin: "qemu" + machine: "pc" + cpu: "max" + memory: "2G" + nic: "user,model=e1000,hostfwd=tcp::2222-:22" + extra_args: !template "-snapshot -drive file=$LG_QEMU_QCOW2_IMAGE,format=qcow2" + ShellDriver: + prompt: " login: " + login_prompt: " login: " + username: "labgrid-user" + login_timeout: 600 + SSHDriver: {} + QEMUStrategy: {} +tools: + qemu: !template "$LG_QEMU_SYSTEM_BIN" +imports: + - bazel.labgrid.strategy diff --git a/e2e/qemu/execute_with_writable_image.sh b/e2e/qemu/execute_with_writable_image.sh new file mode 100755 index 00000000..5562537c --- /dev/null +++ b/e2e/qemu/execute_with_writable_image.sh @@ -0,0 +1,14 @@ +# Remove the first positional argument +# It is the path to this script due to the bazel launcher script +shift 1 + +# Copy the read-only disk image to a temp location and make it writable +DISKTMP=$TMPDIR/booted.qcow2 +cp $LG_QEMU_QCOW2_IMAGE $DISKTMP +chmod 666 $DISKTMP + +# Update the env to pick up the new disk location +LG_QEMU_QCOW2_IMAGE=$DISKTMP + +# Execute the rest of the script +$@ diff --git a/labgrid/toolchain/config/qemu/amd64/BUILD.bazel b/labgrid/toolchain/config/qemu/amd64/BUILD.bazel index 00b1ca8e..fcbc80cc 100644 --- a/labgrid/toolchain/config/qemu/amd64/BUILD.bazel +++ b/labgrid/toolchain/config/qemu/amd64/BUILD.bazel @@ -1,4 +1,6 @@ +load("@rules_python//python:defs.bzl", "py_binary") load("//labgrid/config/toolchain:defs.bzl", "labgrid_config_toolchain") +load("//labgrid/genrule:defs.bzl", "labgrid_genrule") labgrid_config_toolchain( name = "amd64", @@ -20,3 +22,22 @@ labgrid_config_toolchain( toolchains = ["//labgrid/toolchain/qemu-system:resolved"], deps = ["//bazel/labgrid/strategy"], ) + +py_binary( + name = "boot_and_save_image", + srcs = ["boot_and_save_image.py"], + deps = [ + "@rules_labgrid//bazel/labgrid/strategy", + "@rules_labgrid//labgrid:pkg", + ], +) + +labgrid_genrule( + name = "booted_image", + outs = ["booted-vm.qcow2"], + cmd = "$(locations :boot_and_save_image) $@", + platform = "@rules_labgrid//platform:qemu-amd64-linux", + tools = [ + ":boot_and_save_image", + ], +) diff --git a/labgrid/toolchain/config/qemu/amd64/boot_and_save_image.py b/labgrid/toolchain/config/qemu/amd64/boot_and_save_image.py new file mode 100644 index 00000000..ecd04776 --- /dev/null +++ b/labgrid/toolchain/config/qemu/amd64/boot_and_save_image.py @@ -0,0 +1,46 @@ +from os import environ, linesep, chmod +from labgrid import Environment +from contextlib import contextmanager +from sys import stderr, stdout, argv +from pathlib import Path +from shutil import copy +import stat + +from bazel.labgrid.strategy import transition + +config = environ["LG_ENV"] +initial = environ.get("LG_INITIAL_STATE", None) +desired = environ["LG_STATE"] +final = environ.get("BZL_LG_FINAL_STATE", None) + +try: + # FIXME: Convert into context manager + runfiles_dir = environ["RUNFILES_DIR"] + del environ["RUNFILES_DIR"] +except KeyError: + pass + +# Copy the read-only disk image, make it writable, and override the environment variable to point labgrid at it +readonly_disk = Path(environ["LG_QEMU_QCOW2_IMAGE"]) +writable_target = argv[1] +copy(readonly_disk, writable_target) +chmod(writable_target, 0o666) +environ["LG_QEMU_QCOW2_IMAGE"] = writable_target + +# Now, boot-up the Qemu machine as normal +target = Environment(str(environ["LG_ENV"])).get_target() +strategy = target.get_driver("Strategy") + +with transition(strategy, initial, desired, final): + shell = target.get_driver("CommandProtocol") + qemu = target.get_driver("QEMUDriver") + + # Wait until the initial OS setup is complete + out, err, code = shell.run("cloud-init status --wait") + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + + # Issue a command to QEMU to save the VM state into the disk image + qemu.monitor_command("human-monitor-command", {"command-line": "savevm boottag"}) diff --git a/labgrid/toolchain/config/qemu/amd64/config-booted.yaml b/labgrid/toolchain/config/qemu/amd64/config-booted.yaml new file mode 100644 index 00000000..cabbd10c --- /dev/null +++ b/labgrid/toolchain/config/qemu/amd64/config-booted.yaml @@ -0,0 +1,22 @@ +targets: + main: + resources: + NetworkService: + address: "localhost" + port: 2222 + username: "labgrid-user" + password: "labgrid" + drivers: + QEMUDriver: + qemu_bin: "qemu" + machine: "pc" + cpu: "max" + memory: "2G" + nic: "user,model=e1000,hostfwd=tcp::2222-:22" + extra_args: !template "-loadvm boottag -drive file=$LG_QEMU_QCOW2_IMAGE,format=qcow2" + SSHDriver: {} + QEMUSSHStrategy: {} +tools: + qemu: !template "$LG_QEMU_SYSTEM_BIN" +imports: + - bazel.labgrid.strategy -- GitLab