From 0211a1bf523f75f399ef2bcd0d488ff1a0e976a1 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 09:00:02 +0100 Subject: [PATCH 01/22] chore: add `docker` Python module Needed for `DockerDriver` --- python/requirements.in | 1 + python/requirements.txt | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/python/requirements.in b/python/requirements.in index c43e8366..bb7dded9 100644 --- a/python/requirements.in +++ b/python/requirements.in @@ -1 +1,2 @@ labgrid==23.0.6 +docker==7.1.0 diff --git a/python/requirements.txt b/python/requirements.txt index 60bdcf5d..428c467a 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -114,6 +114,10 @@ cryptography==42.0.8 \ --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e # via autobahn +docker==7.1.0 \ + --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \ + --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0 + # via -r python/requirements.in hyperlink==21.0.0 \ --hash=sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b \ --hash=sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4 @@ -295,7 +299,9 @@ pyyaml==6.0.1 \ requests==2.26.0 \ --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 - # via labgrid + # via + # docker + # labgrid six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -307,7 +313,9 @@ txaio==23.1.1 \ urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 - # via requests + # via + # docker + # requests xmodem==0.4.6 \ --hash=sha256:089737298f5738eabc43f2519efdc80b402693768f16383f7013b9e6f8f279d7 \ --hash=sha256:48ba47b68f86419399b8ddf9cdcf7ce27e594a881c287ad5c24ec71c11a79791 -- GitLab From b850dbf52adde4b6c6eaaef6a78d87b9dcf3574e Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 09:03:34 +0100 Subject: [PATCH 02/22] fix: update LabGrid to `24.0a1` --- python/requirements.in | 2 +- python/requirements.txt | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/requirements.in b/python/requirements.in index bb7dded9..8e3438e0 100644 --- a/python/requirements.in +++ b/python/requirements.in @@ -1,2 +1,2 @@ -labgrid==23.0.6 +labgrid==24.0a1 docker==7.1.0 diff --git a/python/requirements.txt b/python/requirements.txt index 428c467a..481a001d 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -136,9 +136,9 @@ jinja2==3.0.2 \ --hash=sha256:827a0e32839ab1600d4eb1c4c33ec5a8edfbc5cb42dafa13b81f182f97784b45 \ --hash=sha256:8569982d3f0889eed11dd620c706d39b60c36d6d25843961f33f77fb6bc6b20c # via labgrid -labgrid==23.0.6 \ - --hash=sha256:0a8a0a7668633babd52ccddc461431b39ec897ce7f165812841a6b0c0130323e \ - --hash=sha256:29990fe1875a3cac96dde25959462a78c3019fad874390142f9199598dbfca48 +labgrid==24.0a1 \ + --hash=sha256:35a277ea13c80ffea7df38454baed9220c3d6dded99719b9cee1a600763689ab \ + --hash=sha256:d03015b3a18f42de14f205170b44a8ac81d06df1d7b900beb53a2a7d3f61e8a1 # via -r python/requirements.in markupsafe==2.1.4 \ --hash=sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69 \ @@ -205,9 +205,7 @@ markupsafe==2.1.4 \ packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 - # via - # labgrid - # pytest + # via pytest pexpect==4.8.0 \ --hash=sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937 \ --hash=sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c -- GitLab From 1331f3c62d7d18ab102cfc12cd15f08fc58b6707 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 08:47:18 +0100 Subject: [PATCH 03/22] feat: expose LabGrid Python package Allows downstream packages to include the `labgrid` Python module into their tests. --- e2e/python/BUILD.bazel | 11 +++++++++++ labgrid/python/BUILD.bazel | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 e2e/python/BUILD.bazel create mode 100644 labgrid/python/BUILD.bazel diff --git a/e2e/python/BUILD.bazel b/e2e/python/BUILD.bazel new file mode 100644 index 00000000..c478ae8e --- /dev/null +++ b/e2e/python/BUILD.bazel @@ -0,0 +1,11 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +build_test( + name = "python", + targets = ["@rules_labgrid//labgrid/python"], +) + +build_test( + name = "pkgs", + targets = ["@rules_labgrid//labgrid/python:{}".format(p) for p in ("labgrid", "docker")], +) diff --git a/labgrid/python/BUILD.bazel b/labgrid/python/BUILD.bazel new file mode 100644 index 00000000..7092e0b8 --- /dev/null +++ b/labgrid/python/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_python//python:defs.bzl", "py_library") + +[ + alias( + name = pkg, + actual = "@pip//{}".format(pkg), + visibility = ["//visibility:public"], + ) + for pkg in ("labgrid", "docker") +] + +py_library( + name = "python", + visibility = ["//visibility:public"], + deps = [ + ":docker", + ":labgrid", + ], +) -- GitLab From ec03dfa37e96cbd7db04870776b175ed8dbb2ff9 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 09:19:25 +0100 Subject: [PATCH 04/22] test(e2e): provide Docker example runner Provides a `py_binary` that can run a command under the `DockerDriver`. This is _not_ hermetic but provides a useful example of how the LabGrid API can be used. --- e2e/MODULE.bazel | 2 +- e2e/docker/BUILD.bazel | 12 +++++ e2e/docker/local-ubuntu.16.04-gnu.yaml | 14 ++++++ e2e/docker/ssh.py | 68 ++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 e2e/docker/BUILD.bazel create mode 100644 e2e/docker/local-ubuntu.16.04-gnu.yaml create mode 100755 e2e/docker/ssh.py diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index 2178e7b6..3531cc87 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -7,6 +7,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") +bazel_dep(name = "rules_python", version = "0.33.2") bazel_dep(name = "rules_labgrid") local_path_override( module_name = "rules_labgrid", @@ -14,7 +15,6 @@ local_path_override( ) bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.33.2", dev_dependency = True) # We have to avoid the `chmod`/`chown`/`id` unhermetic-ness # TODO: remove this when `ignore_root_user_error` is hermetic diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel new file mode 100644 index 00000000..59b640ff --- /dev/null +++ b/e2e/docker/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +# This is *not* hermetic and requires access to `docker`/`ssh` on the host +# Useful as a test with `bazel run docker:ssh` +py_binary( + name = "ssh", + srcs = ["ssh.py"], + args = ["--lg-env=$(location local-ubuntu.16.04-gnu.yaml)"], + data = ["local-ubuntu.16.04-gnu.yaml"], + tags = ["manual"], + deps = ["@rules_labgrid//labgrid/python"], +) diff --git a/e2e/docker/local-ubuntu.16.04-gnu.yaml b/e2e/docker/local-ubuntu.16.04-gnu.yaml new file mode 100644 index 00000000..6239d70c --- /dev/null +++ b/e2e/docker/local-ubuntu.16.04-gnu.yaml @@ -0,0 +1,14 @@ +targets: + main: + resources: + - DockerDaemon: + docker_daemon_url: "unix:///var/run/docker.sock" + drivers: + - DockerDriver: + image_uri: "rastasheep/ubuntu-sshd:16.04" + container_name: "ubuntu-lg-example" + host_config: {"network_mode":"bridge"} + network_services: [{"port":22,"username":"root","password":"root"}] + - DockerStrategy: {} + - SSHDriver: + keyfile: "" diff --git a/e2e/docker/ssh.py b/e2e/docker/ssh.py new file mode 100755 index 00000000..d7eb732f --- /dev/null +++ b/e2e/docker/ssh.py @@ -0,0 +1,68 @@ +#! /usr/bin/env python3 + +from argparse import ArgumentParser +from os import environ, linesep +from pathlib import Path, PurePath +from shlex import join +from sys import argv, stderr, stdout +from typing import Collection, Protocol, Tuple + +from labgrid import Environment + + +class Shell(Protocol): + def run(cmd: str) -> Tuple[Collection[str], Collection[str], int]: ... + + +def shell(config: PurePath) -> Shell: + env = Environment(str(config)) + target = env.get_target() + strategy = target.get_driver("DockerStrategy") + strategy.transition("accessible") + shell = target.get_driver("CommandProtocol") + return shell + + +def ssh(shell: Shell, *args: str) -> int: + out, err, code = shell.run(join(args)) + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + return code + + +def arguments(prsr: ArgumentParser) -> None: + prsr.add_argument( + "program", metavar="PROG", help="The program to run on the device." + ) + prsr.add_argument( + "arguments", metavar="ARG", nargs="*", help="Command to run over SSH." + ) + prsr.add_argument( + "--lg-env", + help="The LabGrid environment configuration.", + dest="config", + default=PurePath(environ.get("LG_ENV", "config.yaml")), + type=PurePath, + ) + + +def main(exe: Path, *args: str) -> int: + prsr = ArgumentParser( + prog=str(exe), description="Runs a command over SSH to a LabGrid Docker device." + ) + + arguments(prsr) + + args = prsr.parse_args(args) + + return ssh(shell(args.config), args.program, *args.arguments) + + +def entry(): + exit(main(Path(argv[0]), *argv[1:])) + + +if __name__ == "__main__": + entry() -- GitLab From 5938ce1e6ec66ae10a60746d88fa67d5adc28050 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 09:35:13 +0100 Subject: [PATCH 05/22] feat: add `executor` toolchain Will be used to provide a binary that runs on the host. --- MODULE.bazel | 5 +++++ e2e/executor/BUILD.bazel | 6 ++++++ labgrid/toolchain/executor/BUILD.bazel | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 e2e/executor/BUILD.bazel create mode 100644 labgrid/toolchain/executor/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index a0b1b688..3dbbbc1b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,11 @@ resolved( toolchain_type = "//labgrid/toolchain/client:type", ) +resolved( + name = "resolved-executor", + toolchain_type = "//labgrid/toolchain/executor:type", +) + register_toolchains("//labgrid/toolchain/...") # We need to pin to Python 3.11 diff --git a/e2e/executor/BUILD.bazel b/e2e/executor/BUILD.bazel new file mode 100644 index 00000000..8ae8f4b5 --- /dev/null +++ b/e2e/executor/BUILD.bazel @@ -0,0 +1,6 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") + +build_test( + name = "type", + targets = ["@rules_labgrid//labgrid/toolchain/executor:type"], +) diff --git a/labgrid/toolchain/executor/BUILD.bazel b/labgrid/toolchain/executor/BUILD.bazel new file mode 100644 index 00000000..39cfd293 --- /dev/null +++ b/labgrid/toolchain/executor/BUILD.bazel @@ -0,0 +1,10 @@ +toolchain_type( + name = "type", + visibility = ["//visibility:public"], +) + +alias( + name = "resolved", + actual = "@resolved-client", + visibility = ["//visibility:public"], +) -- GitLab From a64d826e99bc440cfa4627bcd1f750f2251464d0 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 09:59:46 +0100 Subject: [PATCH 06/22] chore: add `@ape` dependency Will be used for hermetic binaries. --- MODULE.bazel | 1 + MODULE.bazel.lock | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 3dbbbc1b..8e387912 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,6 +8,7 @@ module( bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") bazel_dep(name = "rules_python", version = "0.33.2") +bazel_dep(name = "ape", version = "1.0.0-beta.11") bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0", dev_dependency = True) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index a3af7571..e9d5160e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -5,6 +5,8 @@ "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad", + "https://bcr.bazel.build/modules/ape/1.0.0-beta.11/MODULE.bazel": "39251c0aff369b378b03c07261a7b5b17ba5ca704d022d2aeff35b243239aee5", + "https://bcr.bazel.build/modules/ape/1.0.0-beta.11/source.json": "584928ecdebf3eca482fd69c2f8cd82b227173f7ac46267e44fb168a6628cf51", "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -20,17 +22,20 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", + "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "ea4b3a25a9417a7db57a8a2f9ebdee91d679823c6274b482b817ed128d81c594", "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/source.json": "9d1df0459caefdf41052d360469922a73e219f67c8ce4da0628cc604469822b9", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", - "https://bcr.bazel.build/modules/platforms/0.0.9/source.json": "cd74d854bf16a9e002fb2ca7b1a421f4403cda29f824a765acd3a8c56f8d43e6", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", @@ -63,6 +68,7 @@ "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/MODULE.bazel": "c6e3b5829ac9c0d096d65e1bd06616e42e859ba4ea10c8762b5de4f316bc983b", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/source.json": "421c4e38637937513784e1cc3776caffac21f1998faac48467ab15a5390e9f59", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.9/MODULE.bazel": "9a8edfa6905229b899225489cbd6db36f073d9455c77238bd6ae6c52da5256d8", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", @@ -152,7 +158,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "meSzxn3DUCcYEhq4HQwExWkWtU4EjriRBQLsZN+Q0SU=", + "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, -- GitLab From 139ef983f285783c5da6cf9b2ce577ead384f8ee Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 10:00:58 +0100 Subject: [PATCH 07/22] feat: add `labgrid_genrule` --- labgrid/genrule/BUILD.bazel | 0 labgrid/genrule/defs.bzl | 5 ++ labgrid/genrule/rule.bzl | 95 +++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 labgrid/genrule/BUILD.bazel create mode 100644 labgrid/genrule/defs.bzl create mode 100644 labgrid/genrule/rule.bzl diff --git a/labgrid/genrule/BUILD.bazel b/labgrid/genrule/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/labgrid/genrule/defs.bzl b/labgrid/genrule/defs.bzl new file mode 100644 index 00000000..bd5783d7 --- /dev/null +++ b/labgrid/genrule/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _genrule = "genrule") + +visibility("public") + +labgrid_genrule = _genrule diff --git a/labgrid/genrule/rule.bzl b/labgrid/genrule/rule.bzl new file mode 100644 index 00000000..595660d4 --- /dev/null +++ b/labgrid/genrule/rule.bzl @@ -0,0 +1,95 @@ +load("@bazel_skylib//lib:shell.bzl", "shell") + +visibility("//...") + +DOC = """Executes a Shell command on the host within a LabGrid environment. + +The "@rules_labgrid//labgrid/toolchain/executor:type` toolchain is used to create the environment that the command runs under. + +`@ape//:bash` is used to provide a hermetic, consistent, shell environment for the command. +""" + +ATTRS = { + "cmd": attr.string( + doc = "A Shell command to run on the executor within the LabGrid environment.", + mandatory = True, + # configurable = True, # TODO: Bazel 8+ + ), + "srcs": attr.label_list( + doc = "A list of inputs for this rule, such as files to process.", + cfg = "target", + ), + "tools": attr.label_list( + doc = "A list of tools for this rule, that will run on the executor.", + cfg = "exec", + ), + "outs": attr.output_list( + doc = "Output files generated by this rule.", + mandatory = True, + allow_empty = False, + ), + "_sh": attr.label( + doc = "The Shell interpreter", + default = "@ape//:bash", + executable = True, + cfg = "exec", + ), +} + +def _substitutions(ctx): + outs = [shell.quote(o.path) for o in ctx.outputs.outs] + srcs = [shell.quote(o.path) for o in ctx.files.srcs] + + # https://bazel.build/reference/be/make-variables#predefined_genrule_variables + substitutions = { + "$(OUTS)": " ".join(outs), + "$(SRCS)": " ".join(srcs), + } + + if len(srcs) == 1: + substitutions["$<"] = srcs[0] + elif "$<" in ctx.attr.cmd: + fail("Cannot specify `$<` Make variable with zero or multiple `srcs`") + + if len(outs) == 1: + substitutions["$@"] = outs[0] + elif "$@" in ctx.attr.cmd: + fail("Cannot specify `$@` Make variable with zero or multiple `outs`") + + for key, value in ctx.var.items(): + substitutions["$({})".format(key)] = value + + return substitutions + +def implementation(ctx): + executor = ctx.toolchains["//labgrid/toolchain/executor:type"] + + cmd = ctx.expand_location(ctx.attr.cmd) + for key, value in _substitutions(ctx).items(): + cmd = cmd.replace(key, value) + + args = ctx.actions.args() + args.add("--") + args.add(ctx.executable._sh) + args.add("-c") + args.add(cmd) + + ctx.actions.run( + executable = executor.run, + arguments = [args], + outputs = ctx.outputs.outs, + inputs = [s.files for s in ctx.attr.srcs], + tools = [t.files_to_run for t in ctx.attr.tools] + [ctx.executable._sh], + mnemonic = "LabGridGenrule", + ) + + return DefaultInfo(files = depset(ctx.outputs.outs)) + +labgrid_genrule = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, + toolchains = ["//labgrid/toolchain/executor:type"], +) + +genrule = labgrid_genrule -- GitLab From 62cac00a63f0fb8b2332871378b9ddf66795e329 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 10:04:43 +0100 Subject: [PATCH 08/22] chore: add `bazel_skylib` dependency --- MODULE.bazel | 1 + MODULE.bazel.lock | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8e387912..bd14b9fc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,6 +6,7 @@ module( ], ) +bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") bazel_dep(name = "rules_python", version = "0.33.2") bazel_dep(name = "ape", version = "1.0.0-beta.11") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e9d5160e..b7a6453c 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -19,7 +19,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", - "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", -- GitLab From 4d77e53e0f03b019a592e5e6df3311c9d31721e3 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 10:09:12 +0100 Subject: [PATCH 09/22] test(docker): example of `labgrid_genrule` --- e2e/MODULE.bazel | 1 + e2e/MODULE.bazel.lock | 13 +++++++++++-- e2e/docker/BUILD.bazel | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index 3531cc87..6a33d47b 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -8,6 +8,7 @@ module( bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") bazel_dep(name = "rules_python", version = "0.33.2") +bazel_dep(name = "rules_diff", version = "1.0.0-beta.3") bazel_dep(name = "rules_labgrid") local_path_override( module_name = "rules_labgrid", diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index b7f1e4be..77ed08e3 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -5,6 +5,9 @@ "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad", + "https://bcr.bazel.build/modules/ape/1.0.0-beta.11/MODULE.bazel": "39251c0aff369b378b03c07261a7b5b17ba5ca704d022d2aeff35b243239aee5", + "https://bcr.bazel.build/modules/ape/1.0.0-beta.11/source.json": "584928ecdebf3eca482fd69c2f8cd82b227173f7ac46267e44fb168a6628cf51", + "https://bcr.bazel.build/modules/ape/1.0.0-beta.6/MODULE.bazel": "247bcdbc358206c37ce8bf1c850115002c6560570d96f9ac8a79dd810b18ea85", "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -21,17 +24,20 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/MODULE.bazel": "bced1551849a5d1ca00b985c0d267ab690af840f04c685f2c62f40e92f66fac0", + "https://bcr.bazel.build/modules/download_utils/1.0.0-beta.2/source.json": "0ab7ebbc57f39a7fe96190e01fe9773482bc4e3d465e9cd9b239bb44ad57791d", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "ea4b3a25a9417a7db57a8a2f9ebdee91d679823c6274b482b817ed128d81c594", "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/source.json": "9d1df0459caefdf41052d360469922a73e219f67c8ce4da0628cc604469822b9", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", - "https://bcr.bazel.build/modules/platforms/0.0.9/source.json": "cd74d854bf16a9e002fb2ca7b1a421f4403cda29f824a765acd3a8c56f8d43e6", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", @@ -41,6 +47,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/MODULE.bazel": "4bcae1c5e3c7fa1169f9940f548f7b8b3316944b4367771b168f925b7a9ee74e", + "https://bcr.bazel.build/modules/rules_diff/1.0.0-beta.3/source.json": "b5db3fcd469061f2051188da97345162f294fc59e7fdf477beb306dbe950566a", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362", @@ -64,6 +72,7 @@ "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/MODULE.bazel": "c6e3b5829ac9c0d096d65e1bd06616e42e859ba4ea10c8762b5de4f316bc983b", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/source.json": "421c4e38637937513784e1cc3776caffac21f1998faac48467ab15a5390e9f59", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.9/MODULE.bazel": "9a8edfa6905229b899225489cbd6db36f073d9455c77238bd6ae6c52da5256d8", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", @@ -153,7 +162,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "meSzxn3DUCcYEhq4HQwExWkWtU4EjriRBQLsZN+Q0SU=", + "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel index 59b640ff..fe29d0e4 100644 --- a/e2e/docker/BUILD.bazel +++ b/e2e/docker/BUILD.bazel @@ -1,4 +1,6 @@ load("@rules_python//python:defs.bzl", "py_binary") +load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") +load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") # This is *not* hermetic and requires access to `docker`/`ssh` on the host # Useful as a test with `bazel run docker:ssh` @@ -10,3 +12,17 @@ py_binary( tags = ["manual"], deps = ["@rules_labgrid//labgrid/python"], ) + +labgrid_genrule( + name = "echo", + outs = ["stdout.log"], + cmd = "$(location :ssh) echo 'Hello, world!' > $@", + tools = [":ssh"], +) + +diff_file_test( + name = "test", + a = ":hello-world.txt", + b = ":stdout.log", + tags = ["manual"], +) -- GitLab From 2bb5aa5a16c01d3e9c1b8f3b9798c252fcd17cb2 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 12:52:50 +0100 Subject: [PATCH 10/22] feat: add a LabGrid executor Loads a Python context manager to set up the LabGrid environment before running the provided binary. --- labgrid/executor/BUILD.bazel | 48 +++++++++++++++++++ labgrid/executor/executor.py | 82 ++++++++++++++++++++++++++++++++ labgrid/executor/hello-world.txt | 1 + labgrid/executor/host.py | 25 ++++++++++ labgrid/executor/manager.py | 11 +++++ 5 files changed, 167 insertions(+) create mode 100644 labgrid/executor/BUILD.bazel create mode 100644 labgrid/executor/executor.py create mode 100644 labgrid/executor/hello-world.txt create mode 100644 labgrid/executor/host.py create mode 100644 labgrid/executor/manager.py diff --git a/labgrid/executor/BUILD.bazel b/labgrid/executor/BUILD.bazel new file mode 100644 index 00000000..7e8eaeb5 --- /dev/null +++ b/labgrid/executor/BUILD.bazel @@ -0,0 +1,48 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_test") +load("@toolchain_utils//toolchain/symlink/target:defs.bzl", "toolchain_symlink_target") + +py_library( + name = "manager", + srcs = ["manager.py"], + visibility = ["//visibility:public"], +) + +py_binary( + name = "executor", + srcs = ["executor.py"], + deps = [":manager"], + visibility = ["//visibility:public"], +) + +py_library( + name = "host", + srcs = ["host.py"], + data = ["hello-world.txt"], + deps = [ + ":manager", + "@rules_python//python/runfiles", + ], +) + +py_test( + name = "argument", + srcs = ["executor.py"], + args = [ + "--manager", + "labgrid.executor.host:manager", + "test", + ], + main = "executor.py", + deps = ["host"], +) + +py_test( + name = "envvar", + srcs = ["executor.py"], + args = ["test"], + env = { + "LABGRID_EXECUTOR_MANAGER": "labgrid.executor.host:manager", + }, + main = "executor.py", + deps = ["host"], +) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py new file mode 100644 index 00000000..ab6662c2 --- /dev/null +++ b/labgrid/executor/executor.py @@ -0,0 +1,82 @@ +#! /usr/bin/env python3 + +from __future__ import annotations + +from argparse import ArgumentParser, ArgumentTypeError +from contextlib import AbstractContextManager +from importlib import import_module +from inspect import signature +from os import environ +from pathlib import Path +from sys import argv, stderr + +from labgrid.executor.manager import Manager + + +def load(value: str) -> Manager: + name, sep, attr = value.partition(":") + if sep != ":": + raise ArgumentTypeError(f"`{value}` must be of the form `:`") + + module = import_module(name) + + try: + manager = getattr(module, attr) + except AttributeError as e: + raise ArgumentTypeError(f"`{name}` is missing `{attr}` attribute") from e + + return manager + + +def arguments(prsr: ArgumentParser) -> None: + prsr.add_argument( + "program", + metavar="PROG", + help="The binary to run within the LabGrid environment.", + ) + prsr.add_argument( + "arguments", + metavar="ARG", + nargs="*", + help="Arguments to pass to the binary.", + ) + prsr.add_argument( + "--manager", + help="A manager to load from a Python module, of the form `module.to.load:Manager`.", + type=load, + default=( + load(environ["LABGRID_EXECUTOR_MANAGER"]) + if "LABGRID_EXECUTOR_MANAGER" in environ + else None + ), + ) + + +def main(exe: Path, *args: str) -> int: + prsr = ArgumentParser( + prog=exe.relative_to(Path.cwd()), + description="Performs management of the LabGrid environment before running a binary.", + fromfile_prefix_chars="@", + ) + + arguments(prsr) + + parsed = prsr.parse_args(args) + + if parsed.manager is None: + print("Must provide a manager", file=stderr) + prsr.print_usage(file=stderr) + return 2 + + with parsed.manager() as run: + run(parsed.program, *parsed.arguments) + + return 0 + + +def entry(): + exit(main(Path(argv[0]), *argv[1:])) + + +if __name__ == "__main__": + entry() diff --git a/labgrid/executor/hello-world.txt b/labgrid/executor/hello-world.txt new file mode 100644 index 00000000..af5626b4 --- /dev/null +++ b/labgrid/executor/hello-world.txt @@ -0,0 +1 @@ +Hello, world! diff --git a/labgrid/executor/host.py b/labgrid/executor/host.py new file mode 100644 index 00000000..0936f99b --- /dev/null +++ b/labgrid/executor/host.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import Iterator + +from python.runfiles import Runfiles + +from labgrid.executor.manager import Manager, Run + + +def _run(program: str, *args: str) -> None: + assert program == "test" + assert not args + runfiles = Runfiles.Create() + path = runfiles.Rlocation("_main/labgrid/executor/hello-world.txt") + with open(path) as stream: + print(stream.read()) + + +run: Run = _run + + +@contextmanager +def manager() -> Iterator[Run]: + yield run diff --git a/labgrid/executor/manager.py b/labgrid/executor/manager.py new file mode 100644 index 00000000..723956d4 --- /dev/null +++ b/labgrid/executor/manager.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from contextlib import AbstractContextManager +from typing import TypeAlias + + +class Run: + def __call__(self, program: str, *args: str) -> None: ... + + +Manager: TypeAlias = AbstractContextManager[Run] -- GitLab From 07a56c843f2452c745c561cf4ffb23e3ab8b8adb Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 12:53:59 +0100 Subject: [PATCH 11/22] refactor: remove `labgrid-client` toolchain This can be used directly as a target, there is no need for a toolchain for this. --- MODULE.bazel | 5 ----- e2e/client/BUILD.bazel | 6 ------ labgrid/toolchain/client/BUILD.bazel | 31 ---------------------------- 3 files changed, 42 deletions(-) delete mode 100644 e2e/client/BUILD.bazel delete mode 100644 labgrid/toolchain/client/BUILD.bazel diff --git a/MODULE.bazel b/MODULE.bazel index bd14b9fc..de12139b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,11 +16,6 @@ bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0", dev_dependency = Tr # Register LabGrid toolchains resolved = use_repo_rule("@toolchain_utils//toolchain/resolved:defs.bzl", "toolchain_resolved") -resolved( - name = "resolved-client", - toolchain_type = "//labgrid/toolchain/client:type", -) - resolved( name = "resolved-executor", toolchain_type = "//labgrid/toolchain/executor:type", diff --git a/e2e/client/BUILD.bazel b/e2e/client/BUILD.bazel deleted file mode 100644 index a3190976..00000000 --- a/e2e/client/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") - -build_test( - name = "cli", - targets = ["@rules_labgrid//labgrid/client"], -) diff --git a/labgrid/toolchain/client/BUILD.bazel b/labgrid/toolchain/client/BUILD.bazel deleted file mode 100644 index 27411ebf..00000000 --- a/labgrid/toolchain/client/BUILD.bazel +++ /dev/null @@ -1,31 +0,0 @@ -load("@toolchain_utils//toolchain/info:defs.bzl", "toolchain_info") -load("@toolchain_utils//toolchain/test:defs.bzl", "toolchain_test") - -toolchain_type( - name = "type", - visibility = ["//visibility:public"], -) - -toolchain_info( - name = "info", - tags = ["manual"], - target = "//labgrid/client", -) - -toolchain( - name = "toolchain", - toolchain = ":info", - toolchain_type = ":type", -) - -alias( - name = "resolved", - actual = "@resolved-client", - visibility = ["//visibility:public"], -) - -toolchain_test( - name = "test", - args = ["--help"], - toolchains = [":resolved"], -) -- GitLab From 3e197419286b40ce991db466587253ba462cdae6 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 16:05:46 +0100 Subject: [PATCH 12/22] feat: add `labgrid_transition` rule Provides multi-platform transitions for `labgrid_genrule` and friends. --- labgrid/transition/BUILD.bazel | 0 labgrid/transition/defs.bzl | 5 ++++ labgrid/transition/platform/BUILD.bazel | 0 labgrid/transition/platform/transition.bzl | 15 +++++++++++ labgrid/transition/rule.bzl | 29 ++++++++++++++++++++++ 5 files changed, 49 insertions(+) create mode 100644 labgrid/transition/BUILD.bazel create mode 100644 labgrid/transition/defs.bzl create mode 100644 labgrid/transition/platform/BUILD.bazel create mode 100644 labgrid/transition/platform/transition.bzl create mode 100644 labgrid/transition/rule.bzl diff --git a/labgrid/transition/BUILD.bazel b/labgrid/transition/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/labgrid/transition/defs.bzl b/labgrid/transition/defs.bzl new file mode 100644 index 00000000..0990b226 --- /dev/null +++ b/labgrid/transition/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _transition = "transition") + +visibility("public") + +labgrid_transition = _transition diff --git a/labgrid/transition/platform/BUILD.bazel b/labgrid/transition/platform/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/labgrid/transition/platform/transition.bzl b/labgrid/transition/platform/transition.bzl new file mode 100644 index 00000000..57d60d22 --- /dev/null +++ b/labgrid/transition/platform/transition.bzl @@ -0,0 +1,15 @@ +visibility("//...") + +def implementation(_, attr): + return [ + {"//command_line_option:platforms": str(platform)} + for platform in attr.platforms + ] + +labgrid_transition_platform = transition( + implementation = implementation, + inputs = [], + outputs = ["//command_line_option:platforms"], +) + +platform = labgrid_transition_platform diff --git a/labgrid/transition/rule.bzl b/labgrid/transition/rule.bzl new file mode 100644 index 00000000..baf3366e --- /dev/null +++ b/labgrid/transition/rule.bzl @@ -0,0 +1,29 @@ +load("//labgrid/transition/platform:transition.bzl", _transition = "platform") + +visibility("//...") + +DOC = "" + +ATTRS = { + "platforms": attr.label_list( + doc = "Platforms to transition `srcs` to.", + providers = [platform_common.PlatformInfo], + ), + "srcs": attr.label_list( + doc = "Outputs to transition to `platforms`.", + cfg = _transition, + allow_files = True, + ), +} + +def implementation(ctx): + files = depset(transitive = [s.files for s in ctx.attr.srcs]) + return DefaultInfo(files = files) + +labgrid_transition = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, +) + +transition = labgrid_transition -- GitLab From 7ed0a00bf6085456cd43fe666aa0e40f3b071cf8 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Tue, 9 Jul 2024 16:06:59 +0100 Subject: [PATCH 13/22] feat: add `labgrid_executor` macro Allows creating executor targets easily. --- labgrid/executor/BUILD.bazel | 7 ++++++- labgrid/executor/defs.bzl | 5 +++++ labgrid/executor/macro.bzl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 labgrid/executor/defs.bzl create mode 100644 labgrid/executor/macro.bzl diff --git a/labgrid/executor/BUILD.bazel b/labgrid/executor/BUILD.bazel index 7e8eaeb5..45b91b33 100644 --- a/labgrid/executor/BUILD.bazel +++ b/labgrid/executor/BUILD.bazel @@ -1,6 +1,11 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@toolchain_utils//toolchain/symlink/target:defs.bzl", "toolchain_symlink_target") +exports_files( + ["executor.py"], + visibility = ["//visibility:public"], +) + py_library( name = "manager", srcs = ["manager.py"], @@ -10,8 +15,8 @@ py_library( py_binary( name = "executor", srcs = ["executor.py"], - deps = [":manager"], visibility = ["//visibility:public"], + deps = [":manager"], ) py_library( diff --git a/labgrid/executor/defs.bzl b/labgrid/executor/defs.bzl new file mode 100644 index 00000000..feed49e1 --- /dev/null +++ b/labgrid/executor/defs.bzl @@ -0,0 +1,5 @@ +load(":macro.bzl", _executor = "executor") + +visibility("public") + +labgrid_executor = _executor diff --git a/labgrid/executor/macro.bzl b/labgrid/executor/macro.bzl new file mode 100644 index 00000000..3a26a260 --- /dev/null +++ b/labgrid/executor/macro.bzl @@ -0,0 +1,33 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +visibility("//...") + +def _manager(labels, manager): + if manager: + return manager + + if len(labels) == 1: + label = labels[0] + if label.workspace_name != "": + fail("TODO: Unsupported") + return "{}.{}:manager".format(label.package.replace("/", "."), label.name) + + fail('Unable to determine LabGrid manager, provide `manager = ":"`') + +def labgrid_executor( + *, + name, + deps, + manager = None): + labels = [native.package_relative_label(d) for d in deps] + py_binary( + name = name, + srcs = [Label(":executor.py")], + main = "executor.py", + deps = labels + [Label("@rules_python//python/runfiles")], + env = { + "LABGRID_EXECUTOR_MANAGER": _manager(labels, manager), + }, + ) + +executor = labgrid_executor -- GitLab From d31e01ffff110290a2d658fd063bbcb28515bfdc Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 10 Jul 2024 09:14:28 +0100 Subject: [PATCH 14/22] fix(genrule): forward executor environment variables --- MODULE.bazel | 2 +- MODULE.bazel.lock | 4 ++-- labgrid/genrule/rule.bzl | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index de12139b..8e60e8d2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,7 +7,7 @@ module( ) bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") +bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.12") bazel_dep(name = "rules_python", version = "0.33.2") bazel_dep(name = "ape", version = "1.0.0-beta.11") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index b7a6453c..1ccdc915 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -67,8 +67,8 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", - "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/MODULE.bazel": "c6e3b5829ac9c0d096d65e1bd06616e42e859ba4ea10c8762b5de4f316bc983b", - "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/source.json": "421c4e38637937513784e1cc3776caffac21f1998faac48467ab15a5390e9f59", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.12/MODULE.bazel": "947cf935fa609c91b05bd8c8c1be38b9e10b7bc8949cf3092ee416ed30995078", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.12/source.json": "ef45e5e75c03a386bf7a3db5d51071f419442b4cf03f86f2f1b6e25b0ad1071f", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.9/MODULE.bazel": "9a8edfa6905229b899225489cbd6db36f073d9455c77238bd6ae6c52da5256d8", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", diff --git a/labgrid/genrule/rule.bzl b/labgrid/genrule/rule.bzl index 595660d4..0d276a7a 100644 --- a/labgrid/genrule/rule.bzl +++ b/labgrid/genrule/rule.bzl @@ -78,8 +78,9 @@ def implementation(ctx): executable = executor.run, arguments = [args], outputs = ctx.outputs.outs, - inputs = [s.files for s in ctx.attr.srcs], + inputs = depset(transitive = [s.files for s in ctx.attr.srcs]), tools = [t.files_to_run for t in ctx.attr.tools] + [ctx.executable._sh], + env = executor.env, mnemonic = "LabGridGenrule", ) -- GitLab From 2751a777357976a0adddc19cbaeacfe18fd24e20 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 10 Jul 2024 09:20:54 +0100 Subject: [PATCH 15/22] test(docker): executor for `labgrid_genrule` --- e2e/MODULE.bazel | 5 ++- e2e/MODULE.bazel.lock | 4 +- e2e/docker/BUILD.bazel | 83 +++++++++++++++++++++++++++++++++++++++--- e2e/docker/manager.py | 24 ++++++++++++ e2e/docker/ssh.py | 49 ++++++++++++++----------- e2e/docker/version.txt | 1 + 6 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 e2e/docker/manager.py create mode 100644 e2e/docker/version.txt diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel index 6a33d47b..b4693592 100644 --- a/e2e/MODULE.bazel +++ b/e2e/MODULE.bazel @@ -6,9 +6,10 @@ module( ) bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.11") +bazel_dep(name = "toolchain_utils", version = "1.0.0-beta.12") bazel_dep(name = "rules_python", version = "0.33.2") bazel_dep(name = "rules_diff", version = "1.0.0-beta.3") +bazel_dep(name = "ape", version = "1.0.0-beta.11") bazel_dep(name = "rules_labgrid") local_path_override( module_name = "rules_labgrid", @@ -17,6 +18,8 @@ local_path_override( bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0", dev_dependency = True) +register_toolchains("//...") + # We have to avoid the `chmod`/`chown`/`id` unhermetic-ness # TODO: remove this when `ignore_root_user_error` is hermetic # https://github.com/bazelbuild/rules_python/issues/2016 diff --git a/e2e/MODULE.bazel.lock b/e2e/MODULE.bazel.lock index 77ed08e3..fdd0b1a3 100644 --- a/e2e/MODULE.bazel.lock +++ b/e2e/MODULE.bazel.lock @@ -70,8 +70,8 @@ "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", - "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/MODULE.bazel": "c6e3b5829ac9c0d096d65e1bd06616e42e859ba4ea10c8762b5de4f316bc983b", - "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.11/source.json": "421c4e38637937513784e1cc3776caffac21f1998faac48467ab15a5390e9f59", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.12/MODULE.bazel": "947cf935fa609c91b05bd8c8c1be38b9e10b7bc8949cf3092ee416ed30995078", + "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.12/source.json": "ef45e5e75c03a386bf7a3db5d51071f419442b4cf03f86f2f1b6e25b0ad1071f", "https://bcr.bazel.build/modules/toolchain_utils/1.0.0-beta.9/MODULE.bazel": "9a8edfa6905229b899225489cbd6db36f073d9455c77238bd6ae6c52da5256d8", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel index fe29d0e4..6e70218e 100644 --- a/e2e/docker/BUILD.bazel +++ b/e2e/docker/BUILD.bazel @@ -1,9 +1,23 @@ -load("@rules_python//python:defs.bzl", "py_binary") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") +load("@rules_labgrid//labgrid/executor:defs.bzl", "labgrid_executor") +load("@rules_labgrid//labgrid/transition:defs.bzl", "labgrid_transition") load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") +load("@toolchain_utils//toolchain/info:defs.bzl", "toolchain_info") + +# The goal of this test/example is to use the nut/bolts of `rules_labgrid` +# It exposes _all_ the steps to configure the `labgrid_genrule` +# Usually, higher-level macros are used +# These are _not_ hermetic to avoid implementation complexity. +# To run these rules, the following are needed on the host: +# - `docker` +# - `ssh` +# - `scp` +# The later targets are marked as "manual" so `bazel {build,test} //...` do not pick them up # This is *not* hermetic and requires access to `docker`/`ssh` on the host -# Useful as a test with `bazel run docker:ssh` +# Useful as a test with `bazel run -- docker:ssh /path/to/target/binary argument` +# This is used later within `labgrid_genrule` py_binary( name = "ssh", srcs = ["ssh.py"], @@ -13,16 +27,75 @@ py_binary( deps = ["@rules_labgrid//labgrid/python"], ) +# An execution manager that is responsible for setting up the LabGrid environment +py_library( + name = "manager", + srcs = ["manager.py"], + data = ["local-ubuntu.16.04-gnu.yaml"], + deps = ["@rules_labgrid//labgrid/executor:manager"], +) + +# Create the LabGrid executor binary using our manager +labgrid_executor( + name = "executor", + deps = [":manager"], +) + +# Create toolchain information around the executor +toolchain_info( + name = "info", + target = ":executor", +) + +# A constraint to register the toolchain to +constraint_setting(name = "device") + +# This value would usually be shared in a common definitions module +constraint_value( + name = "constraint", + constraint_setting = ":device", +) + +# A platform that describes the Docker "platform" +platform( + name = "platform", + constraint_values = [":constraint"], +) + +# Provide the toolchain with Bazel +toolchain( + name = "toolchain", + target_compatible_with = [":constraint"], + toolchain = ":info", + toolchain_type = "@rules_labgrid//labgrid/toolchain/executor:type", +) + +# Run within the LabGrid environment +# This shows a few things: +# - It resolves the executor registered above to set up the `LG_ENV` enviroment variable +# - `:ssh` target uses `LG_ENV` to start the Docker container +# - `@ape//:cat` is provided to labgrid_genrule( name = "echo", + srcs = ["@ape//:cat"], outs = ["stdout.log"], - cmd = "$(location :ssh) echo 'Hello, world!' > $@", + cmd = "$(location :ssh) $(location @ape//:cat) /proc/version > $@", + tags = ["manual"], tools = [":ssh"], ) +# Transition the above `genrule` to the Docker platform +labgrid_transition( + name = "transition", + srcs = [":stdout.log"], + platforms = [":platform"], + tags = ["manual"], +) + +# Check that the output of `/proc/version` from within the container is what we expect diff_file_test( name = "test", - a = ":hello-world.txt", - b = ":stdout.log", + a = ":version.txt", + b = ":transition", tags = ["manual"], ) diff --git a/e2e/docker/manager.py b/e2e/docker/manager.py new file mode 100644 index 00000000..92896504 --- /dev/null +++ b/e2e/docker/manager.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from contextlib import contextmanager +from subprocess import run as execute +from typing import Iterator + +from python.runfiles import Runfiles + +from labgrid.executor.manager import Manager, Run + + +def _run(program: str, *args: str) -> None: + runfiles = Runfiles.Create() + config = runfiles.Rlocation("_main/docker/local-ubuntu.16.04-gnu.yaml") + env = {"LG_ENV": config} + execute((program, *args), env=env, check=True) + + +run: Run = _run + + +@contextmanager +def manager() -> Iterator[Run]: + yield run diff --git a/e2e/docker/ssh.py b/e2e/docker/ssh.py index d7eb732f..fa8f7e19 100755 --- a/e2e/docker/ssh.py +++ b/e2e/docker/ssh.py @@ -14,27 +14,12 @@ class Shell(Protocol): def run(cmd: str) -> Tuple[Collection[str], Collection[str], int]: ... -def shell(config: PurePath) -> Shell: - env = Environment(str(config)) - target = env.get_target() - strategy = target.get_driver("DockerStrategy") - strategy.transition("accessible") - shell = target.get_driver("CommandProtocol") - return shell - - -def ssh(shell: Shell, *args: str) -> int: - out, err, code = shell.run(join(args)) - for line in out: - stdout.write(f"{line}{linesep}") - for line in err: - stderr.write(f"{line}{linesep}") - return code - - def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( - "program", metavar="PROG", help="The program to run on the device." + "program", + metavar="PROG", + help="The program to run on the device.", + type=Path, ) prsr.add_argument( "arguments", metavar="ARG", nargs="*", help="Command to run over SSH." @@ -55,9 +40,31 @@ def main(exe: Path, *args: str) -> int: arguments(prsr) - args = prsr.parse_args(args) + parsed = prsr.parse_args(args) - return ssh(shell(args.config), args.program, *args.arguments) + # Start up Docker container with LabGrid + env = Environment(str(parsed.config)) + target = env.get_target() + strategy = target.get_driver("DockerStrategy") + strategy.transition("accessible") + + # Retrieve the communication protocols + shell = target.get_driver("CommandProtocol") + transfer = target.get_driver("FileTransferProtocol") + + # Transfer the provided program over to the Docker image + # TODO: this should be a context manager that removes it later + # TODO: this should use a unique remote name/directory + program = "/tmp/{}".format(parsed.program.name) + transfer.put(parsed.program, program) + + # Run the transferred program + out, err, code = shell.run(join((program, *parsed.arguments))) + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + return code def entry(): diff --git a/e2e/docker/version.txt b/e2e/docker/version.txt new file mode 100644 index 00000000..c0f07114 --- /dev/null +++ b/e2e/docker/version.txt @@ -0,0 +1 @@ +Linux version 6.8.1-arch1-1 (linux@archlinux) (gcc (GCC) 13.2.1 20230801, GNU ld (GNU Binutils) 2.42.0) #1 SMP PREEMPT_DYNAMIC Sat, 16 Mar 2024 17:15:35 +0000 -- GitLab From 77ce7063b342864e3d8e4db62a72f5ef23cd3026 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 10 Jul 2024 11:27:30 +0100 Subject: [PATCH 16/22] test(docker): add Linux constraint to platform/executor --- e2e/docker/BUILD.bazel | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel index 6e70218e..0993cecc 100644 --- a/e2e/docker/BUILD.bazel +++ b/e2e/docker/BUILD.bazel @@ -59,13 +59,19 @@ constraint_value( # A platform that describes the Docker "platform" platform( name = "platform", - constraint_values = [":constraint"], + constraint_values = [ + ":constraint", + "@toolchain_utils//toolchain/constraint/os:linux", + ], ) # Provide the toolchain with Bazel toolchain( name = "toolchain", - target_compatible_with = [":constraint"], + target_compatible_with = [ + ":constraint", + "@toolchain_utils//toolchain/constraint/os:linux", + ], toolchain = ":info", toolchain_type = "@rules_labgrid//labgrid/toolchain/executor:type", ) -- GitLab From b5e9217614d28e11860425cf8d6abdebe851f2d1 Mon Sep 17 00:00:00 2001 From: David Brown Date: Wed, 10 Jul 2024 11:52:43 +0000 Subject: [PATCH 17/22] test(docker): use /etc/os-release instead of /proc/version --- e2e/docker/BUILD.bazel | 2 +- e2e/docker/version.txt | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel index 0993cecc..6dc48f79 100644 --- a/e2e/docker/BUILD.bazel +++ b/e2e/docker/BUILD.bazel @@ -85,7 +85,7 @@ labgrid_genrule( name = "echo", srcs = ["@ape//:cat"], outs = ["stdout.log"], - cmd = "$(location :ssh) $(location @ape//:cat) /proc/version > $@", + cmd = "$(location :ssh) $(location @ape//:cat) /etc/os-release > $@", tags = ["manual"], tools = [":ssh"], ) diff --git a/e2e/docker/version.txt b/e2e/docker/version.txt index c0f07114..c167782a 100644 --- a/e2e/docker/version.txt +++ b/e2e/docker/version.txt @@ -1 +1,11 @@ -Linux version 6.8.1-arch1-1 (linux@archlinux) (gcc (GCC) 13.2.1 20230801, GNU ld (GNU Binutils) 2.42.0) #1 SMP PREEMPT_DYNAMIC Sat, 16 Mar 2024 17:15:35 +0000 +NAME="Ubuntu" +VERSION="16.04.4 LTS (Xenial Xerus)" +ID=ubuntu +ID_LIKE=debian +PRETTY_NAME="Ubuntu 16.04.4 LTS" +VERSION_ID="16.04" +HOME_URL="http://www.ubuntu.com/" +SUPPORT_URL="http://help.ubuntu.com/" +BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/" +VERSION_CODENAME=xenial +UBUNTU_CODENAME=xenial -- GitLab From 27d2f0382bc6873021af02a5c6466f2bd189e2dc Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Thu, 11 Jul 2024 13:02:00 +0100 Subject: [PATCH 18/22] fix(executor): workaround running APE binaries > If you use zsh and have trouble running APE programs try `sh -c ./prog` > or simply upgrade to zsh 5.9+ (since we patched it two years ago). > The same is the case for Python subprocess, old versions of fish, etc. https://github.com/jart/cosmopolitan?tab=readme-ov-file#shells --- labgrid/executor/executor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index ab6662c2..7f35fb02 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -8,6 +8,8 @@ from importlib import import_module from inspect import signature from os import environ from pathlib import Path +from platform import machine, system +from shutil import which from sys import argv, stderr from labgrid.executor.manager import Manager @@ -68,8 +70,15 @@ def main(exe: Path, *args: str) -> int: prsr.print_usage(file=stderr) return 2 + cmd = (parsed.program, *parsed.arguments) + + # A workaround for invoking APE binaries on Apple Silicon + # https://github.com/jart/cosmopolitan?tab=readme-ov-file#shells + if machine() == "arm64" and system() == "Darwin": + cmd = (which("sh"), *cmd) + with parsed.manager() as run: - run(parsed.program, *parsed.arguments) + run(*cmd) return 0 -- GitLab From d8e89377a6c879e340c7aa703b4d969bf4f7029e Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Thu, 11 Jul 2024 13:33:51 +0100 Subject: [PATCH 19/22] fix(executor): catch `CalledProcessError` --- labgrid/executor/executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index 7f35fb02..a9cfa8bb 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -10,6 +10,7 @@ from os import environ from pathlib import Path from platform import machine, system from shutil import which +from subprocess import CalledProcessError from sys import argv, stderr from labgrid.executor.manager import Manager @@ -78,7 +79,10 @@ def main(exe: Path, *args: str) -> int: cmd = (which("sh"), *cmd) with parsed.manager() as run: - run(*cmd) + try: + run(*cmd) + except CalledProcessError as e: + return e.returncode return 0 -- GitLab From 6f2f54792f6c6303ab4de3474c4eaaf60cec1ada Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Thu, 11 Jul 2024 13:34:13 +0100 Subject: [PATCH 20/22] fix(executor): catch `KeyboardInterrupt` --- labgrid/executor/executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index a9cfa8bb..b7da6f89 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -83,6 +83,8 @@ def main(exe: Path, *args: str) -> int: run(*cmd) except CalledProcessError as e: return e.returncode + except KeyboardInterrupt: + return 130 return 0 -- GitLab From 227f5e908e270ca91723d4e18bf9d2dd0608edb4 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 17 Jul 2024 10:01:59 +0100 Subject: [PATCH 21/22] fix(executor): use `label.repo_name` --- MODULE.bazel | 2 +- labgrid/executor/macro.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8e60e8d2..3ea721db 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,7 +2,7 @@ module( name = "rules_labgrid", version = "0.0.0", bazel_compatibility = [ - ">=7.0.0", + ">=7.1.0", ], ) diff --git a/labgrid/executor/macro.bzl b/labgrid/executor/macro.bzl index 3a26a260..477b6f1b 100644 --- a/labgrid/executor/macro.bzl +++ b/labgrid/executor/macro.bzl @@ -8,7 +8,7 @@ def _manager(labels, manager): if len(labels) == 1: label = labels[0] - if label.workspace_name != "": + if label.repo_name != "": fail("TODO: Unsupported") return "{}.{}:manager".format(label.package.replace("/", "."), label.name) -- GitLab From 44fb104140d7f99eb70915418e7d64341a7616ae Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Mon, 15 Jul 2024 14:23:07 +0100 Subject: [PATCH 22/22] refactor(executor): return data rather than a `run` function This removes complexity in the context manager. It consolidates the control of the execution into the executor. BREAKING CHANGE: LabGrid context managers now return `labgrid.manager#Data` rather than `Run` functions. --- e2e/docker/manager.py | 16 ++++------------ labgrid/executor/BUILD.bazel | 5 +++-- labgrid/executor/executor.py | 6 +++--- labgrid/executor/host.py | 19 ++++++------------- labgrid/executor/manager.py | 15 +++++++++++---- 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/e2e/docker/manager.py b/e2e/docker/manager.py index 92896504..cca44f05 100644 --- a/e2e/docker/manager.py +++ b/e2e/docker/manager.py @@ -1,24 +1,16 @@ from __future__ import annotations from contextlib import contextmanager -from subprocess import run as execute from typing import Iterator from python.runfiles import Runfiles -from labgrid.executor.manager import Manager, Run +from labgrid.executor.manager import Data, Manager -def _run(program: str, *args: str) -> None: +@contextmanager +def manager() -> Iterator[Run]: runfiles = Runfiles.Create() config = runfiles.Rlocation("_main/docker/local-ubuntu.16.04-gnu.yaml") env = {"LG_ENV": config} - execute((program, *args), env=env, check=True) - - -run: Run = _run - - -@contextmanager -def manager() -> Iterator[Run]: - yield run + yield Data(env=env) diff --git a/labgrid/executor/BUILD.bazel b/labgrid/executor/BUILD.bazel index 45b91b33..e6124a46 100644 --- a/labgrid/executor/BUILD.bazel +++ b/labgrid/executor/BUILD.bazel @@ -35,7 +35,8 @@ py_test( args = [ "--manager", "labgrid.executor.host:manager", - "test", + "echo", + "$${DATA}" ], main = "executor.py", deps = ["host"], @@ -44,7 +45,7 @@ py_test( py_test( name = "envvar", srcs = ["executor.py"], - args = ["test"], + args = ["echo", "$${DATA}"], env = { "LABGRID_EXECUTOR_MANAGER": "labgrid.executor.host:manager", }, diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index b7da6f89..c205278a 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -10,7 +10,7 @@ from os import environ from pathlib import Path from platform import machine, system from shutil import which -from subprocess import CalledProcessError +from subprocess import CalledProcessError, run from sys import argv, stderr from labgrid.executor.manager import Manager @@ -78,9 +78,9 @@ def main(exe: Path, *args: str) -> int: if machine() == "arm64" and system() == "Darwin": cmd = (which("sh"), *cmd) - with parsed.manager() as run: + with parsed.manager() as data: try: - run(*cmd) + run(cmd, env=data.env, check=True) except CalledProcessError as e: return e.returncode except KeyboardInterrupt: diff --git a/labgrid/executor/host.py b/labgrid/executor/host.py index 0936f99b..2a2c5558 100644 --- a/labgrid/executor/host.py +++ b/labgrid/executor/host.py @@ -5,21 +5,14 @@ from typing import Iterator from python.runfiles import Runfiles -from labgrid.executor.manager import Manager, Run +from labgrid.executor.manager import Manager, Data -def _run(program: str, *args: str) -> None: - assert program == "test" - assert not args +@contextmanager +def manager() -> Iterator[Run]: runfiles = Runfiles.Create() path = runfiles.Rlocation("_main/labgrid/executor/hello-world.txt") with open(path) as stream: - print(stream.read()) - - -run: Run = _run - - -@contextmanager -def manager() -> Iterator[Run]: - yield run + data = stream.read() + env = {"DATA": data} + yield Data(env=env) diff --git a/labgrid/executor/manager.py b/labgrid/executor/manager.py index 723956d4..a78fc4c9 100644 --- a/labgrid/executor/manager.py +++ b/labgrid/executor/manager.py @@ -1,11 +1,18 @@ from __future__ import annotations from contextlib import AbstractContextManager -from typing import TypeAlias +from dataclasses import dataclass, field +from typing import Protocol, TypeAlias -class Run: - def __call__(self, program: str, *args: str) -> None: ... +@dataclass(frozen=True) +class Data: + env: Mapping[str, str] = field(default_factory=dict) -Manager: TypeAlias = AbstractContextManager[Run] +class DataProtocol(Protocol): + @property + def env(self) -> Mapping[str, str]: ... + + +Manager: TypeAlias = AbstractContextManager[DataProtocol] -- GitLab