From d33b5e35fa3f570cefcfd7e82a0e1901d82a86a8 Mon Sep 17 00:00:00 2001 From: Xuming Meng Date: Sat, 12 Oct 2024 13:13:29 +0200 Subject: [PATCH] feat(config): accept multiple managers Managers and deps in toolchain are merged into one default info and py info respectively. Passthough manager is removed. Py test for multiple manager is added. --- labgrid/config/BUILD.bazel | 6 +-- labgrid/config/deps.bzl | 2 +- labgrid/config/{manager.bzl => managers.bzl} | 6 +-- labgrid/config/rule.bzl | 41 +++++++++++++++----- labgrid/config/toolchain/macro.bzl | 4 +- labgrid/executor/BUILD.bazel | 18 ++++++++- labgrid/executor/args.bzl | 16 ++++++-- labgrid/executor/executor.py | 26 +++++-------- labgrid/executor/host.py | 6 ++- labgrid/manager/BUILD.bazel | 9 ----- labgrid/manager/passthrough.py | 14 ------- 11 files changed, 83 insertions(+), 65 deletions(-) rename labgrid/config/{manager.bzl => managers.bzl} (83%) delete mode 100644 labgrid/manager/BUILD.bazel delete mode 100644 labgrid/manager/passthrough.py diff --git a/labgrid/config/BUILD.bazel b/labgrid/config/BUILD.bazel index 10f07c1a..36c525c3 100644 --- a/labgrid/config/BUILD.bazel +++ b/labgrid/config/BUILD.bazel @@ -1,7 +1,7 @@ load(":data.bzl", "data") load(":deps.bzl", "deps") load(":env.bzl", "env") -load(":manager.bzl", "manager") +load(":managers.bzl", "managers") env( name = "env", @@ -13,8 +13,8 @@ data( visibility = ["//:__subpackages__"], ) -manager( - name = "manager", +managers( + name = "managers", visibility = ["//:__subpackages__"], ) diff --git a/labgrid/config/deps.bzl b/labgrid/config/deps.bzl index 3866e918..e0e33c55 100644 --- a/labgrid/config/deps.bzl +++ b/labgrid/config/deps.bzl @@ -15,7 +15,7 @@ deps = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - provides = [PyInfo], + provides = [DefaultInfo, PyInfo], toolchains = ["//labgrid/toolchain/config:type"], cfg = _cfg, ) diff --git a/labgrid/config/manager.bzl b/labgrid/config/managers.bzl similarity index 83% rename from labgrid/config/manager.bzl rename to labgrid/config/managers.bzl index b4ac3b06..df1bb66e 100644 --- a/labgrid/config/manager.bzl +++ b/labgrid/config/managers.bzl @@ -9,13 +9,13 @@ ATTRS = {} def implementation(ctx): toolchain = ctx.toolchains["//labgrid/toolchain/config:type"] - return toolchain.manager + return list(toolchain.managers) -manager = rule( +managers = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - provides = [PyInfo], + provides = [DefaultInfo, PyInfo], toolchains = ["//labgrid/toolchain/config:type"], cfg = _cfg, ) diff --git a/labgrid/config/rule.bzl b/labgrid/config/rule.bzl index 97733b1a..3751ead0 100644 --- a/labgrid/config/rule.bzl +++ b/labgrid/config/rule.bzl @@ -69,19 +69,13 @@ ATTRS = { providers = [DefaultInfo], cfg = "exec", ), - "manager": attr.label( + "managers": attr.label_list( doc = "A context manager that `//labgrid/executor` will use that sets up the LabGrid environment.", providers = [[DefaultInfo, PyInfo]], cfg = "exec", ), } -def _forward(target): - return target[DefaultInfo], target[PyInfo] - -def _flatten(iterable): - return [e for x in iterable for e in x] - def _expand(ctx, data, value): value = value.replace("$(location", "$(rlocationpath") value = ctx.expand_location(value, targets = data) @@ -104,16 +98,45 @@ def _runfile(value, separator = "~"): return value +def _merge_py_info(targets): + transitive_sources = [] + imports = [] + for target in targets: + p = target[PyInfo] + transitive_sources.extend(p.transitive_sources.to_list() if p.transitive_sources else []) + imports.extend(p.imports.to_list() if p.imports else []) + return PyInfo( + transitive_sources = depset(transitive_sources), + imports = depset(imports), + ) + +def _merge_default_info(targets): + files = [] + data_runfiles = [] + default_runfiles = [] + for target in targets: + d = target[DefaultInfo] + files.extend(d.files.to_list() if d.files else []) + if d.data_runfiles: + data_runfiles.append(d.data_runfiles) + if d.default_runfiles: + default_runfiles.append(d.default_runfiles) + return DefaultInfo( + files = depset(files), + data_runfiles = data_runfiles[0].merge_all(data_runfiles) if data_runfiles else None, + default_runfiles = default_runfiles[0].merge_all(default_runfiles) if default_runfiles else None, + ) + def implementation(ctx): data = ctx.attr.data + [ctx.attr.src] + ctx.attr.toolchains env = {k: _expand(ctx, data, v) for k, v in ctx.attr.env.items()} default = DefaultInfo(files = depset([ctx.file.src])) toolchain = platform_common.ToolchainInfo( src = ctx.file.src, - deps = _flatten([_forward(d) for d in ctx.attr.deps]), + deps = [_merge_default_info(ctx.attr.deps), _merge_py_info(ctx.attr.deps)], env = env, data = depset(transitive = [d.files for d in data]), - manager = _forward(ctx.attr.manager), + managers = [_merge_default_info(ctx.attr.managers), _merge_py_info(ctx.attr.managers)], ) return [default, toolchain] diff --git a/labgrid/config/toolchain/macro.bzl b/labgrid/config/toolchain/macro.bzl index 3928a677..13c015d8 100644 --- a/labgrid/config/toolchain/macro.bzl +++ b/labgrid/config/toolchain/macro.bzl @@ -2,7 +2,7 @@ load("@rules_labgrid//labgrid/config:defs.bzl", "labgrid_config") visibility("//...") -def labgrid_config_toolchain(*, name, src, state, target_compatible_with, manager = "@rules_labgrid//labgrid/manager:passthrough", deps = (), env = {}, data = [], toolchains = []): +def labgrid_config_toolchain(*, name, src, state, target_compatible_with, managers = [], deps = [], env = {}, data = [], toolchains = []): src = native.package_relative_label(src) labgrid_config( name = "{}-config".format(name), @@ -14,7 +14,7 @@ def labgrid_config_toolchain(*, name, src, state, target_compatible_with, manage }, data = data, toolchains = toolchains, - manager = native.package_relative_label(manager), + managers = [native.package_relative_label(m) for m in managers], ) constraints = [native.package_relative_label(c) for c in target_compatible_with] diff --git a/labgrid/executor/BUILD.bazel b/labgrid/executor/BUILD.bazel index b43b480e..db6fcbec 100644 --- a/labgrid/executor/BUILD.bazel +++ b/labgrid/executor/BUILD.bazel @@ -14,7 +14,7 @@ py_binary( visibility = ["//:__subpackages__"], deps = [ "//bazel/labgrid/executor", - "//labgrid/config:manager", + "//labgrid/config:managers", "@rules_python//python/runfiles", ], ) @@ -42,3 +42,19 @@ py_test( main = "executor.py", deps = ["host"], ) + +py_test( + name = "multiple-managers", + size = "small", + srcs = ["executor.py"], + args = [ + "--manager", + "labgrid.executor.host:manager", + "--manager", + "labgrid.executor.host:manager", + "echo", + "$${DATA}", + ], + main = "executor.py", + deps = ["host"], +) \ No newline at end of file diff --git a/labgrid/executor/args.bzl b/labgrid/executor/args.bzl index ecee5a99..138aad82 100644 --- a/labgrid/executor/args.bzl +++ b/labgrid/executor/args.bzl @@ -5,10 +5,10 @@ visibility("//labgrid/executor/...") DOC = "Creates an args file to be passed to `//labgrid/executor`" ATTRS = { - "_manager": attr.label( - allow_single_file = True, + "_manager": attr.label_list( + allow_files = True, doc = "The manager to use", - default = "//labgrid/config:manager", + default = ["//labgrid/config:managers"], cfg = "exec", ), "_env": attr.label( @@ -18,10 +18,18 @@ ATTRS = { ), } +def _import_path(short_path): + path = short_path.removesuffix(".py") + parts = path.split("/") + parts = parts[2:] if parts[0] == ".." else parts + return ".".join(parts) + def implementation(ctx): output = ctx.actions.declare_file("executor-args.txt") - lines = ["--manager", ctx.file._manager.short_path] + lines = [] + for m in ctx.files._manager: + lines.extend(["--manager", "{}:{}".format(_import_path(m.short_path), "manager")]) for name, value in ctx.attr._env[EnvironmentInfo].environment.items(): lines.extend(["--env", name, value]) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index 9cc59d60..b593cf60 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -3,7 +3,7 @@ from __future__ import annotations from argparse import ArgumentParser, ArgumentTypeError -from contextlib import AbstractContextManager +from contextlib import AbstractContextManager, ExitStack from importlib import import_module from inspect import signature from os import environ @@ -15,19 +15,11 @@ from sys import argv, stderr from python.runfiles import Runfiles -from bazel.labgrid.executor.manager import Manager +from bazel.labgrid.executor.manager import Data, Manager def load(value: str) -> Manager: - name, sep, attr = value.partition(":") - if sep != ":": - path = PurePath(value).with_suffix("") - parts = path.parts - if parts[0] == "..": - parts = parts[2:] - name = ".".join(parts) - attr = "manager" - + name, _, attr = value.partition(":") module = import_module(name) try: @@ -53,6 +45,7 @@ def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( "--manager", help="A manager to load from a Python module, of the form `module.to.load:Manager`.", + action="append", type=load, ) prsr.add_argument( @@ -76,11 +69,6 @@ def main(exe: Path, *args: str) -> int: parsed = prsr.parse_args(args) - if parsed.manager is None: - print("Must provide a manager", file=stderr) - prsr.print_usage(file=stderr) - return 2 - cmd = (parsed.program, *parsed.arguments) # A workaround for invoking APE binaries on Apple Silicon @@ -88,7 +76,11 @@ def main(exe: Path, *args: str) -> int: if machine() == "arm64" and system() == "Darwin": cmd = (which("sh"), *cmd) - with parsed.manager() as data: + data = Data(env={}) + with ExitStack() as stack: + if parsed.manager: + for manager in parsed.manager: + data = stack.enter_context(manager(data)) env = {k: resolve_runfile(v) for k, v in parsed.env} env |= {k: v for k, v in data.env.items() if k != "RUNFILES_DIR"} try: diff --git a/labgrid/executor/host.py b/labgrid/executor/host.py index d638d9e4..4beb0fac 100644 --- a/labgrid/executor/host.py +++ b/labgrid/executor/host.py @@ -9,10 +9,12 @@ from bazel.labgrid.executor.manager import Data, Manager @contextmanager -def manager() -> Iterator[Run]: +def manager(input_data: Data) -> Iterator[Run]: runfiles = Runfiles.Create() path = runfiles.Rlocation("_main/labgrid/executor/hello-world.txt") with open(path) as stream: data = stream.read() - env = {"DATA": data} + env = { + "DATA": input_data.env.get("DATA", "") + "\n" + data, + } yield Data(env=env) diff --git a/labgrid/manager/BUILD.bazel b/labgrid/manager/BUILD.bazel deleted file mode 100644 index 943a15d8..00000000 --- a/labgrid/manager/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@rules_python//python:defs.bzl", "py_library") - -py_library( - name = "passthrough", - srcs = ["passthrough.py"], - tags = ["manual"], - visibility = ["//visibility:public"], - deps = ["@rules_labgrid//bazel/labgrid/executor"], -) diff --git a/labgrid/manager/passthrough.py b/labgrid/manager/passthrough.py deleted file mode 100644 index cf7bb4c8..00000000 --- a/labgrid/manager/passthrough.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from contextlib import contextmanager -from typing import Iterator - -from bazel.labgrid.executor.manager import Data - - -@contextmanager -def manager() -> Iterator[Data]: - """ - A passthrough LabGrid context manager - """ - yield Data(env={}) -- GitLab