From 573fbafdb9e7cbeced83f95922e2198cdbe3dc3c Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 10:06:03 +0000 Subject: [PATCH 01/12] refactor(//labgrid/run): extract `__main__.py` --- labgrid/run/BUILD.bazel | 7 +- labgrid/run/__main__.py | 260 ++++++++++++++++++++++++++++++++++++++++ labgrid/run/run.py | 253 +------------------------------------- 3 files changed, 269 insertions(+), 251 deletions(-) create mode 100644 labgrid/run/__main__.py diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index f55b312e..82f173ad 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -19,11 +19,16 @@ TOOLS = [ py_binary( name = "run", - srcs = ["run.py"], + srcs = [ + "__main__.py", + "run.py", + ], data = [ ":{}".format(tool) for tool in TOOLS ], + imports = ["."], + main = "__main__.py", tags = ["manual"], visibility = ["//visibility:public"], deps = [ diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py new file mode 100644 index 00000000..751c0486 --- /dev/null +++ b/labgrid/run/__main__.py @@ -0,0 +1,260 @@ +#! /usr/bin/env python3 +from __future__ import annotations + +import logging +from argparse import REMAINDER, Action, ArgumentParser, ArgumentTypeError, Namespace +from dataclasses import replace +from os import environ, getenv +from pathlib import Path, PurePath +from string import Template +from subprocess import CalledProcessError +from sys import argv, stderr +from typing import ( + Any, + Final, + MutableMapping, + Optional, + Sequence, + Text, + TypeVar, + Union, +) + +from python.runfiles import Runfiles + +from run import FileTransfer, State, run + +T = TypeVar("T") +LOG_DEFAULT = logging.ERROR + + +def arguments(prsr: ArgumentParser) -> None: + prsr.add_argument( + "program", + metavar="PROG", + help="The program to run on the device.", + type=resolve, + ) + prsr.add_argument( + "arguments", metavar="ARG", nargs=REMAINDER, help="Command to run over SSH." + ) + prsr.add_argument( + "--config", + help="The LabGrid configuration.", + default=PurePath(environ["LG_ENV"]), + type=PurePath, + ) + prsr.add_argument( + "--initial-state", + help="The state to transition the LabGrid strategy to initially", + dest="state.initial", + action=StateAction, + default=environ.get("LG_INITIAL_STATE", None), + ) + prsr.add_argument( + "--state", + help="The state to transition the LabGrid strategy to", + dest="state.desired", + action=StateAction, + default=environ["LG_STATE"], + ) + prsr.add_argument( + "--final-state", + help="The state to transition the LabGrid strategy to at the end", + dest="state.final", + action=StateAction, + default=environ.get("BZL_LG_FINAL_STATE", None), + ) + prsr.add_argument( + "--env", + help=( + "The environment variables to be passed to the target for execution of the program. " + "Suports VAR=VAL syntax or uses the host environment variable if a single key is" + "specified." + ), + action="append", + required=False, + default=[], + metavar="KEY[=VALUE]", + type=env, + ) + prsr.add_argument( + "--get", + help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `REMOTE` can have a trailing `?` to indicate it's optional. `LOCAL` performs environment variable substitution. `LOCAL` can be omitted, which will use the same path for remote and local.", + type=get, + metavar="REMOTE[?][:LOCAL]", + action="append", + default=[], + ) + prsr.add_argument( + "--put", + help="Transfer files to the target before execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` performs environment variable substitution. `REMOTE` can be omitted, which will use the same path for remote and local.", + type=put, + metavar="LOCAL[:REMOTE]", + action="append", + default=[], + ) + prsr.add_argument( + "--log-level", + action=LogLevelAction, + help=f"Verbosity level of logging. Uses standard python logging levels (e.g. WARNING, INFO, DEBUG), the default is {logging.getLevelName(LOG_DEFAULT)}. Can also be set with LOG_LEVEL environment variable", + default=LogLevelAction.coerce( + environ.get("LOG_LEVEL", logging.getLevelName(LOG_DEFAULT)) + ), + dest="log_level", + ) + + +def resolve(value: str) -> Path: + runfiles = Runfiles.Create() + resolved = Path(runfiles.Rlocation(value)) + if not resolved.exists(): + return Path(value) + return resolved + + +class StateAction(Action): + __default: Final[MutableMapping] = {} + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: str | None, + const: Optional[T] = None, + help: Optional[str] = None, + ): + dest, found, attr = dest.partition(".") + if not found: + raise ValueError("`dest` must contain `.`") + self.attr = attr + super().__init__( + option_strings, + dest, + nargs=0, + const=const, + default=default, + type=None, + choices=None, + required=False, + help=help, + metavar=None, + ) + + @property + def default(self) -> State: + return State(**self.__default) + + @default.setter + def default(self, value: str | None) -> None: + self.__default[self.attr] = value + + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[Text] = None, + ) -> None: + assert isinstance(values, str) + try: + state = getattr(namespace, self.dest) + except AttributeError: + state = self.default + state = replace(state, **{self.attr: values}) + setattr(namespace, self.dest, state) + + +class LogLevelAction(Action): + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[Text] = None, + ) -> None: + assert isinstance(values, str) + setattr(namespace, self.dest, LogLevelAction.coerce(values)) + + def coerce(value: str) -> int: + # The Logging module converts strings to level integers with this (unintuitively named) function + log_level = logging.getLevelName(value) + if isinstance(log_level, str): + # getLevelName will return a string if the level was not recognised + raise ArgumentTypeError(f"Unsupported verbosity envvar: '{value}'") + + return log_level + + +def env(arg: str) -> str: + key, sep, value = arg.partition("=") + if not sep: + value = getenv(key) + + if value is None: + raise ArgumentTypeError(f"Env var {key} does not exist in host environment") + else: + return f"{key}={value}" + + return arg + + +def get(value: str) -> FileTransfer: + # FIXME: Extract `BazelArgumentParser` to handle quoting + remote, found, local = value.removeprefix("'").removesuffix("'").partition(":") + optional = remote.endswith("?") + if optional: + remote = remote.removesuffix("?") + if found: + local = Template(local).substitute(environ) + else: + local = remote + return FileTransfer(PurePath(remote), PurePath(local), optional) + + +def put(value: str) -> FileTransfer: + # FIXME: Extract `BazelArgumentParser` to handle quoting + local, found, remote = value.removeprefix("'").removesuffix("'").partition(":") + if found: + local = Template(local).substitute(environ) + else: + remote = local + return FileTransfer(PurePath(remote), PurePath(local), False) + + +def main(exe: Path, *args: str) -> int: + prsr = ArgumentParser( + prog=str(exe), description="Runs a command on a LabGrid device." + ) + + arguments(prsr) + parsed = prsr.parse_args(args) + + try: + return run( + parsed.program, + *parsed.arguments, + state=parsed.state, + config=parsed.config, + get=parsed.get, + put=parsed.put, + env=parsed.env, + log_level=parsed.log_level, + ) + except CalledProcessError as e: + print(f"fatal: subprocess failed: {e}", file=stderr) + if e.stdout is not None: + print(e.stdout, file=stderr) + if e.stderr is not None: + print(e.stderr, file=stderr) + return e.returncode + except KeyboardInterrupt: + return 130 + + +def entry(): + exit(main(Path(argv[0]), *argv[1:])) + + +if __name__ == "__main__": + entry() diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 53cd1b05..442eea5c 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -1,25 +1,13 @@ -#! /usr/bin/env python3 from __future__ import annotations -from typing import Any import logging -from argparse import REMAINDER, Action, ArgumentParser, ArgumentTypeError, Namespace -from dataclasses import dataclass, replace -from os import environ, linesep, getenv +from dataclasses import dataclass +from os import environ, linesep from pathlib import Path, PurePath from shlex import join -from string import Template -from subprocess import CalledProcessError -from sys import argv, stderr, stdout +from sys import stderr, stdout from typing import ( - Final, Iterator, - MutableMapping, - Optional, - Sequence, - Text, - TypeVar, - Union, ) from python.runfiles import Runfiles @@ -30,10 +18,6 @@ from labgrid import Environment from labgrid.logging import basicConfig, StepLogger from labgrid.driver.exception import ExecutionError -T = TypeVar("T") -logger = logging.getLogger(__name__) -LOG_DEFAULT = logging.ERROR - class RunfileNotFoundError(FileNotFoundError): pass @@ -56,13 +40,6 @@ class Tools: mv: Path = runfile("ape/ape/assimilate/mv/mv") -def resolve(value: str) -> Path: - try: - return runfile(value) - except RunfileNotFoundError: - return Path(value) - - @dataclass(frozen=True) class State: desired: str @@ -70,192 +47,6 @@ class State: final: str | None = None -class StateAction(Action): - __default: Final[MutableMapping] = {} - - def __init__( - self, - option_strings: Sequence[str], - dest: str, - default: str | None, - const: Optional[T] = None, - help: Optional[str] = None, - ): - dest, found, attr = dest.partition(".") - if not found: - raise ValueError("`dest` must contain `.`") - self.attr = attr - super().__init__( - option_strings, - dest, - nargs=0, - const=const, - default=default, - type=None, - choices=None, - required=False, - help=help, - metavar=None, - ) - - @property - def default(self) -> State: - return State(**self.__default) - - @default.setter - def default(self, value: str | None) -> None: - self.__default[self.attr] = value - - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[Text] = None, - ) -> None: - assert isinstance(values, str) - try: - state = getattr(namespace, self.dest) - except AttributeError: - state = self.default - state = replace(state, **{self.attr: values}) - setattr(namespace, self.dest, state) - - -class LogLevelAction(Action): - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[Text] = None, - ) -> None: - assert isinstance(values, str) - setattr(namespace, self.dest, LogLevelAction.coerce(values)) - - def coerce(value: str) -> int: - # The Logging module converts strings to level integers with this (unintuitively named) function - log_level = logging.getLevelName(value) - if isinstance(log_level, str): - # getLevelName will return a string if the level was not recognised - raise ArgumentTypeError(f"Unsupported verbosity envvar: '{value}'") - - return log_level - - -def env(arg: str) -> str: - key, sep, value = arg.partition("=") - if not sep: - value = getenv(key) - - if value is None: - raise ArgumentTypeError(f"Env var {key} does not exist in host environment") - else: - return f"{key}={value}" - - return arg - - -def arguments(prsr: ArgumentParser) -> None: - prsr.add_argument( - "program", - metavar="PROG", - help="The program to run on the device.", - type=resolve, - ) - prsr.add_argument( - "arguments", metavar="ARG", nargs=REMAINDER, help="Command to run over SSH." - ) - prsr.add_argument( - "--config", - help="The LabGrid configuration.", - default=PurePath(environ["LG_ENV"]), - type=PurePath, - ) - prsr.add_argument( - "--initial-state", - help="The state to transition the LabGrid strategy to initially", - dest="state.initial", - action=StateAction, - default=environ.get("LG_INITIAL_STATE", None), - ) - prsr.add_argument( - "--state", - help="The state to transition the LabGrid strategy to", - dest="state.desired", - action=StateAction, - default=environ["LG_STATE"], - ) - prsr.add_argument( - "--final-state", - help="The state to transition the LabGrid strategy to at the end", - dest="state.final", - action=StateAction, - default=environ.get("BZL_LG_FINAL_STATE", None), - ) - prsr.add_argument( - "--env", - help=( - "The environment variables to be passed to the target for execution of the program. " - "Suports VAR=VAL syntax or uses the host environment variable if a single key is" - "specified." - ), - action="append", - required=False, - default=[], - metavar="KEY[=VALUE]", - type=env, - ) - prsr.add_argument( - "--get", - help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `REMOTE` can have a trailing `?` to indicate it's optional. `LOCAL` performs environment variable substitution. `LOCAL` can be omitted, which will use the same path for remote and local.", - type=get, - metavar="REMOTE[?][:LOCAL]", - action="append", - default=[], - ) - prsr.add_argument( - "--put", - help="Transfer files to the target before execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` performs environment variable substitution. `REMOTE` can be omitted, which will use the same path for remote and local.", - type=put, - metavar="LOCAL[:REMOTE]", - action="append", - default=[], - ) - prsr.add_argument( - "--log-level", - action=LogLevelAction, - help=f"Verbosity level of logging. Uses standard python logging levels (e.g. WARNING, INFO, DEBUG), the default is {logging.getLevelName(LOG_DEFAULT)}. Can also be set with LOG_LEVEL environment variable", - default=LogLevelAction.coerce( - environ.get("LOG_LEVEL", logging.getLevelName(LOG_DEFAULT)) - ), - dest="log_level", - ) - - -def get(value: str) -> FileTransfer: - # FIXME: Extract `BazelArgumentParser` to handle quoting - remote, found, local = value.removeprefix("'").removesuffix("'").partition(":") - optional = remote.endswith("?") - if optional: - remote = remote.removesuffix("?") - if found: - local = Template(local).substitute(environ) - else: - local = remote - return FileTransfer(PurePath(remote), PurePath(local), optional) - - -def put(value: str) -> FileTransfer: - # FIXME: Extract `BazelArgumentParser` to handle quoting - local, found, remote = value.removeprefix("'").removesuffix("'").partition(":") - if found: - local = Template(local).substitute(environ) - else: - remote = local - return FileTransfer(PurePath(remote), PurePath(local), False) - - @dataclass(frozen=True) class FileTransfer: remote: PurePath @@ -363,41 +154,3 @@ def run( raise e return code - - -def main(exe: Path, *args: str) -> int: - prsr = ArgumentParser( - prog=str(exe), description="Runs a command on a LabGrid device." - ) - - arguments(prsr) - parsed = prsr.parse_args(args) - - try: - return run( - parsed.program, - *parsed.arguments, - state=parsed.state, - config=parsed.config, - get=parsed.get, - put=parsed.put, - env=parsed.env, - log_level=parsed.log_level, - ) - except CalledProcessError as e: - print(f"fatal: subprocess failed: {e}", file=stderr) - if e.stdout is not None: - print(e.stdout, file=stderr) - if e.stderr is not None: - print(e.stderr, file=stderr) - return e.returncode - except KeyboardInterrupt: - return 130 - - -def entry(): - exit(main(Path(argv[0]), *argv[1:])) - - -if __name__ == "__main__": - entry() -- GitLab From 2712c2234492f798a582fe1715097ba62f768368 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 13:12:30 +0000 Subject: [PATCH 02/12] refactor(//labgrid/run): remove state flags The state environment variables should be an implementation detail of the `run` library. --- labgrid/run/__main__.py | 79 +---------------------------------------- labgrid/run/run.py | 8 ++--- 2 files changed, 5 insertions(+), 82 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index 751c0486..bfe5e254 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging from argparse import REMAINDER, Action, ArgumentParser, ArgumentTypeError, Namespace -from dataclasses import replace from os import environ, getenv from pathlib import Path, PurePath from string import Template @@ -11,8 +10,6 @@ from subprocess import CalledProcessError from sys import argv, stderr from typing import ( Any, - Final, - MutableMapping, Optional, Sequence, Text, @@ -22,7 +19,7 @@ from typing import ( from python.runfiles import Runfiles -from run import FileTransfer, State, run +from run import FileTransfer, run T = TypeVar("T") LOG_DEFAULT = logging.ERROR @@ -44,27 +41,6 @@ def arguments(prsr: ArgumentParser) -> None: default=PurePath(environ["LG_ENV"]), type=PurePath, ) - prsr.add_argument( - "--initial-state", - help="The state to transition the LabGrid strategy to initially", - dest="state.initial", - action=StateAction, - default=environ.get("LG_INITIAL_STATE", None), - ) - prsr.add_argument( - "--state", - help="The state to transition the LabGrid strategy to", - dest="state.desired", - action=StateAction, - default=environ["LG_STATE"], - ) - prsr.add_argument( - "--final-state", - help="The state to transition the LabGrid strategy to at the end", - dest="state.final", - action=StateAction, - default=environ.get("BZL_LG_FINAL_STATE", None), - ) prsr.add_argument( "--env", help=( @@ -113,58 +89,6 @@ def resolve(value: str) -> Path: return resolved -class StateAction(Action): - __default: Final[MutableMapping] = {} - - def __init__( - self, - option_strings: Sequence[str], - dest: str, - default: str | None, - const: Optional[T] = None, - help: Optional[str] = None, - ): - dest, found, attr = dest.partition(".") - if not found: - raise ValueError("`dest` must contain `.`") - self.attr = attr - super().__init__( - option_strings, - dest, - nargs=0, - const=const, - default=default, - type=None, - choices=None, - required=False, - help=help, - metavar=None, - ) - - @property - def default(self) -> State: - return State(**self.__default) - - @default.setter - def default(self, value: str | None) -> None: - self.__default[self.attr] = value - - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[Text] = None, - ) -> None: - assert isinstance(values, str) - try: - state = getattr(namespace, self.dest) - except AttributeError: - state = self.default - state = replace(state, **{self.attr: values}) - setattr(namespace, self.dest, state) - - class LogLevelAction(Action): def __call__( self, @@ -234,7 +158,6 @@ def main(exe: Path, *args: str) -> int: return run( parsed.program, *parsed.arguments, - state=parsed.state, config=parsed.config, get=parsed.get, put=parsed.put, diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 442eea5c..535263e6 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -42,9 +42,9 @@ class Tools: @dataclass(frozen=True) class State: - desired: str - initial: str | None = None - final: str | None = None + desired: str = environ["LG_STATE"] + initial: str | None = environ.get("LG_INITIAL_STATE", None) + final: str | None = environ.get("BZL_LG_FINAL_STATE", None) @dataclass(frozen=True) @@ -72,7 +72,7 @@ def run( program: Path, *arguments: str, config: Path, - state: State, + state: State = State(), get: Iterator[FileTransfer], put: Iterator[FileTransfer], env: Iterator[str], -- GitLab From 4fd8061264dec4ce2a1ef697fb19f2e0325a0a9e Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 13:18:22 +0000 Subject: [PATCH 03/12] refactor(//labgrid/run): remove config flag --- labgrid/run/__main__.py | 7 ------- labgrid/run/run.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index bfe5e254..4b29c301 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -35,12 +35,6 @@ def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( "arguments", metavar="ARG", nargs=REMAINDER, help="Command to run over SSH." ) - prsr.add_argument( - "--config", - help="The LabGrid configuration.", - default=PurePath(environ["LG_ENV"]), - type=PurePath, - ) prsr.add_argument( "--env", help=( @@ -158,7 +152,6 @@ def main(exe: Path, *args: str) -> int: return run( parsed.program, *parsed.arguments, - config=parsed.config, get=parsed.get, put=parsed.put, env=parsed.env, diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 535263e6..0d628d58 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -71,7 +71,7 @@ class FileTransfer: def run( program: Path, *arguments: str, - config: Path, + config: Path = PurePath(environ["LG_ENV"]), state: State = State(), get: Iterator[FileTransfer], put: Iterator[FileTransfer], -- GitLab From ae11003c35858bd4dfd9b632565aeedcf29f8089 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 13:36:54 +0000 Subject: [PATCH 04/12] refactor(//labgrid/run): remove logging flag --- labgrid/run/__main__.py | 45 +---------------------------------------- labgrid/run/run.py | 12 ++++++++++- 2 files changed, 12 insertions(+), 45 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index 4b29c301..d433e3a9 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -1,29 +1,17 @@ #! /usr/bin/env python3 from __future__ import annotations -import logging -from argparse import REMAINDER, Action, ArgumentParser, ArgumentTypeError, Namespace +from argparse import REMAINDER, ArgumentParser, ArgumentTypeError from os import environ, getenv from pathlib import Path, PurePath from string import Template from subprocess import CalledProcessError from sys import argv, stderr -from typing import ( - Any, - Optional, - Sequence, - Text, - TypeVar, - Union, -) from python.runfiles import Runfiles from run import FileTransfer, run -T = TypeVar("T") -LOG_DEFAULT = logging.ERROR - def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( @@ -64,15 +52,6 @@ def arguments(prsr: ArgumentParser) -> None: action="append", default=[], ) - prsr.add_argument( - "--log-level", - action=LogLevelAction, - help=f"Verbosity level of logging. Uses standard python logging levels (e.g. WARNING, INFO, DEBUG), the default is {logging.getLevelName(LOG_DEFAULT)}. Can also be set with LOG_LEVEL environment variable", - default=LogLevelAction.coerce( - environ.get("LOG_LEVEL", logging.getLevelName(LOG_DEFAULT)) - ), - dest="log_level", - ) def resolve(value: str) -> Path: @@ -83,27 +62,6 @@ def resolve(value: str) -> Path: return resolved -class LogLevelAction(Action): - def __call__( - self, - parser: ArgumentParser, - namespace: Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[Text] = None, - ) -> None: - assert isinstance(values, str) - setattr(namespace, self.dest, LogLevelAction.coerce(values)) - - def coerce(value: str) -> int: - # The Logging module converts strings to level integers with this (unintuitively named) function - log_level = logging.getLevelName(value) - if isinstance(log_level, str): - # getLevelName will return a string if the level was not recognised - raise ArgumentTypeError(f"Unsupported verbosity envvar: '{value}'") - - return log_level - - def env(arg: str) -> str: key, sep, value = arg.partition("=") if not sep: @@ -155,7 +113,6 @@ def main(exe: Path, *args: str) -> int: get=parsed.get, put=parsed.put, env=parsed.env, - log_level=parsed.log_level, ) except CalledProcessError as e: print(f"fatal: subprocess failed: {e}", file=stderr) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 0d628d58..3066cc56 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -67,6 +67,16 @@ class FileTransfer: return path +def log_level(value): + # The Logging module converts strings to level integers with this (unintuitively named) function + log_level = logging.getLevelName(value) + if isinstance(log_level, str): + # getLevelName will return a string if the level was not recognised + raise ValueError(f"Unsupported value for LOG_LEVEL: '{value}'") + + return log_level + + # TODO: move this out to `bazel.labgrid.run` on day to allow downstream to re-use def run( program: Path, @@ -76,7 +86,7 @@ def run( get: Iterator[FileTransfer], put: Iterator[FileTransfer], env: Iterator[str], - log_level: int, + log_level: int = log_level(environ.get("LOG_LEVEL", "ERROR")), tools: Tools = Tools(), ) -> int: runfiles_dir = None -- GitLab From 1c8cadeffaac1d951dc0a0d0bc0cb931ef2d6738 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 15:20:17 +0000 Subject: [PATCH 05/12] refactor: simplify `FileTransfer` constructor --- labgrid/run/__main__.py | 6 +++--- labgrid/run/run.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index d433e3a9..589595b0 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -3,7 +3,7 @@ from __future__ import annotations from argparse import REMAINDER, ArgumentParser, ArgumentTypeError from os import environ, getenv -from pathlib import Path, PurePath +from pathlib import Path from string import Template from subprocess import CalledProcessError from sys import argv, stderr @@ -85,7 +85,7 @@ def get(value: str) -> FileTransfer: local = Template(local).substitute(environ) else: local = remote - return FileTransfer(PurePath(remote), PurePath(local), optional) + return FileTransfer(remote, local, optional) def put(value: str) -> FileTransfer: @@ -95,7 +95,7 @@ def put(value: str) -> FileTransfer: local = Template(local).substitute(environ) else: remote = local - return FileTransfer(PurePath(remote), PurePath(local), False) + return FileTransfer(remote, local) def main(exe: Path, *args: str) -> int: diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 3066cc56..e3bd4d53 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -53,6 +53,11 @@ class FileTransfer: local: PurePath optional: bool + def __init__(self, remote: str, local: str, optional=False): + object.__setattr__(self, "remote", PurePath(remote)) + object.__setattr__(self, "local", PurePath(local)) + object.__setattr__(self, "optional", optional) + def resolve(self, remote: PurePath) -> FileTransfer: return FileTransfer( remote=self.join(remote, self.remote), -- GitLab From a16cf59d9aaf70ff0da8f7a15219d6b018e69764 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 15:30:13 +0000 Subject: [PATCH 06/12] refactor: move `State` under `bazel.labgrid.strategy` --- bazel/labgrid/strategy/__init__.py | 4 ++-- bazel/labgrid/strategy/transition.py | 31 ++++++++++++++++++++++------ labgrid/run/run.py | 12 ++--------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/bazel/labgrid/strategy/__init__.py b/bazel/labgrid/strategy/__init__.py index b1f21abb..7994b5dd 100644 --- a/bazel/labgrid/strategy/__init__.py +++ b/bazel/labgrid/strategy/__init__.py @@ -1,6 +1,6 @@ from .localhost import LocalhostStrategy from .qemu import QEMUStrategy from .ssh import SSHStrategy -from .transition import transition +from .transition import transition, State -__all__ = ["LocalhostStrategy", "QEMUStrategy", "SSHStrategy", "transition"] +__all__ = ["LocalhostStrategy", "QEMUStrategy", "SSHStrategy", "transition", "State"] diff --git a/bazel/labgrid/strategy/transition.py b/bazel/labgrid/strategy/transition.py index c7d3bf08..e8153183 100644 --- a/bazel/labgrid/strategy/transition.py +++ b/bazel/labgrid/strategy/transition.py @@ -1,13 +1,32 @@ from contextlib import contextmanager +from dataclasses import dataclass +from os import environ + +from labgrid.strategy import Strategy + + +@dataclass(frozen=True) +class State: + desired: str + initial: str | None + final: str | None + + @staticmethod + def from_env() -> "State": + return State( + desired=environ["LG_STATE"], + initial=environ.get("LG_INITIAL_STATE", None), + final=environ.get("BZL_LG_FINAL_STATE", None), + ) @contextmanager -def transition(strategy, initial, desired, final): +def transition(strategy: Strategy, state: State): try: - if initial is not None: - strategy.force(initial) - strategy.transition(desired) + if state.initial is not None: + strategy.force(state.initial) + strategy.transition(state.desired) yield finally: - if final is not None: - strategy.transition(final) + if state.final is not None: + strategy.transition(state.final) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index e3bd4d53..bfb1ffaf 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -12,7 +12,7 @@ from typing import ( from python.runfiles import Runfiles -from bazel.labgrid.strategy import transition +from bazel.labgrid.strategy import transition, State from bazel.labgrid.util.target import TemporaryDirectory from labgrid import Environment from labgrid.logging import basicConfig, StepLogger @@ -40,13 +40,6 @@ class Tools: mv: Path = runfile("ape/ape/assimilate/mv/mv") -@dataclass(frozen=True) -class State: - desired: str = environ["LG_STATE"] - initial: str | None = environ.get("LG_INITIAL_STATE", None) - final: str | None = environ.get("BZL_LG_FINAL_STATE", None) - - @dataclass(frozen=True) class FileTransfer: remote: PurePath @@ -87,7 +80,6 @@ def run( program: Path, *arguments: str, config: Path = PurePath(environ["LG_ENV"]), - state: State = State(), get: Iterator[FileTransfer], put: Iterator[FileTransfer], env: Iterator[str], @@ -111,7 +103,7 @@ def run( # Start up Docker container with LabGrid target = Environment(str(config)).get_target() strategy = target.get_driver("Strategy") - with transition(strategy, state.initial, state.desired, state.final): + with transition(strategy, State.from_env()): # Retrieve the communication protocols shell = target.get_driver("CommandProtocol") transfer = target.get_driver("FileTransferProtocol") -- GitLab From b5c38f0f502ae71f73ada227c59a0b2efcaba819 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Mon, 24 Mar 2025 16:44:17 +0000 Subject: [PATCH 07/12] refactor(//labgrid/run): extract `runner` context manager This makes the contents `run` very readable. --- labgrid/run/run.py | 126 +++++++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 50 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index bfb1ffaf..03935fa1 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from contextlib import contextmanager from dataclasses import dataclass from os import environ, linesep from pathlib import Path, PurePath @@ -45,17 +46,22 @@ class FileTransfer: remote: PurePath local: PurePath optional: bool + # FIXME: Decide whether or not to transfer runfiles implicitly instead of using a flag + include_runfiles: bool - def __init__(self, remote: str, local: str, optional=False): + def __init__(self, remote: str, local: str, optional=False, include_runfiles=False): object.__setattr__(self, "remote", PurePath(remote)) object.__setattr__(self, "local", PurePath(local)) object.__setattr__(self, "optional", optional) + object.__setattr__(self, "include_runfiles", include_runfiles) + # FIXME: return `(remote, local)` tuple to distinguish from unresolved instance def resolve(self, remote: PurePath) -> FileTransfer: return FileTransfer( remote=self.join(remote, self.remote), local=self.join(Path.cwd(), self.local), optional=self.optional, + include_runfiles=self.include_runfiles, ) @staticmethod @@ -65,7 +71,7 @@ class FileTransfer: return path -def log_level(value): +def _log_level(value): # The Logging module converts strings to level integers with this (unintuitively named) function log_level = logging.getLevelName(value) if isinstance(log_level, str): @@ -79,13 +85,28 @@ def log_level(value): def run( program: Path, *arguments: str, - config: Path = PurePath(environ["LG_ENV"]), get: Iterator[FileTransfer], put: Iterator[FileTransfer], env: Iterator[str], - log_level: int = log_level(environ.get("LOG_LEVEL", "ERROR")), - tools: Tools = Tools(), ) -> int: + with runner() as r: + uploads = put + [FileTransfer(program.name, program, include_runfiles=True)] + cmd = join((f"./{program.name}", *arguments)) + downloads = get + + r.put(uploads) + code = r.run(cmd, env) + r.get(downloads, code) + + return code + + +@contextmanager +def runner(): + config = PurePath(environ["LG_ENV"]) + log_level = _log_level(environ.get("LOG_LEVEL", "ERROR")) + tools = Tools() + runfiles_dir = None # Allow any Bazel binaries used in the configuration to work try: @@ -100,7 +121,6 @@ def run( if log_level <= logging.DEBUG: StepLogger.start() - # Start up Docker container with LabGrid target = Environment(str(config)).get_target() strategy = target.get_driver("Strategy") with transition(strategy, State.from_env()): @@ -115,49 +135,55 @@ def run( tools.mkdir, tools.mv, ) as temp: - # Transfer the provided program over to the device - remote = temp / program.name - transfer.put(f"{program}", f"{remote}") - - # Transfer program runfiles - # TODO: implement a `BazelFileTransfer` that transfers runfiles implicitly in `put` - # Make that the default file transfer implementation so _any_ file transfer automatically gets the runfiles as well - src_runfiles = program.with_suffix(".runfiles") - dest_runfiles = remote.with_suffix(".runfiles") - if src_runfiles.exists(): - transfer.put(f"{src_runfiles}", f"{dest_runfiles}") - elif runfiles_dir is not None: - transfer.put(f"{runfiles_dir}", f"{dest_runfiles}") + r = Runner(temp, transfer, shell, runfiles_dir) # Transfer 'env' binary over to the device - transfer.put(f"{tools.env}", f"{temp / tools.env.name}") - - # Transfer user defined files - for unresolved in put: - resolved = unresolved.resolve(temp) - temp.mkdir(resolved.remote.parent) - transfer.put(resolved.local, resolved.remote) - - # Construct the CLI using env, the env vars and then the program and args. - cmd = join((f"{remote}", *arguments)) - cmd = f"cd {temp} && {temp / tools.env.name} {' '.join(env)} {cmd}" - - # Run the transferred program - out, err, code = shell.run(cmd) - for line in out: - stdout.write(f"{line}{linesep}") - for line in err: - stderr.write(f"{line}{linesep}") - - # Copy the files back from the target - - for unresolved in get: - resolved = unresolved.resolve(temp) - # FIXME: Check if file exists with `@ape//ape:test` instead of assuming any error is due to a missing file - try: - transfer.get(resolved.remote, resolved.local) - except ExecutionError as e: - if not resolved.optional and code == 0: - raise e - - return code + r.put([FileTransfer(r.exec_path / tools.env.name, tools.env)]) + + yield r + + +class Runner: + def __init__(self, exec_path, transfer, shell, runfiles_dir): + self.exec_path = exec_path + self.transfer = transfer + self.shell = shell + self.runfiles_dir = runfiles_dir + + def put(self, uploads: Iterator[FileTransfer]): + for unresolved in uploads: + resolved = unresolved.resolve(self.exec_path) + self.exec_path.mkdir(resolved.remote.parent) + self.transfer.put(resolved.local, resolved.remote) + + if resolved.include_runfiles: + src_runfiles = resolved.local.with_suffix(".runfiles") + dest_runfiles = resolved.remote.with_suffix(".runfiles") + if Path(src_runfiles).exists(): + self.transfer.put(f"{src_runfiles}", f"{dest_runfiles}") + elif self.runfiles_dir is not None: + self.transfer.put(f"{self.runfiles_dir}", f"{dest_runfiles}") + + def get(self, downloads: Iterator[FileTransfer], code: int): + for transfer in downloads: + resolved = transfer.resolve(self.exec_path) + # FIXME: Check if file exists with `@ape//ape:test` instead of assuming any error is due to a missing file + try: + self.transfer.get(resolved.remote, resolved.local) + except ExecutionError as e: + if not resolved.optional and code == 0: + raise e + + # FIXME: Make `env` a `Mapping[str, str]` + def run(self, cmd: str, env: Iterator[str]) -> int: + # Construct the CLI using env, the env vars and then the program and args. + cmd = f"cd {self.exec_path} && {self.exec_path / 'env'} {' '.join(env)} {cmd}" + + # Run the transferred program + out, err, code = self.shell.run(cmd) + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + + return code -- GitLab From 584a8b5191864eb85eb185db916438d3b8b18775 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Tue, 25 Mar 2025 08:55:25 +0000 Subject: [PATCH 08/12] fix(//labgrid/run): make `env` a `Mapping[str, str]` --- labgrid/run/__main__.py | 10 +++------- labgrid/run/run.py | 9 +++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index 589595b0..6177bb4f 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -62,17 +62,13 @@ def resolve(value: str) -> Path: return resolved -def env(arg: str) -> str: +def env(arg: str) -> tuple[str, str]: key, sep, value = arg.partition("=") if not sep: value = getenv(key) - if value is None: raise ArgumentTypeError(f"Env var {key} does not exist in host environment") - else: - return f"{key}={value}" - - return arg + return (key, value) def get(value: str) -> FileTransfer: @@ -112,7 +108,7 @@ def main(exe: Path, *args: str) -> int: *parsed.arguments, get=parsed.get, put=parsed.put, - env=parsed.env, + env=dict(parsed.env), ) except CalledProcessError as e: print(f"fatal: subprocess failed: {e}", file=stderr) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 03935fa1..d43634c1 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -9,6 +9,7 @@ from shlex import join from sys import stderr, stdout from typing import ( Iterator, + Mapping, ) from python.runfiles import Runfiles @@ -87,7 +88,7 @@ def run( *arguments: str, get: Iterator[FileTransfer], put: Iterator[FileTransfer], - env: Iterator[str], + env: Mapping[str, str], ) -> int: with runner() as r: uploads = put + [FileTransfer(program.name, program, include_runfiles=True)] @@ -174,10 +175,10 @@ class Runner: if not resolved.optional and code == 0: raise e - # FIXME: Make `env` a `Mapping[str, str]` - def run(self, cmd: str, env: Iterator[str]) -> int: + def run(self, cmd: str, env: Mapping[str, str]) -> int: # Construct the CLI using env, the env vars and then the program and args. - cmd = f"cd {self.exec_path} && {self.exec_path / 'env'} {' '.join(env)} {cmd}" + name_values = " ".join(f"{k}={v}" for k, v in env.items()) + cmd = f"cd {self.exec_path} && {self.exec_path / 'env'} {name_values} {cmd}" # Run the transferred program out, err, code = self.shell.run(cmd) -- GitLab From b9535446ffe8f14dafcc1f5d8730c8ab0acc97be Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Tue, 25 Mar 2025 09:05:04 +0000 Subject: [PATCH 09/12] refactor(//labgrid/run): move resolution close to usage --- labgrid/run/run.py | 50 +++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index d43634c1..db5b7e58 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -56,21 +56,6 @@ class FileTransfer: object.__setattr__(self, "optional", optional) object.__setattr__(self, "include_runfiles", include_runfiles) - # FIXME: return `(remote, local)` tuple to distinguish from unresolved instance - def resolve(self, remote: PurePath) -> FileTransfer: - return FileTransfer( - remote=self.join(remote, self.remote), - local=self.join(Path.cwd(), self.local), - optional=self.optional, - include_runfiles=self.include_runfiles, - ) - - @staticmethod - def join(base: PurePath, path: PurePath) -> PurePath: - if not path.is_absolute(): - return base / path - return path - def _log_level(value): # The Logging module converts strings to level integers with this (unintuitively named) function @@ -152,27 +137,27 @@ class Runner: self.runfiles_dir = runfiles_dir def put(self, uploads: Iterator[FileTransfer]): - for unresolved in uploads: - resolved = unresolved.resolve(self.exec_path) - self.exec_path.mkdir(resolved.remote.parent) - self.transfer.put(resolved.local, resolved.remote) - - if resolved.include_runfiles: - src_runfiles = resolved.local.with_suffix(".runfiles") - dest_runfiles = resolved.remote.with_suffix(".runfiles") + for upload in uploads: + remote, local = self._resolve(upload) + self.exec_path.mkdir(remote.parent) + self.transfer.put(local, remote) + + if upload.include_runfiles: + src_runfiles = local.with_suffix(".runfiles") + dest_runfiles = remote.with_suffix(".runfiles") if Path(src_runfiles).exists(): self.transfer.put(f"{src_runfiles}", f"{dest_runfiles}") elif self.runfiles_dir is not None: self.transfer.put(f"{self.runfiles_dir}", f"{dest_runfiles}") def get(self, downloads: Iterator[FileTransfer], code: int): - for transfer in downloads: - resolved = transfer.resolve(self.exec_path) + for download in downloads: + remote, local = self._resolve(download) # FIXME: Check if file exists with `@ape//ape:test` instead of assuming any error is due to a missing file try: - self.transfer.get(resolved.remote, resolved.local) + self.transfer.get(remote, local) except ExecutionError as e: - if not resolved.optional and code == 0: + if not download.optional and code == 0: raise e def run(self, cmd: str, env: Mapping[str, str]) -> int: @@ -188,3 +173,14 @@ class Runner: stderr.write(f"{line}{linesep}") return code + + def _resolve(self, transfer: FileTransfer) -> tuple[str, str]: + remote = self._join(self.exec_path, transfer.remote) + local = self._join(Path.cwd(), transfer.local) + return (remote, local) + + @staticmethod + def _join(base: PurePath, path: PurePath) -> PurePath: + if not path.is_absolute(): + return base / path + return path -- GitLab From d36bb5ef5dd8525709f6831bffef0fffa3c34996 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Tue, 25 Mar 2025 13:48:21 +0000 Subject: [PATCH 10/12] refactor: extract `runfiles` library --- bazel/python/runfiles/BUILD.bazel | 11 +++++++++++ bazel/python/runfiles/__init__.py | 3 +++ bazel/python/runfiles/runfiles.py | 15 +++++++++++++++ labgrid/executor/BUILD.bazel | 4 ++-- labgrid/executor/executor.py | 15 +-------------- labgrid/executor/host.py | 5 ++--- labgrid/run/BUILD.bazel | 2 +- labgrid/run/__main__.py | 11 +++++------ labgrid/run/run.py | 14 +------------- 9 files changed, 41 insertions(+), 39 deletions(-) create mode 100644 bazel/python/runfiles/BUILD.bazel create mode 100644 bazel/python/runfiles/__init__.py create mode 100644 bazel/python/runfiles/runfiles.py diff --git a/bazel/python/runfiles/BUILD.bazel b/bazel/python/runfiles/BUILD.bazel new file mode 100644 index 00000000..35526b81 --- /dev/null +++ b/bazel/python/runfiles/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "runfiles", + srcs = [ + "__init__.py", + "runfiles.py", + ], + visibility = ["//:__subpackages__"], + deps = ["@rules_python//python/runfiles"], +) diff --git a/bazel/python/runfiles/__init__.py b/bazel/python/runfiles/__init__.py new file mode 100644 index 00000000..724ec359 --- /dev/null +++ b/bazel/python/runfiles/__init__.py @@ -0,0 +1,3 @@ +from .runfiles import runfile, RunfileNotFoundError + +__all__ = ["runfile", "RunfileNotFoundError"] diff --git a/bazel/python/runfiles/runfiles.py b/bazel/python/runfiles/runfiles.py new file mode 100644 index 00000000..3ebbb9a6 --- /dev/null +++ b/bazel/python/runfiles/runfiles.py @@ -0,0 +1,15 @@ +from pathlib import Path + +from python.runfiles import Runfiles + + +class RunfileNotFoundError(FileNotFoundError): + pass + + +def runfile(path: Path) -> Path: + runfiles = Runfiles.Create() + resolved = runfiles.Rlocation(path) + if resolved and Path(resolved).exists(): + return Path(resolved) + raise RunfileNotFoundError(path) diff --git a/labgrid/executor/BUILD.bazel b/labgrid/executor/BUILD.bazel index fdc6f791..1effed64 100644 --- a/labgrid/executor/BUILD.bazel +++ b/labgrid/executor/BUILD.bazel @@ -18,8 +18,8 @@ py_binary( visibility = ["//:__subpackages__"], deps = [ "//bazel/labgrid/executor", + "//bazel/python/runfiles", "//labgrid/config:managers", - "@rules_python//python/runfiles", ], ) @@ -29,7 +29,7 @@ py_library( data = ["hello-world.txt"], deps = [ "//bazel/labgrid/executor", - "@rules_python//python/runfiles", + "//bazel/python/runfiles", ], ) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index 9cd9c1b0..467d9027 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -12,21 +12,8 @@ from shutil import which from subprocess import CalledProcessError, run from sys import argv -from python.runfiles import Runfiles - from bazel.labgrid.executor.manager import Data, Manager - - -class RunfileNotFoundError(FileNotFoundError): - pass - - -def runfile(path: Path) -> Path: - runfiles = Runfiles.Create() - resolved = runfiles.Rlocation(path) - if resolved and Path(resolved).exists(): - return Path(resolved) - raise RunfileNotFoundError(path) +from bazel.python.runfiles import runfile, RunfileNotFoundError def resolve(value: str) -> str: diff --git a/labgrid/executor/host.py b/labgrid/executor/host.py index d966d24f..0b3b913b 100644 --- a/labgrid/executor/host.py +++ b/labgrid/executor/host.py @@ -3,15 +3,14 @@ from __future__ import annotations from contextlib import contextmanager from typing import Iterator -from python.runfiles import Runfiles from bazel.labgrid.executor.manager import Data +from bazel.python.runfiles import runfile @contextmanager def manager(input_data: Data) -> Iterator[Data]: - runfiles = Runfiles.Create() - path = runfiles.Rlocation("_main/labgrid/executor/hello-world.txt") + path = runfile("_main/labgrid/executor/hello-world.txt") with open(path) as stream: data = stream.read() env = { diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index 82f173ad..bee5c9fd 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -34,8 +34,8 @@ py_binary( deps = [ "//bazel/labgrid/strategy", "//bazel/labgrid/util:target", + "//bazel/python/runfiles", "//labgrid:pkg", "//labgrid/config:deps", - "@rules_python//python/runfiles", ], ) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index 6177bb4f..fe7b46a8 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -8,10 +8,10 @@ from string import Template from subprocess import CalledProcessError from sys import argv, stderr -from python.runfiles import Runfiles - from run import FileTransfer, run +from bazel.python.runfiles import RunfileNotFoundError, runfile + def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( @@ -55,11 +55,10 @@ def arguments(prsr: ArgumentParser) -> None: def resolve(value: str) -> Path: - runfiles = Runfiles.Create() - resolved = Path(runfiles.Rlocation(value)) - if not resolved.exists(): + try: + return runfile(value) + except RunfileNotFoundError: return Path(value) - return resolved def env(arg: str) -> tuple[str, str]: diff --git a/labgrid/run/run.py b/labgrid/run/run.py index db5b7e58..7aea1b78 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -12,27 +12,15 @@ from typing import ( Mapping, ) -from python.runfiles import Runfiles from bazel.labgrid.strategy import transition, State from bazel.labgrid.util.target import TemporaryDirectory +from bazel.python.runfiles import runfile from labgrid import Environment from labgrid.logging import basicConfig, StepLogger from labgrid.driver.exception import ExecutionError -class RunfileNotFoundError(FileNotFoundError): - pass - - -def runfile(path: str) -> Path: - runfiles = Runfiles.Create() - resolved = Path(runfiles.Rlocation(path)) - if not resolved.exists(): - raise RunfileNotFoundError(f"runfile not found: {path}") - return resolved - - @dataclass class Tools: mktemp: Path = runfile("ape/ape/assimilate/mktemp/mktemp") -- GitLab From b44481f5d1b6ce19a5cf5873153416718906d095 Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Tue, 25 Mar 2025 14:47:11 +0000 Subject: [PATCH 11/12] feat: extract `runner` library --- bazel/labgrid/runner/BUILD.bazel | 35 +++++++ bazel/labgrid/runner/__init__.py | 3 + bazel/labgrid/runner/runner.py | 152 +++++++++++++++++++++++++++++++ labgrid/run/BUILD.bazel | 29 +----- labgrid/run/__main__.py | 3 +- labgrid/run/run.py | 149 +----------------------------- 6 files changed, 195 insertions(+), 176 deletions(-) create mode 100644 bazel/labgrid/runner/BUILD.bazel create mode 100644 bazel/labgrid/runner/__init__.py create mode 100644 bazel/labgrid/runner/runner.py diff --git a/bazel/labgrid/runner/BUILD.bazel b/bazel/labgrid/runner/BUILD.bazel new file mode 100644 index 00000000..5c0e7121 --- /dev/null +++ b/bazel/labgrid/runner/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_labgrid//labgrid/config:transition.bzl", "labgrid_config_transition") +load("@rules_python//python:defs.bzl", "py_library") + +TOOLS = [ + "mktemp", + "mkdir", + "rm", + "env", + "mv", +] + +[ + labgrid_config_transition( + name = tool, + srcs = ["@ape//ape:{}".format(tool)], + ) + for tool in TOOLS +] + +py_library( + name = "runner", + srcs = [ + "__init__.py", + "runner.py", + ], + data = [":{}".format(tool) for tool in TOOLS], + visibility = ["//visibility:public"], + deps = [ + "//bazel/labgrid/strategy", + "//bazel/labgrid/util:target", + "//bazel/python/runfiles", + "//labgrid:pkg", + "//labgrid/config:deps", + ], +) diff --git a/bazel/labgrid/runner/__init__.py b/bazel/labgrid/runner/__init__.py new file mode 100644 index 00000000..31f77f5b --- /dev/null +++ b/bazel/labgrid/runner/__init__.py @@ -0,0 +1,3 @@ +from .runner import FileTransfer, runner + +__all__ = ["FileTransfer", "runner"] diff --git a/bazel/labgrid/runner/runner.py b/bazel/labgrid/runner/runner.py new file mode 100644 index 00000000..a2f88d36 --- /dev/null +++ b/bazel/labgrid/runner/runner.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import logging +from contextlib import contextmanager +from dataclasses import dataclass +from os import environ, linesep +from pathlib import Path, PurePath +from sys import stderr, stdout +from typing import ( + Iterator, + Mapping, +) + +from bazel.labgrid.strategy import State, transition +from bazel.labgrid.util.target import TemporaryDirectory +from bazel.python.runfiles import runfile +from labgrid import Environment +from labgrid.driver.exception import ExecutionError +from labgrid.logging import StepLogger, basicConfig + + +@dataclass +class _Tools: + mktemp: Path = runfile("ape/ape/assimilate/mktemp/mktemp") + rm: Path = runfile("ape/ape/assimilate/rm/rm") + mkdir: Path = runfile("ape/ape/assimilate/mkdir/mkdir") + env: Path = runfile("ape/ape/assimilate/env/env") + mv: Path = runfile("ape/ape/assimilate/mv/mv") + + +@dataclass(frozen=True) +class FileTransfer: + remote: PurePath + local: PurePath + optional: bool + # FIXME: Decide whether or not to transfer runfiles implicitly instead of using a flag + include_runfiles: bool + + def __init__(self, remote: str, local: str, optional=False, include_runfiles=False): + object.__setattr__(self, "remote", PurePath(remote)) + object.__setattr__(self, "local", PurePath(local)) + object.__setattr__(self, "optional", optional) + object.__setattr__(self, "include_runfiles", include_runfiles) + + +def _log_level(value): + # The Logging module converts strings to level integers with this (unintuitively named) function + log_level = logging.getLevelName(value) + if isinstance(log_level, str): + # getLevelName will return a string if the level was not recognised + raise ValueError(f"Unsupported value for LOG_LEVEL: '{value}'") + + return log_level + + +@contextmanager +def runner(): + config = PurePath(environ["LG_ENV"]) + log_level = _log_level(environ.get("LOG_LEVEL", "ERROR")) + tools = _Tools() + + runfiles_dir = None + # Allow any Bazel binaries used in the configuration to work + try: + # FIXME: Convert into context manager + runfiles_dir = environ["RUNFILES_DIR"] + del environ["RUNFILES_DIR"] + except KeyError: + pass + + # Activating the StepLogger requires a DEBUG level of verbosity + basicConfig(level=log_level) + if log_level <= logging.DEBUG: + StepLogger.start() + + target = Environment(str(config)).get_target() + strategy = target.get_driver("Strategy") + with transition(strategy, State.from_env()): + # Retrieve the communication protocols + shell = target.get_driver("CommandProtocol") + transfer = target.get_driver("FileTransferProtocol") + with TemporaryDirectory( + shell, + transfer, + tools.mktemp, + tools.rm, + tools.mkdir, + tools.mv, + ) as temp: + r = Runner(temp, transfer, shell, runfiles_dir) + + # Transfer 'env' binary over to the device + r.put([FileTransfer(r.exec_path / tools.env.name, tools.env)]) + + yield r + + +class Runner: + def __init__(self, exec_path, transfer, shell, runfiles_dir): + self.exec_path = exec_path + self.transfer = transfer + self.shell = shell + self.runfiles_dir = runfiles_dir + + def put(self, uploads: Iterator[FileTransfer]): + for upload in uploads: + remote, local = self._resolve(upload) + self.exec_path.mkdir(remote.parent) + self.transfer.put(local, remote) + + if upload.include_runfiles: + src_runfiles = local.with_suffix(".runfiles") + dest_runfiles = remote.with_suffix(".runfiles") + if Path(src_runfiles).exists(): + self.transfer.put(f"{src_runfiles}", f"{dest_runfiles}") + elif self.runfiles_dir is not None: + self.transfer.put(f"{self.runfiles_dir}", f"{dest_runfiles}") + + def get(self, downloads: Iterator[FileTransfer], code: int): + for download in downloads: + remote, local = self._resolve(download) + # FIXME: Check if file exists with `@ape//ape:test` instead of assuming any error is due to a missing file + try: + self.transfer.get(remote, local) + except ExecutionError as e: + if not download.optional and code == 0: + raise e + + def run(self, cmd: str, env: Mapping[str, str]) -> int: + # Construct the CLI using env, the env vars and then the program and args. + name_values = " ".join(f"{k}={v}" for k, v in env.items()) + cmd = f"cd {self.exec_path} && {self.exec_path / 'env'} {name_values} {cmd}" + + # Run the transferred program + out, err, code = self.shell.run(cmd) + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") + + return code + + def _resolve(self, transfer: FileTransfer) -> tuple[str, str]: + remote = self._join(self.exec_path, transfer.remote) + local = self._join(Path.cwd(), transfer.local) + return (remote, local) + + @staticmethod + def _join(base: PurePath, path: PurePath) -> PurePath: + if not path.is_absolute(): + return base / path + return path diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index bee5c9fd..09e5125f 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -1,41 +1,14 @@ -load("@rules_labgrid//labgrid/config:transition.bzl", "labgrid_config_transition") load("@rules_python//python:defs.bzl", "py_binary") -TOOLS = [ - "mktemp", - "mkdir", - "rm", - "env", - "mv", -] - -[ - labgrid_config_transition( - name = tool, - srcs = ["@ape//ape:{}".format(tool)], - ) - for tool in TOOLS -] - py_binary( name = "run", srcs = [ "__main__.py", "run.py", ], - data = [ - ":{}".format(tool) - for tool in TOOLS - ], imports = ["."], main = "__main__.py", tags = ["manual"], visibility = ["//visibility:public"], - deps = [ - "//bazel/labgrid/strategy", - "//bazel/labgrid/util:target", - "//bazel/python/runfiles", - "//labgrid:pkg", - "//labgrid/config:deps", - ], + deps = ["//bazel/labgrid/runner"], ) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index fe7b46a8..b9f66952 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -8,8 +8,9 @@ from string import Template from subprocess import CalledProcessError from sys import argv, stderr -from run import FileTransfer, run +from run import run +from bazel.labgrid.runner import FileTransfer from bazel.python.runfiles import RunfileNotFoundError, runfile diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 7aea1b78..8317d0b1 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -1,61 +1,15 @@ from __future__ import annotations -import logging -from contextlib import contextmanager -from dataclasses import dataclass -from os import environ, linesep -from pathlib import Path, PurePath +from pathlib import Path from shlex import join -from sys import stderr, stdout from typing import ( Iterator, Mapping, ) +from bazel.labgrid.runner import FileTransfer, runner -from bazel.labgrid.strategy import transition, State -from bazel.labgrid.util.target import TemporaryDirectory -from bazel.python.runfiles import runfile -from labgrid import Environment -from labgrid.logging import basicConfig, StepLogger -from labgrid.driver.exception import ExecutionError - -@dataclass -class Tools: - mktemp: Path = runfile("ape/ape/assimilate/mktemp/mktemp") - rm: Path = runfile("ape/ape/assimilate/rm/rm") - mkdir: Path = runfile("ape/ape/assimilate/mkdir/mkdir") - env: Path = runfile("ape/ape/assimilate/env/env") - mv: Path = runfile("ape/ape/assimilate/mv/mv") - - -@dataclass(frozen=True) -class FileTransfer: - remote: PurePath - local: PurePath - optional: bool - # FIXME: Decide whether or not to transfer runfiles implicitly instead of using a flag - include_runfiles: bool - - def __init__(self, remote: str, local: str, optional=False, include_runfiles=False): - object.__setattr__(self, "remote", PurePath(remote)) - object.__setattr__(self, "local", PurePath(local)) - object.__setattr__(self, "optional", optional) - object.__setattr__(self, "include_runfiles", include_runfiles) - - -def _log_level(value): - # The Logging module converts strings to level integers with this (unintuitively named) function - log_level = logging.getLevelName(value) - if isinstance(log_level, str): - # getLevelName will return a string if the level was not recognised - raise ValueError(f"Unsupported value for LOG_LEVEL: '{value}'") - - return log_level - - -# TODO: move this out to `bazel.labgrid.run` on day to allow downstream to re-use def run( program: Path, *arguments: str, @@ -73,102 +27,3 @@ def run( r.get(downloads, code) return code - - -@contextmanager -def runner(): - config = PurePath(environ["LG_ENV"]) - log_level = _log_level(environ.get("LOG_LEVEL", "ERROR")) - tools = Tools() - - runfiles_dir = None - # Allow any Bazel binaries used in the configuration to work - try: - # FIXME: Convert into context manager - runfiles_dir = environ["RUNFILES_DIR"] - del environ["RUNFILES_DIR"] - except KeyError: - pass - - # Activating the StepLogger requires a DEBUG level of verbosity - basicConfig(level=log_level) - if log_level <= logging.DEBUG: - StepLogger.start() - - target = Environment(str(config)).get_target() - strategy = target.get_driver("Strategy") - with transition(strategy, State.from_env()): - # Retrieve the communication protocols - shell = target.get_driver("CommandProtocol") - transfer = target.get_driver("FileTransferProtocol") - with TemporaryDirectory( - shell, - transfer, - tools.mktemp, - tools.rm, - tools.mkdir, - tools.mv, - ) as temp: - r = Runner(temp, transfer, shell, runfiles_dir) - - # Transfer 'env' binary over to the device - r.put([FileTransfer(r.exec_path / tools.env.name, tools.env)]) - - yield r - - -class Runner: - def __init__(self, exec_path, transfer, shell, runfiles_dir): - self.exec_path = exec_path - self.transfer = transfer - self.shell = shell - self.runfiles_dir = runfiles_dir - - def put(self, uploads: Iterator[FileTransfer]): - for upload in uploads: - remote, local = self._resolve(upload) - self.exec_path.mkdir(remote.parent) - self.transfer.put(local, remote) - - if upload.include_runfiles: - src_runfiles = local.with_suffix(".runfiles") - dest_runfiles = remote.with_suffix(".runfiles") - if Path(src_runfiles).exists(): - self.transfer.put(f"{src_runfiles}", f"{dest_runfiles}") - elif self.runfiles_dir is not None: - self.transfer.put(f"{self.runfiles_dir}", f"{dest_runfiles}") - - def get(self, downloads: Iterator[FileTransfer], code: int): - for download in downloads: - remote, local = self._resolve(download) - # FIXME: Check if file exists with `@ape//ape:test` instead of assuming any error is due to a missing file - try: - self.transfer.get(remote, local) - except ExecutionError as e: - if not download.optional and code == 0: - raise e - - def run(self, cmd: str, env: Mapping[str, str]) -> int: - # Construct the CLI using env, the env vars and then the program and args. - name_values = " ".join(f"{k}={v}" for k, v in env.items()) - cmd = f"cd {self.exec_path} && {self.exec_path / 'env'} {name_values} {cmd}" - - # Run the transferred program - out, err, code = self.shell.run(cmd) - for line in out: - stdout.write(f"{line}{linesep}") - for line in err: - stderr.write(f"{line}{linesep}") - - return code - - def _resolve(self, transfer: FileTransfer) -> tuple[str, str]: - remote = self._join(self.exec_path, transfer.remote) - local = self._join(Path.cwd(), transfer.local) - return (remote, local) - - @staticmethod - def _join(base: PurePath, path: PurePath) -> PurePath: - if not path.is_absolute(): - return base / path - return path -- GitLab From 4849ac54b0f16aafc9e5373576eadd3f51df884d Mon Sep 17 00:00:00 2001 From: Alex Tercete Date: Thu, 27 Mar 2025 11:39:05 +0000 Subject: [PATCH 12/12] refactor: rename variables --- labgrid/run/__main__.py | 6 ++++-- labgrid/run/run.py | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/labgrid/run/__main__.py b/labgrid/run/__main__.py index b9f66952..621254e3 100644 --- a/labgrid/run/__main__.py +++ b/labgrid/run/__main__.py @@ -44,6 +44,7 @@ def arguments(prsr: ArgumentParser) -> None: metavar="REMOTE[?][:LOCAL]", action="append", default=[], + dest="downloads", ) prsr.add_argument( "--put", @@ -52,6 +53,7 @@ def arguments(prsr: ArgumentParser) -> None: metavar="LOCAL[:REMOTE]", action="append", default=[], + dest="uploads", ) @@ -106,8 +108,8 @@ def main(exe: Path, *args: str) -> int: return run( parsed.program, *parsed.arguments, - get=parsed.get, - put=parsed.put, + downloads=parsed.downloads, + uploads=parsed.uploads, env=dict(parsed.env), ) except CalledProcessError as e: diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 8317d0b1..e7a04dbc 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -13,14 +13,13 @@ from bazel.labgrid.runner import FileTransfer, runner def run( program: Path, *arguments: str, - get: Iterator[FileTransfer], - put: Iterator[FileTransfer], + downloads: Iterator[FileTransfer], + uploads: Iterator[FileTransfer], env: Mapping[str, str], ) -> int: with runner() as r: - uploads = put + [FileTransfer(program.name, program, include_runfiles=True)] + uploads += [FileTransfer(program.name, program, include_runfiles=True)] cmd = join((f"./{program.name}", *arguments)) - downloads = get r.put(uploads) code = r.run(cmd, env) -- GitLab