From f9c9859a5437895b77cf0e4bb10fe22a89d693df Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 27 Oct 2024 11:51:52 +0200 Subject: [PATCH 1/8] feat: Create isolated remote execution root --- labgrid/run/BUILD.bazel | 2 ++ labgrid/run/run.py | 60 +++++++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index fe0048d7..cad53802 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -3,10 +3,12 @@ load("@rules_python//python:defs.bzl", "py_binary") py_binary( name = "run", srcs = ["run.py"], + data = ["@ape//ape:mktemp"], tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//labgrid:pkg", "//labgrid/config:deps", + "@rules_python//python/runfiles", ], ) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 5d0a236b..b93f72c5 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -6,8 +6,10 @@ from pathlib import Path, PurePath from shlex import join from sys import argv, stderr, stdout from typing import Collection, Protocol, Tuple - -from labgrid import Environment +from python.runfiles import Runfiles +from labgrid import Environment,Target +from subprocess import PIPE, run +from contextlib import contextmanager class Shell(Protocol): @@ -41,9 +43,28 @@ def arguments(prsr: ArgumentParser) -> None: default=environ["BZL_LG_FINAL_STATE"], ) +@contextmanager +def create_temp_dir(shell:Target,transfer: Target,runfiles: Runfiles): + # Transfer mktemp + mktemp = runfiles.Rlocation("ape/ape/assimilate/mktemp.ape/mktemp") + transfer.put(mktemp,"/tmp/mktemp") + + out, _, code = shell.run("/tmp/mktemp -d") + if code: + raise Exception("Failed to create a temporary directory") + temp_dir = out[0] + yield temp_dir + + _, _, code = shell.run(f"rm -r {temp_dir}") + if code: + raise Exception("Unable to delete the temporary directory") + + def main(exe: Path, *args: str) -> int: # Allow subprocesses to have their own runfiles + runfiles = Runfiles.Create() + del environ["RUNFILES_DIR"] prsr = ArgumentParser( @@ -65,24 +86,23 @@ def main(exe: Path, *args: str) -> int: shell = target.get_driver("CommandProtocol") transfer = target.get_driver("FileTransferProtocol") - # Transfer the provided program over to the Docker image - # TODO: this should be a context manager that removes it later - # TODO: this should use a unique remote name/directory - program = "/tmp/{}".format(parsed.program.name) - transfer.put(parsed.program, program) - - # Transfer runfiles - src_runfiles = parsed.program.with_suffix(".runfiles") - dest_runfiles = Path(program).with_suffix(".runfiles") - if src_runfiles.exists(): - transfer.put(src_runfiles, dest_runfiles) - - # Run the transferred program - out, err, code = shell.run(join((program, *parsed.arguments))) - for line in out: - stdout.write(f"{line}{linesep}") - for line in err: - stderr.write(f"{line}{linesep}") + with create_temp_dir(shell,transfer,runfiles) as temp_dir: + # Transfer the provided program over to the Docker image + program = "{}/{}".format(temp_dir,parsed.program.name) + transfer.put(parsed.program, program) + + # Transfer runfiles + src_runfiles = parsed.program.with_suffix(".runfiles") + dest_runfiles = Path(program).with_suffix(".runfiles") + if src_runfiles.exists(): + transfer.put(src_runfiles, dest_runfiles) + + # Run the transferred program + out, err, code = shell.run(join((program, *parsed.arguments))) + for line in out: + stdout.write(f"{line}{linesep}") + for line in err: + stderr.write(f"{line}{linesep}") finally: strategy.transition(parsed.final_state) -- GitLab From fb0cdd123fe154fcab3dd41457e8c086dd7cc6ab Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 27 Oct 2024 12:07:34 +0200 Subject: [PATCH 2/8] feat: adding log information --- labgrid/run/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index b93f72c5..4d5d4b00 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -49,9 +49,9 @@ def create_temp_dir(shell:Target,transfer: Target,runfiles: Runfiles): mktemp = runfiles.Rlocation("ape/ape/assimilate/mktemp.ape/mktemp") transfer.put(mktemp,"/tmp/mktemp") - out, _, code = shell.run("/tmp/mktemp -d") + out, err, code = shell.run("/tmp/mktemp -d") if code: - raise Exception("Failed to create a temporary directory") + raise Exception(f"Failed to create a temporary directory - error code:{code} error: {err}") temp_dir = out[0] yield temp_dir -- GitLab From 478e75cf472e81ccf27411a8681d3718b9ef1a8c Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 12:19:14 +0200 Subject: [PATCH 3/8] feat: hardcoded selected amd64 --- labgrid/run/BUILD.bazel | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index cad53802..c08f54b3 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -1,9 +1,24 @@ +load("@rules_labgrid//labgrid/transition:defs.bzl", "labgrid_transition") load("@rules_python//python:defs.bzl", "py_binary") +platform( + name = "platform", + constraint_values = [ + "@toolchain_utils//toolchain/constraint/os:linux", + "@toolchain_utils//toolchain/constraint/cpu:amd64", + ], +) + +labgrid_transition( + name = "new_tool", + srcs = ["@ape//ape:mktemp"], + platforms = [":platform"], +) + py_binary( name = "run", srcs = ["run.py"], - data = ["@ape//ape:mktemp"], + data = [":new_tool"], tags = ["manual"], visibility = ["//visibility:public"], deps = [ -- GitLab From 9ce37d3540c2dbc96a546e052c33bd17e1f149ef Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 19:50:08 +0200 Subject: [PATCH 4/8] feat: adding labgrid_transition --- labgrid/config_transition/BUILD.bazel | 6 ++++++ labgrid/config_transition/defs.bzl | 5 +++++ labgrid/config_transition/rule.bzl | 25 +++++++++++++++++++++++++ labgrid/run/BUILD.bazel | 24 +++++++++++++++--------- 4 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 labgrid/config_transition/BUILD.bazel create mode 100644 labgrid/config_transition/defs.bzl create mode 100644 labgrid/config_transition/rule.bzl diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel new file mode 100644 index 00000000..239bd02b --- /dev/null +++ b/labgrid/config_transition/BUILD.bazel @@ -0,0 +1,6 @@ +load(":defs.bzl", "labgrid_config_transition") + +labgrid_config_transition( + name = "test", + srcs = ["@ape//ape:mktemp"], +) diff --git a/labgrid/config_transition/defs.bzl b/labgrid/config_transition/defs.bzl new file mode 100644 index 00000000..085380c6 --- /dev/null +++ b/labgrid/config_transition/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _transition = "transition") + +visibility("public") + +labgrid_config_transition = _transition diff --git a/labgrid/config_transition/rule.bzl b/labgrid/config_transition/rule.bzl new file mode 100644 index 00000000..17ca4ebc --- /dev/null +++ b/labgrid/config_transition/rule.bzl @@ -0,0 +1,25 @@ +load("//labgrid/cfg:unstore.bzl", _cfg = "unstore") + +visibility("//...") + +DOC = "" + +ATTRS = { + "srcs": attr.label_list( + doc = "Outputs to transition to `platforms`.", + cfg = _cfg, + allow_files = True, + ), +} + +def implementation(ctx): + files = depset(transitive = [s.files for s in ctx.attr.srcs]) + return DefaultInfo(files = files) + +labgrid_config_transition = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, +) + +transition = labgrid_config_transition diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index c08f54b3..894faed0 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -1,18 +1,24 @@ +load("@rules_labgrid//labgrid/config_transition:defs.bzl", "labgrid_config_transition") load("@rules_labgrid//labgrid/transition:defs.bzl", "labgrid_transition") load("@rules_python//python:defs.bzl", "py_binary") -platform( - name = "platform", - constraint_values = [ - "@toolchain_utils//toolchain/constraint/os:linux", - "@toolchain_utils//toolchain/constraint/cpu:amd64", - ], -) +# platform( +# name = "platform", +# constraint_values = [ +# "@toolchain_utils//toolchain/constraint/os:linux", +# "@toolchain_utils//toolchain/constraint/cpu:amd64", +# ], +# ) + +# labgrid_transition( +# name = "new_tool", +# srcs = ["@ape//ape:mktemp"], +# platforms = [":platform"], +# ) -labgrid_transition( +labgrid_config_transition( name = "new_tool", srcs = ["@ape//ape:mktemp"], - platforms = [":platform"], ) py_binary( -- GitLab From 04a297eba2237f3cf9198746f961d3bf0abfe183 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 19:58:35 +0200 Subject: [PATCH 5/8] feat: removing comments --- labgrid/config_transition/BUILD.bazel | 6 ------ labgrid/run/BUILD.bazel | 15 --------------- 2 files changed, 21 deletions(-) diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel index 239bd02b..e69de29b 100644 --- a/labgrid/config_transition/BUILD.bazel +++ b/labgrid/config_transition/BUILD.bazel @@ -1,6 +0,0 @@ -load(":defs.bzl", "labgrid_config_transition") - -labgrid_config_transition( - name = "test", - srcs = ["@ape//ape:mktemp"], -) diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index 894faed0..cb098615 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -1,21 +1,6 @@ load("@rules_labgrid//labgrid/config_transition:defs.bzl", "labgrid_config_transition") -load("@rules_labgrid//labgrid/transition:defs.bzl", "labgrid_transition") load("@rules_python//python:defs.bzl", "py_binary") -# platform( -# name = "platform", -# constraint_values = [ -# "@toolchain_utils//toolchain/constraint/os:linux", -# "@toolchain_utils//toolchain/constraint/cpu:amd64", -# ], -# ) - -# labgrid_transition( -# name = "new_tool", -# srcs = ["@ape//ape:mktemp"], -# platforms = [":platform"], -# ) - labgrid_config_transition( name = "new_tool", srcs = ["@ape//ape:mktemp"], -- GitLab From 9fd748529228c9c8066326580e8a0a158349ddc6 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 20:25:16 +0200 Subject: [PATCH 6/8] feat: changing the name target name --- labgrid/run/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index cb098615..d113672c 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -2,14 +2,14 @@ load("@rules_labgrid//labgrid/config_transition:defs.bzl", "labgrid_config_trans load("@rules_python//python:defs.bzl", "py_binary") labgrid_config_transition( - name = "new_tool", + name = "mktemp", srcs = ["@ape//ape:mktemp"], ) py_binary( name = "run", srcs = ["run.py"], - data = [":new_tool"], + data = [":mktemp"], tags = ["manual"], visibility = ["//visibility:public"], deps = [ -- GitLab From e23fe050a22d0af32de3998b5f0bcf0e0af46274 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Wed, 30 Oct 2024 12:31:23 +0200 Subject: [PATCH 7/8] feat: code review changes --- bazel/labgrid/target_subprocess/BUILD.bazel | 7 +++ .../target_subprocess/target_subprocess.py | 18 +++++++ .../rule.bzl => config/transition.bzl} | 10 ++-- labgrid/config_transition/BUILD.bazel | 0 labgrid/config_transition/defs.bzl | 5 -- labgrid/run/BUILD.bazel | 19 ++++++- labgrid/run/run.py | 51 +++++++++++++------ 7 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 bazel/labgrid/target_subprocess/BUILD.bazel create mode 100644 bazel/labgrid/target_subprocess/target_subprocess.py rename labgrid/{config_transition/rule.bzl => config/transition.bzl} (54%) delete mode 100644 labgrid/config_transition/BUILD.bazel delete mode 100644 labgrid/config_transition/defs.bzl diff --git a/bazel/labgrid/target_subprocess/BUILD.bazel b/bazel/labgrid/target_subprocess/BUILD.bazel new file mode 100644 index 00000000..57b7bbaf --- /dev/null +++ b/bazel/labgrid/target_subprocess/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "target_subprocess", + srcs = ["target_subprocess.py"], + visibility = ["//visibility:public"], +) diff --git a/bazel/labgrid/target_subprocess/target_subprocess.py b/bazel/labgrid/target_subprocess/target_subprocess.py new file mode 100644 index 00000000..9607efc4 --- /dev/null +++ b/bazel/labgrid/target_subprocess/target_subprocess.py @@ -0,0 +1,18 @@ +from subprocess import CalledProcessError, CompletedProcess +from shlex import join +from labgrid.protocol.commandprotocol import CommandProtocol + +class TargetSubprocess: + def __init__(self, shell: CommandProtocol): + self.__shell = shell + + @property + def shell(self) -> CommandProtocol: + return self.__shell + + def run(self,cmd,args: tuple[str, ...] = None, *, check: bool = False, text: bool = True) -> CompletedProcess: + assert text, "Non-text target execution is not supported" + out, err, code = self.shell.run(cmd) + if check and code: + raise CalledProcessError(cmd=cmd,returncode=code, output=out, stderr=err) + return CompletedProcess(returncode=code, stdout=out, stderr=err, args=args) diff --git a/labgrid/config_transition/rule.bzl b/labgrid/config/transition.bzl similarity index 54% rename from labgrid/config_transition/rule.bzl rename to labgrid/config/transition.bzl index 17ca4ebc..fa33ebee 100644 --- a/labgrid/config_transition/rule.bzl +++ b/labgrid/config/transition.bzl @@ -2,11 +2,11 @@ load("//labgrid/cfg:unstore.bzl", _cfg = "unstore") visibility("//...") -DOC = "" +DOC = "Transitions a target to the stored LabGrid target platform." ATTRS = { "srcs": attr.label_list( - doc = "Outputs to transition to `platforms`.", + doc = "Outputs to transition to the stored LabGrid target platform.", cfg = _cfg, allow_files = True, ), @@ -14,12 +14,12 @@ ATTRS = { def implementation(ctx): files = depset(transitive = [s.files for s in ctx.attr.srcs]) - return DefaultInfo(files = files) + runfiles = ctx.runfiles() + runfiles = runfiles.merge_all([s.default_runfiles for s in ctx.attr.srcs]) + return DefaultInfo(files = files, runfiles = runfiles) labgrid_config_transition = rule( doc = DOC, attrs = ATTRS, implementation = implementation, ) - -transition = labgrid_config_transition diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/labgrid/config_transition/defs.bzl b/labgrid/config_transition/defs.bzl deleted file mode 100644 index 085380c6..00000000 --- a/labgrid/config_transition/defs.bzl +++ /dev/null @@ -1,5 +0,0 @@ -load(":rule.bzl", _transition = "transition") - -visibility("public") - -labgrid_config_transition = _transition diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index d113672c..f98975fc 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_labgrid//labgrid/config_transition:defs.bzl", "labgrid_config_transition") +load("@rules_labgrid//labgrid/config:transition.bzl", "labgrid_config_transition") load("@rules_python//python:defs.bzl", "py_binary") labgrid_config_transition( @@ -6,13 +6,28 @@ labgrid_config_transition( srcs = ["@ape//ape:mktemp"], ) +labgrid_config_transition( + name = "mkdir", + srcs = ["@ape//ape:mkdir"], +) + +labgrid_config_transition( + name = "rm", + srcs = ["@ape//ape:rm"], +) + py_binary( name = "run", srcs = ["run.py"], - data = [":mktemp"], + data = [ + ":mkdir", + ":mktemp", + ":rm", + ], tags = ["manual"], visibility = ["//visibility:public"], deps = [ + "//bazel/labgrid/target_subprocess", "//labgrid:pkg", "//labgrid/config:deps", "@rules_python//python/runfiles", diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 4d5d4b00..b23fa044 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -1,5 +1,5 @@ #! /usr/bin/env python3 - +from typing import Iterator, Mapping from argparse import ArgumentParser from os import environ, linesep from pathlib import Path, PurePath @@ -10,7 +10,10 @@ from python.runfiles import Runfiles from labgrid import Environment,Target from subprocess import PIPE, run from contextlib import contextmanager - +import uuid +from bazel.labgrid.target_subprocess.target_subprocess import TargetSubprocess +from labgrid.protocol.commandprotocol import CommandProtocol +from labgrid.protocol.filetransferprotocol import FileTransferProtocol class Shell(Protocol): def run(cmd: str) -> Tuple[Collection[str], Collection[str], int]: ... @@ -44,20 +47,38 @@ def arguments(prsr: ArgumentParser) -> None: ) @contextmanager -def create_temp_dir(shell:Target,transfer: Target,runfiles: Runfiles): +def TargetTemporaryDirectory(shell:CommandProtocol,transfer: FileTransferProtocol,runfiles: Runfiles)-> Iterator[PurePath]: # Transfer mktemp mktemp = runfiles.Rlocation("ape/ape/assimilate/mktemp.ape/mktemp") - transfer.put(mktemp,"/tmp/mktemp") - - out, err, code = shell.run("/tmp/mktemp -d") - if code: - raise Exception(f"Failed to create a temporary directory - error code:{code} error: {err}") - temp_dir = out[0] - yield temp_dir - - _, _, code = shell.run(f"rm -r {temp_dir}") - if code: - raise Exception("Unable to delete the temporary directory") + mktemp_remote = f"/tmp/{uuid.getnode()}" #avoid race-condition if there are parallel executions + transfer.put(mktemp, mktemp_remote) + target = TargetSubprocess(shell) + result = target.run(cmd=f"{mktemp_remote} -d",check=True) + root = PurePath(result.stdout[0].rstrip()) + + tools = root / "tools" + execroot = root / "execroot" + # Transfer mkdir + mkdir = runfiles.Rlocation("ape/ape/assimilate/mkdir.ape/mkdir") + transfer.put(mkdir,f"{root}") + # Create tools dir + target.run(cmd=f"{root / 'mkdir'} {tools}",check=True) + # Create execroot dir + target.run(cmd=f"{root / 'mkdir'} {execroot}",check=True) + # Transfer rm + rm = runfiles.Rlocation("ape/ape/assimilate/rm.ape/rm") + transfer.put(rm, f"{tools}") + + ##cleanup + + # Delete mkdir + target.run(cmd=f"{tools / 'rm'} {root / 'mkdir'}",check=True) + # Delete mktemp + target.run(cmd=f"{tools / 'rm'} {mktemp_remote}",check=True) + + yield execroot + + target.run(cmd=f"{tools / 'rm'} -rf {root}",check=True) @@ -86,7 +107,7 @@ def main(exe: Path, *args: str) -> int: shell = target.get_driver("CommandProtocol") transfer = target.get_driver("FileTransferProtocol") - with create_temp_dir(shell,transfer,runfiles) as temp_dir: + with TargetTemporaryDirectory(shell,transfer,runfiles) as temp_dir: # Transfer the provided program over to the Docker image program = "{}/{}".format(temp_dir,parsed.program.name) transfer.put(parsed.program, program) -- GitLab From 1291832f0087795838a57a3f9c4c3d536d5b4652 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Wed, 30 Oct 2024 15:03:21 +0200 Subject: [PATCH 8/8] feat: code review changes --- .../util}/BUILD.bazel | 4 +- bazel/labgrid/protocol/util/target.py | 67 ++++++++++++++++++ .../target_subprocess/target_subprocess.py | 18 ----- labgrid/run/BUILD.bazel | 2 +- labgrid/run/run.py | 68 +++++-------------- labgrid/transition/rule.bzl | 4 +- 6 files changed, 89 insertions(+), 74 deletions(-) rename bazel/labgrid/{target_subprocess => protocol/util}/BUILD.bazel (61%) create mode 100644 bazel/labgrid/protocol/util/target.py delete mode 100644 bazel/labgrid/target_subprocess/target_subprocess.py diff --git a/bazel/labgrid/target_subprocess/BUILD.bazel b/bazel/labgrid/protocol/util/BUILD.bazel similarity index 61% rename from bazel/labgrid/target_subprocess/BUILD.bazel rename to bazel/labgrid/protocol/util/BUILD.bazel index 57b7bbaf..70cbb96f 100644 --- a/bazel/labgrid/target_subprocess/BUILD.bazel +++ b/bazel/labgrid/protocol/util/BUILD.bazel @@ -1,7 +1,7 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( - name = "target_subprocess", - srcs = ["target_subprocess.py"], + name = "target", + srcs = ["target.py"], visibility = ["//visibility:public"], ) diff --git a/bazel/labgrid/protocol/util/target.py b/bazel/labgrid/protocol/util/target.py new file mode 100644 index 00000000..1ec7b420 --- /dev/null +++ b/bazel/labgrid/protocol/util/target.py @@ -0,0 +1,67 @@ +import uuid +from contextlib import contextmanager +from pathlib import Path, PurePath +from shlex import join +from subprocess import CalledProcessError, CompletedProcess +from typing import Iterator, Mapping + +from labgrid.protocol.commandprotocol import CommandProtocol +from labgrid.protocol.filetransferprotocol import FileTransferProtocol + + +class Subprocess: + def __init__(self, shell: CommandProtocol): + self.__shell = shell + + @property + def shell(self) -> CommandProtocol: + return self.__shell + + def run( + self, args: tuple[str, ...], *, check: bool = False, text: bool = False + ) -> CompletedProcess: + assert text, "Non-text target execution is not supported" + out, err, code = self.shell.run(args) + if check and code: + raise CalledProcessError(cmd=args, returncode=code, output=out, stderr=err) + return CompletedProcess(returncode=code, stdout=out, stderr=err, args=args) + + +@contextmanager +def TemporaryDirectory( + shell: CommandProtocol, + transfer: FileTransferProtocol, + mktemp: Path, + rm: Path, + mkdir: Path, +) -> Iterator[PurePath]: + # Transfer mktemp + mktemp_remote = ( + f"/tmp/{uuid.getnode()}" + ) # avoid race-condition if there are parallel executions + transfer.put(mktemp, mktemp_remote) + target = Subprocess(shell) + result = target.run(f"{mktemp_remote} -d", check=True, text=True) + root = PurePath(result.stdout[0].rstrip()) + + tools = root / "tools" + execroot = root / "execroot" + # Transfer mkdir + transfer.put(mkdir, f"{root}") + # Create tools dir + target.run(f"{root / 'mkdir'} {tools}", check=True, text=True) + # Create execroot dir + target.run(f"{root / 'mkdir'} {execroot}", check=True, text=True) + # Transfer rm + transfer.put(rm, f"{tools}") + + ##cleanup + + # Delete mkdir + target.run(f"{tools / 'rm'} {root / 'mkdir'}", check=True, text=True) + # Delete mktemp + target.run(f"{tools / 'rm'} {mktemp_remote}", check=True, text=True) + + yield execroot + + target.run(f"{tools / 'rm'} -rf {root}", check=True, text=True) diff --git a/bazel/labgrid/target_subprocess/target_subprocess.py b/bazel/labgrid/target_subprocess/target_subprocess.py deleted file mode 100644 index 9607efc4..00000000 --- a/bazel/labgrid/target_subprocess/target_subprocess.py +++ /dev/null @@ -1,18 +0,0 @@ -from subprocess import CalledProcessError, CompletedProcess -from shlex import join -from labgrid.protocol.commandprotocol import CommandProtocol - -class TargetSubprocess: - def __init__(self, shell: CommandProtocol): - self.__shell = shell - - @property - def shell(self) -> CommandProtocol: - return self.__shell - - def run(self,cmd,args: tuple[str, ...] = None, *, check: bool = False, text: bool = True) -> CompletedProcess: - assert text, "Non-text target execution is not supported" - out, err, code = self.shell.run(cmd) - if check and code: - raise CalledProcessError(cmd=cmd,returncode=code, output=out, stderr=err) - return CompletedProcess(returncode=code, stdout=out, stderr=err, args=args) diff --git a/labgrid/run/BUILD.bazel b/labgrid/run/BUILD.bazel index f98975fc..80f170ab 100644 --- a/labgrid/run/BUILD.bazel +++ b/labgrid/run/BUILD.bazel @@ -27,7 +27,7 @@ py_binary( tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//bazel/labgrid/target_subprocess", + "//bazel/labgrid/protocol/util:target", "//labgrid:pkg", "//labgrid/config:deps", "@rules_python//python/runfiles", diff --git a/labgrid/run/run.py b/labgrid/run/run.py index b23fa044..9027dce0 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -1,30 +1,26 @@ #! /usr/bin/env python3 -from typing import Iterator, Mapping from argparse import ArgumentParser from os import environ, linesep from pathlib import Path, PurePath from shlex import join +from subprocess import PIPE, run from sys import argv, stderr, stdout from typing import Collection, Protocol, Tuple + from python.runfiles import Runfiles -from labgrid import Environment,Target -from subprocess import PIPE, run -from contextlib import contextmanager -import uuid -from bazel.labgrid.target_subprocess.target_subprocess import TargetSubprocess -from labgrid.protocol.commandprotocol import CommandProtocol -from labgrid.protocol.filetransferprotocol import FileTransferProtocol + +from bazel.labgrid.protocol.util.target import TemporaryDirectory +from labgrid import Environment, Target + class Shell(Protocol): - def run(cmd: str) -> Tuple[Collection[str], Collection[str], int]: ... + def run(cmd: str) -> Tuple[Collection[str], Collection[str], int]: + ... def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( - "program", - metavar="PROG", - help="The program to run on the device.", - type=Path, + "program", metavar="PROG", help="The program to run on the device.", type=Path ) prsr.add_argument( "arguments", metavar="ARG", nargs="*", help="Command to run over SSH." @@ -46,46 +42,11 @@ def arguments(prsr: ArgumentParser) -> None: default=environ["BZL_LG_FINAL_STATE"], ) -@contextmanager -def TargetTemporaryDirectory(shell:CommandProtocol,transfer: FileTransferProtocol,runfiles: Runfiles)-> Iterator[PurePath]: - # Transfer mktemp - mktemp = runfiles.Rlocation("ape/ape/assimilate/mktemp.ape/mktemp") - mktemp_remote = f"/tmp/{uuid.getnode()}" #avoid race-condition if there are parallel executions - transfer.put(mktemp, mktemp_remote) - target = TargetSubprocess(shell) - result = target.run(cmd=f"{mktemp_remote} -d",check=True) - root = PurePath(result.stdout[0].rstrip()) - - tools = root / "tools" - execroot = root / "execroot" - # Transfer mkdir - mkdir = runfiles.Rlocation("ape/ape/assimilate/mkdir.ape/mkdir") - transfer.put(mkdir,f"{root}") - # Create tools dir - target.run(cmd=f"{root / 'mkdir'} {tools}",check=True) - # Create execroot dir - target.run(cmd=f"{root / 'mkdir'} {execroot}",check=True) - # Transfer rm - rm = runfiles.Rlocation("ape/ape/assimilate/rm.ape/rm") - transfer.put(rm, f"{tools}") - - ##cleanup - - # Delete mkdir - target.run(cmd=f"{tools / 'rm'} {root / 'mkdir'}",check=True) - # Delete mktemp - target.run(cmd=f"{tools / 'rm'} {mktemp_remote}",check=True) - - yield execroot - - target.run(cmd=f"{tools / 'rm'} -rf {root}",check=True) - - def main(exe: Path, *args: str) -> int: # Allow subprocesses to have their own runfiles runfiles = Runfiles.Create() - + del environ["RUNFILES_DIR"] prsr = ArgumentParser( @@ -106,10 +67,13 @@ def main(exe: Path, *args: str) -> int: # Retrieve the communication protocols shell = target.get_driver("CommandProtocol") transfer = target.get_driver("FileTransferProtocol") - - with TargetTemporaryDirectory(shell,transfer,runfiles) as temp_dir: + # utils + mktemp = runfiles.Rlocation("ape/ape/assimilate/mktemp.ape/mktemp") + mkdir = runfiles.Rlocation("ape/ape/assimilate/mkdir.ape/mkdir") + rm = runfiles.Rlocation("ape/ape/assimilate/rm.ape/rm") + with TemporaryDirectory(shell, transfer, mktemp, rm, mkdir) as temp_dir: # Transfer the provided program over to the Docker image - program = "{}/{}".format(temp_dir,parsed.program.name) + program = "{}/{}".format(temp_dir, parsed.program.name) transfer.put(parsed.program, program) # Transfer runfiles diff --git a/labgrid/transition/rule.bzl b/labgrid/transition/rule.bzl index baf3366e..9e098b6f 100644 --- a/labgrid/transition/rule.bzl +++ b/labgrid/transition/rule.bzl @@ -18,7 +18,9 @@ ATTRS = { def implementation(ctx): files = depset(transitive = [s.files for s in ctx.attr.srcs]) - return DefaultInfo(files = files) + runfiles = ctx.runfiles() + runfiles = runfiles.merge_all([s.default_runfiles for s in ctx.attr.srcs]) + return DefaultInfo(files = files, runfiles = runfiles) labgrid_transition = rule( doc = DOC, -- GitLab