From 99ac9b75230053e8a35c8ee523d53995523846eb Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 19:50:08 +0200 Subject: [PATCH 01/16] feat: adding labgrid_transition --- labgrid/config_transition/BUILD.bazel | 6 ++++++ labgrid/config_transition/defs.bzl | 5 +++++ labgrid/config_transition/rule.bzl | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 labgrid/config_transition/BUILD.bazel create mode 100644 labgrid/config_transition/defs.bzl create mode 100644 labgrid/config_transition/rule.bzl diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel new file mode 100644 index 00000000..239bd02b --- /dev/null +++ b/labgrid/config_transition/BUILD.bazel @@ -0,0 +1,6 @@ +load(":defs.bzl", "labgrid_config_transition") + +labgrid_config_transition( + name = "test", + srcs = ["@ape//ape:mktemp"], +) diff --git a/labgrid/config_transition/defs.bzl b/labgrid/config_transition/defs.bzl new file mode 100644 index 00000000..085380c6 --- /dev/null +++ b/labgrid/config_transition/defs.bzl @@ -0,0 +1,5 @@ +load(":rule.bzl", _transition = "transition") + +visibility("public") + +labgrid_config_transition = _transition diff --git a/labgrid/config_transition/rule.bzl b/labgrid/config_transition/rule.bzl new file mode 100644 index 00000000..17ca4ebc --- /dev/null +++ b/labgrid/config_transition/rule.bzl @@ -0,0 +1,25 @@ +load("//labgrid/cfg:unstore.bzl", _cfg = "unstore") + +visibility("//...") + +DOC = "" + +ATTRS = { + "srcs": attr.label_list( + doc = "Outputs to transition to `platforms`.", + cfg = _cfg, + allow_files = True, + ), +} + +def implementation(ctx): + files = depset(transitive = [s.files for s in ctx.attr.srcs]) + return DefaultInfo(files = files) + +labgrid_config_transition = rule( + doc = DOC, + attrs = ATTRS, + implementation = implementation, +) + +transition = labgrid_config_transition -- GitLab From 60f741ea3f5846e4c08ff9f64a41f7fb67ebb619 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 19:58:35 +0200 Subject: [PATCH 02/16] feat: removing comments --- labgrid/config_transition/BUILD.bazel | 6 ------ 1 file changed, 6 deletions(-) diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel index 239bd02b..e69de29b 100644 --- a/labgrid/config_transition/BUILD.bazel +++ b/labgrid/config_transition/BUILD.bazel @@ -1,6 +0,0 @@ -load(":defs.bzl", "labgrid_config_transition") - -labgrid_config_transition( - name = "test", - srcs = ["@ape//ape:mktemp"], -) -- GitLab From 1569236a1b4484e9cc4ab8ec96b2b6b9bdd774db Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 28 Oct 2024 09:00:49 +0200 Subject: [PATCH 03/16] feat: labgrid_run - support transferring files back from device --- e2e/qemu/BUILD.bazel | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 69a6e7a2..0adbe65d 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -12,6 +12,11 @@ platform( labgrid_run( name = "cat", srcs = ["@ape//ape:cat"], + # outs = [ + # "some_output.txt", + # "stdout.log", + # ], + #cmd = "$(location @ape//ape:cat) /etc/os-release --get_remote_exe_root output.txt:$(location some_output.txt) > $(location stdout.log)", debug rempve it outs = ["stdout.log"], cmd = "$(location @ape//ape:cat) /etc/os-release > $@", platform = ":platform", -- GitLab From a18f1f8716bf6b0cceab0f1bd47eec485e3faa44 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Thu, 31 Oct 2024 10:46:19 +0200 Subject: [PATCH 04/16] feat: code review changes --- e2e/qemu/BUILD.bazel | 19 ++++++++++--------- labgrid/config_transition/BUILD.bazel | 0 labgrid/config_transition/defs.bzl | 5 ----- labgrid/config_transition/rule.bzl | 25 ------------------------- 4 files changed, 10 insertions(+), 39 deletions(-) delete mode 100644 labgrid/config_transition/BUILD.bazel delete mode 100644 labgrid/config_transition/defs.bzl delete mode 100644 labgrid/config_transition/rule.bzl diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 0adbe65d..1a3826a6 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") +load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") +#load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") platform( name = "platform", @@ -9,17 +10,17 @@ platform( ], ) -labgrid_run( +labgrid_genrule( name = "cat", srcs = ["@ape//ape:cat"], - # outs = [ - # "some_output.txt", - # "stdout.log", - # ], - #cmd = "$(location @ape//ape:cat) /etc/os-release --get_remote_exe_root output.txt:$(location some_output.txt) > $(location stdout.log)", debug rempve it - outs = ["stdout.log"], - cmd = "$(location @ape//ape:cat) /etc/os-release > $@", + outs = [ + # "output.txt", #debug remove it + "stdout.log", + ], + #cmd = "$(location @rules_labgrid//labgrid/run) --get output.txt:$(location output.txt) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", #debug remove it + cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", platform = ":platform", + tools = ["@rules_labgrid//labgrid/run"], ) diff_file_test( diff --git a/labgrid/config_transition/BUILD.bazel b/labgrid/config_transition/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/labgrid/config_transition/defs.bzl b/labgrid/config_transition/defs.bzl deleted file mode 100644 index 085380c6..00000000 --- a/labgrid/config_transition/defs.bzl +++ /dev/null @@ -1,5 +0,0 @@ -load(":rule.bzl", _transition = "transition") - -visibility("public") - -labgrid_config_transition = _transition diff --git a/labgrid/config_transition/rule.bzl b/labgrid/config_transition/rule.bzl deleted file mode 100644 index 17ca4ebc..00000000 --- a/labgrid/config_transition/rule.bzl +++ /dev/null @@ -1,25 +0,0 @@ -load("//labgrid/cfg:unstore.bzl", _cfg = "unstore") - -visibility("//...") - -DOC = "" - -ATTRS = { - "srcs": attr.label_list( - doc = "Outputs to transition to `platforms`.", - cfg = _cfg, - allow_files = True, - ), -} - -def implementation(ctx): - files = depset(transitive = [s.files for s in ctx.attr.srcs]) - return DefaultInfo(files = files) - -labgrid_config_transition = rule( - doc = DOC, - attrs = ATTRS, - implementation = implementation, -) - -transition = labgrid_config_transition -- GitLab From ec33e13ce030394f31a65c7965b316eb75e72860 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Thu, 31 Oct 2024 11:23:30 +0200 Subject: [PATCH 05/16] refactor: rebase --- labgrid/run/run.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 93162a9d..f7559545 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -11,6 +11,8 @@ from shlex import join from subprocess import CalledProcessError from sys import argv, stderr, stdout +from typing import Collection, Protocol, Tuple +import os from python.runfiles import Runfiles from bazel.labgrid.strategy import transition @@ -77,9 +79,38 @@ def arguments(prsr: ArgumentParser) -> None: "--mkdir", help="A `mkdir` binary for the target.", type=runfile, - default=runfile("ape/ape/assimilate/mkdir.ape/mkdir"), + default=runfile("ape/ape/assimilate/mkdir.ape/mkdir") + ) + prsr.add_argument( + "--get", + help="Transfer files from target at end of execution. Usage is `--get :`. Relative paths are resolved to the execution root on both the local and remote. A shorthand `--get=` is available which uses the same path for remote and local.", + type=RemoteLocalPathArgument(), + required=False ) +class RemoteLocalPath: + def __init__(self,remote: Path,local: Path) -> None: + self.remote = remote + self.local = local + def convert_to_abs_path(self,remote_base_path,local_base_path=Path.cwd()): + if not self.remote.is_absolute(): + self.remote = remote_base_path / self.remote + if not self.local.is_absolute(): + self.local = local_base_path / self.local + +class RemoteLocalPathArgument: + def __call__(self,value): + try: + remote_local_list = [] + for remote_local in value.split(): + if ":" in remote_local: + remote_path,local_path = value.split(":") + remote_local_list.append(RemoteLocalPath(Path(remote_path),Path(local_path))) + else: + remote_local_list.append(RemoteLocalPath(Path(remote_local),Path(remote_local))) + return remote_local_list + except: + print('error parsing --get flag') @dataclass class State: @@ -102,6 +133,7 @@ def run( mktemp: Path, mkdir: Path, rm: Path, + get:RemoteLocalPathArgument, ) -> int: # Allow any Bazel binaries used in the configuration to work try: @@ -135,6 +167,14 @@ def run( stdout.write(f"{line}{linesep}") for line in err: stderr.write(f"{line}{linesep}") + + + if code == 0: #copy the files back from the target + if get: + shell.run(f"touch /{temp_dir}/output.txt") #debug remove it + for remote_local in parsed.get: + remote_local.convert_to_abs_path(temp_dir) + transfer.get(remote_local.remote,remote_local.local) return code @@ -157,6 +197,7 @@ def main(exe: Path, *args: str) -> int: mktemp=parsed.mktemp, rm=parsed.rm, mkdir=parsed.mkdir, + get=parsed.get, ) except CalledProcessError as e: print(f"fatal: subprocess failed: {e}", file=stderr) -- GitLab From 33ebc5607dc6e6c6b1ad7ce6e11a1fb43009d3bf Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Thu, 31 Oct 2024 15:14:47 +0200 Subject: [PATCH 06/16] refactor: rebase --- e2e/qemu/BUILD.bazel | 6 +++--- labgrid/run/run.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 1a3826a6..3c041062 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -14,11 +14,11 @@ labgrid_genrule( name = "cat", srcs = ["@ape//ape:cat"], outs = [ - # "output.txt", #debug remove it + "output.txt", #debug remove it "stdout.log", ], - #cmd = "$(location @rules_labgrid//labgrid/run) --get output.txt:$(location output.txt) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", #debug remove it - cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", + cmd = "$(location @rules_labgrid//labgrid/run) --get output.txt:$(location output.txt) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", #debug remove it + #cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", platform = ":platform", tools = ["@rules_labgrid//labgrid/run"], ) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index f7559545..73aca3bf 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -171,9 +171,9 @@ def run( if code == 0: #copy the files back from the target if get: - shell.run(f"touch /{temp_dir}/output.txt") #debug remove it - for remote_local in parsed.get: - remote_local.convert_to_abs_path(temp_dir) + shell.run(f"touch /{temp}/output.txt") #debug remove it + for remote_local in get: + remote_local.convert_to_abs_path(temp) transfer.get(remote_local.remote,remote_local.local) return code -- GitLab From 6d43804a9221171f3e8f88659ed60a751b49113a Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Thu, 31 Oct 2024 16:46:32 +0200 Subject: [PATCH 07/16] refactor: removing debug code --- e2e/qemu/BUILD.bazel | 9 ++------- labgrid/run/run.py | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 3c041062..ec59755f 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") -#load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") platform( name = "platform", @@ -13,12 +12,8 @@ platform( labgrid_genrule( name = "cat", srcs = ["@ape//ape:cat"], - outs = [ - "output.txt", #debug remove it - "stdout.log", - ], - cmd = "$(location @rules_labgrid//labgrid/run) --get output.txt:$(location output.txt) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", #debug remove it - #cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", + outs = ["stdout.log"], + cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", platform = ":platform", tools = ["@rules_labgrid//labgrid/run"], ) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 73aca3bf..daa02283 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -171,7 +171,6 @@ def run( if code == 0: #copy the files back from the target if get: - shell.run(f"touch /{temp}/output.txt") #debug remove it for remote_local in get: remote_local.convert_to_abs_path(temp) transfer.get(remote_local.remote,remote_local.local) -- GitLab From e0ef86145833b073567e59b349918cf4af568992 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 3 Nov 2024 15:08:45 +0200 Subject: [PATCH 08/16] feat: code review changes --- labgrid/run/run.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index daa02283..79aca701 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -84,7 +84,7 @@ def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( "--get", help="Transfer files from target at end of execution. Usage is `--get :`. Relative paths are resolved to the execution root on both the local and remote. A shorthand `--get=` is available which uses the same path for remote and local.", - type=RemoteLocalPathArgument(), + action=RemoteLocalPathArgument, required=False ) @@ -98,19 +98,18 @@ class RemoteLocalPath: if not self.local.is_absolute(): self.local = local_base_path / self.local -class RemoteLocalPathArgument: - def __call__(self,value): - try: - remote_local_list = [] - for remote_local in value.split(): - if ":" in remote_local: - remote_path,local_path = value.split(":") - remote_local_list.append(RemoteLocalPath(Path(remote_path),Path(local_path))) - else: - remote_local_list.append(RemoteLocalPath(Path(remote_local),Path(remote_local))) - return remote_local_list - except: - print('error parsing --get flag') +class RemoteLocalPathArgument(Action): + def __init__(self,option_strings,dest,**kwargs): + super().__init__(option_strings,dest,**kwargs) + self.remote_local_list = [] + def __call__(self,parser,namespace,values,option_string=None): + for remote_local in values.split(): + if ":" in remote_local: + remote_path,local_path = values.split(":") + self.remote_local_list.append(RemoteLocalPath(Path(remote_path),Path(local_path))) + else: + self.remote_local_list.append(RemoteLocalPath(Path(remote_local),Path(remote_local))) + setattr(namespace,self.dest,self.remote_local_list) @dataclass class State: -- GitLab From 652100331e61058538bc2c8eecc976d87c20e677 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 3 Nov 2024 15:39:58 +0200 Subject: [PATCH 09/16] feat: code review changes --- labgrid/run/run.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 79aca701..522c0c20 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -83,9 +83,10 @@ def arguments(prsr: ArgumentParser) -> None: ) prsr.add_argument( "--get", - help="Transfer files from target at end of execution. Usage is `--get :`. Relative paths are resolved to the execution root on both the local and remote. A shorthand `--get=` is available which uses the same path for remote and local.", + help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` can be omitted, which will use the same path for remote and local.", action=RemoteLocalPathArgument, - required=False + required=False, + metavar="REMOTE[:LOCAL]" ) class RemoteLocalPath: @@ -104,11 +105,11 @@ class RemoteLocalPathArgument(Action): self.remote_local_list = [] def __call__(self,parser,namespace,values,option_string=None): for remote_local in values.split(): - if ":" in remote_local: - remote_path,local_path = values.split(":") - self.remote_local_list.append(RemoteLocalPath(Path(remote_path),Path(local_path))) + remote, found, local = remote_local.partition(":") + if found: + self.remote_local_list.append(RemoteLocalPath(Path(remote),Path(local))) else: - self.remote_local_list.append(RemoteLocalPath(Path(remote_local),Path(remote_local))) + self.remote_local_list.append(RemoteLocalPath(Path(remote),Path(remote))) setattr(namespace,self.dest,self.remote_local_list) @dataclass -- GitLab From e6130778b77c7f79e3e55d616cea12e097947809 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 3 Nov 2024 15:45:15 +0200 Subject: [PATCH 10/16] feat: code review changes --- labgrid/run/run.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 522c0c20..a84ae0e0 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -84,12 +84,12 @@ def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( "--get", help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` can be omitted, which will use the same path for remote and local.", - action=RemoteLocalPathArgument, + action=PathMappingArgument, required=False, metavar="REMOTE[:LOCAL]" ) -class RemoteLocalPath: +class PathMapping: def __init__(self,remote: Path,local: Path) -> None: self.remote = remote self.local = local @@ -99,7 +99,7 @@ class RemoteLocalPath: if not self.local.is_absolute(): self.local = local_base_path / self.local -class RemoteLocalPathArgument(Action): +class PathMappingArgument(Action): def __init__(self,option_strings,dest,**kwargs): super().__init__(option_strings,dest,**kwargs) self.remote_local_list = [] @@ -107,11 +107,12 @@ class RemoteLocalPathArgument(Action): for remote_local in values.split(): remote, found, local = remote_local.partition(":") if found: - self.remote_local_list.append(RemoteLocalPath(Path(remote),Path(local))) + self.remote_local_list.append(PathMapping(Path(remote),Path(local))) else: - self.remote_local_list.append(RemoteLocalPath(Path(remote),Path(remote))) + self.remote_local_list.append(PathMapping(Path(remote),Path(remote))) setattr(namespace,self.dest,self.remote_local_list) + @dataclass class State: initial: str | None = environ.get("LG_INITIAL_STATE", None) @@ -133,7 +134,7 @@ def run( mktemp: Path, mkdir: Path, rm: Path, - get:RemoteLocalPathArgument, + get:PathMappingArgument, ) -> int: # Allow any Bazel binaries used in the configuration to work try: -- GitLab From e6e182ca6c895bbe2f2a6167a051866a2a9d5524 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 3 Nov 2024 16:07:59 +0200 Subject: [PATCH 11/16] feat: code review changes --- labgrid/run/run.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index a84ae0e0..91da4c13 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -89,15 +89,10 @@ def arguments(prsr: ArgumentParser) -> None: metavar="REMOTE[:LOCAL]" ) -class PathMapping: - def __init__(self,remote: Path,local: Path) -> None: - self.remote = remote - self.local = local - def convert_to_abs_path(self,remote_base_path,local_base_path=Path.cwd()): - if not self.remote.is_absolute(): - self.remote = remote_base_path / self.remote - if not self.local.is_absolute(): - self.local = local_base_path / self.local + +def __convert_to_abs_path(path,base_path=Path.cwd()): + if not path.is_absolute(): + return base_path / path class PathMappingArgument(Action): def __init__(self,option_strings,dest,**kwargs): @@ -107,9 +102,9 @@ class PathMappingArgument(Action): for remote_local in values.split(): remote, found, local = remote_local.partition(":") if found: - self.remote_local_list.append(PathMapping(Path(remote),Path(local))) + self.remote_local_list.append((Path(remote),Path(local))) else: - self.remote_local_list.append(PathMapping(Path(remote),Path(remote))) + self.remote_local_list.append((Path(remote),Path(remote))) setattr(namespace,self.dest,self.remote_local_list) @@ -172,9 +167,8 @@ def run( if code == 0: #copy the files back from the target if get: - for remote_local in get: - remote_local.convert_to_abs_path(temp) - transfer.get(remote_local.remote,remote_local.local) + for remote, local in get: + transfer.get(__convert_to_abs_path(remote,temp),__convert_to_abs_path(local)) return code -- GitLab From a3df9b68bd8e7cc15495ea188cc8485f742e36d0 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Sun, 3 Nov 2024 19:17:21 +0200 Subject: [PATCH 12/16] feat: code review changes --- e2e/qemu/BUILD.bazel | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index ec59755f..39b99fbe 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") -load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") +load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") platform( name = "platform", @@ -9,13 +9,12 @@ platform( ], ) -labgrid_genrule( +labgrid_run( name = "cat", srcs = ["@ape//ape:cat"], outs = ["stdout.log"], - cmd = "$(location @rules_labgrid//labgrid/run) -- $(location @ape//ape:cat) /etc/os-release > $(location stdout.log)", + cmd = "-- $(location @ape//ape:cat) /etc/os-release > $@", platform = ":platform", - tools = ["@rules_labgrid//labgrid/run"], ) diff_file_test( -- GitLab From b687a834634eae164cfa6a57dde3c32157a0731d Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Mon, 4 Nov 2024 12:49:40 +0200 Subject: [PATCH 13/16] feat: black+isort --- labgrid/run/run.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 91da4c13..50c9b89d 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -2,7 +2,8 @@ from __future__ import annotations import argparse -from argparse import ArgumentParser +import os +from argparse import Action, ArgumentParser from contextlib import contextmanager from dataclasses import dataclass from os import environ, linesep @@ -10,9 +11,8 @@ from pathlib import Path, PurePath from shlex import join from subprocess import CalledProcessError from sys import argv, stderr, stdout - from typing import Collection, Protocol, Tuple -import os + from python.runfiles import Runfiles from bazel.labgrid.strategy import transition @@ -79,33 +79,35 @@ def arguments(prsr: ArgumentParser) -> None: "--mkdir", help="A `mkdir` binary for the target.", type=runfile, - default=runfile("ape/ape/assimilate/mkdir.ape/mkdir") + default=runfile("ape/ape/assimilate/mkdir.ape/mkdir"), ) prsr.add_argument( "--get", help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` can be omitted, which will use the same path for remote and local.", action=PathMappingArgument, required=False, - metavar="REMOTE[:LOCAL]" + metavar="REMOTE[:LOCAL]", ) -def __convert_to_abs_path(path,base_path=Path.cwd()): +def __convert_to_abs_path(path, base_path=Path.cwd()): if not path.is_absolute(): return base_path / path + class PathMappingArgument(Action): - def __init__(self,option_strings,dest,**kwargs): - super().__init__(option_strings,dest,**kwargs) + def __init__(self, option_strings, dest, **kwargs): + super().__init__(option_strings, dest, **kwargs) self.remote_local_list = [] - def __call__(self,parser,namespace,values,option_string=None): + + def __call__(self, parser, namespace, values, option_string=None): for remote_local in values.split(): remote, found, local = remote_local.partition(":") if found: - self.remote_local_list.append((Path(remote),Path(local))) + self.remote_local_list.append((Path(remote), Path(local))) else: - self.remote_local_list.append((Path(remote),Path(remote))) - setattr(namespace,self.dest,self.remote_local_list) + self.remote_local_list.append((Path(remote), Path(remote))) + setattr(namespace, self.dest, self.remote_local_list) @dataclass @@ -129,7 +131,7 @@ def run( mktemp: Path, mkdir: Path, rm: Path, - get:PathMappingArgument, + get: PathMappingArgument, ) -> int: # Allow any Bazel binaries used in the configuration to work try: @@ -163,12 +165,14 @@ def run( stdout.write(f"{line}{linesep}") for line in err: stderr.write(f"{line}{linesep}") - - if code == 0: #copy the files back from the target + if code == 0: # copy the files back from the target if get: for remote, local in get: - transfer.get(__convert_to_abs_path(remote,temp),__convert_to_abs_path(local)) + transfer.get( + __convert_to_abs_path(remote, temp), + __convert_to_abs_path(local), + ) return code -- GitLab From 2dc0beab3549e9fbe942dbe306c7173d1099d6a3 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Tue, 5 Nov 2024 09:09:25 +0200 Subject: [PATCH 14/16] feat: code review changes --- e2e/docker/BUILD.bazel | 35 ++++++++++++++++++++++++----------- e2e/qemu/BUILD.bazel | 2 +- labgrid/executor/executor.py | 1 - labgrid/run/run.py | 20 +++++++++++--------- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel index 21e9f3b9..5a6dbe3c 100644 --- a/e2e/docker/BUILD.bazel +++ b/e2e/docker/BUILD.bazel @@ -1,8 +1,9 @@ load("@rules_diff//diff/file/test:defs.bzl", "diff_file_test") load("@rules_labgrid//labgrid/config/toolchain:defs.bzl", "labgrid_config_toolchain") +load("@rules_labgrid//labgrid/genrule:defs.bzl", "labgrid_genrule") load("@rules_labgrid//labgrid/run:defs.bzl", "labgrid_run") -load("@rules_python//python:defs.bzl", "py_binary") load("@rules_labgrid//labgrid/state:defs.bzl", "labgrid_state") +load("@rules_python//python:defs.bzl", "py_binary") # A constraint to register the toolchain to constraint_setting(name = "device") @@ -66,14 +67,26 @@ labgrid_run( ], ) -# Check that the output of `/proc/version` from within the container is what we expect -diff_file_test( - name = "test", - size = "small", - a = ":version.txt", - b = ":stdout.log", - tags = [ - "manual", - "requires-docker", - ], +labgrid_genrule( + name = "copy", + srcs = ["@ape//ape:cp"], + outs = ["output.txt"], + cmd = "$(location @rules_labgrid//labgrid/run) --get output.txt:$(location output.txt) $(location @ape//ape:cp) /etc/os-release output.txt", + platform = ":platform", + tools = ["@rules_labgrid//labgrid/run"], ) + +[ + # Check that the output of `/proc/version` from within the container is what we expect + diff_file_test( + name = "test-{textfile}".format(textfile = textfile), + size = "small", + a = ":version.txt", + b = textfile, + tags = [ + "manual", + "requires-docker", + ], + ) + for textfile in ("output.txt", "stdout.log") +] diff --git a/e2e/qemu/BUILD.bazel b/e2e/qemu/BUILD.bazel index 39b99fbe..69a6e7a2 100644 --- a/e2e/qemu/BUILD.bazel +++ b/e2e/qemu/BUILD.bazel @@ -13,7 +13,7 @@ labgrid_run( name = "cat", srcs = ["@ape//ape:cat"], outs = ["stdout.log"], - cmd = "-- $(location @ape//ape:cat) /etc/os-release > $@", + cmd = "$(location @ape//ape:cat) /etc/os-release > $@", platform = ":platform", ) diff --git a/labgrid/executor/executor.py b/labgrid/executor/executor.py index 49d5e075..409a1702 100644 --- a/labgrid/executor/executor.py +++ b/labgrid/executor/executor.py @@ -77,7 +77,6 @@ def main(exe: Path, *args: str) -> int: args = (f"@{baked}", *args) parsed = prsr.parse_args(args) - cmd = (parsed.program, *parsed.arguments) # A workaround for invoking APE binaries on Apple Silicon diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 50c9b89d..5019b0ea 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -87,12 +87,14 @@ def arguments(prsr: ArgumentParser) -> None: action=PathMappingArgument, required=False, metavar="REMOTE[:LOCAL]", + default=[] ) -def __convert_to_abs_path(path, base_path=Path.cwd()): +def __convert_to_abs_path(base_path, path): if not path.is_absolute(): return base_path / path + return path class PathMappingArgument(Action): @@ -160,19 +162,20 @@ def run( if src_runfiles.exists(): transfer.put(f"{src_runfiles}", f"{dest_runfiles}") # Run the transferred program - out, err, code = shell.run(join((f"{remote}", *arguments))) + out, err, code = shell.run( + f"cd {temp} && " + join((f"{remote}", *arguments)) + ) for line in out: stdout.write(f"{line}{linesep}") for line in err: stderr.write(f"{line}{linesep}") if code == 0: # copy the files back from the target - if get: - for remote, local in get: - transfer.get( - __convert_to_abs_path(remote, temp), - __convert_to_abs_path(local), - ) + for remote, local in get: + transfer.get( + __convert_to_abs_path(temp, remote), + __convert_to_abs_path(Path.cwd(), local), + ) return code @@ -183,7 +186,6 @@ def main(exe: Path, *args: str) -> int: prsr.set_defaults(state=State()) arguments(prsr) - parsed = prsr.parse_args(args) try: -- GitLab From 31c5cf3c6ab2e570b0e830c4897eca3790219d83 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Tue, 5 Nov 2024 15:22:25 +0200 Subject: [PATCH 15/16] feat: code review changes --- labgrid/run/run.py | 61 +++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index 5019b0ea..cf42a5e5 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse import os -from argparse import Action, ArgumentParser +from argparse import Action, ArgumentParser, REMAINDER from contextlib import contextmanager from dataclasses import dataclass from os import environ, linesep @@ -11,7 +11,7 @@ from pathlib import Path, PurePath from shlex import join from subprocess import CalledProcessError from sys import argv, stderr, stdout -from typing import Collection, Protocol, Tuple +from typing import Collection, Protocol, Tuple, Iterator from python.runfiles import Runfiles @@ -37,7 +37,7 @@ def arguments(prsr: ArgumentParser) -> None: "program", metavar="PROG", help="The program to run on the device.", type=Path ) prsr.add_argument( - "arguments", metavar="ARG", nargs="*", help="Command to run over SSH." + "arguments", metavar="ARG", nargs=REMAINDER, help="Command to run over SSH." ) prsr.add_argument( "--config", @@ -84,32 +84,37 @@ def arguments(prsr: ArgumentParser) -> None: prsr.add_argument( "--get", help="Transfer files from target at end of execution. Relative paths are resolved to the execution root on both the local and remote. `LOCAL` can be omitted, which will use the same path for remote and local.", - action=PathMappingArgument, - required=False, + type=get, metavar="REMOTE[:LOCAL]", + action="append", default=[] ) -def __convert_to_abs_path(base_path, path): - if not path.is_absolute(): - return base_path / path - return path +def get(value:str) -> Get: + remote, found, local = value.partition(":") + if found: + return Get(PurePath(remote), PurePath(local)) + return Get(PurePath(remote), PurePath(remote)) -class PathMappingArgument(Action): - def __init__(self, option_strings, dest, **kwargs): - super().__init__(option_strings, dest, **kwargs) - self.remote_local_list = [] +@dataclass(frozen=True) +class Get: + remote: PurePath + local: PurePath + + def resolve(self,remote: PurePath) -> Get: + return Get( + remote = self.join(remote,self.remote), + local = self.join(environ.get("BAZEL_GEN_DIR", Path.cwd()),self.local) + ) + + @staticmethod + def join(base: PurePath, path: PurePath) -> PurePath: + if not path.is_absolute(): + return base / path + return path - def __call__(self, parser, namespace, values, option_string=None): - for remote_local in values.split(): - remote, found, local = remote_local.partition(":") - if found: - self.remote_local_list.append((Path(remote), Path(local))) - else: - self.remote_local_list.append((Path(remote), Path(remote))) - setattr(namespace, self.dest, self.remote_local_list) @dataclass @@ -133,14 +138,13 @@ def run( mktemp: Path, mkdir: Path, rm: Path, - get: PathMappingArgument, + get: Iterator[Get], ) -> int: # Allow any Bazel binaries used in the configuration to work try: del environ["RUNFILES_DIR"] except KeyError: pass - # Start up Docker container with LabGrid env = Environment(str(config)) target = env.get_target() @@ -171,11 +175,12 @@ def run( stderr.write(f"{line}{linesep}") if code == 0: # copy the files back from the target - for remote, local in get: - transfer.get( - __convert_to_abs_path(temp, remote), - __convert_to_abs_path(Path.cwd(), local), - ) + + for unresolved in get: + resolved = unresolved.resolve(temp) + transfer.get(resolved.remote, resolved.local) + + return code -- GitLab From 3d1413b8c0f692c8e4469b34a267362d230420f3 Mon Sep 17 00:00:00 2001 From: Idan Saadon Date: Tue, 5 Nov 2024 15:27:15 +0200 Subject: [PATCH 16/16] feat: black+isort --- labgrid/run/run.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/labgrid/run/run.py b/labgrid/run/run.py index cf42a5e5..71584b76 100644 --- a/labgrid/run/run.py +++ b/labgrid/run/run.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse import os -from argparse import Action, ArgumentParser, REMAINDER +from argparse import REMAINDER, Action, ArgumentParser from contextlib import contextmanager from dataclasses import dataclass from os import environ, linesep @@ -11,7 +11,7 @@ from pathlib import Path, PurePath from shlex import join from subprocess import CalledProcessError from sys import argv, stderr, stdout -from typing import Collection, Protocol, Tuple, Iterator +from typing import Collection, Iterator, Protocol, Tuple from python.runfiles import Runfiles @@ -87,28 +87,28 @@ def arguments(prsr: ArgumentParser) -> None: type=get, metavar="REMOTE[:LOCAL]", action="append", - default=[] + default=[], ) - -def get(value:str) -> Get: +def get(value: str) -> Get: remote, found, local = value.partition(":") if found: return Get(PurePath(remote), PurePath(local)) return Get(PurePath(remote), PurePath(remote)) + @dataclass(frozen=True) class Get: remote: PurePath local: PurePath - def resolve(self,remote: PurePath) -> Get: + def resolve(self, remote: PurePath) -> Get: return Get( - remote = self.join(remote,self.remote), - local = self.join(environ.get("BAZEL_GEN_DIR", Path.cwd()),self.local) + remote=self.join(remote, self.remote), + local=self.join(environ.get("BAZEL_GEN_DIR", Path.cwd()), self.local), ) - + @staticmethod def join(base: PurePath, path: PurePath) -> PurePath: if not path.is_absolute(): @@ -116,7 +116,6 @@ class Get: return path - @dataclass class State: initial: str | None = environ.get("LG_INITIAL_STATE", None) @@ -180,7 +179,6 @@ def run( resolved = unresolved.resolve(temp) transfer.get(resolved.remote, resolved.local) - return code -- GitLab