From 25ef7842a20fe0ab558388114e08d088d4ec1301 Mon Sep 17 00:00:00 2001
From: Matt Clarkson <matthew.clarkson@arm.com>
Date: Wed, 11 Sep 2024 17:06:45 +0100
Subject: [PATCH 1/3] fix: make Python hub name unique

The extension requires that hub names are unique across
the whole Bazel dependency graph.
---
 MODULE.bazel | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/MODULE.bazel b/MODULE.bazel
index c567e14a..73eb05d2 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -48,11 +48,11 @@ dev.toolchain(
 # Download PyPI dependencies
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
-    hub_name = "pip",
+    hub_name = "rules_labgrid-pip",
     python_version = "3.11",
     requirements_lock = "//python:requirements.txt",
 )
-use_repo(pip, "pip")
+use_repo(pip, pip = "rules_labgrid-pip")
 
 # Download Debian QEMU
 download_archive = use_repo_rule("@download_utils//download/archive:defs.bzl", "download_archive")
-- 
GitLab


From 4eadae350e8ae4d74606ce99812e652aa3ad8b1b Mon Sep 17 00:00:00 2001
From: Matt Clarkson <matthew.clarkson@arm.com>
Date: Wed, 11 Sep 2024 17:05:26 +0100
Subject: [PATCH 2/3] refactor: move `docker` Python module in end-to-end tests

The `docker` module is _only_ needed for the `DockerDriver`.

This patch moves the download, resolution and dependency of
the `docker` module into the end-to-end tests.

Then the execution only pays the cost to download packages
that are needed for execution.
---
 e2e/MODULE.bazel            | 13 +++++++++++++
 e2e/docker/BUILD.bazel      |  5 ++++-
 e2e/python/BUILD.bazel      | 17 ++++++++++-------
 e2e/python/requirements.in  |  1 +
 e2e/python/requirements.txt | 32 ++++++++++++++++++++++++++++++++
 labgrid/python/BUILD.bazel  | 18 ++----------------
 python/requirements.in      |  1 -
 python/requirements.txt     | 12 ++----------
 8 files changed, 64 insertions(+), 35 deletions(-)
 create mode 100644 e2e/python/requirements.in
 create mode 100644 e2e/python/requirements.txt

diff --git a/e2e/MODULE.bazel b/e2e/MODULE.bazel
index 0473e352..a90786bb 100644
--- a/e2e/MODULE.bazel
+++ b/e2e/MODULE.bazel
@@ -21,6 +21,10 @@ bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0", dev_dependency = Tr
 
 register_toolchains("//...")
 
+# We need to pin to Python 3.11
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+use_repo(python, "python_versions")
+
 # We have to avoid the `chmod`/`chown`/`id` unhermetic-ness
 # TODO: remove this when `ignore_root_user_error` is hermetic
 # https://github.com/bazelbuild/rules_python/issues/2016
@@ -31,3 +35,12 @@ dev.toolchain(
     ignore_root_user_error = True,
     python_version = "3.11",
 )
+
+# Download PyPI dependencies
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip",
+    python_version = "3.11",
+    requirements_lock = "//python:requirements.txt",
+)
+use_repo(pip, "pip")
diff --git a/e2e/docker/BUILD.bazel b/e2e/docker/BUILD.bazel
index 9a9e299c..d174c1ff 100644
--- a/e2e/docker/BUILD.bazel
+++ b/e2e/docker/BUILD.bazel
@@ -24,7 +24,10 @@ py_binary(
     args = ["--lg-env=$(location local-ubuntu.16.04-gnu.yaml)"],
     data = ["local-ubuntu.16.04-gnu.yaml"],
     tags = ["manual"],
-    deps = ["@rules_labgrid//labgrid/python"],
+    deps = [
+        "@pip//docker",
+        "@rules_labgrid//labgrid/python",
+    ],
 )
 
 # An execution manager that is responsible for setting up the LabGrid environment
diff --git a/e2e/python/BUILD.bazel b/e2e/python/BUILD.bazel
index c478ae8e..69adeca6 100644
--- a/e2e/python/BUILD.bazel
+++ b/e2e/python/BUILD.bazel
@@ -1,11 +1,14 @@
-load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@python_versions//3.11:defs.bzl", "compile_pip_requirements")
 
-build_test(
-    name = "python",
-    targets = ["@rules_labgrid//labgrid/python"],
+compile_pip_requirements(
+    name = "requirements",
+    src = "requirements.in",
+    requirements_txt = "requirements.txt",
+    tags = ["manual"],
 )
 
-build_test(
-    name = "pkgs",
-    targets = ["@rules_labgrid//labgrid/python:{}".format(p) for p in ("labgrid", "docker")],
+alias(
+    name = "bootstrap",
+    actual = "@rules_python//python/config_settings:bootstrap_impl",
+    visibility = ["//visibility:public"],
 )
diff --git a/e2e/python/requirements.in b/e2e/python/requirements.in
new file mode 100644
index 00000000..0fb37929
--- /dev/null
+++ b/e2e/python/requirements.in
@@ -0,0 +1 @@
+docker==7.1.0
diff --git a/e2e/python/requirements.txt b/e2e/python/requirements.txt
new file mode 100644
index 00000000..79c5d1c1
--- /dev/null
+++ b/e2e/python/requirements.txt
@@ -0,0 +1,32 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //python:requirements.update
+#
+certifi==2024.6.2 \
+    --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \
+    --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56
+    # via requests
+charset-normalizer==2.0.12 \
+    --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \
+    --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df
+    # via requests
+docker==7.1.0 \
+    --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \
+    --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0
+    # via -r python/requirements.in
+idna==3.7 \
+    --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
+    --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
+    # via requests
+requests==2.26.0 \
+    --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
+    --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
+    # via docker
+urllib3==1.26.19 \
+    --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \
+    --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429
+    # via
+    #   docker
+    #   requests
diff --git a/labgrid/python/BUILD.bazel b/labgrid/python/BUILD.bazel
index 7092e0b8..d1dcfbf9 100644
--- a/labgrid/python/BUILD.bazel
+++ b/labgrid/python/BUILD.bazel
@@ -1,19 +1,5 @@
-load("@rules_python//python:defs.bzl", "py_library")
-
-[
-    alias(
-        name = pkg,
-        actual = "@pip//{}".format(pkg),
-        visibility = ["//visibility:public"],
-    )
-    for pkg in ("labgrid", "docker")
-]
-
-py_library(
+alias(
     name = "python",
+    actual = "@pip//labgrid",
     visibility = ["//visibility:public"],
-    deps = [
-        ":docker",
-        ":labgrid",
-    ],
 )
diff --git a/python/requirements.in b/python/requirements.in
index 8e3438e0..6cf539cb 100644
--- a/python/requirements.in
+++ b/python/requirements.in
@@ -1,2 +1 @@
 labgrid==24.0a1
-docker==7.1.0
diff --git a/python/requirements.txt b/python/requirements.txt
index 481a001d..c19a2721 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -114,10 +114,6 @@ cryptography==42.0.8 \
     --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \
     --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e
     # via autobahn
-docker==7.1.0 \
-    --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \
-    --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0
-    # via -r python/requirements.in
 hyperlink==21.0.0 \
     --hash=sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b \
     --hash=sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4
@@ -297,9 +293,7 @@ pyyaml==6.0.1 \
 requests==2.26.0 \
     --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
     --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
-    # via
-    #   docker
-    #   labgrid
+    # via labgrid
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
@@ -311,9 +305,7 @@ txaio==23.1.1 \
 urllib3==1.26.19 \
     --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \
     --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429
-    # via
-    #   docker
-    #   requests
+    # via requests
 xmodem==0.4.6 \
     --hash=sha256:089737298f5738eabc43f2519efdc80b402693768f16383f7013b9e6f8f279d7 \
     --hash=sha256:48ba47b68f86419399b8ddf9cdcf7ce27e594a881c287ad5c24ec71c11a79791
-- 
GitLab


From cbb3eb1957f3b3513640fb99434cb7997a22b819 Mon Sep 17 00:00:00 2001
From: Matt Clarkson <matthew.clarkson@arm.com>
Date: Thu, 12 Sep 2024 12:12:51 +0100
Subject: [PATCH 3/3] refactor: move `labgrid` module into `//labgrid/python`

Moving the download as close to the publically
visible alias as possible.

This will scale better as the project grows.

We will need, likely, many upstream PyPI packages.
Having them all in one `pip` hub means that the download is verbose
when using the Bazel targets.

Having multiple hubs means that only the packages needed
for a target are downloaded.
---
 MODULE.bazel                                |  6 +++---
 labgrid/python/BUILD.bazel                  | 21 ++++++++++++++++++++-
 {python => labgrid/python}/requirements.in  |  0
 {python => labgrid/python}/requirements.txt |  0
 labgrid/python/test.py                      | 18 ++++++++++++++++++
 python/BUILD.bazel                          |  9 ---------
 6 files changed, 41 insertions(+), 13 deletions(-)
 rename {python => labgrid/python}/requirements.in (100%)
 rename {python => labgrid/python}/requirements.txt (100%)
 create mode 100644 labgrid/python/test.py

diff --git a/MODULE.bazel b/MODULE.bazel
index 73eb05d2..a0b321de 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -48,11 +48,11 @@ dev.toolchain(
 # Download PyPI dependencies
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
-    hub_name = "rules_labgrid-pip",
+    hub_name = "rules_labgrid-labgrid-python",
     python_version = "3.11",
-    requirements_lock = "//python:requirements.txt",
+    requirements_lock = "//labgrid/python:requirements.txt",
 )
-use_repo(pip, pip = "rules_labgrid-pip")
+use_repo(pip, "rules_labgrid-labgrid-python")
 
 # Download Debian QEMU
 download_archive = use_repo_rule("@download_utils//download/archive:defs.bzl", "download_archive")
diff --git a/labgrid/python/BUILD.bazel b/labgrid/python/BUILD.bazel
index d1dcfbf9..3e4a1a7b 100644
--- a/labgrid/python/BUILD.bazel
+++ b/labgrid/python/BUILD.bazel
@@ -1,5 +1,24 @@
+load("@rules_python//python:defs.bzl", "py_test")
+load("@python_versions//3.11:defs.bzl", "compile_pip_requirements")
+
+# TODO: generate for more Python interpreter versions and platforms
+# `rules_uv` would be an easier way to do this
+compile_pip_requirements(
+    name = "requirements",
+    src = "requirements.in",
+    requirements_txt = "requirements.txt",
+    tags = ["manual"],
+)
+
+# TODO: we need this alias to select on Python interpreter version
 alias(
     name = "python",
-    actual = "@pip//labgrid",
+    actual = "@rules_labgrid-labgrid-python//labgrid",
     visibility = ["//visibility:public"],
 )
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    deps = [":python"],
+)
diff --git a/python/requirements.in b/labgrid/python/requirements.in
similarity index 100%
rename from python/requirements.in
rename to labgrid/python/requirements.in
diff --git a/python/requirements.txt b/labgrid/python/requirements.txt
similarity index 100%
rename from python/requirements.txt
rename to labgrid/python/requirements.txt
diff --git a/labgrid/python/test.py b/labgrid/python/test.py
new file mode 100644
index 00000000..1bff2fa9
--- /dev/null
+++ b/labgrid/python/test.py
@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from importlib import import_module
+from os import environ
+from pathlib import Path
+from unittest import TestCase, main
+
+
+class ModuleTestCase(TestCase):
+    def test_load(self):
+        module = import_module("labgrid")
+        name = "rules_labgrid-labgrid-python"
+        # Validate that `labgrid` is loaded from our `pip.parse` hub
+        self.assertTrue(name in module.__file__)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
index 69adeca6..1a2e495d 100644
--- a/python/BUILD.bazel
+++ b/python/BUILD.bazel
@@ -1,12 +1,3 @@
-load("@python_versions//3.11:defs.bzl", "compile_pip_requirements")
-
-compile_pip_requirements(
-    name = "requirements",
-    src = "requirements.in",
-    requirements_txt = "requirements.txt",
-    tags = ["manual"],
-)
-
 alias(
     name = "bootstrap",
     actual = "@rules_python//python/config_settings:bootstrap_impl",
-- 
GitLab