From ecbf96e5146d2bd982a003f04fdc3d07406ce7b5 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Fri, 17 Jan 2025 10:52:00 +0000 Subject: [PATCH 1/5] docs(readme): provide more concise getting start guides --- README.md | 310 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 245 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 7680331..75a16d8 100644 --- a/README.md +++ b/README.md @@ -4,132 +4,312 @@ ## Getting Started -Add the following to `MODULE.bazel`: +Add the module following to `MODULE.bazel`: ```py -which = use_repo_rule("@toolchain_utils//toolchain/local/which:defs.bzl", "toolchain_local_which") -which( - name = "which-echo", -) - -resolved = use_repo_rule("@toolchain_utils//toolchain/resolved:defs.bzl", "toolchain_resolved") -resolved( - name = "resolved-echo", - toolchain_type = "//toolchain/echo:type", -) +bazel_dep(name = "toolchain_utils", version = "<...>") ``` -Create a `toolchain/echo/BUILD.bazel` with the following: +### Register a built target + +The `toolchain_info` provides a way to create the `ToolchainInfo` provider around a executable Bazel target. ```py -load("@toolchain_utils//toolchain/test:defs.bzl", "toolchain_test") +# Assuming a ruleset named `rules_placeholder` and a tool named `binary` +# This would be in `//placeholder/toolchain/binary/BUILD.bazel` +load("@toolchain_utils//toolchain/info:defs.bzl", "toolchain_info") -# The `toolchain/echo:type` for registration +# Used for registration. Public so that downstream can register other toolchains toolchain_type( name = "type", visibility = ["//visibility:public"], ) -# Register the `local` binary as a toolchain -# No `exec_compatible_with` constraints are needed as a local binary is always compatible with the execution platform +# Create toolchain information around a binary target +# This could be a `go_binary`, `py_binary`, etc. +# The theoretical binary generates code that targets `amd64-linux-gnu` +# It will always be built for the current execution platform +toolchain_info( + name = "info-amd64-linux-gnu", + target = "//placeholder/binary:arm64-linux-gnu", + # We want a consistent `$(BINARY)` Make variable + # This will default to `basename.upper()` + variable = "BINARY", +) + +# Register the toolchain +# The `target_compatible_with` describes the code generated by the toolchain +# No `exec_compatible_with` as Bazel will build the binary for the execution platform toolchain( - name = "local", - toolchain = "@which-echo", + name = "built-amd64-linux-gnu", + toolchain = ":info", toolchain_type = ":type", + target_compatible_with = [ + "@toolchain_utils//toolchain/constraint/cpu:amd64", + "@toolchain_utils//toolchain/constraint/os:linux", + "@toolchain_utils//toolchain/constraint/libc:gnu", + ], ) +``` -# Run the resolved toolchain with: -# bazel run -- //toolchain/echo:resolved -alias( - name = "resolved", - actual = "@resolved-echo", -) +The toolchain can be implicitly registered by the current module in `MODULE.bazel`: -# Performs a execution test of the binary -# Validates it works on the current platform -toolchain_test( - name = "test", - args = ["Hello, world!"], - stdout = ":hello-world.txt", - toolchains = [":resolved"], +```py +register_toolchains("//placeholder/toolchain/binary:local-amd64-linux-gnu") +# Or use a recursive registration +# register_toolchains("//placeholder/toolchain/...") +``` + +### Register a local binary on `PATH` + +It is often useful for debugging or quick setup to use a binary defined on `PATH` as the toolchain binary. + +The project provides a repository rule to detect a binary on `PATH`. Add the following to `MODULE.bazel`: + +```py +# Assuming a binary named `amd64-linux-gnu-binary` on `PATH` +which = use_repo_rule("@toolchain_utils//toolchain/local/which:defs.bzl", "toolchain_local_which") +which( + name = "which-amd64-linux-gnu-binary", ) ``` -To create a hermetic toolchain from a built target: +The repository rule provides a `toolchain_info` target that can be registered against a toolchain type: ```py -load("@toolchain_utils//toolchain/info/target:defs.bzl", "toolchain_info") +# Assuming a ruleset named `rules_placeholder` and a tool named `binary` +# This would be in `//placeholder/toolchain/binary/BUILD.bazel` -# Wrap the binary with `ToolchainInfo` provider -toolchain_info( - name = "info", - target = ":built", +# Used for registration. Public so that downstream can register other toolchains +toolchain_type( + name = "type", + visibility = ["//visibility:public"], ) -# Register the hermetic toolchain +# Register the local toolchain +# The `target_compatible_with` describes the code generated by the toolchain +# No `exec_compatible_with` as Bazel will build the binary for the execution platform toolchain( - name = "hermetic", - toolchain = ":info", + name = "local-amd64-linux-gnu", + toolchain = "@which-amd64-linux-gnu-binary", toolchain_type = ":type", + exec_compatible_with = [ + "@toolchain_utils//toolchain/constraint/cpu:local", + "@toolchain_utils//toolchain/constraint/os:local", + "@toolchain_utils//toolchain/constraint/libc:local", + ], + target_compatible_with = [ + "@toolchain_utils//toolchain/constraint/cpu:amd64", + "@toolchain_utils//toolchain/constraint/os:linux", + "@toolchain_utils//toolchain/constraint/libc:gnu", + ], +) + +``` + +The toolchain can be implicitly registered by the current module in `MODULE.bazel`: + +```py +register_toolchains("//placeholder/toolchain/binary:local-amd64-linux-gnu") +# Or use a recursive registration +# register_toolchains("//placeholder/toolchain/...") +``` + +### Register downloaded binaries + +A common use-case is to download pre-built binaries for different execution architectures and register them as a Bazel toolchain. + +The binaries will need to be downloaded in `MODULE.bazel`: + +```py +bazel_dep(name = "download_utils", version = "<...>") + +# Download a binary that runs on `amd64-linux-gnu` +# It will generate code for `arm64-windows-ucrt` +download_file = use_repo_rule("@download_utils//download/file:defs.bzl", "download_file") +download_file( + name = "binary-amd64-linux-gnu", + output = "arm64-windows-ucrt-binary", + executable = True, + urls = ["https://some.thing/amd64-linux-gnu/arm64-windows-ucrt-binary"], ) ``` -To create a hermetic toolchain from a downloaded target: +The downloaded binary can now be registered: ```py -load("@toolchain_utils//toolchain/info/target:defs.bzl", "toolchain_info") +# Assuming a ruleset named `rules_placeholder` and a tool named `binary` +# This would be in `//placeholder/toolchain/binary/BUILD.bazel` + +# Used for registration. Public so that downstream can register other toolchains +toolchain_type( + name = "type", + visibility = ["//visibility:public"], +) +``` + +To make organisation easier, it can be useful to create a Bazel package for each execution architecture. + +``` +# This would be in `//placeholder/toolchain/binary/amd64-linux-gnu/BUILD.bazel` +# Where `amd64-linux-gnu` is the execution architecture these toolchains work on -# Create the necessary toolchain providers around the downloaded target +# Create toolchain information around the downloaded binary toolchain_info( - name = "echo-arm64-linux-gnu", - target = ":downloaded-echo-arm64-linux-gnu" + name = "info-arm64-windows-ucrt", + target = "@binary-amd64-linux-gnu//:arm64-windows-ucrt-binary", + # We want a consistent `$(BINARY)` Make variable + # This will default to `basename.upper()` + variable = "BINARY", ) -# Register with the correct contraints +# Register the downloaded toolchain +# The `target_compatible_with` describes the code generated by the toolchain +# The `exec_compatible_with` describes which execution platform the tool can run on toolchain( - name = "arm64-linux", - toolchain = ":echo-arm64-linux-gnu", - toolchain_type = ":type", - # Use constraints to signify what host machines the toolchain is compatible with + name = "downloaded-arm64-linux-gnu", + toolchain = ":info-arm64-linux-gnu", + toolchain_type = "//placholder/toolchain/binary:type", exec_compatible_with = [ - "@toolchain_utils//toolchain/constraint/cpu:arm64", + "@toolchain_utils//toolchain/constraint/cpu:amd64", "@toolchain_utils//toolchain/constraint/os:linux", - # Bazel _assumes_ `glibc` for Linux - # "@toolchain_utils//toolchain/constraint/libc:gnu", + "@toolchain_utils//toolchain/constraint/libc:gnu", + ], + target_compatible_with = [ + "@toolchain_utils//toolchain/constraint/cpu:arm64", + "@toolchain_utils//toolchain/constraint/os:windows", + "@toolchain_utils//toolchain/constraint/libc:ucrt", ], ) + ``` -## Usage +The toolchain can be implicitly registered by the current module in `MODULE.bazel`: + +```py +register_toolchains("//placeholder/toolchain/binary/amd64-linux-gnu:downloaded-amd64-windows-ucrt") +# Or use a recursive registration +# register_toolchains("//placeholder/toolchain/...") +``` -### Toolchain +### Use a toolchain in a Bazel `rule` Declare the usage of the toolchain in a rule definition: ```py def implementation(ctx): - toolchain = ctx.toolchains["//toolchain/echo:type"] - print(toolchain.executable) - print(toolchain.default.files) - print(toolchain.default.runfiles) - print(toolchain.run) + toolchain = ctx.toolchains["//placeholder/toolchain/binary:type"] + + # The `ToolchainInfo` generated by `toolchain_info` is always the same shape + + # `toolchain.default` is the `DefaultInfo` of the wrapped tool + # Use the `DefaultInfo` to forward runfiles through rules, if needed + + # `toolchain.run` can be passed to `ctx.actions.run` to execute the tool + output = ctx.actions.declare_file(ctx.label.name) + args = ctx.actions.args() + args.add(output) + ctx.actions.run( + outputs = [output], + # inputs = [], # likely need some inputs to the rule + executable = toolchain.run, + arguments = [args], + ) + return DefaultInfo(files = depset([output])) example = rule( implementation = implementation, - toolchains = ["//toolchain/echo:type"], + toolchains = ["//placeholder/toolchain/binary:type"], +) +``` + +### Provide a resolved toolchain Make variable + +Due to a [quirk] in Bazel, to retrieve the resolved toolchain as a Make variable another `rule` implementation must perform the toolchain resolution. + +The project provides a repository rule the implements the boilerplate necessary for this. + +Use the `resolved` repository rule in `MODULE.bazel`: + +```py +resolved = use_repo_rule("@toolchain_utils//toolchain/resolved:defs.bzl", "toolchain_resolved") +resolved( + name = "resolved-binary", + toolchain_type = "//placholder/toolchain/binary:type", +) +``` + +A `alias` can then be provided to expose the resolved Make variable to downstream users: + +```py +# Assuming a ruleset named `rules_placeholder` and a tool named `binary` +# This would be in `//placeholder/toolchain/binary/BUILD.bazel` + +# Used for registration. Public so that downstream can register other toolchains +toolchain_type( + name = "type", + visibility = ["//visibility:public"], +) + +# Provides `TemplateVariableInfo` so that the resolved toolchain can be used as Make variable +alias( + name = "resolved", + actual = "@resolved-binary", + visibility = ["//visibility:public"], ) ``` -### Variable +### Use a resolved toolchain in a `genrule` + +A resolved toolchain Make variable can be used in any rule that expands Make variables. -Pass the resolved toolchain to a rule that supports variables: +Commonly, this functionality is used in `genrule` targets: ```py genrule( - toolchains = ["//toolchain/echo:resolved"] + name = "generate", + outputs = ["stdout.log"], + cmd = "$(BINARY) --help > $@", + toolchains = ["@rules_placeholder//placeholder/toolchain/binary:resolved"], +) +``` + +### Test a resolved toolchain + +When developing a toolchain is can be useful to do simple testing of the toolchain resolution. + +This can be performed with `toolchain_test` and a resolved toolchain target. + +```py +# Assuming a ruleset named `rules_placeholder` and a tool named `binary` +# This would be in `//placeholder/toolchain/binary/BUILD.bazel` +load("@toolchain_utils//toolchain/test:defs.bzl", "toolchain_test") + +# Used for registration. Public so that downstream can register other toolchains +toolchain_type( + name = "type", + visibility = ["//visibility:public"], +) + +# Assuming that some `toolchain` targets are added here and registered + +# Provides `TemplateVariableInfo` so that the resolved toolchain can be used as Make variable +alias( + name = "resolved", + actual = "@resolved-binary", + visibility = ["//visibility:public"], +) + +# Add a simple test to ensure that the toolchain is resolved, exits with zero and outputs _something_ to `stdout` +toolchain_test( + name = "test", + args = ["--version"], + toolchains = [":resolved"], ) ``` +The `toolchain_test` can provide `diff` on `stdout`/`stderr`. See the documentation for the rule for more insight. + ## Hermeticity ### POSIX @@ -147,4 +327,4 @@ The `toolchain_test` uses the `FC.exe` binary to compare `stdout`/`stderr` of to Effectively, the ruleset is hermetic. [launcher-cs]: toolchain/launcher/launcher.cs -[resolved]: https://github.com/bazelbuild/bazel/issues/14009 +[quirk]: https://github.com/bazelbuild/bazel/issues/14009 -- GitLab From e20a28d373337abe9733dfda3ea7ad392a206554 Mon Sep 17 00:00:00 2001 From: Matthew Clarkson Date: Fri, 17 Jan 2025 13:44:18 +0000 Subject: [PATCH 2/5] docs(readme): move code comments into Markdown --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 75a16d8..80dcba6 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ toolchain_type( To make organisation easier, it can be useful to create a Bazel package for each execution architecture. -``` +```py # This would be in `//placeholder/toolchain/binary/amd64-linux-gnu/BUILD.bazel` # Where `amd64-linux-gnu` is the execution architecture these toolchains work on -- GitLab From 52e8583ab6a53670b6ab30979a742d260ec2694d Mon Sep 17 00:00:00 2001 From: Matthew Clarkson Date: Fri, 17 Jan 2025 13:44:45 +0000 Subject: [PATCH 3/5] docs(readme): spelling correction --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80dcba6..468775d 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ example = rule( Due to a [quirk] in Bazel, to retrieve the resolved toolchain as a Make variable another `rule` implementation must perform the toolchain resolution. -The project provides a repository rule the implements the boilerplate necessary for this. +The project provides a repository rule that implements the boilerplate necessary for this. Use the `resolved` repository rule in `MODULE.bazel`: -- GitLab From 13a6823f3bd6cc214357d50b1d698a66ef95dc11 Mon Sep 17 00:00:00 2001 From: Matthew Clarkson Date: Fri, 17 Jan 2025 13:45:34 +0000 Subject: [PATCH 4/5] docs(readme): correct example target label --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 468775d..391e740 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ toolchain_info( # No `exec_compatible_with` as Bazel will build the binary for the execution platform toolchain( name = "built-amd64-linux-gnu", - toolchain = ":info", + toolchain = ":info-amd64-linux-gnu", toolchain_type = ":type", target_compatible_with = [ "@toolchain_utils//toolchain/constraint/cpu:amd64", -- GitLab From 446242c818d96940e398eb36fe4b11592b03cdc4 Mon Sep 17 00:00:00 2001 From: Matthew Clarkson Date: Fri, 17 Jan 2025 13:47:31 +0000 Subject: [PATCH 5/5] docs(readme): clarify local toolchain constraints --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 391e740..50c9f2c 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ toolchain_type( # Register the local toolchain # The `target_compatible_with` describes the code generated by the toolchain -# No `exec_compatible_with` as Bazel will build the binary for the execution platform +# `exec_compatible_with` set to local constraints. +# It uses a local symlink path so is not available for remote execution. toolchain( name = "local-amd64-linux-gnu", toolchain = "@which-amd64-linux-gnu-binary", -- GitLab