From a13cf8abf2fab0b15cbe041cfc34176d03ec0f86 Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 26 Jun 2024 10:49:08 +0100 Subject: [PATCH 1/2] fix: support Windows Needed to switch to piping the data into `tar`. The `PATH` override does not work on Windows. This has the added benefit that we do not depend on any `tar` specific flags. --- tar/codec.bzl | 17 ---------- tar/extract/rule.bzl | 23 ++++---------- tar/path.bzl | 22 ------------- tar/pipe/BUILD.bazel | 8 +++++ tar/pipe/nt.tmpl.bat | 27 ++++++++++++++++ tar/pipe/posix.tmpl.sh | 31 ++++++++++++++++++ tar/pipe/subrule.bzl | 72 ++++++++++++++++++++++++++++++++++++++++++ tar/unpack/rule.bzl | 25 +++++---------- 8 files changed, 153 insertions(+), 72 deletions(-) delete mode 100644 tar/codec.bzl delete mode 100644 tar/path.bzl create mode 100644 tar/pipe/BUILD.bazel create mode 100644 tar/pipe/nt.tmpl.bat create mode 100644 tar/pipe/posix.tmpl.sh create mode 100644 tar/pipe/subrule.bzl diff --git a/tar/codec.bzl b/tar/codec.bzl deleted file mode 100644 index a481a07..0000000 --- a/tar/codec.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load(":path.bzl", "path") - -visibility("//...") - -MAP = { - "gz": ("-z", "gzip", "@rules_gzip//gzip/toolchain/pigz:type"), - "bz2": ("-j", "bzip2", "@rules_bzip2//bzip2/toolchain/bzip2:type"), - "xz": ("-J", "xz", "@rules_xz//xz/toolchain/xz:type"), - "zst": ("--zstd", "zstd", "@rules_zstd//zstd/toolchain/zstd:type"), -} - -def codec(ctx, src): - if src.extension not in MAP: - return (), (), {} - arg, basename, toolchain = MAP[src.extension] - tools, env = path(ctx, {basename: toolchain}) - return (arg,), tools, env diff --git a/tar/extract/rule.bzl b/tar/extract/rule.bzl index 21fddab..00f4e4c 100644 --- a/tar/extract/rule.bzl +++ b/tar/extract/rule.bzl @@ -1,4 +1,4 @@ -load("//tar:codec.bzl", "codec") +load("//tar/pipe:subrule.bzl", _ATTRS = "ATTRS", _tar = "implementation", _toolchains = "toolchains") visibility("//tar/...") @@ -18,7 +18,7 @@ tar_extract( The `outs` labels `:some/member.txt` and `:some/other/member.txt` can be used in subsequent rules. """ -ATTRS = { +ATTRS = _ATTRS | { "src": attr.label( doc = "An archive to extract decleared files from.", mandatory = True, @@ -35,26 +35,23 @@ def _name(label): return label.name def implementation(ctx): - tar = ctx.toolchains["//tar/toolchain/tar:type"] - - flags, tools, env = codec(ctx, ctx.file.src) + # TODO: switch this to a subrule, when available + tar, tools = _tar(ctx, ctx.file.src, _template = ctx.file._template) args = ctx.actions.args() args.add("-C").add("{}/{}".format(ctx.bin_dir.path, ctx.label.package)) - args.add("-xvf").add(ctx.file.src.path) - args.add_all(flags) + args.add("-x") args.add_all(ctx.attr.outs, map_each = _name) ctx.actions.run( outputs = ctx.outputs.outs, inputs = [ctx.file.src], arguments = [args], - executable = tar.run, + executable = tar, tools = tools, mnemonic = "TarExtract", progress_message = "extracting %{input} files", toolchain = "//tar/toolchain/tar:type", - env = env, ) return DefaultInfo(files = depset(ctx.outputs.outs)) @@ -63,13 +60,7 @@ tar_extract = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - toolchains = [ - "//tar/toolchain/tar:type", - "@rules_zstd//zstd/toolchain/zstd:type", - "@rules_bzip2//bzip2/toolchain/bzip2:type", - "@rules_gzip//gzip/toolchain/pigz:type", - "@rules_xz//xz/toolchain/xz:type", - ], + toolchains = _toolchains, ) extract = tar_extract diff --git a/tar/path.bzl b/tar/path.bzl deleted file mode 100644 index a372ce7..0000000 --- a/tar/path.bzl +++ /dev/null @@ -1,22 +0,0 @@ -visibility("//...") - -def path(ctx, links): - tools = [] - - for basename, type in links.items(): - symlink = ctx.actions.declare_file("{}.bin/{}".format(ctx.label.name, basename)) - toolchain = ctx.toolchains[type] - - ctx.actions.symlink( - output = symlink, - target_file = toolchain.executable, - is_executable = True, - ) - tools.append(symlink) - tools.append(toolchain.run) - - env = { - "PATH": tools[0].dirname, - } - - return tools, env diff --git a/tar/pipe/BUILD.bazel b/tar/pipe/BUILD.bazel new file mode 100644 index 0000000..d986794 --- /dev/null +++ b/tar/pipe/BUILD.bazel @@ -0,0 +1,8 @@ +alias( + name = "template", + actual = select({ + "@toolchain_utils//toolchain/constraint/os:windows": ":nt.tmpl.bat", + "//conditions:default": ":posix.tmpl.sh", + }), + visibility = ["//:__subpackages__"], +) diff --git a/tar/pipe/nt.tmpl.bat b/tar/pipe/nt.tmpl.bat new file mode 100644 index 0000000..f001ee0 --- /dev/null +++ b/tar/pipe/nt.tmpl.bat @@ -0,0 +1,27 @@ +@echo off + +:: Enable Batch extensions +verify other 2>nul +setlocal EnableExtensions +if errorlevel 1 ( + echo>&2.Failed to enable extensions + exit /b 120 +) + +:: Bazel substitutions +for /f usebackq %%a in ('{{tar}}') do set "TAR=%%~fa" +for /f usebackq %%a in ('{{src}}') do set "SRC=%%~a" +for /f usebackq %%a in ('{{decompress}}') do set "DECOMPRESS=%%~a" + +:: Run `tar` +if not exist "%DECOMPRESS%" ( + "%TAR%" -f "%SRC%" %* + exit /b "%ERRORLEVEL%" +) + +:: Convert `DECOMPRESS` into a Windows path to allow execution +for /f usebackq %%a in ('%DECOMPRESS%') do set "DECOMPRESS=%%~fa" + +:: Run the decompression +"%DECOMPRESS%" -cd "%SRC%" | "%TAR%" %* +exit /b "%ERRORLEVEL%" \ No newline at end of file diff --git a/tar/pipe/posix.tmpl.sh b/tar/pipe/posix.tmpl.sh new file mode 100644 index 0000000..eccfdf9 --- /dev/null +++ b/tar/pipe/posix.tmpl.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env sh + +# Strict Shell +set -o errexit +set -o nounset + +# Bazel substitutions +TAR="{{tar}}" +DECOMPRESS="{{decompress}}" +SRC="{{src}}" +readonly TAR DECOMPRESS SRC + +# Run `tar` +if test ! -x "${DECOMPRESS}"; then + "${TAR}" -f "${SRC}" "${@}" + exit 0 +fi + +# Workaround no POSIX `pipefail` +readonly FAIL=".fail.$$" +{ "${DECOMPRESS}" -fcd "${SRC}" || echo "$?" >"${FAIL}"; } | "${TAR}" "${@}" +if test ! -s "${FAIL}"; then + exit 0 +fi + +while IFS= read -r STATUS; do + exit "${STATUS}" +done <"${FAIL}" + +printf >&2 'Pipefail workaround failed\n' +exit 1 diff --git a/tar/pipe/subrule.bzl b/tar/pipe/subrule.bzl new file mode 100644 index 0000000..7bb0c9d --- /dev/null +++ b/tar/pipe/subrule.bzl @@ -0,0 +1,72 @@ +visibility("//...") + +DOC = """Provides a `tar` binary that can perform decompression based on an input file. + +```py +def implementation(ctx): + tar, tools = tar_pipe(ctx.file.src) + # Pass `tar`/`tools` to `ctx.actions.run` + +something = rule( + implementation = implementation, + subrules = [tar_pipe], +) +``` +""" + +ATTRS = { + "_template": attr.label( + doc = "The pipe template.", + allow_single_file = True, + default = ":template", + ), +} + +toolchains = ( + "//tar/toolchain/tar:type", + "@rules_zstd//zstd/toolchain/zstd:type", + "@rules_bzip2//bzip2/toolchain/bzip2:type", + "@rules_gzip//gzip/toolchain/pigz:type", + "@rules_xz//xz/toolchain/xz:type", +) + +MAP = { + "tar": None, + "gz": "@rules_gzip//gzip/toolchain/pigz:type", + "bz2": "@rules_bzip2//bzip2/toolchain/bzip2:type", + "xz": "@rules_xz//xz/toolchain/xz:type", + "zst": "@rules_zstd//zstd/toolchain/zstd:type", +} + +def implementation(ctx, src, *, _template): + tar = ctx.toolchains["//tar/toolchain/tar:type"] + tools = (tar.run,) + + substitutions = ctx.actions.template_dict() + substitutions.add("{{tar}}", tar.executable.path) + substitutions.add("{{src}}", src.path) + kind = MAP[src.extension] + if kind != None: + decompress = ctx.toolchains[kind] + tools += (decompress.run,) + substitutions.add("{{decompress}}", decompress.executable.path) + + rendered = ctx.actions.declare_file("{}.tar.pipe/tar.{}".format(ctx.label.name, _template.extension)) + ctx.actions.expand_template( + template = _template, + output = rendered, + computed_substitutions = substitutions, + is_executable = True, + ) + + return rendered, tools + +# TODO: enable `subrule` when they become non-experimental, use `bazel_features` to detect +#tar_pipe = subrule( +# # doc = DOC, +# implementation = implementation, +# attrs = ATTRS, +# toolchains = toolchains, +#) + +#pipe = tar_pipe diff --git a/tar/unpack/rule.bzl b/tar/unpack/rule.bzl index 844af3b..064f736 100644 --- a/tar/unpack/rule.bzl +++ b/tar/unpack/rule.bzl @@ -1,4 +1,4 @@ -load("//tar:codec.bzl", "codec") +load("//tar/pipe:subrule.bzl", _ATTRS = "ATTRS", _tar = "implementation", _toolchains = "toolchains") visibility("//tar/...") @@ -13,7 +13,7 @@ tar_unpack( ``` """ -ATTRS = { +ATTRS = _ATTRS | { "src": attr.label( doc = "An archive to extract into a declared directory (`TreeArtifact`).", mandatory = True, @@ -22,26 +22,23 @@ ATTRS = { } def implementation(ctx): - tar = ctx.toolchains["//tar/toolchain/tar:type"] + # TODO: switch this to a subrule, when available + tar, tools = _tar(ctx, ctx.file.src, _template = ctx.file._template) - output_dir = ctx.actions.declare_directory(ctx.attr.name) - - flags, tools, env = codec(ctx, ctx.file.src) + output_dir = ctx.actions.declare_directory(ctx.label.name) args = ctx.actions.args() - args.add("-xvf").add(ctx.file.src.path) + args.add("-x") args.add("-C").add(output_dir.path) - args.add_all(flags) ctx.actions.run( outputs = [output_dir], inputs = [ctx.file.src], arguments = [args], - executable = tar.run, + executable = tar, tools = tools, mnemonic = "TarUnpack", progress_message = "unpacking %{input}", - env = env, ) depset_files = depset([output_dir]) @@ -51,13 +48,7 @@ tar_unpack = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - toolchains = [ - "//tar/toolchain/tar:type", - "@rules_zstd//zstd/toolchain/zstd:type", - "@rules_bzip2//bzip2/toolchain/bzip2:type", - "@rules_gzip//gzip/toolchain/pigz:type", - "@rules_xz//xz/toolchain/xz:type", - ], + toolchains = _toolchains, ) unpack = tar_unpack -- GitLab From 0d26bd3d490c4994e5f5d9202266194c987afbfb Mon Sep 17 00:00:00 2001 From: Matt Clarkson Date: Wed, 26 Jun 2024 12:06:03 +0100 Subject: [PATCH 2/2] docs(readme): add rules example --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index f619c0d..8278401 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,31 @@ Add the following to `MODULE.bazel`: bazel_dep(name="rules_tar", version="0.0.0") ``` +Use the unpacking and extraction rules: + +```py +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/extract:defs.bzl", "tar_extract") + +tar_unpack( + name = "unpack", + src = "some.tar.bz2", +) + +tar_extract( + name = "extract", + src = "some.tar.zst", + outs = [ + "a/specific/member.txt", + "another/member.txt", + ] +) +``` + +> Note: use [rules_pkg] to pack archives. +> +> This ruleset _may_ support hermetic packing without a Python toolchain in the future. + # Hermeticity The module is entirely hermetic, using `@ape//:tar` as the default toolchain. @@ -31,3 +56,4 @@ common --registry=https://gitlab.arm.com/bazel/rules_tar/-/releases/v1.0.0-alpha Then a GitLab release version can be used in `bazel_dep`. [bcr]: https://registry.bazel.build/ +[rules_pkg]: https://github.com/bazelbuild/rules_pkg -- GitLab