diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ad9321e255411af95be0d6d33f46811b64bea92..b9e8e189e4225bc8e6657604b4ebf1803dd9bf3c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,87 +1,13 @@ include: - - component: "${CI_SERVER_HOST}/ci/component/bazelisk/bazelisk@v1.0.0-beta.5" + - component: "${CI_SERVER_HOST}/ci/component/bazelisk/ruleset@v1.0.0" inputs: - variables: | - CI_PROJECT_DIR - CI_PROJECT_ID - CI_REMOTE_EXECUTOR - CI_REMOTE_CACHE - CI_REMOTE_HEADER - CI_REMOTE_JOBS - CI_BES_RESULTS_URL - CI_BES_BACKEND - CI_PROJECT_URL - CI_COMMIT_REF_NAME - CI_COMMIT_SHA - CI_SERVER_HOST - CI_LOCKFILE_MODE + versions: + - 7.1.0 + - 7.x + - last_rc + configs: + - local default: tags: - arm64 - -.test: - extends: .bazelisk - cache: - - !reference [.bazelisk, cache] - - key: "bazel-cache-${CI_PROJECT_ID}" - paths: - - ".cache/bazel/disk" - - ".cache/bazel/repo" - variables: - CI_LOCKFILE_MODE: error - CONFIG: local - script: - - (cd "${ROOT}"; bazelisk test --config="${CONFIG}" //...) - -config: - extends: .test - parallel: - matrix: - - ROOT: - - . - - e2e - CONFIG: - - local - - remote - -version: - extends: .test - variables: - CI_LOCKFILE_MODE: "off" - parallel: - matrix: - - ROOT: - - . - - e2e - USE_BAZEL_VERSION: - - 7.1.0 - - 7.x - - last_rc - -# TODO: switch this out for `rules_semantic_release` -semantic-release: - extends: .test - stage: .post - needs: - - config - - version - image: node:lts - cache: - key: - prefix: "node" - files: - - package-lock.json - paths: - - node_modules - - .cache/npm - before_script: - - npm config --location project set cache "${CI_PROJECT_DIR}/.cache/npm" - - npm ci --prefer-offline - - !reference [.bazelisk, before_script] - script: - - npx semantic-release - rules: - - if: $CI_COMMIT_TAG - when: never - - if: $CI_COMMIT_REF_PROTECTED == "true" diff --git a/MODULE.bazel b/MODULE.bazel index 77492a4896d7000356b7bcd0ba344d8ecc8d2c3a..323dee79997364c5ac6f5fd3e3378e47af3472d1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -44,4 +44,8 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_bmatcuk_doublestar_v4", + "com_github_dsnet_compress", + "com_github_h2non_filetype", + "com_github_klauspost_compress", + "com_github_ulikunitz_xz", ) diff --git a/README.md b/README.md index 66167aa2f79db506510896c2f1cf14fec71157d0..3b0604236a006eca846a1a6d9ae13e458dc5ea7a 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ bazel_dep(name="rules_tar", version="0.0.0") Use the unpacking and extraction rules: ```py -load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/concatenate:defs.bzl", "tar_concatenate") +load("@rules_tar//tar/filter:defs.bzl", "tar_filter") load("@rules_tar//tar/extract:defs.bzl", "tar_extract") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") tar_unpack( name = "unpack", @@ -29,6 +31,23 @@ tar_extract( "another/member.txt", ] ) + +tar_concatenate( + name = "concatenate.tar.zst", + srcs = [ + "some.tar.gz", + "someother.tar.bzip2", + "something.tar.zst", + ], +) + +tar_filter( + name = "filtered.tar.zst", + srcs = "some.tar.xz", + patterns = [ + "**/*.txt", # Remove all `.txt` members + ], +) ``` > Note: use [rules_pkg] to pack archives. diff --git a/e2e/concatenate/bzip2/BUILD.bazel b/e2e/concatenate/bzip2/BUILD.bazel index 970f34ea164ee649a91d9877399872332e91c82b..e1adf47d31da5d220bb216fd577e8e8ee6004b66 100644 --- a/e2e/concatenate/bzip2/BUILD.bazel +++ b/e2e/concatenate/bzip2/BUILD.bazel @@ -8,7 +8,6 @@ tar_concatenate( "//fixture:txt.tar.bz2", "//fixture/nested:txt.tar.bz2", ], - compress = "@rules_tar//tar/compress:bzip2", ) tar_unpack( diff --git a/e2e/concatenate/compress/bzip2/BUILD.bazel b/e2e/concatenate/compress/bzip2/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..970f34ea164ee649a91d9877399872332e91c82b --- /dev/null +++ b/e2e/concatenate/compress/bzip2/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_diff//diff/directory/test:defs.bzl", "diff_directory_test") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/concatenate:defs.bzl", "tar_concatenate") + +tar_concatenate( + name = "concatenated.tar.bz2", + srcs = [ + "//fixture:txt.tar.bz2", + "//fixture/nested:txt.tar.bz2", + ], + compress = "@rules_tar//tar/compress:bzip2", +) + +tar_unpack( + name = "unpack", + src = ":concatenated.tar.bz2", +) + +diff_directory_test( + name = "diff", + size = "small", + a = "//fixture/directory:all", + b = ":unpack", +) diff --git a/e2e/concatenate/compress/gzip/BUILD.bazel b/e2e/concatenate/compress/gzip/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..046912526661a769aaa7829a94135e97f7575db8 --- /dev/null +++ b/e2e/concatenate/compress/gzip/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_diff//diff/directory/test:defs.bzl", "diff_directory_test") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/concatenate:defs.bzl", "tar_concatenate") + +tar_concatenate( + name = "concatenated.tar.gz", + srcs = [ + "//fixture:txt.tar.gz", + "//fixture/nested:txt.tar.gz", + ], + compress = "@rules_tar//tar/compress:gzip", +) + +tar_unpack( + name = "unpack", + src = ":concatenated.tar.gz", +) + +diff_directory_test( + name = "diff", + size = "small", + a = "//fixture/directory:all", + b = ":unpack", +) diff --git a/e2e/concatenate/compress/xz/BUILD.bazel b/e2e/concatenate/compress/xz/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..fc9037d10e5ef5c91d4b8fcc8277e9011e99f8af --- /dev/null +++ b/e2e/concatenate/compress/xz/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_diff//diff/directory/test:defs.bzl", "diff_directory_test") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/concatenate:defs.bzl", "tar_concatenate") + +tar_concatenate( + name = "concatenated.tar.xz", + srcs = [ + "//fixture:txt.tar.xz", + "//fixture/nested:txt.tar.xz", + ], + compress = "@rules_tar//tar/compress:xz", +) + +tar_unpack( + name = "unpack", + src = ":concatenated.tar.xz", +) + +diff_directory_test( + name = "diff", + size = "small", + a = "//fixture/directory:all", + b = ":unpack", +) diff --git a/e2e/concatenate/compress/zstd/BUILD.bazel b/e2e/concatenate/compress/zstd/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..6519d6ea180136da63ed9f5564f4b2a7ac3df3fe --- /dev/null +++ b/e2e/concatenate/compress/zstd/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_diff//diff/directory/test:defs.bzl", "diff_directory_test") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/concatenate:defs.bzl", "tar_concatenate") + +tar_concatenate( + name = "concatenated.tar.zst", + srcs = [ + "//fixture:txt.tar.zst", + "//fixture/nested:txt.tar.zst", + ], + compress = "@rules_tar//tar/compress:zstd", +) + +tar_unpack( + name = "unpack", + src = ":concatenated.tar.zst", +) + +diff_directory_test( + name = "diff", + size = "small", + a = "//fixture/directory:all", + b = ":unpack", +) diff --git a/e2e/concatenate/gzip/BUILD.bazel b/e2e/concatenate/gzip/BUILD.bazel index 046912526661a769aaa7829a94135e97f7575db8..eabf52640bc4cf207668edf9f01fe2ff7d13e610 100644 --- a/e2e/concatenate/gzip/BUILD.bazel +++ b/e2e/concatenate/gzip/BUILD.bazel @@ -8,7 +8,6 @@ tar_concatenate( "//fixture:txt.tar.gz", "//fixture/nested:txt.tar.gz", ], - compress = "@rules_tar//tar/compress:gzip", ) tar_unpack( diff --git a/e2e/concatenate/xz/BUILD.bazel b/e2e/concatenate/xz/BUILD.bazel index fc9037d10e5ef5c91d4b8fcc8277e9011e99f8af..2d10d1f10e84e55c38076bf6a273e1d9034196d0 100644 --- a/e2e/concatenate/xz/BUILD.bazel +++ b/e2e/concatenate/xz/BUILD.bazel @@ -8,7 +8,6 @@ tar_concatenate( "//fixture:txt.tar.xz", "//fixture/nested:txt.tar.xz", ], - compress = "@rules_tar//tar/compress:xz", ) tar_unpack( diff --git a/e2e/concatenate/zstd/BUILD.bazel b/e2e/concatenate/zstd/BUILD.bazel index 64bc8982b2b504721f1ff8c6ee15a3845e078497..8382c93cf814c048b3d5aafb00a54cd36599a928 100644 --- a/e2e/concatenate/zstd/BUILD.bazel +++ b/e2e/concatenate/zstd/BUILD.bazel @@ -8,7 +8,6 @@ tar_concatenate( "//fixture:txt.tar.zst", "//fixture/nested:txt.tar.zst", ], - compress = "@rules_tar//tar/compress:zstd", ) tar_unpack( @@ -21,6 +20,4 @@ diff_directory_test( size = "small", a = "//fixture/directory:all", b = ":unpack", - # FIXME: not sure why `zstd` decompression is failing - tags = ["manual"], ) diff --git a/e2e/filter/BUILD.bazel b/e2e/filter/BUILD.bazel index 3cf0c9978b0da8830bd1d36b6b29499bdd44d685..da70d0a6d130fb25cae0966d90305676a6ae3cc9 100644 --- a/e2e/filter/BUILD.bazel +++ b/e2e/filter/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_tar//tar/filter:defs.bzl", "tar_filter") tar_filter( name = "filtered.tar", src = "//fixture:fixture.tar", - patterns = ["fixture/*.txt"], + patterns = ["fixture/nested/*.txt"], ) tar_unpack( diff --git a/e2e/filter/bzip2/BUILD.bazel b/e2e/filter/bzip2/BUILD.bazel index 0fca25e4824edda116d181c33b82a88a3269455e..9fa22be22d439f1a46395382b04ec343328620b2 100644 --- a/e2e/filter/bzip2/BUILD.bazel +++ b/e2e/filter/bzip2/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_tar//tar/filter:defs.bzl", "tar_filter") tar_filter( name = "filtered.tar.bz2", src = "//fixture:fixture.tar.bz2", - patterns = ["fixture/*.txt"], + patterns = ["fixture/nested/*.txt"], ) tar_unpack( diff --git a/e2e/filter/gzip/BUILD.bazel b/e2e/filter/gzip/BUILD.bazel index e1eea4bfdc8b34324d7517f9d5869ff629f0638e..6423b70ee1c23c0d5fcbc23489bd145860b2902e 100644 --- a/e2e/filter/gzip/BUILD.bazel +++ b/e2e/filter/gzip/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_tar//tar/filter:defs.bzl", "tar_filter") tar_filter( name = "filtered.tar.gz", src = "//fixture:fixture.tar.gz", - patterns = ["fixture/*.txt"], + patterns = ["fixture/nested/*.txt"], ) tar_unpack( diff --git a/e2e/filter/negated/BUILD.bazel b/e2e/filter/negated/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..17de95f114acdbb3c9ad5001d30b4c70e8586b89 --- /dev/null +++ b/e2e/filter/negated/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_diff//diff/directory/test:defs.bzl", "diff_directory_test") +load("@rules_tar//tar/unpack:defs.bzl", "tar_unpack") +load("@rules_tar//tar/filter:defs.bzl", "tar_filter") + +tar_filter( + name = "filtered.tar", + src = "//fixture:fixture.tar", + patterns = [ + "!fixture/hello-world.txt", + "!fixture/goodbye-world.txt", + "**/*", + ], +) + +tar_unpack( + name = "unpack", + src = ":filtered.tar", +) + +diff_directory_test( + name = "diff", + size = "small", + a = "//fixture/directory:txt", + b = ":unpack", +) diff --git a/e2e/filter/xz/BUILD.bazel b/e2e/filter/xz/BUILD.bazel index 4728babb3d4fcd72e3755fb7e7d24638a96e8ae6..87cb5f7122b890c258fd89af6bc2e7763cf0cba3 100644 --- a/e2e/filter/xz/BUILD.bazel +++ b/e2e/filter/xz/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_tar//tar/filter:defs.bzl", "tar_filter") tar_filter( name = "filtered.tar.xz", src = "//fixture:fixture.tar.xz", - patterns = ["fixture/*.txt"], + patterns = ["fixture/nested/*.txt"], ) tar_unpack( diff --git a/e2e/filter/zstd/BUILD.bazel b/e2e/filter/zstd/BUILD.bazel index 5d3a4a1252cc2c48802376bc50ecca65f64c6666..60535d47d4b9cb5125e3fafd7400d7c9a16db20f 100644 --- a/e2e/filter/zstd/BUILD.bazel +++ b/e2e/filter/zstd/BUILD.bazel @@ -5,7 +5,7 @@ load("@rules_tar//tar/filter:defs.bzl", "tar_filter") tar_filter( name = "filtered.tar.zst", src = "//fixture:fixture.tar.zst", - patterns = ["fixture/*.txt"], + patterns = ["fixture/nested/*.txt"], ) tar_unpack( diff --git a/go.mod b/go.mod index 825715016580841ef8349bd3e7187643b55cb6f6..91d1c4aa50119c198a119f27354f060504054ac6 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,11 @@ module gitlab.arm.com/bazel/rules_tar/v1 go 1.21.3 -require github.com/bmatcuk/doublestar/v4 v4.6.1 +require ( + github.com/bmatcuk/doublestar/v4 v4.6.1 + github.com/h2non/filetype v1.1.3 + github.com/klauspost/compress v1.17.10 + github.com/ulikunitz/xz v0.5.12 + github.com/dsnet/compress v0.0.1 +) + diff --git a/go.sum b/go.sum index 9eaac920e8546dd2de493872016f2ba951f6afcd..5539941f32fea56414aa8b77b54bafd1ef6a9ba1 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,14 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/tar/BUILD.bazel b/tar/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/tar/compress/BUILD.bazel b/tar/compress/BUILD.bazel index 4a38424a090b2020bb3cc16a81d94979215e2cac..c196c2df275f9304f168490e8f221f820ff5f529 100644 --- a/tar/compress/BUILD.bazel +++ b/tar/compress/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_go//go:def.bzl", "go_library") + alias( name = "cat", actual = "@rules_coreutils//coreutils/toolchain/cat:resolved", @@ -27,3 +29,15 @@ alias( actual = "@rules_zstd//zstd/toolchain/zstd:resolved", visibility = ["//visibility:public"], ) + +go_library( + name = "compress", + srcs = ["compress.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/compress", + visibility = ["//:__subpackages__"], + deps = [ + "@com_github_dsnet_compress//bzip2:go_default_library", + "@com_github_klauspost_compress//zstd:go_default_library", + "@com_github_ulikunitz_xz//:go_default_library", + ], +) diff --git a/tar/compress/compress.go b/tar/compress/compress.go new file mode 100644 index 0000000000000000000000000000000000000000..ead81ae165fc4dd0d8e26806e532cce3886673c1 --- /dev/null +++ b/tar/compress/compress.go @@ -0,0 +1,108 @@ +package compress + +import ( + "compress/gzip" + "fmt" + "io" + "os/exec" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +type subprocess struct { + stdin io.WriteCloser + wait chan error +} + +func NewSubprocess(w io.Writer, value string) (io.WriteCloser, error) { + cmd := exec.Command(value) + cmd.Stdout = w + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + + if err = cmd.Start(); err != nil { + return nil, err + } + + wait := make(chan error, 1) + go func() { + wait <- cmd.Wait() + }() + + return subprocess{stdin, wait}, nil +} + +func (s subprocess) Write(b []byte) (int, error) { + return s.stdin.Write(b) +} + +func (s subprocess) Close() error { + if err := s.stdin.Close(); err != nil { + return err + } + + if err := <-s.wait; err != nil { + return err + } + + return nil +} + +type writeCloser struct { + wrapped io.Closer + compress io.WriteCloser +} + +func (w writeCloser) Write(b []byte) (int, error) { + return w.compress.Write(b) +} + +func (w writeCloser) Close() error { + if err := w.compress.Close(); err != nil { + return err + } + return w.wrapped.Close() +} + +type passthrough struct { + w io.Writer +} + +func (p passthrough) Write(b []byte) (int, error) { + return p.w.Write(b) +} + +func (w passthrough) Close() error { + return nil +} + +func extension(w io.WriteCloser, ext string) (io.WriteCloser, error) { + switch ext { + case ".tar": + return passthrough{w}, nil + case ".gz", ".tar.gz", ".tgz": + return gzip.NewWriterLevel(w, gzip.BestCompression) + case ".bz2", ".tar.bz2": + return bzip2.NewWriter(w, nil) + case ".xz", ".tar.xz": + return xz.NewWriter(w) + case ".zst", ".tar.zst": + return zstd.NewWriter(w) + default: + return nil, fmt.Errorf("unknown extension `%s`", ext) + } +} + +func NewExtension(w io.WriteCloser, ext string) (io.WriteCloser, error) { + c, err := extension(w, ext) + if err != nil { + return nil, err + } + + return writeCloser{w, c}, nil +} diff --git a/tar/concatenate/BUILD.bazel b/tar/concatenate/BUILD.bazel index f2fea36e2e9d424db2dd1546230368ea3dc5c4d7..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/tar/concatenate/BUILD.bazel +++ b/tar/concatenate/BUILD.bazel @@ -1,23 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -alias( - name = "template", - actual = select({ - "//conditions:default": ":posix.tmpl.sh", - }), - visibility = ["//:__subpackages__"], -) - -go_binary( - name = "concatenate", - embed = [":concatenate_lib"], - pure = "on", - visibility = ["//visibility:public"], -) - -go_library( - name = "concatenate_lib", - srcs = ["main.go"], - importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/concatenate", - visibility = ["//visibility:private"], -) diff --git a/tar/concatenate/main.go b/tar/concatenate/main.go deleted file mode 100644 index d17375970592ef51e5c7ffcd629b34e4bdf1c5fb..0000000000000000000000000000000000000000 --- a/tar/concatenate/main.go +++ /dev/null @@ -1,318 +0,0 @@ -package main - -import ( - "archive/tar" - "bufio" - "flag" - "fmt" - "io" - "log" - "os" - "path" - "syscall" -) - -type Input interface { - Read(b []byte) (int, error) - Peek(n int) ([]byte, error) - Discard(n int) (int, error) - Close() error -} - -type FileInput struct { - f *os.File - b *bufio.Reader -} - -func NewFileInput(f *os.File) FileInput { - return FileInput{f, bufio.NewReader(f)} -} - -func (i FileInput) Read(b []byte) (int, error) { - return i.b.Read(b) -} - -func (i FileInput) Peek(n int) ([]byte, error) { - return i.b.Peek(n) -} - -func (i FileInput) Discard(n int) (int, error) { - return i.b.Discard(n) -} - -func (i FileInput) Close() error { - return i.f.Close() -} - -type StdInput struct { - b *bufio.Reader -} - -func NewStdInput() StdInput { - return StdInput{bufio.NewReader(os.Stdin)} -} - -func (i StdInput) Read(b []byte) (int, error) { - return i.b.Read(b) -} - -func (i StdInput) Peek(n int) ([]byte, error) { - return i.b.Peek(n) -} - -func (i StdInput) Discard(n int) (int, error) { - return i.b.Discard(n) -} - -func (i StdInput) Close() error { - return nil -} - -func open(v string) (Input, error) { - if v == "-" { - return NewStdInput(), nil - } - - f, err := os.Open(v) - - switch e := err.(type) { - case *os.PathError: - if e.Err == syscall.ENOENT { - if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { - f, err = os.Open(path.Join(bwd, v)) - } - } - } - - if err != nil { - return nil, err - } - - return NewFileInput(f), nil -} - -type Skip interface { - Read(b []byte) (int, error) - Peek(n int) ([]byte, error) - Discard(n int) (int, error) -} - -func skip(f Skip) error { - for { - buf, err := f.Peek(512) - - if err == io.EOF { - return err - } - - if err != nil { - return err - } - - for i, b := range buf { - if b != 0 { - if _, err := f.Discard(i); err != nil { - return err - } - return nil - } - } - - if _, err := f.Discard(512); err != nil { - return err - } - } - - return nil -} - -type Output interface { - Write(b []byte) (int, error) - Close() error -} - -type FileOutput struct { - f *os.File -} - -func NewFileOutput(f *os.File) FileOutput { - return FileOutput{f} -} - -func (i FileOutput) Write(b []byte) (int, error) { - return i.f.Write(b) -} - -func (i FileOutput) Close() error { - return i.f.Close() -} - -type StdOutput struct{} - -func NewStdOutput() StdOutput { - return StdOutput{} -} - -func (i StdOutput) Write(b []byte) (int, error) { - return os.Stdout.Write(b) -} - -func (i StdOutput) Close() error { - return nil -} - -type OutputFlag struct { - o Output -} - -func NewStdOutputFlag() OutputFlag { - return OutputFlag{NewStdOutput()} -} - -func (f OutputFlag) Write(b []byte) (int, error) { - return f.o.Write(b) -} - -func (f OutputFlag) Close() error { - return f.o.Close() -} - -func (f *OutputFlag) Set(value string) error { - if value == "-" { - f.o = NewStdOutput() - } - - if !path.IsAbs(value) { - if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { - value = path.Join(bwd, value) - } - } - - x, err := os.OpenFile(value, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - f.o = NewFileOutput(x) - return nil -} - -func (f OutputFlag) String() string { - switch f.o.(type) { - case *StdOutput: - return "-" - case *FileOutput: - return "" - } - return "" -} - -type Duplicate int - -const ( - DuplicateError Duplicate = 0 - DuplicateSkip Duplicate = 1 - DuplicateContinue Duplicate = 2 -) - -func (d *Duplicate) Set(value string) error { - switch value { - case "error": - *d = DuplicateError - case "skip": - *d = DuplicateSkip - case "continue": - *d = DuplicateContinue - default: - return fmt.Errorf("Invalid duplicate value: %s", value) - } - return nil -} - -func (d Duplicate) String() string { - switch d { - case DuplicateError: - return "error" - case DuplicateSkip: - return "skip" - case DuplicateContinue: - return "continue" - } - return "unknown" -} - -type Flags struct { - Output OutputFlag - Duplicate Duplicate -} - -func main() { - flags := Flags{NewStdOutputFlag(), DuplicateError} - flag.Var(&flags.Output, "output", "Output location for concatenated `tar`.") - flag.Var(&flags.Duplicate, "duplicate", "Operation when duplicate archive member names are encountered.") - flag.Parse() - - w := tar.NewWriter(flags.Output) - defer w.Close() - - args := flag.Args() - if len(args) == 0 { - args = []string{"-"} - } - - paths := make(map[string]struct{}) - for _, archive := range args { - f, err := open(archive) - if err != nil { - log.Fatal(err) - } - defer f.Close() - - for { - r := tar.NewReader(f) - for { - h, err := r.Next() - if err == io.EOF { - break - } - if err != nil { - log.Fatal(err) - } - if _, exists := paths[h.Name]; exists { - switch flags.Duplicate { - case DuplicateError: - log.Fatalf("Name already exists in a previous archive: %s", h.Name) - case DuplicateSkip: - continue - case DuplicateContinue: - default: - log.Fatalf("Unknown duplicate handling: %d", flags.Duplicate) - } - } - paths[h.Name] = struct{}{} - if err := w.WriteHeader(h); err != nil { - log.Fatal(err) - } - if _, err := io.Copy(w, r); err != nil { - log.Fatal(err) - } - } - - err := skip(f) - if err == io.EOF { - break - } else if err == nil { - continue - } - log.Fatal(err) - } - - if err := f.Close(); err != nil { - log.Fatal(err) - } - } - - if err := w.Close(); err != nil { - log.Fatal(err) - } -} diff --git a/tar/concatenate/posix.tmpl.sh b/tar/concatenate/posix.tmpl.sh deleted file mode 100644 index 815b5940435bb11c3d7f798a2ba030e37184b070..0000000000000000000000000000000000000000 --- a/tar/concatenate/posix.tmpl.sh +++ /dev/null @@ -1,70 +0,0 @@ -#! /usr/bin/env sh - -# Strict Shell -set -o errexit -set -o nounset - -# Bazel replacements -CONCATENATE="{{concatenate}}" -CAT="{{cat}}" -GZIP="{{gzip}}" -BZIP2="{{bzip2}}" -XZ="{{xz}}" -ZSTD="{{zstd}}" -COMPRESS="{{compress}}" -readonly CONCATENATE CAT GZIP BZIP2 XZ ZSTD COMPRESS - -# Parse the output -test "--output" = "${1}" -shift -OUTPUT="${1}" -readonly OUTPUT -shift - -# Parse duplicate handling -test "--duplicate" = "${1}" -shift -DUPLICATE="${1}" -readonly DUPLICATE -shift - -# Concatenate -readonly FAIL=".fail.$$" -{ - for TAR in "${@}"; do - case "${TAR}" in - *".tar") - "${CAT}" "${TAR}" - ;; - *".tar.gz") - "${GZIP}" -fcd "${TAR}" - ;; - *".tar.bz2") - "${BZIP2}" -fcd "${TAR}" - ;; - *".tar.xz") - "${XZ}" -fcd "${TAR}" - ;; - *".tar.zst") - "${ZSTD}" -fcd "${TAR}" - ;; - *) - echo >&2 "Unsupported: ${TAR}" - exit 2 - ;; - esac - done -} | - { "${CONCATENATE}" --duplicate "${DUPLICATE}" || echo "$?" >"${FAIL}"; } | - { "${COMPRESS}" || echo "$?" >"${FAIL}"; } >"${OUTPUT}" - -if test ! -s "${FAIL}"; then - exit 0 -fi - -while IFS= read -r STATUS; do - exit "${STATUS}" -done <"${FAIL}" - -printf >&2 'Pipefail workaround failed\n' -exit 1 diff --git a/tar/concatenate/rule.bzl b/tar/concatenate/rule.bzl index 0fb60c99f15c85ee85e6ed09b00cd87474c110ee..ea73154503a137e8a269d04c4c6986b08343ef60 100644 --- a/tar/concatenate/rule.bzl +++ b/tar/concatenate/rule.bzl @@ -19,7 +19,6 @@ ATTRS = { ), "compress": attr.label( doc = "A compression binary.", - default = "//tar/compress:cat", executable = True, cfg = "exec", ), @@ -28,84 +27,46 @@ ATTRS = { default = "error", values = ["error", "skip", "continue"], ), - "_template": attr.label( - doc = "The concatenation binary.", - default = ":template", - allow_single_file = [".sh"], - ), - "_bash": attr.label( - doc = "The Bash interpreter to use.", - default = "@ape//ape:bash", + "_tool": attr.label( + doc = "The `tar` tool.", + default = "//tar/tool", executable = True, cfg = "exec", ), } def implementation(ctx): - concatenate = ctx.toolchains["//tar/toolchain/concatenate:type"] - cat = ctx.toolchains["@rules_coreutils//coreutils/toolchain/cat:type"] - gzip = ctx.toolchains["@rules_gzip//gzip/toolchain/gzip:type"] - bzip2 = ctx.toolchains["@rules_bzip2//bzip2/toolchain/bzip2:type"] - xz = ctx.toolchains["@rules_xz//xz/toolchain/xz:type"] - zstd = ctx.toolchains["@rules_zstd//zstd/toolchain/zstd:type"] - - rendered = ctx.actions.declare_file("{}.{}".format(ctx.label.name, ctx.file._template.extension)) - substitutions = ctx.actions.template_dict() - substitutions.add("{{concatenate}}", concatenate.executable.path) - substitutions.add("{{cat}}", cat.executable.path) - substitutions.add("{{gzip}}", gzip.executable.path) - substitutions.add("{{bzip2}}", bzip2.executable.path) - substitutions.add("{{xz}}", xz.executable.path) - substitutions.add("{{zstd}}", zstd.executable.path) - substitutions.add("{{compress}}", ctx.executable.compress.path) - ctx.actions.expand_template( - output = rendered, - template = ctx.file._template, - computed_substitutions = substitutions, - is_executable = True, - ) + archive = ctx.actions.declare_file(ctx.label.name) - output = ctx.actions.declare_file(ctx.label.name) + output = archive.path + compress = ctx.executable.compress + tools = () + if compress: + output = "{}:{}".format(output, compress.path) + tools = (compress,) srcs = depset(transitive = [s[DefaultInfo].files for s in ctx.attr.srcs]) args = ctx.actions.args() - args.add(rendered) - args.add("--output", output.path) + args.add("--output", output) args.add("--duplicate", ctx.attr.duplicate) args.add_all(srcs) ctx.actions.run( - outputs = [output], - inputs = depset([rendered], transitive = [srcs]), + outputs = [archive], + inputs = srcs, arguments = [args], - executable = ctx.executable._bash, - tools = [ - concatenate.run, - cat.run, - gzip.run, - bzip2.run, - xz.run, - zstd.run, - ctx.attr.compress.files_to_run, - ], + executable = ctx.executable._tool, + tools = tools, mnemonic = "TarConcatenate", progress_message = "Concatenating into %{output}", ) - return DefaultInfo(files = depset([output])) + return DefaultInfo(files = depset([archive])) tar_concatenate = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - toolchains = [ - "@rules_coreutils//coreutils/toolchain/cat:type", - "@rules_gzip//gzip/toolchain/gzip:type", - "@rules_bzip2//bzip2/toolchain/bzip2:type", - "@rules_xz//xz/toolchain/xz:type", - "@rules_zstd//zstd/toolchain/zstd:type", - "//tar/toolchain/concatenate:type", - ], ) concatenate = tar_concatenate diff --git a/tar/decompress/BUILD.bazel b/tar/decompress/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..4a3dbc84c1573719c8d1c6e9ca163c81207bc6c1 --- /dev/null +++ b/tar/decompress/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "decompress", + srcs = ["decompress.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/decompress", + visibility = ["//:__subpackages__"], + deps = [ + "@com_github_dsnet_compress//bzip2:go_default_library", + "@com_github_h2non_filetype//:go_default_library", + "@com_github_klauspost_compress//zstd:go_default_library", + "@com_github_ulikunitz_xz//:go_default_library", + ], +) diff --git a/tar/decompress/decompress.go b/tar/decompress/decompress.go new file mode 100644 index 0000000000000000000000000000000000000000..f6fa336b74cf6463e3898ed55ec11fc19862c8a8 --- /dev/null +++ b/tar/decompress/decompress.go @@ -0,0 +1,64 @@ +package decompress + +import ( + "bufio" + "compress/gzip" + "fmt" + "io" + + "github.com/dsnet/compress/bzip2" + "github.com/h2non/filetype" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +func NewMagic(r io.Reader) (*bufio.Reader, error) { + reader := bufio.NewReader(r) + + if buf, err := reader.Peek(3); err == nil && filetype.IsExtension(buf, "gz") { + decompressor, err := gzip.NewReader(reader) + if err != nil { + return nil, err + } + return bufio.NewReader(decompressor), err + } + + if buf, err := reader.Peek(3); err == nil && filetype.IsExtension(buf, "bz2") { + decompressor, err := bzip2.NewReader(reader, nil) + if err != nil { + return nil, err + } + return bufio.NewReader(decompressor), err + } + + if buf, err := reader.Peek(6); err == nil && filetype.IsExtension(buf, "xz") { + decompressor, err := xz.NewReader(reader) + if err != nil { + return nil, err + } + return bufio.NewReader(decompressor), err + } + + if buf, err := reader.Peek(4); err == nil && filetype.IsExtension(buf, "zst") { + decompressor, err := zstd.NewReader(reader) + if err != nil { + return nil, err + } + return bufio.NewReader(decompressor), err + } + + if buf, err := reader.Peek(262); err == nil && filetype.IsExtension(buf, "tar") { + return reader, err + } + + kind, err := filetype.MatchReader(reader) + if err != nil { + return nil, err + } + + if kind.MIME.Value == "" { + kind.MIME.Value = "unknown" + } + + return nil, fmt.Errorf("unknown file `%s`", kind.MIME.Value) +} diff --git a/tar/duplicate/BUILD.bazel b/tar/duplicate/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..8f1408e9447e303377904e3ecb27c9eeecc6c530 --- /dev/null +++ b/tar/duplicate/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "duplicate", + srcs = ["duplicate.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/duplicate", + visibility = ["//:__subpackages__"], +) diff --git a/tar/duplicate/duplicate.go b/tar/duplicate/duplicate.go new file mode 100644 index 0000000000000000000000000000000000000000..b3b6c561d14d1f6c91d88008478e8324d35586fb --- /dev/null +++ b/tar/duplicate/duplicate.go @@ -0,0 +1,43 @@ +package duplicate + +import ( + "fmt" +) + +type Flag int + +const ( + Error Flag = 0 + Skip Flag = 1 + Continue Flag = 2 +) + +func NewFlag() Flag { + return Error +} + +func (f *Flag) Set(value string) error { + switch value { + case "error": + *f = Error + case "skip": + *f = Skip + case "continue": + *f = Continue + default: + return fmt.Errorf("Invalid duplicate value: %s", value) + } + return nil +} + +func (f Flag) String() string { + switch f { + case Error: + return "error" + case Skip: + return "skip" + case Continue: + return "continue" + } + return "unknown" +} diff --git a/tar/filter/BUILD.bazel b/tar/filter/BUILD.bazel index 6a93af6c892e8fc77816e03f100310e40d65f00c..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/tar/filter/BUILD.bazel +++ b/tar/filter/BUILD.bazel @@ -1,24 +0,0 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") - -alias( - name = "template", - actual = select({ - "//conditions:default": ":posix.tmpl.sh", - }), - visibility = ["//:__subpackages__"], -) - -go_binary( - name = "filter", - embed = [":filter_lib"], - pure = "on", - visibility = ["//visibility:public"], -) - -go_library( - name = "filter_lib", - srcs = ["main.go"], - importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/filter", - visibility = ["//visibility:private"], - deps = ["@com_github_bmatcuk_doublestar_v4//:go_default_library"], -) diff --git a/tar/filter/main.go b/tar/filter/main.go deleted file mode 100644 index 103cc3bb63fc36c1e5cc19248089110b8554c451..0000000000000000000000000000000000000000 --- a/tar/filter/main.go +++ /dev/null @@ -1,273 +0,0 @@ -package main - -import ( - "archive/tar" - "bufio" - "flag" - "fmt" - "io" - "log" - "os" - "path" - "strings" - "syscall" - - "github.com/bmatcuk/doublestar/v4" -) - -type Input interface { - Read(b []byte) (int, error) - Peek(n int) ([]byte, error) - Discard(n int) (int, error) - Close() error -} - -type FileInput struct { - f *os.File - b *bufio.Reader -} - -func NewFileInput(f *os.File) FileInput { - return FileInput{f, bufio.NewReader(f)} -} - -func (i FileInput) Read(b []byte) (int, error) { - return i.b.Read(b) -} - -func (i FileInput) Peek(n int) ([]byte, error) { - return i.b.Peek(n) -} - -func (i FileInput) Discard(n int) (int, error) { - return i.b.Discard(n) -} - -func (i FileInput) Close() error { - return i.f.Close() -} - -type StdInput struct { - b *bufio.Reader -} - -func NewStdInput() StdInput { - return StdInput{bufio.NewReader(os.Stdin)} -} - -func (i StdInput) Read(b []byte) (int, error) { - return i.b.Read(b) -} - -func (i StdInput) Peek(n int) ([]byte, error) { - return i.b.Peek(n) -} - -func (i StdInput) Discard(n int) (int, error) { - return i.b.Discard(n) -} - -func (i StdInput) Close() error { - return nil -} - -func open(v string) (Input, error) { - if v == "-" { - return NewStdInput(), nil - } - - f, err := os.Open(v) - - switch e := err.(type) { - case *os.PathError: - if e.Err == syscall.ENOENT { - if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { - f, err = os.Open(path.Join(bwd, v)) - } - } - } - - if err != nil { - return nil, err - } - - return NewFileInput(f), nil -} - -type Output interface { - Write(b []byte) (int, error) - Close() error -} - -type FileOutput struct { - f *os.File -} - -func NewFileOutput(f *os.File) FileOutput { - return FileOutput{f} -} - -func (i FileOutput) Write(b []byte) (int, error) { - return i.f.Write(b) -} - -func (i FileOutput) Close() error { - return i.f.Close() -} - -type StdOutput struct{} - -func NewStdOutput() StdOutput { - return StdOutput{} -} - -func (i StdOutput) Write(b []byte) (int, error) { - return os.Stdout.Write(b) -} - -func (i StdOutput) Close() error { - return nil -} - -type OutputFlag struct { - o Output -} - -func NewStdOutputFlag() OutputFlag { - return OutputFlag{NewStdOutput()} -} - -func (f OutputFlag) Write(b []byte) (int, error) { - return f.o.Write(b) -} - -func (f OutputFlag) Close() error { - return f.o.Close() -} - -func (f *OutputFlag) Set(value string) error { - if value == "-" { - f.o = NewStdOutput() - } - - if !path.IsAbs(value) { - if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { - value = path.Join(bwd, value) - } - } - - x, err := os.OpenFile(value, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - f.o = NewFileOutput(x) - return nil -} - -func (f OutputFlag) String() string { - switch f.o.(type) { - case *StdOutput: - return "-" - case *FileOutput: - return "" - } - return "" -} - -type PatternsFlag struct { - patterns []string -} - -func NewPatternsFlag() PatternsFlag { - return PatternsFlag{} -} - -func (p *PatternsFlag) Set(value string) error { - if !doublestar.ValidatePattern(strings.TrimPrefix(value, "!")) { - return fmt.Errorf("Invalid pattern: %s", value) - } - p.patterns = append(p.patterns, value) - return nil -} - -func (p PatternsFlag) String() string { - return strings.Join(p.patterns, ",") -} - -func (p PatternsFlag) Match(name string) (bool, error) { - for _, pattern := range p.patterns { - after, found := strings.CutPrefix(pattern, "!") - match, err := doublestar.Match(after, name) - if err != nil { - return match, err - } - if match { - return !found, err - } - } - return false, nil -} - -type Flags struct { - Output OutputFlag - Patterns PatternsFlag -} - -func main() { - flags := Flags{NewStdOutputFlag(), NewPatternsFlag()} - flag.Var(&flags.Output, "output", "Output location for filtered `tar`.") - flag.Var(&flags.Patterns, "pattern", "A glob pattern to filter archive members.") - flag.Parse() - - w := tar.NewWriter(flags.Output) - defer w.Close() - - args := flag.Args() - switch len(args) { - case 0: - args = []string{"-"} - default: - log.Fatalf("Invalid number of archives: %s", args) - case 1: - } - - for _, archive := range args { - f, err := open(archive) - if err != nil { - log.Fatal(err) - } - defer f.Close() - - r := tar.NewReader(f) - for { - h, err := r.Next() - if err == io.EOF { - break - } - - match, err := flags.Patterns.Match(h.Name) - if err != nil { - log.Fatal(err) - } - if !match { - continue - } - - if err := w.WriteHeader(h); err != nil { - log.Fatal(err) - } - if _, err := io.Copy(w, r); err != nil { - log.Fatal(err) - } - } - - if err := f.Close(); err != nil { - log.Fatal(err) - } - } - - if err := w.Close(); err != nil { - log.Fatal(err) - } -} diff --git a/tar/filter/posix.tmpl.sh b/tar/filter/posix.tmpl.sh deleted file mode 100644 index d19df896636e3c079e0fb47039323e36ff99c4cb..0000000000000000000000000000000000000000 --- a/tar/filter/posix.tmpl.sh +++ /dev/null @@ -1,70 +0,0 @@ -#! /usr/bin/env bash - -# Strict Shell -set -o errexit -set -o nounset -set -o pipefail - -# Bazel replacements -FILTER="{{filter}}" -CAT="{{cat}}" -GZIP="{{gzip}}" -BZIP2="{{bzip2}}" -XZ="{{xz}}" -ZSTD="{{zstd}}" -COMPRESS="{{compress}}" -readonly FILTER CAT GZIP BZIP2 XZ ZSTD COMPRESS - -# Parse the output -test "--output" = "${1}" -shift -OUTPUT="${1}" -readonly OUTPUT -shift - -# Parse the patterns -declare -a ARGS -while test $# != 0; do - case "${1}" in - "--pattern="*) - ARGS+=("${1}") - ;; - "--pattern") - ARGS+=("${1}" "${2}") - shift - ;; - *) - break - ;; - esac - shift -done - -# Concatenate -{ - for TAR in "${@}"; do - case "${TAR}" in - *".tar") - "${CAT}" "${TAR}" - ;; - *".tar.gz") - "${GZIP}" -fcd "${TAR}" - ;; - *".tar.bz2") - "${BZIP2}" -fcd "${TAR}" - ;; - *".tar.xz") - "${XZ}" -fcd "${TAR}" - ;; - *".tar.zst") - "${ZSTD}" -fcd "${TAR}" - ;; - *) - echo >&2 "Unsupported: ${TAR}" - exit 2 - ;; - esac - done -} | - "${FILTER}" "${ARGS[@]}" | - "${COMPRESS}" >"${OUTPUT}" diff --git a/tar/filter/rule.bzl b/tar/filter/rule.bzl index 6e7c3cd8f19e356d3f7655aed954aa4d369496a2..17792b39df1b3cbcf1e10fb3861973de2842593a 100644 --- a/tar/filter/rule.bzl +++ b/tar/filter/rule.bzl @@ -3,6 +3,7 @@ visibility("//tar/...") DOC = """Filters an archive members. ```py +# Remove all `*.txt` files tar_filter( name = "filtered.tar", src = ":archive.tar", @@ -13,11 +14,12 @@ tar_filter( Filters can be negated: ```py +# Keep all `.md` files but remove everything else tar_filter( name = "filtered.tar.zst", src = ":archive.tar.xz", compress = "@rules_tar//tar/compress:zstd", - patterns = ["!**/*.md", "**/*"], + patterns = ["**/*.md", "!**/*"], ) ``` """ @@ -30,93 +32,54 @@ ATTRS = { ), "compress": attr.label( doc = "A compression binary.", - default = "//tar/compress:cat", executable = True, cfg = "exec", ), "patterns": attr.string_list( - doc = "Glob patterns to use as filters for archive member names.", + doc = "Glob patterns of archive member names to remove. Can be negated with `!`.", mandatory = True, allow_empty = False, ), - "_template": attr.label( - doc = "The executable script template.", - default = ":template", - allow_single_file = [".sh"], - ), - "_bash": attr.label( - doc = "The Bash interpreter to use.", - default = "@ape//ape:bash", + "_tool": attr.label( + doc = "The `tar` tool.", + default = "//tar/tool", executable = True, cfg = "exec", ), } def implementation(ctx): - filter = ctx.toolchains["//tar/toolchain/filter:type"] - cat = ctx.toolchains["@rules_coreutils//coreutils/toolchain/cat:type"] - gzip = ctx.toolchains["@rules_gzip//gzip/toolchain/gzip:type"] - bzip2 = ctx.toolchains["@rules_bzip2//bzip2/toolchain/bzip2:type"] - xz = ctx.toolchains["@rules_xz//xz/toolchain/xz:type"] - zstd = ctx.toolchains["@rules_zstd//zstd/toolchain/zstd:type"] - - rendered = ctx.actions.declare_file("{}.{}".format(ctx.label.name, ctx.file._template.extension)) - substitutions = ctx.actions.template_dict() - substitutions.add("{{filter}}", filter.executable.path) - substitutions.add("{{cat}}", cat.executable.path) - substitutions.add("{{gzip}}", gzip.executable.path) - substitutions.add("{{bzip2}}", bzip2.executable.path) - substitutions.add("{{xz}}", xz.executable.path) - substitutions.add("{{zstd}}", zstd.executable.path) - substitutions.add("{{compress}}", ctx.executable.compress.path) - ctx.actions.expand_template( - output = rendered, - template = ctx.file._template, - computed_substitutions = substitutions, - is_executable = True, - ) + archive = ctx.actions.declare_file(ctx.label.name) - output = ctx.actions.declare_file(ctx.label.name) + output = archive.path + compress = ctx.executable.compress + tools = () + if compress: + output = "{}:{}".format(output, compress.path) + tools = (compress,) - srcs = depset(transitive = [ctx.attr.src[DefaultInfo].files]) + srcs = depset([ctx.file.src]) args = ctx.actions.args() - args.add(rendered) - args.add("--output", output.path) - args.add_all(ctx.attr.patterns, format_each = "--pattern=%s") + args.add("--output", output) + args.add_all(ctx.attr.patterns, format_each = "--filter=%s") args.add_all(srcs) ctx.actions.run( - outputs = [output], - inputs = depset([rendered], transitive = [srcs]), + outputs = [archive], + inputs = srcs, arguments = [args], - executable = ctx.executable._bash, - tools = [ - filter.run, - cat.run, - gzip.run, - bzip2.run, - xz.run, - zstd.run, - ctx.attr.compress.files_to_run, - ], + executable = ctx.executable._tool, + tools = tools, mnemonic = "TarFilter", progress_message = "Filtering %{input} to %{output}", ) - return DefaultInfo(files = depset([output])) + return DefaultInfo(files = depset([archive])) tar_filter = rule( doc = DOC, attrs = ATTRS, implementation = implementation, - toolchains = [ - "@rules_coreutils//coreutils/toolchain/cat:type", - "@rules_gzip//gzip/toolchain/gzip:type", - "@rules_bzip2//bzip2/toolchain/bzip2:type", - "@rules_xz//xz/toolchain/xz:type", - "@rules_zstd//zstd/toolchain/zstd:type", - "//tar/toolchain/filter:type", - ], ) filter = tar_filter diff --git a/tar/filters/BUILD.bazel b/tar/filters/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..0941f11ee516f4c0cb8c12c533422d50eb040409 --- /dev/null +++ b/tar/filters/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "filters", + srcs = ["filters.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/filters", + visibility = ["//:__subpackages__"], + deps = ["@com_github_bmatcuk_doublestar_v4//:go_default_library"], +) diff --git a/tar/filters/filters.go b/tar/filters/filters.go new file mode 100644 index 0000000000000000000000000000000000000000..87a40a31bdfaaeb3da1dcc4e778615e0348eace7 --- /dev/null +++ b/tar/filters/filters.go @@ -0,0 +1,42 @@ +package filters + +import ( + "fmt" + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +type Flag struct { + filters []string +} + +func NewFlag() Flag { + return Flag{} +} + +func (f *Flag) Set(value string) error { + if !doublestar.ValidatePattern(strings.TrimPrefix(value, "!")) { + return fmt.Errorf("Invalid filter: %s", value) + } + f.filters = append(f.filters, value) + return nil +} + +func (f Flag) String() string { + return strings.Join(f.filters, ",") +} + +func (f Flag) Match(name string) (bool, error) { + for _, filter := range f.filters { + after, found := strings.CutPrefix(filter, "!") + match, err := doublestar.Match(after, name) + if err != nil { + return match, err + } + if match { + return found, err + } + } + return true, nil +} diff --git a/tar/input/BUILD.bazel b/tar/input/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..3eea9c2d0f455bf7a65124dd6b38064cb697e06d --- /dev/null +++ b/tar/input/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "input", + srcs = ["input.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/input", + visibility = ["//:__subpackages__"], + deps = ["//tar/decompress"], +) diff --git a/tar/input/input.go b/tar/input/input.go new file mode 100644 index 0000000000000000000000000000000000000000..0fd82f8259e01413af48cf3b720343c1c01794ef --- /dev/null +++ b/tar/input/input.go @@ -0,0 +1,93 @@ +package input + +import ( + "bufio" + "io" + "os" + "path" + "syscall" + + "gitlab.arm.com/bazel/rules_tar/v1/tar/decompress" +) + +type readPeekDiscardCloser interface { + io.ReadCloser + Peek(n int) ([]byte, error) + Discard(n int) (int, error) +} + +type file struct { + f *os.File + b *bufio.Reader +} + +func newFile(f *os.File) (readPeekDiscardCloser, error) { + reader, err := decompress.NewMagic(f) + if err != nil { + return nil, err + } + return file{f, reader}, nil +} + +func (i file) Read(b []byte) (int, error) { + return i.b.Read(b) +} + +func (i file) Peek(n int) ([]byte, error) { + return i.b.Peek(n) +} + +func (i file) Discard(n int) (int, error) { + return i.b.Discard(n) +} + +func (i file) Close() error { + return i.f.Close() +} + +type stdin struct { + b *bufio.Reader +} + +func newStdin() readPeekDiscardCloser { + return stdin{bufio.NewReader(os.Stdin)} +} + +func (i stdin) Read(b []byte) (int, error) { + return i.b.Read(b) +} + +func (i stdin) Peek(n int) ([]byte, error) { + return i.b.Peek(n) +} + +func (i stdin) Discard(n int) (int, error) { + return i.b.Discard(n) +} + +func (i stdin) Close() error { + return nil +} + +func Open(v string) (readPeekDiscardCloser, error) { + if v == "-" { + return newStdin(), nil + } + + f, err := os.Open(v) + + switch e := err.(type) { + case *os.PathError: + if e.Err == syscall.ENOENT { + if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { + f, err = os.Open(path.Join(bwd, v)) + } + } + } + + if err != nil { + return nil, err + } + + return newFile(f) +} diff --git a/tar/output/BUILD.bazel b/tar/output/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..0d2c99872911f5c9bf2bd6f92df8cfa4d73e3f56 --- /dev/null +++ b/tar/output/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "output", + srcs = ["output.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/output", + visibility = ["//:__subpackages__"], + deps = ["//tar/compress"], +) diff --git a/tar/output/output.go b/tar/output/output.go new file mode 100644 index 0000000000000000000000000000000000000000..4974d178fc2bd12e8f593d8eb04808ce38a99800 --- /dev/null +++ b/tar/output/output.go @@ -0,0 +1,101 @@ +package output + +import ( + "os" + "path" + "strings" + + "gitlab.arm.com/bazel/rules_tar/v1/tar/compress" +) + +type output interface { + Write(b []byte) (int, error) + Close() error +} + +func newFile(value string) (output, error) { + if !path.IsAbs(value) { + if bwd, ok := os.LookupEnv("BUILD_WORKING_DIRECTORY"); ok { + value = path.Join(bwd, value) + } + } + + f, err := os.OpenFile(value, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + + return f, nil +} + +type stdout struct{} + +func newStdout() output { + return stdout{} +} + +func (i stdout) Write(b []byte) (int, error) { + return os.Stdout.Write(b) +} + +func (i stdout) Close() error { + return nil +} + +func newOutput(value string) (output, error) { + if value == "-" { + return newStdout(), nil + } + + o, err := newFile(value) + if err != nil { + return nil, err + } + + return o, nil +} + +type Flag struct { + o output + v string +} + +func NewFlag() Flag { + return Flag{newStdout(), "-"} +} + +func (f Flag) Write(b []byte) (int, error) { + return f.o.Write(b) +} + +func (f Flag) Close() error { + return f.o.Close() +} + +func (f *Flag) Set(value string) error { + value, binary, found := strings.Cut(value, ":") + + o, err := newOutput(value) + if err != nil { + return err + } + + if found { + o, err = compress.NewSubprocess(o, binary) + } else { + o, err = compress.NewExtension(o, path.Ext(value)) + } + + if err != nil { + return err + } + + f.o = o + f.v = value + + return nil +} + +func (f Flag) String() string { + return f.v +} diff --git a/tar/skip/BUILD.bazel b/tar/skip/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..8c0baef5c0c6a3f6782833e0502a382e9ff847e5 --- /dev/null +++ b/tar/skip/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "skip", + srcs = ["skip.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/skip", + visibility = ["//:__subpackages__"], +) diff --git a/tar/skip/skip.go b/tar/skip/skip.go new file mode 100644 index 0000000000000000000000000000000000000000..82a012853eb3f4a4be042c86ba139373260f49dc --- /dev/null +++ b/tar/skip/skip.go @@ -0,0 +1,40 @@ +package skip + +import ( + "io" +) + +type readPeekDiscarder interface { + io.Reader + Peek(n int) ([]byte, error) + Discard(n int) (int, error) +} + +func Seek(f readPeekDiscarder) error { + for { + buf, err := f.Peek(512) + + if err == io.EOF { + return err + } + + if err != nil { + return err + } + + for i, b := range buf { + if b != 0 { + if _, err := f.Discard(i); err != nil { + return err + } + return nil + } + } + + if _, err := f.Discard(512); err != nil { + return err + } + } + + return nil +} diff --git a/tar/tool/BUILD.bazel b/tar/tool/BUILD.bazel new file mode 100644 index 0000000000000000000000000000000000000000..781613ed78935de10ceb0b89cec4baf719aae593 --- /dev/null +++ b/tar/tool/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "tool_lib", + srcs = ["main.go"], + importpath = "gitlab.arm.com/bazel/rules_tar/v1/tar/tool", + visibility = ["//visibility:private"], + deps = [ + "//tar/duplicate", + "//tar/filters", + "//tar/input", + "//tar/output", + "//tar/skip", + "@com_github_bmatcuk_doublestar_v4//:go_default_library", + ], +) + +go_binary( + name = "tool", + embed = [":tool_lib"], + visibility = ["//visibility:public"], +) diff --git a/tar/tool/main.go b/tar/tool/main.go new file mode 100644 index 0000000000000000000000000000000000000000..f3d227ec09acaa255ae7de5318c2651b3525d140 --- /dev/null +++ b/tar/tool/main.go @@ -0,0 +1,106 @@ +package main + +import ( + "archive/tar" + "flag" + "io" + "log" + + "gitlab.arm.com/bazel/rules_tar/v1/tar/duplicate" + "gitlab.arm.com/bazel/rules_tar/v1/tar/filters" + "gitlab.arm.com/bazel/rules_tar/v1/tar/input" + "gitlab.arm.com/bazel/rules_tar/v1/tar/output" + "gitlab.arm.com/bazel/rules_tar/v1/tar/skip" +) + +type Flags struct { + Output output.Flag + Duplicate duplicate.Flag + Filters filters.Flag +} + +func main() { + flags := Flags{output.NewFlag(), duplicate.NewFlag(), filters.NewFlag()} + flag.Var(&flags.Output, "output", "Output location for concatenated `tar`.") + flag.Var(&flags.Duplicate, "duplicate", "Operation when duplicate archive member names are encountered.") + flag.Var(&flags.Filters, "filter", "A glob pattern to filter archive members.") + flag.Parse() + defer flags.Output.Close() + + w := tar.NewWriter(flags.Output) + defer w.Close() + + args := flag.Args() + if len(args) == 0 { + args = []string{"-"} + } + + paths := make(map[string]struct{}) + for _, archive := range args { + f, err := input.Open(archive) + if err != nil { + log.Fatalf("failed to open archive `%s`: %s", archive, err) + } + defer f.Close() + + for { + r := tar.NewReader(f) + for { + h, err := r.Next() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("failed to advance to next archive `%s` member: %s", archive, err) + } + + match, err := flags.Filters.Match(h.Name) + if err != nil { + log.Fatalf("failed to match filter on `%s#%s`: %s", archive, h.Name, err) + } + if !match { + continue + } + + if _, exists := paths[h.Name]; exists { + switch flags.Duplicate { + case duplicate.Error: + log.Fatalf("name already exists in a previous archive: %s", h.Name) + case duplicate.Skip: + continue + case duplicate.Continue: + default: + log.Fatalf("unknown duplicate handling: %d", flags.Duplicate) + } + } + paths[h.Name] = struct{}{} + if err := w.WriteHeader(h); err != nil { + log.Fatalf("failed to write header for `%s#%s`: %s", archive, h.Name, err) + } + if _, err := io.Copy(w, r); err != nil { + log.Fatalf("failed to copy member for `%s#%s`: %s", archive, h.Name, err) + } + } + + err := skip.Seek(f) + if err == io.EOF { + break + } else if err == nil { + continue + } + log.Fatalf("failed to skip to next archive: %s", err) + } + + if err := f.Close(); err != nil { + log.Fatalf("failed to close archive `%s`: %s", archive, err) + } + } + + if err := w.Close(); err != nil { + log.Fatalf("failed to close writer: %s", err) + } + + if err := flags.Output.Close(); err != nil { + log.Fatalf("failed to close output stream: %s", err) + } +} diff --git a/tar/toolchain/concatenate/BUILD.bazel b/tar/toolchain/concatenate/BUILD.bazel deleted file mode 100644 index cf66e67f6d721ad2a249ab17ede672706bf95eee..0000000000000000000000000000000000000000 --- a/tar/toolchain/concatenate/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@toolchain_utils//toolchain/info:defs.bzl", "toolchain_info") -load("@toolchain_utils//toolchain/test:defs.bzl", "toolchain_test") - -toolchain_type( - name = "type", - visibility = ["//visibility:public"], -) - -toolchain_info( - name = "info", - target = "//tar/concatenate", -) - -toolchain( - name = "concatenate", - toolchain = ":info", - toolchain_type = ":type", -) - -alias( - name = "resolved", - actual = "@resolved-concatenate", - visibility = ["//visibility:public"], -) - -toolchain_test( - name = "test", - args = ["--help"], - stderr = "@toolchain_utils//toolchain/test:non-empty", - stdout = "@toolchain_utils//toolchain/test:empty", - toolchains = [":resolved"], -) diff --git a/tar/toolchain/filter/BUILD.bazel b/tar/toolchain/filter/BUILD.bazel deleted file mode 100644 index e09df30b9f2a1521765dd1f9815036e65e3ffc32..0000000000000000000000000000000000000000 --- a/tar/toolchain/filter/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@toolchain_utils//toolchain/info:defs.bzl", "toolchain_info") -load("@toolchain_utils//toolchain/test:defs.bzl", "toolchain_test") - -toolchain_type( - name = "type", - visibility = ["//visibility:public"], -) - -toolchain_info( - name = "info", - target = "//tar/filter", -) - -toolchain( - name = "filter", - toolchain = ":info", - toolchain_type = ":type", -) - -alias( - name = "resolved", - actual = "@resolved-filter", - visibility = ["//visibility:public"], -) - -toolchain_test( - name = "test", - args = ["--help"], - stderr = "@toolchain_utils//toolchain/test:non-empty", - stdout = "@toolchain_utils//toolchain/test:empty", - toolchains = [":resolved"], -)